Code-first video & image, rendered natively on WebGPU.
Speed up audio mixing for long timelines
Claudefa5c2fd1624d
diff --git a/packages/compositions/src/utils/audio-extractor.test.ts b/packages/compositions/src/utils/audio-extractor.test.tsindex 7fea9406..8753c402 100644--- a/packages/compositions/src/utils/audio-extractor.test.ts+++ b/packages/compositions/src/utils/audio-extractor.test.ts@@ -1,8 +1,19 @@+import { execFileSync } from "node:child_process";import fs from "node:fs/promises";+import os from "node:os";import path from "node:path";import { describe, expect, it } from "vitest";import { decodeAudioSource } from "./audio-extractor.js";+const hasFfmpeg = (() => {+ try {+ execFileSync("ffmpeg", ["-version"], { stdio: "ignore" });+ return true;+ } catch {+ return false;+ }+})();+function encodeTestWav(samples: Float32Array,sampleRate = 48000,@@ -72,4 +83,28 @@ describe("Audio Extractor: decodeAudioSource", () => {expect(maxAbs).toBeGreaterThan(0.7);expect(maxAbs).toBeLessThanOrEqual(0.76);});++ it.skipIf(!hasFfmpeg)(+ "decodes a source longer than a few minutes in full",+ async () => {+ // 48 kHz stereo float is 384 KB/s: five minutes is past any small+ // output cap a decoder pipe might have.+ const file = path.join(+ await fs.mkdtemp(path.join(os.tmpdir(), "audio-extractor-")),+ "long.mp3",+ );+ execFileSync("ffmpeg", [+ ...["-v", "error", "-y", "-f", "lavfi"],+ ...["-i", "sine=frequency=440:sample_rate=48000:duration=300"],+ ...["-ac", "2", "-c:a", "libmp3lame", "-b:a", "64k", file],+ ]);++ const result = await decodeAudioSource(file);+ expect(result?.channels.length).toBe(2);+ expect(result?.channels[0]?.length ?? 0).toBeGreaterThanOrEqual(+ 299 * (result?.sampleRate ?? 0),+ );+ },+ 120_000,+ );});diff --git a/packages/compositions/src/utils/audio-extractor.ts b/packages/compositions/src/utils/audio-extractor.tsindex c604e8e6..fe14e0ad 100644--- a/packages/compositions/src/utils/audio-extractor.ts+++ b/packages/compositions/src/utils/audio-extractor.ts@@ -116,6 +116,40 @@ export interface AudioClip {export async function decodeAudioSource(url: string,): Promise<{ channels: Float32Array[]; sampleRate: number } | null> {+ const decoded = await decodeAudioRange(url);+ return decoded+ ? { channels: decoded.channels, sampleRate: decoded.sampleRate }+ : null;+}++/**+ * Seconds decoded ahead of a requested range: a compressed stream (MP3, AAC)+ * decoded from mid-file only matches a whole-file decode once the decoder has+ * warmed up on the packets before it.+ */+const DECODE_PREROLL_SEC = 0.5;++/**+ * A decoded stretch of a source. `channels[c][i]` is sample `startIndex + i`+ * of a whole-file decode, so callers index it as they would the whole file.+ * Samples in [validFromSec, validToSec) match a whole-file decode exactly.+ */+interface DecodedAudioRange {+ channels: Float32Array[];+ sampleRate: number;+ startIndex: number;+ validFromSec: number;+ validToSec: number;+}++/**+ * Decodes [range.startSec, range.endSec) of a source (the whole file without+ * a range), so a short clip from a long file decodes only what it plays.+ */+async function decodeAudioRange(+ url: string,+ range?: { startSec: number; endSec: number },+): Promise<DecodedAudioRange | null> {mediaLogger.debug(`[AudioExtractor] decodeAudioSource started for url: ${url}`,);@@ -161,7 +195,14 @@ export async function decodeAudioSource(for (let c = 0; c < audioBuffer.numberOfChannels; c++) {channels.push(new Float32Array(audioBuffer.getChannelData(c)));}- return { channels, sampleRate: audioBuffer.sampleRate };+ // decodeAudioData only decodes whole files.+ return {+ channels,+ sampleRate: audioBuffer.sampleRate,+ startIndex: 0,+ validFromSec: 0,+ validToSec: Number.POSITIVE_INFINITY,+ };} finally {void audioCtx.close();mediaLogger.debug(`[AudioExtractor] AudioContext closed for ${url}`);@@ -187,17 +228,22 @@ export async function decodeAudioSource(}const sink = new AudioSampleSink(track);- const generator = sink.samples(0);+ const decodeFromSec = range+ ? Math.max(0, range.startSec - DECODE_PREROLL_SEC)+ : 0;+ const generator = sink.samples(decodeFromSec, range?.endSec);const decodedPlanes: Float32Array[][] = [];let totalFrames = 0;let sampleRate = 48000;let numChannels = 1;+ let firstTimestamp: number | null = null;mediaLogger.debug(`[AudioExtractor] Starting mediabunny generator loop for ${url}...`,);for await (const sample of generator) {+ if (firstTimestamp === null) firstTimestamp = sample.timestamp;sampleRate = sample.sampleRate;numChannels = sample.numberOfChannels;const sampleFrames = sample.numberOfFrames;@@ -279,7 +325,29 @@ export async function decodeAudioSource(return merged;});- return { channels, sampleRate };+ // A whole-file decode starts at the sample samples(0) yields first;+ // a range is placed on that same index.+ let startIndex = 0;+ if (decodeFromSec > 0 && firstTimestamp !== null) {+ const origin = await sink.getSample(0);+ const originTimestamp =+ origin?.timestamp ?? (await track.getFirstTimestamp());+ origin?.close();+ startIndex = Math.round(+ (firstTimestamp - originTimestamp) * sampleRate,+ );+ }+ const endedEarly =+ !range || (startIndex + totalFrames) / sampleRate < range.endSec;++ return {+ channels,+ sampleRate,+ startIndex,+ validFromSec: decodeFromSec > 0 && range ? range.startSec : 0,+ validToSec:+ endedEarly || !range ? Number.POSITIVE_INFINITY : range.endSec,+ };} catch (err) {if (err instanceof Error &&@@ -289,10 +357,32 @@ export async function decodeAudioSource(}if (typeof process !== "undefined" && process.versions?.node) {try {- const { execSync } = await import("node:child_process");- const stdout = execSync(- `ffmpeg -v error -i "${url}" -f f32le -ar 48000 -ac 2 pipe:1`,- { maxBuffer: 100 * 1024 * 1024 },+ const { execFileSync } = await import("node:child_process");+ // ffmpeg's input -ss decodes from before the seek point and discards+ // up to it, so the range starts on an exact 48 kHz sample.+ const ffStartIndex = range+ ? Math.round(Math.max(0, range.startSec) * 48000)+ : 0;+ const ffStartSec = ffStartIndex / 48000;+ const stdout = execFileSync(+ "ffmpeg",+ [+ "-v",+ "error",+ ...(ffStartSec > 0 ? ["-ss", String(ffStartSec)] : []),+ "-i",+ url,+ ...(range ? ["-t", String(range.endSec - ffStartSec)] : []),+ "-f",+ "f32le",+ "-ar",+ "48000",+ "-ac",+ "2",+ "pipe:1",+ ],+ // 48 kHz stereo f32 is 384 KB/s: a fixed cap drops long sources.+ { maxBuffer: Number.POSITIVE_INFINITY },);const totalSamples = stdout.byteLength / 4;const numFrames = totalSamples / 2;@@ -307,7 +397,16 @@ export async function decodeAudioSource(left[i] = interleaved[i * 2];right[i] = interleaved[i * 2 + 1];}- return { channels: [left, right], sampleRate: 48000 };+ const ffEndedEarly =+ !range || ffStartSec + numFrames / 48000 < range.endSec;+ return {+ channels: [left, right],+ sampleRate: 48000,+ startIndex: ffStartIndex,+ validFromSec: ffStartSec,+ validToSec:+ ffEndedEarly || !range ? Number.POSITIVE_INFINITY : range.endSec,+ };} catch (ffmpegErr) {mediaLogger.warn({ ffmpegErr },@@ -342,6 +441,8 @@ export async function decodeAudioSource(/*** Resamples a PCM buffer using linear interpolation to target sample rate & speed.+ * `trimOffsetSamples` is a whole-file index; `inputStartIndex` is the+ * whole-file index of `inputChannel[0]` when it holds only a range.*/async function resampleAndStretchPCM(inputChannel: Float32Array,@@ -350,6 +451,7 @@ async function resampleAndStretchPCM(speed: number,outputLength: number,trimOffsetSamples: number,+ inputStartIndex = 0,): Promise<Float32Array> {const output = new Float32Array(outputLength);const resampleRatio = sourceSampleRate / targetSampleRate;@@ -361,7 +463,9 @@ async function resampleAndStretchPCM(if (i > 0 && i % YIELD_INTERVAL === 0) {await yieldToMain();}- const srcIndex = trimOffsetSamples + i * factor;+ // Subtracting the integer start after the sum keeps the position exact,+ // so a range resamples bit-for-bit as the whole file would.+ const srcIndex = trimOffsetSamples + i * factor - inputStartIndex;const indexFloor = Math.floor(srcIndex);const indexCeil = Math.ceil(srcIndex);@@ -387,6 +491,63 @@ async function resampleAndStretchPCM(return output;}+/** Seconds decoded past a clip's last source sample, for interpolation. */+const RANGE_MARGIN_SEC = 0.05;++/** Processors that only scale what is already there, so silence stays silence. */+const GAIN_ONLY_PROCESSORS = new Set(["Compositor", "CompositorLayer"]);++/** Adds `src` into `dest` starting at `destStart` (which may be negative). */+function addInto(dest: Float32Array, src: Float32Array, destStart: number) {+ const from = Math.max(0, -destStart);+ const to = Math.min(src.length, dest.length - destStart);+ for (let i = from; i < to; i++) {+ dest[destStart + i] += src[i];+ }+}++/**+ * A node's stereo mix, allocated on first write. A child that covers the+ * whole bus while it is still silent is taken over instead of copied (adding+ * into silence changes nothing), so a chain of single-child nodes passes one+ * buffer up instead of allocating and summing a full-length copy per level.+ */+class MixBus {+ channels: Float32Array[] | null = null;++ constructor(readonly length: number) {}++ /** The bus channels, allocated (silent) if nothing was written yet. */+ get(): Float32Array[] {+ this.channels ??= [+ new Float32Array(this.length),+ new Float32Array(this.length),+ ];+ return this.channels;+ }++ /** Whether `src` written at `start` could replace the bus outright. */+ canAdopt(src: Float32Array[], start: number): boolean {+ return (+ this.channels === null &&+ start === 0 &&+ src.every((ch) => ch.length === this.length)+ );+ }++ /** Mixes `src` into the bus starting at sample `start`. */+ add(src: Float32Array[], start: number): void {+ if (this.canAdopt(src, start) && src[0] !== src[1]) {+ this.channels = src;+ return;+ }+ const dest = this.get();+ for (let c = 0; c < 2; c++) {+ addInto(dest[c], src[c], start);+ }+ }+}+/*** Recursively parses the VirtualMediaData tree and mixes all active audio tracks* into a single unified stereo PCM stream (Float32Array channels).@@ -413,16 +574,59 @@ export async function mixAudioTracks(`[AudioExtractor] Composition total duration: ${durationMs}ms (${totalDurationSec}s). Total mixed samples to allocate: ${totalTargetSamples}`,);- // Output mixed buffer is stereo- const outputLeft = new Float32Array(totalTargetSamples);- const outputRight = new Float32Array(totalTargetSamples);- const mixedChannels = [outputLeft, outputRight];+ // Decoded ranges per source, so clips sharing a stretch of a file decode it once+ const decodedCache = new Map<string, DecodedAudioRange[]>();+ const undecodable = new Set<string>();++ async function getDecodedRange(+ url: string,+ startSec: number,+ endSec: number,+ ): Promise<DecodedAudioRange | null> {+ if (undecodable.has(url)) return null;+ let ranges = decodedCache.get(url);+ const hit = ranges?.find(+ (r) => r.validFromSec <= startSec && r.validToSec >= endSec,+ );+ if (hit) return hit;+ const decoded = await decodeAudioRange(url, { startSec, endSec });+ if (!decoded) {+ undecodable.add(url);+ return null;+ }+ if (!ranges) {+ ranges = [];+ decodedCache.set(url, ranges);+ }+ ranges.push(decoded);+ return decoded;+ }- // Map to cache decoded sources so we don't decode multiple clips of the same file twice- const decodedCache = new Map<- string,- { channels: Float32Array[]; sampleRate: number } | null- >();+ // A subtree without a source or a sound-producing processor mixes to+ // silence (text, images, shapes), so it is skipped instead of being+ // allocated, processed and summed at full length.+ const audibleCache = new WeakMap<VirtualMediaData, boolean>();+ function isAudible(node: VirtualMediaData): boolean {+ const cached = audibleCache.get(node);+ if (cached !== undefined) return cached;+ const op = node.operation;+ let audible = false;+ if (op?.op === "source") {+ const mediaType = getMediaType(node);+ audible = mediaType === "Audio" || mediaType === "Video";+ } else if (+ op?.op &&+ !GAIN_ONLY_PROCESSORS.has(op.op) &&+ audioRegistry.get(op.op)+ ) {+ audible = true;+ }+ if (!audible && node.children) {+ audible = node.children.some((child) => child && isAudible(child));+ }+ audibleCache.set(node, audible);+ return audible;+ }async function traverse(node: VirtualMediaData,@@ -432,7 +636,7 @@ export async function mixAudioTracks(inheritedSpeed: number,inheritedDurationFrames: number | null,): Promise<Float32Array[] | null> {- if (!node) return null;+ if (!node || !isAudible(node)) return null;const op = node.operation;if (!op) {const durationMs = getActiveMediaMetadata(node)?.durationMs ?? 0;@@ -449,9 +653,7 @@ export async function mixAudioTracks();if (localSamplesCount <= 0) return null;- const localLeft = new Float32Array(localSamplesCount);- const localRight = new Float32Array(localSamplesCount);- const localChannels = [localLeft, localRight];+ const bus = new MixBus(localSamplesCount);for (const child of node.children) {const childChannels = await traverse(@@ -469,19 +671,10 @@ export async function mixAudioTracks(const childStartSample = Math.round((childStartFrame / fps) * targetSampleRate,);- for (let c = 0; c < 2; c++) {- const dest = localChannels[c];- const src = childChannels[c];- const len = Math.min(src.length, dest.length - childStartSample);- for (let i = 0; i < len; i++) {- if (childStartSample + i >= 0) {- dest[childStartSample + i] += src[i];- }- }- }+ bus.add(childChannels, childStartSample);}}- return localChannels;+ return bus.channels;}const volume =@@ -525,9 +718,7 @@ export async function mixAudioTracks();if (localSamplesCount <= 0) return null;- const localLeft = new Float32Array(localSamplesCount);- const localRight = new Float32Array(localSamplesCount);- const localChannels = [localLeft, localRight];+ const bus = new MixBus(localSamplesCount);const processSegment = async (seg: { startSec: number; endSec?: number } | null,@@ -574,54 +765,76 @@ export async function mixAudioTracks((Number(params.trimStartSec) || 0) + totalSeekSec,);- // Load and decode from cache or file- let decoded = decodedCache.get(params.sourceUrl);- if (decoded === undefined) {- decoded = await decodeAudioSource(params.sourceUrl);- decodedCache.set(params.sourceUrl, decoded);- }+ const clipDurationSec = actualDurationFrames / fps;+ const clipSamplesCount = Math.min(+ localSamplesCount - segmentStartSample,+ Math.ceil(clipDurationSec * targetSampleRate),+ );++ // Decode (or reuse) only the stretch of the source this clip plays+ const decoded =+ clipSamplesCount > 0 && segmentStartSample < localSamplesCount+ ? await getDecodedRange(+ params.sourceUrl,+ effectiveTrimSec,+ effectiveTrimSec ++ (clipSamplesCount / targetSampleRate) * finalSpeed ++ RANGE_MARGIN_SEC,+ )+ : null;if (decoded) {- const clipDurationSec = actualDurationFrames / fps;- const clipSamplesCount = Math.min(- localSamplesCount - segmentStartSample,- Math.ceil(clipDurationSec * targetSampleRate),+ const trimOffsetSamples = Math.round(+ effectiveTrimSec * decoded.sampleRate,);- if (- clipSamplesCount > 0 &&- segmentStartSample < localSamplesCount- ) {- const trimOffsetSamples = Math.round(- effectiveTrimSec * decoded.sampleRate,- );-- // Resample left & right channels- const srcLeft = decoded.channels[0];- const srcRight =- decoded.channels.length > 1- ? decoded.channels[1]- : decoded.channels[0]; // fallback to mono-- const stretchedLeft = await resampleAndStretchPCM(- srcLeft,- decoded.sampleRate,- targetSampleRate,- finalSpeed,- clipSamplesCount,- trimOffsetSamples,- );-- const stretchedRight = await resampleAndStretchPCM(- srcRight,- decoded.sampleRate,- targetSampleRate,- finalSpeed,- clipSamplesCount,- trimOffsetSamples,- );+ // Resample left & right channels+ const stretchedLeft = await resampleAndStretchPCM(+ decoded.channels[0],+ decoded.sampleRate,+ targetSampleRate,+ finalSpeed,+ clipSamplesCount,+ trimOffsetSamples,+ decoded.startIndex,+ );+ // A mono source plays the same samples on both sides+ const stretchedRight =+ decoded.channels.length > 1+ ? await resampleAndStretchPCM(+ decoded.channels[1],+ decoded.sampleRate,+ targetSampleRate,+ finalSpeed,+ clipSamplesCount,+ trimOffsetSamples,+ decoded.startIndex,+ )+ : stretchedLeft;++ const stretched = [stretchedLeft, stretchedRight];+ if (bus.canAdopt(stretched, segmentStartSample)) {+ // The clip fills the silent bus: scale it in place and keep it+ if (volume !== 1) {+ for (let i = 0; i < clipSamplesCount; i++) {+ stretchedLeft[i] *= volume;+ }+ if (stretchedRight !== stretchedLeft) {+ for (let i = 0; i < clipSamplesCount; i++) {+ stretchedRight[i] *= volume;+ }+ }+ }+ bus.channels = [+ stretchedLeft,+ stretchedRight === stretchedLeft+ ? stretchedLeft.slice()+ : stretchedRight,+ ];+ } else {// Add to local mix with volume scaling+ const [localLeft, localRight] = bus.get();for (let i = 0; i < clipSamplesCount; i++) {const outIndex = segmentStartSample + i;localLeft[outIndex] += stretchedLeft[i] * volume;@@ -651,17 +864,7 @@ export async function mixAudioTracks(const childStartSample = Math.round((childStartFrame / fps) * targetSampleRate,);- const destStartSample = segmentStartSample + childStartSample;- for (let c = 0; c < 2; c++) {- const dest = localChannels[c];- const src = childChannels[c];- const len = Math.min(src.length, dest.length - destStartSample);- for (let i = 0; i < len; i++) {- if (destStartSample + i >= 0) {- dest[destStartSample + i] += src[i];- }- }- }+ bus.add(childChannels, segmentStartSample + childStartSample);}}}@@ -681,12 +884,14 @@ export async function mixAudioTracks(segmentStartSample < localSamplesCount &&segmentStartSample >= 0) {- const slicedChannels = localChannels.map((ch) =>- ch.subarray(- segmentStartSample,- segmentStartSample + clipSamplesCount,- ),- );+ const slicedChannels = bus+ .get()+ .map((ch) =>+ ch.subarray(+ segmentStartSample,+ segmentStartSample + clipSamplesCount,+ ),+ );// The layer's own clock: its in-point moves the source, not its// animation, and a CompositorLayer's keyframes count from its// own first frame (as its picture's do), not the composition's.@@ -729,12 +934,23 @@ export async function mixAudioTracks(await processSegment(null, 0);}- return localChannels;+ return bus.channels;}const normalized = normalizeTimeline(virtualMedia, fps);const rootChannels = await traverse(normalized, 0, 0, 1, 1, null);- if (rootChannels) {++ // Output mixed buffer is stereo; a root mix of the same length is used as is+ const mixedChannels =+ rootChannels?.every((ch) => ch.length === totalTargetSamples) &&+ rootChannels[0] !== rootChannels[1]+ ? rootChannels+ : [+ new Float32Array(totalTargetSamples),+ new Float32Array(totalTargetSamples),+ ];+ const [outputLeft, outputRight] = mixedChannels;+ if (rootChannels && mixedChannels !== rootChannels) {for (let c = 0; c < 2; c++) {const dest = mixedChannels[c];const src = rootChannels[c];diff --git a/packages/core/src/signals/index.ts b/packages/core/src/signals/index.tsindex dce89e78..6494153a 100644--- a/packages/core/src/signals/index.ts+++ b/packages/core/src/signals/index.ts@@ -316,6 +316,14 @@ export function signal<T>(initialValue: T): WritableSignal<T> {return new StateSignalImpl<T>(initialValue);}+/**+ * Whether `val` is a plain value signal (from `signal()`): it reads the same+ * whatever frame context it is given, so one read stands for every frame.+ */+export function isValueSignal(val: unknown): val is WritableSignal<unknown> {+ return val instanceof StateSignalImpl;+}+export function computed<T>(fn: () => T): Signal<T> {return new ComputedImpl<T>(fn);}diff --git a/packages/core/src/signals/signals.test.ts b/packages/core/src/signals/signals.test.tsindex 15d796d4..cd2d3472 100644--- a/packages/core/src/signals/signals.test.ts+++ b/packages/core/src/signals/signals.test.ts@@ -4,6 +4,7 @@ import {frameArraySignal,frameSignal,isSignal,+ isValueSignal,Signal,signal,updateClockSignals,@@ -174,4 +175,12 @@ describe("Signals Reactive Core", () => {});expect(valAtHalfSec).toBe(20);});++ it("tells plain value signals from frame-dependent ones", () => {+ expect(isValueSignal(signal(0.5))).toBe(true);+ expect(isValueSignal(computed(() => 1))).toBe(false);+ expect(isValueSignal(Signal.programmatic((ctx) => ctx.time))).toBe(false);+ expect(isValueSignal(frameArraySignal([1, 2, 3]))).toBe(false);+ expect(isValueSignal(0.5)).toBe(false);+ });});diff --git a/packages/framefields/src/audio-mix-ranges.test.ts b/packages/framefields/src/audio-mix-ranges.test.tsnew file mode 100644index 00000000..985d17e3--- /dev/null+++ b/packages/framefields/src/audio-mix-ranges.test.ts@@ -0,0 +1,151 @@+/**+ * The mix decodes only the stretch of a source each clip plays and skips+ * silent layers. Both must leave the mix sample-for-sample what a whole-file+ * decode gives, including for compressed sources decoded from mid-file.+ */+import { execFileSync } from "node:child_process";+import fs from "node:fs/promises";+import os from "node:os";+import path from "node:path";+import { decodeAudioSource } from "@framefields/compositions";+import { beforeAll, describe, expect, it } from "vitest";+import { Composition, encodeStereoWav, Layer } from "./index.js";+import { HeadlessMediaRenderer } from "./renderer/index.js";++const SR = 48000;+const FPS = 30;+const SOURCE_SEC = 40;+const TRIM_SEC = 31.5;++const hasFfmpeg = (() => {+ try {+ execFileSync("ffmpeg", ["-version"], { stdio: "ignore" });+ return true;+ } catch {+ return false;+ }+})();++/** Seeded noise, distinct per channel, quiet enough that no codec peak trips the mix's auto-gain. */+function noise(seconds: number, seed: number): Float32Array {+ const out = new Float32Array(Math.round(seconds * SR));+ let s = seed;+ for (let i = 0; i < out.length; i++) {+ s = (s * 1103515245 + 12345) >>> 0;+ out[i] = ((s / 2 ** 32) * 2 - 1) * 0.3;+ }+ return out;+}++let dir = "";+const sources: Record<string, string> = {};++beforeAll(async () => {+ dir = await fs.mkdtemp(path.join(os.tmpdir(), "audio-mix-ranges-"));+ sources.wav = path.join(dir, "source.wav");+ await fs.writeFile(+ sources.wav,+ encodeStereoWav([noise(SOURCE_SEC, 1), noise(SOURCE_SEC, 2)], {+ sampleRate: SR,+ }),+ );+ if (hasFfmpeg) {+ for (const [ext, codec] of [+ ["mp3", "libmp3lame"],+ ["m4a", "aac"],+ ]) {+ sources[ext] = path.join(dir, `source.${ext}`);+ execFileSync("ffmpeg", [+ ...["-v", "error", "-y", "-i", sources.wav],+ ...["-c:a", codec, "-b:a", "128k", sources[ext]],+ ]);+ }+ }+}, 120_000);++async function mix(comp: Composition) {+ return new HeadlessMediaRenderer().renderAudio(comp.toVirtualMedia(), {+ fps: FPS,+ sampleRate: SR,+ });+}++describe("audio mix source ranges", () => {+ for (const ext of ["wav", "mp3", "m4a"]) {+ it.skipIf(ext !== "wav" && !hasFfmpeg)(+ `plays a clip late in a long ${ext} source as the whole file decodes it`,+ async () => {+ const comp = new Composition({+ width: 64,+ height: 64,+ fps: FPS,+ durationMs: 3000,+ });+ comp.addAudio(+ Layer.audio(sources[ext], {+ trimStartSec: TRIM_SEC,+ startFrame: 15,+ durationFrames: 60,+ }),+ );+ const { channels } = await mix(comp);++ const whole = await decodeAudioSource(sources[ext]);+ expect(whole?.sampleRate).toBe(SR);+ const from = Math.round(TRIM_SEC * SR);+ const start = Math.round((15 / FPS) * SR);+ const length = Math.round((60 / FPS) * SR);+ for (let c = 0; c < 2; c++) {+ const out = channels[c];+ const src = whole?.channels[c] as Float32Array;+ let mismatches = 0;+ for (let i = 0; i < length; i++) {+ // The mix's soft limiter is a plain tanh at this level+ if (out[start + i] !== Math.fround(Math.tanh(src[from + i]))) {+ mismatches++;+ }+ }+ expect(mismatches).toBe(0);+ expect(out[start - 1]).toBe(0);+ expect(out[start + length]).toBe(0);+ }+ },+ 60_000,+ );+ }++ it("leaves the mix unchanged by silent visual layers", async () => {+ const build = (withVisuals: boolean) => {+ const comp = new Composition({+ width: 64,+ height: 64,+ fps: FPS,+ durationMs: 2000,+ });+ if (withVisuals) {+ for (let i = 0; i < 4; i++) comp.add(Layer.text(`t${i}`));+ }+ comp.addAudio(+ Layer.audio(sources.wav, { trimStartSec: 3, durationFrames: 45 }),+ );+ comp.addAudio(+ Layer.audio(sources.wav, {+ trimStartSec: 20,+ startFrame: 30,+ volume: 0.5,+ }),+ );+ return comp;+ };+ const plain = await mix(build(false));+ const withVisuals = await mix(build(true));+ for (let c = 0; c < 2; c++) {+ expect(withVisuals.channels[c]).toEqual(plain.channels[c]);+ }+ // Both clips are heard where they play+ expect(Math.abs(plain.channels[0][SR / 2])).toBeGreaterThan(0);+ expect(Math.abs(plain.channels[0][Math.round(1.5 * SR)])).toBeGreaterThan(+ 0,+ );+ }, 60_000);+});diff --git a/packages/webgpu-renderers/src/audio/webgpu-audio-processor.ts b/packages/webgpu-renderers/src/audio/webgpu-audio-processor.tsindex 594386f2..24279329 100644--- a/packages/webgpu-renderers/src/audio/webgpu-audio-processor.ts+++ b/packages/webgpu-renderers/src/audio/webgpu-audio-processor.ts@@ -1,4 +1,8 @@-import { isSignal, type VirtualMediaData } from "@framefields/core";+import {+ isSignal,+ isValueSignal,+ type VirtualMediaData,+} from "@framefields/core";import { shaderStore } from "../shader-store.js";import { signalRegistry } from "../signals/signal-registry.js";@@ -284,22 +288,27 @@ export class WebGPUAudioProcessor {const signalData = new Float32Array(numSamples);const durationSec =durationSeconds > 0 ? durationSeconds : numSamples / _sampleRate;- for (let s = 0; s < numSamples; s++) {- const t = elapsedSeconds + s / _sampleRate;- const f = t * fps;- const progress = durationSec > 0 ? t / durationSec : 0;- signalData[s] =- Number(- directVal.get({- frame: f,- fps,- time: t,- duration: durationSec * fps,- durationMs: durationSec * 1000,- progress,- deltaTime: 1 / _sampleRate,- }),- ) || 0;+ if (isValueSignal(directVal)) {+ // A plain value reads the same at every sample: read it once+ signalData.fill(Number(directVal.get()) || 0);+ } else {+ for (let s = 0; s < numSamples; s++) {+ const t = elapsedSeconds + s / _sampleRate;+ const f = t * fps;+ const progress = durationSec > 0 ? t / durationSec : 0;+ signalData[s] =+ Number(+ directVal.get({+ frame: f,+ fps,+ time: t,+ duration: durationSec * fps,+ durationMs: durationSec * 1000,+ progress,+ deltaTime: 1 / _sampleRate,+ }),+ ) || 0;+ }}const buf = device.createBuffer({size: Math.max(16, Math.ceil(signalData.byteLength / 16) * 16),@@ -400,6 +409,35 @@ export class WebGPUAudioProcessor {// 3. Process in chunks to prevent GPU driver timeout (TDR) and browser freezingconst CHUNK_SIZE = 65536;+ // Chunks submitted before the oldest is read back. The queue runs them in+ // order (the state buffer carries across), so waiting on each readback+ // before submitting the next only adds a GPU round trip per chunk.+ const MAX_CHUNKS_IN_FLIGHT = 8;+ const pending: {+ stagingBuffer: GPUBuffer;+ offset: number;+ chunkSamples: number;+ }[] = [];++ const readBackOldest = async () => {+ const chunk = pending.shift();+ if (!chunk) return;+ const { stagingBuffer, offset, chunkSamples } = chunk;+ await stagingBuffer.mapAsync(GPUMapMode.READ);+ const mappedRange = new Float32Array(stagingBuffer.getMappedRange());++ for (let c = 0; c < numChannels; c++) {+ channels[c].set(+ mappedRange.subarray(c * chunkSamples, (c + 1) * chunkSamples),+ offset,+ );+ }++ stagingBuffer.unmap();+ stagingBuffer.destroy();++ await yieldToMain();+ };try {for (let offset = 0; offset < numSamples; offset += CHUNK_SIZE) {@@ -572,22 +610,19 @@ export class WebGPUAudioProcessor {device.queue.submit([commandEncoder.finish()]);- // Map & Read back- await stagingBuffer.mapAsync(GPUMapMode.READ);- const mappedRange = new Float32Array(stagingBuffer.getMappedRange());-- for (let c = 0; c < numChannels; c++) {- channels[c].set(- mappedRange.subarray(c * chunkSamples, (c + 1) * chunkSamples),- offset,- );+ // Read back later; the next chunk's writes queue behind this one+ pending.push({ stagingBuffer, offset, chunkSamples });+ if (pending.length >= MAX_CHUNKS_IN_FLIGHT) {+ await readBackOldest();}-- stagingBuffer.unmap();-- await yieldToMain();+ }+ while (pending.length > 0) {+ await readBackOldest();}} finally {+ for (const { stagingBuffer } of pending) {+ stagingBuffer.destroy();+ }for (const b of createdBuffers) {b.destroy();}