Skip to content

Code-first video & image, rendered natively on WebGPU.

10.5 KB
Raw Blame
/**
 * The mix decodes only the stretch of a source each clip plays and skips
 * silent layers. Both must leave the mix sample-for-sample what a whole-file
 * decode gives, including for compressed sources decoded from mid-file.
 */
import { execFileSync } from "node:child_process";
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { decodeAudioSource } from "@framefields/compositions";
import { beforeAll, describe, expect, it } from "vitest";
import { Composition, encodeStereoWav, Layer } from "./index.js";
import { HeadlessMediaRenderer } from "./renderer/index.js";

const SR = 48000;
const FPS = 30;
const SOURCE_SEC = 40;
const TRIM_SEC = 31.5;

const hasFfmpeg = (() => {
	try {
		execFileSync("ffmpeg", ["-version"], { stdio: "ignore" });
		return true;
	} catch {
		return false;
	}
})();

/** Seeded noise, distinct per channel, quiet enough that no codec peak trips the mix's auto-gain. */
function noise(seconds: number, seed: number): Float32Array {
	const out = new Float32Array(Math.round(seconds * SR));
	let s = seed;
	for (let i = 0; i < out.length; i++) {
		s = (s * 1103515245 + 12345) >>> 0;
		out[i] = ((s / 2 ** 32) * 2 - 1) * 0.3;
	}
	return out;
}

let dir = "";
const sources: Record<string, string> = {};

beforeAll(async () => {
	dir = await fs.mkdtemp(path.join(os.tmpdir(), "audio-mix-ranges-"));
	sources.wav = path.join(dir, "source.wav");
	await fs.writeFile(
		sources.wav,
		encodeStereoWav([noise(SOURCE_SEC, 1), noise(SOURCE_SEC, 2)], {
			sampleRate: SR,
		}),
	);
	sources.short = path.join(dir, "short.wav");
	await fs.writeFile(
		sources.short,
		encodeStereoWav([noise(1, 1), noise(1, 2)], { sampleRate: SR }),
	);
	if (hasFfmpeg) {
		for (const [ext, codec] of [
			["mp3", "libmp3lame"],
			["m4a", "aac"],
		]) {
			sources[ext] = path.join(dir, `source.${ext}`);
			execFileSync("ffmpeg", [
				...["-v", "error", "-y", "-i", sources.wav],
				...["-c:a", codec, "-b:a", "128k", sources[ext]],
			]);
		}
	}
}, 120_000);

async function mix(comp: Composition) {
	return new HeadlessMediaRenderer().renderAudio(comp.toVirtualMedia(), {
		fps: FPS,
		sampleRate: SR,
	});
}

describe("audio mix source ranges", () => {
	for (const ext of ["wav", "mp3", "m4a"]) {
		it.skipIf(ext !== "wav" && !hasFfmpeg)(
			`plays a clip late in a long ${ext} source as the whole file decodes it`,
			async () => {
				const comp = new Composition({
					width: 64,
					height: 64,
					fps: FPS,
					durationMs: 3000,
				});
				comp.addAudio(
					Layer.audio(sources[ext], {
						trimStartSec: TRIM_SEC,
						startFrame: 15,
						durationFrames: 60,
					}),
				);
				const { channels } = await mix(comp);

				const whole = await decodeAudioSource(sources[ext]);
				expect(whole?.sampleRate).toBe(SR);
				const from = Math.round(TRIM_SEC * SR);
				const start = Math.round((15 / FPS) * SR);
				const length = Math.round((60 / FPS) * SR);
				for (let c = 0; c < 2; c++) {
					const out = channels[c];
					const src = whole?.channels[c] as Float32Array;
					let mismatches = 0;
					for (let i = 0; i < length; i++) {
						// The mix's soft limiter is a plain tanh at this level
						if (out[start + i] !== Math.fround(Math.tanh(src[from + i]))) {
							mismatches++;
						}
					}
					expect(mismatches).toBe(0);
					expect(out[start - 1]).toBe(0);
					expect(out[start + length]).toBe(0);
				}
			},
			60_000,
		);
	}

	it("leaves the mix unchanged by silent visual layers", async () => {
		const build = (withVisuals: boolean) => {
			const comp = new Composition({
				width: 64,
				height: 64,
				fps: FPS,
				durationMs: 2000,
			});
			if (withVisuals) {
				for (let i = 0; i < 4; i++) comp.add(Layer.text(`t${i}`));
			}
			comp.addAudio(
				Layer.audio(sources.wav, { trimStartSec: 3, durationFrames: 45 }),
			);
			comp.addAudio(
				Layer.audio(sources.wav, {
					trimStartSec: 20,
					startFrame: 30,
					volume: 0.5,
				}),
			);
			return comp;
		};
		const plain = await mix(build(false));
		const withVisuals = await mix(build(true));
		for (let c = 0; c < 2; c++) {
			expect(withVisuals.channels[c]).toEqual(plain.channels[c]);
		}
		// Both clips are heard where they play
		expect(Math.abs(plain.channels[0][SR / 2])).toBeGreaterThan(0);
		expect(Math.abs(plain.channels[0][Math.round(1.5 * SR)])).toBeGreaterThan(
			0,
		);
	}, 60_000);
});

describe("media time remapping", () => {
	it("plays a constant negative rate backward", async () => {
		const comp = new Composition({
			width: 64,
			height: 64,
			fps: FPS,
			durationFrames: 30,
		});
		comp.addAudio(
			Layer.audio(sources.short, {
				durationFrames: 30,
				timeRemap: { kind: "rate", sourceStartSec: 0.9, rate: -0.5 },
			}),
		);
		const { channels } = await mix(comp);
		const whole = await decodeAudioSource(sources.short);
		if (!whole) throw new Error("source audio did not decode");
		for (const i of [0, 12000, 24000, 36000]) {
			const sourceIndex = Math.round(0.9 * SR - i / 2);
			expect(channels[0][i]).toBeCloseTo(
				Math.tanh(whole.channels[0][sourceIndex]),
				5,
			);
		}
	});
	it("keeps reverse samples continuous across bounded decode chunks", async () => {
		const comp = new Composition({
			width: 64,
			height: 64,
			fps: FPS,
			durationFrames: 90,
		});
		comp.addAudio(
			Layer.audio(sources.short, {
				durationFrames: 90,
				timeRemap: { kind: "rate", sourceStartSec: 0.9, rate: -0.25 },
			}),
		);
		const { channels } = await mix(comp);
		const whole = await decodeAudioSource(sources.short);
		if (!whole) throw new Error("source audio did not decode");
		for (const i of [2 * SR - 4, 2 * SR - 1, 2 * SR, 2 * SR + 1, 2 * SR + 4]) {
			const position = 0.9 * SR - i * 0.25;
			const index = Math.floor(position);
			const expected =
				whole.channels[0][index] +
				(whole.channels[0][index + 1] - whole.channels[0][index]) *
					(position - index);
			expect(channels[0][i]).toBeCloseTo(Math.tanh(expected), 5);
		}
	});
	it("plays reverse audio through direction changes and smooth ramps", async () => {
		const comp = new Composition({
			width: 64,
			height: 64,
			fps: FPS,
			durationFrames: 90,
		});
		comp.addAudio(
			Layer.audio(sources.short, {
				durationFrames: 90,
				timeRemap: {
					kind: "keyframes",
					keyframes: [
						{ frame: 0, sourceSec: 0.1, interpolation: "smooth" },
						{ frame: 30, sourceSec: 0.8 },
						{ frame: 60, sourceSec: 0.2 },
						{ frame: 90, sourceSec: 0.2 },
					],
				},
			}),
		);
		const { channels } = await mix(comp);
		const whole = await decodeAudioSource(sources.short);
		if (!whole) throw new Error("source audio did not decode");
		for (const [output, source] of [
			[0, 0.1],
			[0.5, 0.45],
			[1, 0.8],
			[1.5, 0.5],
			[1.9, 0.26],
		]) {
			const sourceIndex = source * SR;
			const index = Math.floor(sourceIndex);
			const expected =
				whole.channels[0][index] +
				(whole.channels[0][index + 1] - whole.channels[0][index]) *
					(sourceIndex - index);
			expect(channels[0][Math.round(output * SR)]).toBeCloseTo(
				Math.tanh(expected),
				4,
			);
		}
		for (const sample of [2 * SR, 2.5 * SR, 3 * SR - 1])
			expect(channels[0][sample]).toBe(0);
	});
	it("resolves nested layer start times without offsetting source time", async () => {
		const comp = new Composition({
			width: 64,
			height: 64,
			fps: FPS,
			durationFrames: 90,
		});
		comp.add(
			Layer.box({
				startFrame: 15,
				durationFrames: 75,
				children: [
					Layer.audio(sources.short, {
						startFrame: 15,
						durationFrames: 30,
						timeRemap: { kind: "rate", sourceStartSec: 0, rate: 1 },
					}),
				],
			}),
		);
		const { channels } = await mix(comp);
		const whole = await decodeAudioSource(sources.short);
		if (!whole) throw new Error("source audio did not decode");
		expect(channels[0][SR - 1]).toBe(0);
		for (const i of [0, 1000, 24000])
			expect(channels[0][SR + i]).toBeCloseTo(
				Math.tanh(whole.channels[0][i]),
				5,
			);
	});
	it("keeps slow audio audible beyond the original source duration", async () => {
		const comp = new Composition({
			width: 64,
			height: 64,
			fps: FPS,
			durationFrames: 60,
		});
		comp.addAudio(
			Layer.audio(sources.short, {
				durationFrames: 60,
				timeRemap: { kind: "rate", sourceStartSec: 0, rate: 0.5 },
			}),
		);
		const { channels } = await mix(comp);
		const whole = await decodeAudioSource(sources.short);
		if (!whole) throw new Error("source audio did not decode");
		for (const i of [48000, 72000, 90000]) {
			expect(channels[0][i]).toBeCloseTo(
				Math.tanh(whole.channels[0][i / 2]),
				5,
			);
		}
	});
	it("does not truncate retimed audio when source metadata has a shorter duration", async () => {
		const comp = new Composition({
			width: 64,
			height: 64,
			fps: FPS,
			durationFrames: 60,
		});
		comp.addAudio(
			Layer.audio(sources.short, {
				durationFrames: 60,
				timeRemap: { kind: "rate", sourceStartSec: 0, rate: 0.5 },
			}),
		);
		const vm = comp.toVirtualMedia();
		vm.children[0].children[0].metadata.durationMs = 1000;
		const { channels } = await new HeadlessMediaRenderer().renderAudio(vm, {
			fps: FPS,
			sampleRate: SR,
		});
		const whole = await decodeAudioSource(sources.short);
		if (!whole) throw new Error("source audio did not decode");
		expect(channels[0][72000]).toBeCloseTo(
			Math.tanh(whole.channels[0][36000]),
			5,
		);
	});
	it("maps source samples at sub-frame rates and outputs silence for holds", async () => {
		const comp = new Composition({
			width: 64,
			height: 64,
			fps: FPS,
			durationFrames: 90,
		});
		comp.addAudio(
			Layer.audio(sources.wav, {
				startFrame: 15,
				durationFrames: 60,
				timeRemap: {
					kind: "keyframes",
					keyframes: [
						{ frame: 0, sourceSec: 2 },
						{ frame: 15, sourceSec: 2.25 },
						{ frame: 30, sourceSec: 2.25 },
						{ frame: 60, sourceSec: 3.25 },
					],
				},
			}),
		);
		const { channels } = await mix(comp);
		const whole = await decodeAudioSource(sources.wav);
		if (!whole) throw new Error("source audio did not decode");
		const base = SR / 2;
		for (const i of [0, 2, 200, 6000, 12000, 23998]) {
			const source = 2 * SR + i * 0.5;
			const index = Math.floor(source);
			const expected =
				whole.channels[0][index] +
				(whole.channels[0][index + 1] - whole.channels[0][index]) *
					(source - index);
			expect(channels[0][base + i]).toBeCloseTo(Math.tanh(expected), 5);
		}
		for (const i of [24000, 30000, 47999])
			expect(channels[0][base + i]).toBe(0);
		for (const i of [48000, 50000, 70000]) {
			const index = Math.round(2.25 * SR + (i - 48000));
			expect(channels[0][base + i]).toBeCloseTo(
				Math.tanh(whole.channels[0][index]),
				5,
			);
		}
	});
});