Skip to content

Write HTML. Render video. Built for agents.

fix(engine): mix any number of audio clips by summing them in stages (#6120)

Miguel Ángel3cdae3804e5f
diff --git a/packages/engine/src/services/audioMixer.manyTracks.test.ts b/packages/engine/src/services/audioMixer.manyTracks.test.ts
new file mode 100644
index 0000000000..3c91a5d14a
--- /dev/null
+++ b/packages/engine/src/services/audioMixer.manyTracks.test.ts
@@ -0,0 +1,135 @@
+import { spawnSync } from "node:child_process";
+import { copyFileSync, mkdtempSync, readdirSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+import { getFfmpegBinary } from "../utils/ffmpegBinaries.js";
+import { MAX_MIX_INPUTS, MIXED_AUDIO_FILENAME, processCompositionAudio } from "./audioMixer.js";
+
+const HAS_FFMPEG = spawnSync(getFfmpegBinary(), ["-version"], { encoding: "utf-8" }).status === 0;
+const tempDirs: string[] = [];
+
+function meanVolumeDb(path: string): number {
+ const result = spawnSync(
+ getFfmpegBinary(),
+ ["-nostdin", "-hide_banner", "-i", path, "-af", "volumedetect", "-f", "null", "-"],
+ { encoding: "utf-8" },
+ );
+ const match = result.stderr.match(/mean_volume:\s*(-?[\d.]+) dB/);
+ if (result.status !== 0 || !match?.[1]) throw new Error(`Could not measure: ${result.stderr}`);
+ return Number(match[1]);
+}
+
+function peakDb(path: string): number {
+ const result = spawnSync(
+ getFfmpegBinary(),
+ ["-nostdin", "-hide_banner", "-i", path, "-af", "astats", "-f", "null", "-"],
+ { encoding: "utf-8" },
+ );
+ const match = result.stderr.match(/Peak level dB:\s*(-?[\d.]+)/);
+ if (result.status !== 0 || !match?.[1]) throw new Error(`Could not measure: ${result.stderr}`);
+ return Number(match[1]);
+}
+
+function writeTone(path: string, seconds: number, gain: number): void {
+ const result = spawnSync(
+ getFfmpegBinary(),
+ [
+ "-nostdin",
+ "-v",
+ "error",
+ "-f",
+ "lavfi",
+ "-i",
+ `sine=frequency=440:duration=${seconds}:sample_rate=48000`,
+ "-af",
+ `volume=${gain}`,
+ "-c:a",
+ "pcm_s16le",
+ path,
+ ],
+ { encoding: "utf-8" },
+ );
+ if (result.status !== 0) throw new Error(`Could not write tone: ${result.stderr}`);
+}
+
+const clip = (i: number, start: number, end: number, groupId?: string) => ({
+ id: `c${i}`,
+ src: `a${i}.wav`,
+ start,
+ end,
+ mediaStart: 0,
+ layer: i,
+ volume: 1,
+ type: "audio" as const,
+ ...(groupId ? { groupId } : {}),
+});
+
+describe.skipIf(!HAS_FFMPEG)("a mix with more clips than one ffmpeg graph can hold", () => {
+ afterEach(() => {
+ for (const dir of tempDirs.splice(0)) rmSync(dir, { recursive: true, force: true });
+ });
+
+ async function mix(
+ count: number,
+ groupId: string | undefined,
+ shape: { stackGain: number } | "sequence",
+ ) {
+ const projectDir = mkdtempSync(join(tmpdir(), "hf-many-"));
+ const workDir = mkdtempSync(join(tmpdir(), "hf-many-work-"));
+ tempDirs.push(projectDir, workDir);
+ const stacked = shape !== "sequence";
+ writeTone(join(projectDir, "a.wav"), stacked ? 2 : 0.2, stacked ? shape.stackGain : 1);
+ for (let i = 0; i < count; i++)
+ copyFileSync(join(projectDir, "a.wav"), join(projectDir, `a${i}.wav`));
+ const clips = Array.from({ length: count }, (_, i) =>
+ stacked ? clip(i, 0, 2, groupId) : clip(i, i * 0.05, (i + 1) * 0.05, groupId),
+ );
+ const out = join(projectDir, MIXED_AUDIO_FILENAME);
+ const sum = join(projectDir, "sum.wav");
+ const result = await processCompositionAudio(
+ clips,
+ projectDir,
+ workDir,
+ out,
+ stacked ? 2 : count * 0.05,
+ undefined,
+ { keepEncoderInputAt: sum },
+ );
+ return { result, out, sum };
+ }
+
+ it("mixes 200 clips in one group at the level of a short group", async () => {
+ const small = await mix(8, "sfx", "sequence");
+ const many = await mix(200, "sfx", "sequence");
+ expect(many.result.failures ?? []).toEqual([]);
+ expect(many.result.success).toBe(true);
+ expect(Math.abs(meanVolumeDb(many.out) - meanVolumeDb(small.out))).toBeLessThan(1);
+ }, 120_000);
+
+ it("mixes 200 ungrouped clips at the level of a short mix", async () => {
+ const small = await mix(8, undefined, "sequence");
+ const many = await mix(200, undefined, "sequence");
+ expect(many.result.failures ?? []).toEqual([]);
+ expect(many.result.success).toBe(true);
+ expect(Math.abs(meanVolumeDb(many.out) - meanVolumeDb(small.out))).toBeLessThan(1);
+ expect(readdirSync(dirname(many.out)).filter((f) => f.startsWith(".mix-batches-"))).toEqual([]);
+ }, 120_000);
+
+ it.each([
+ ["ungrouped", undefined],
+ ["grouped", "sfx"],
+ ])(
+ "sums %s clips past the slice size at the gain the flat mix gives",
+ async (_name, groupId) => {
+ // Stacked coherent clips add, so 120 at gain g and 60 at gain 2g have the same sum.
+ // 60 is one flat graph; 120 is staged. The float sums must peak alike.
+ const flat = await mix(60, groupId, { stackGain: 0.1 });
+ const staged = await mix(MAX_MIX_INPUTS + 24, groupId, { stackGain: 0.05 });
+ expect(flat.result.success).toBe(true);
+ expect(staged.result.success).toBe(true);
+ expect(Math.abs(peakDb(staged.sum) - peakDb(flat.sum))).toBeLessThan(0.1);
+ },
+ 120_000,
+ );
+});
diff --git a/packages/engine/src/services/audioMixer.test.ts b/packages/engine/src/services/audioMixer.test.ts
index de45b80fcd..e2bcdfbbc8 100644
--- a/packages/engine/src/services/audioMixer.test.ts
+++ b/packages/engine/src/services/audioMixer.test.ts
@@ -83,7 +83,7 @@ vi.mock("../utils/ffprobe.js", async (importOriginal) => {
import { RATE_RANGE } from "@hyperframes/core/audio-automation";
import { AudioFxRenderError } from "./audioFxRender.js";
-import { parseAudioElements, processCompositionAudio } from "./audioMixer.js";
+import { MAX_MIX_INPUTS, parseAudioElements, processCompositionAudio } from "./audioMixer.js";
import { encodeAac } from "./aacEncode.js";
describe("parseAudioElements strict literal timing", () => {
@@ -1093,6 +1093,140 @@ describe("processCompositionAudio", () => {
expect(runFfmpegMock).toHaveBeenCalledTimes(2);
});
+ it("keeps how long a killed group mix ran and ffmpeg's last words", async () => {
+ const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
+ const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
+ tempDirs.push(baseDir, workDir);
+ writeFileSync(join(baseDir, "voice.wav"), "stub");
+
+ const streamLines = Array.from(
+ { length: 400 },
+ (_, i) =>
+ ` Stream #${i}:0 -> #0:${i} (pcm_s16le (native) -> atrim (graph 0 input from stream ${i}:0))`,
+ );
+ runFfmpegMock
+ .mockResolvedValueOnce({
+ success: true,
+ durationMs: 1,
+ stderr: "",
+ exitCode: 0,
+ terminationReason: "exit",
+ })
+ .mockResolvedValueOnce({
+ success: false,
+ durationMs: 530_000,
+ stderr: [...streamLines, "Exiting normally, received signal 15."].join("\n"),
+ exitCode: 255,
+ signal: null,
+ terminationReason: "exit",
+ failureReason: "external_interruption",
+ });
+
+ const result = await processCompositionAudio(
+ [
+ {
+ id: "voice",
+ src: "voice.wav",
+ start: 0,
+ end: 2,
+ mediaStart: 0,
+ layer: 0,
+ volume: 1,
+ type: "audio",
+ groupId: "sfx",
+ },
+ ],
+ baseDir,
+ workDir,
+ join(baseDir, "out.m4a"),
+ 2,
+ );
+
+ const failure = result.failures?.find((f) => f.elementId === "sfx");
+ expect(failure?.reason).toBe("external_interruption");
+ expect(failure?.subprocess).toMatchObject({ exitCode: 255 });
+ expect(failure?.detail).toContain("ran 530000 ms");
+ expect(failure?.detail).toContain("Exiting normally, received signal 15.");
+ });
+
+ it.each([
+ ["ungrouped", undefined],
+ ["grouped", "sfx"],
+ ])(
+ "never hands one ffmpeg more inputs than a graph holds, %s",
+ async (_name, groupId) => {
+ const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
+ const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
+ tempDirs.push(baseDir, workDir);
+ const trackCount = 2_000;
+ const elements = Array.from({ length: trackCount }, (_, i) => {
+ writeFileSync(join(baseDir, `clip-${i}.wav`), "stub");
+ return {
+ id: `clip-${i}`,
+ src: `clip-${i}.wav`,
+ start: i * 0.1,
+ end: i * 0.1 + 0.5,
+ mediaStart: 0,
+ layer: i,
+ volume: 1,
+ type: "audio" as const,
+ ...(groupId ? { groupId } : {}),
+ };
+ });
+ runFfmpegMock.mockClear();
+
+ const result = await processCompositionAudio(
+ elements,
+ baseDir,
+ workDir,
+ join(baseDir, "out.m4a"),
+ trackCount * 0.1 + 0.5,
+ );
+
+ // The mocked ffmpeg writes no files, so only the ungrouped mix runs to the end.
+ if (!groupId) expect(result.success).toBe(true);
+ const inputsPerCall = runFfmpegMock.mock.calls.map(
+ (call) => (call[0] as string[]).filter((arg) => arg === "-i").length,
+ );
+ expect(Math.max(...inputsPerCall)).toBeLessThanOrEqual(MAX_MIX_INPUTS);
+ },
+ 60_000,
+ );
+
+ it("lets a mix's ffmpeg deadline grow with the film length", async () => {
+ const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
+ const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
+ tempDirs.push(baseDir, workDir);
+ writeFileSync(join(baseDir, "voice.wav"), "stub");
+ const voice = {
+ id: "voice",
+ src: "voice.wav",
+ start: 0,
+ end: 2,
+ mediaStart: 0,
+ layer: 0,
+ volume: 1,
+ type: "audio" as const,
+ };
+ const mixTimeout = async (totalDuration: number) => {
+ runFfmpegMock.mockClear();
+ await processCompositionAudio(
+ [voice],
+ baseDir,
+ workDir,
+ join(baseDir, "out.m4a"),
+ totalDuration,
+ );
+ const mixCall = runFfmpegMock.mock.calls.find((call) =>
+ (call[0] as string[]).includes("-filter_complex_script"),
+ );
+ return (mixCall?.[1] as { timeout?: number } | undefined)?.timeout;
+ };
+
+ expect(await mixTimeout(60)).toBe(300_000);
+ expect(await mixTimeout(36_000)).toBe(3_600_000);
+ });
+
it("preserves and classifies unsupported FFmpeg filter failures", async () => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
@@ -1466,21 +1600,22 @@ describe("processCompositionAudio", () => {
expect(result.success).toBe(true);
expect(result.tracksProcessed).toBe(trackCount);
- const mixArgs = runFfmpegMock.mock.calls.at(-1)?.[0] as string[];
- expect(mixArgs).toContain("-filter_complex_script");
- expect(mixArgs).not.toContain("-filter_complex");
-
- // The only things that scale with track count are the -i pairs (short,
- // fixed-size each) and the filter SCRIPT FILE's content (off the command
- // line entirely) — not the args array's own total character length.
- const argsLength = mixArgs.join(" ").length;
- expect(argsLength).toBeLessThan(20_000);
+ // 150 tracks exceed one graph: two slice sums, the fold of those two, then the final mix.
+ const mixCalls = runFfmpegMock.mock.calls.slice(-4).map((call) => call[0] as string[]);
+ for (const mixArgs of mixCalls) {
+ expect(mixArgs).toContain("-filter_complex_script");
+ expect(mixArgs).not.toContain("-filter_complex");
+ // Only the -i pairs and the filter script's content scale with track count.
+ expect(mixArgs.join(" ").length).toBeLessThan(20_000);
+ }
- const filter = capturedFilterScripts.at(-1);
- expect(filter).toContain(`amix=inputs=${trackCount}`);
+ const [first, second, folded, last] = capturedFilterScripts.slice(-4);
+ expect(first).toContain(`amix=inputs=${MAX_MIX_INPUTS}`);
+ expect(second).toContain(`amix=inputs=${trackCount - MAX_MIX_INPUTS}`);
+ expect(folded).toContain("amix=inputs=2");
+ expect(last).toContain("amix=inputs=1");
// Each track is trimmed to its authored clip; only the mixed output is padded.
- expect((filter?.match(/atrim=/g) ?? []).length).toBe(trackCount + 1);
- expect((filter?.match(/apad,/g) ?? []).length).toBe(1);
+ expect((last?.match(/apad,/g) ?? []).length).toBe(1);
// 150 real files + 150 existence checks: ~60ms on Linux, but well past the 5s
// default on the Windows lane, where each file create is orders slower.
}, 30_000);
diff --git a/packages/engine/src/services/audioMixer.ts b/packages/engine/src/services/audioMixer.ts
index 908b381486..7b0c233910 100644
--- a/packages/engine/src/services/audioMixer.ts
+++ b/packages/engine/src/services/audioMixer.ts
@@ -392,7 +392,11 @@ function boundedDetail(message: string, maxLength = 2_000): string {
/(^|[\s"'(])\/.+?(?=:\s[A-Z]|\s(?:ENOENT|EACCES|EPERM)\b|\r?$)/gm,
"$1<redacted-path>",
);
- return redacted.length <= maxLength ? redacted : `${redacted.slice(0, maxLength - 1)}…`;
+ // Keep both ends: the head names the failure, the tail holds ffmpeg's last words.
+ const head = Math.floor(maxLength / 4);
+ return redacted.length <= maxLength
+ ? redacted
+ : `${redacted.slice(0, head)}…${redacted.slice(-(maxLength - head - 1))}`;
}
function probeFailure(message: string, elementId: string): AudioProcessingFailure {
@@ -486,9 +490,10 @@ function ffmpegFailure(
elementId,
subprocess: audioSubprocessEvidence(result),
detail: boundedDetail(
- result.error?.message
- ? `${describeFfmpegFailure(result, timeoutMs)}: ${result.error.message}`
- : describeFfmpegFailure(result, timeoutMs),
+ `(ran ${result.durationMs} ms, limit ${timeoutMs} ms) ` +
+ (result.error?.message
+ ? `${describeFfmpegFailure(result, timeoutMs)}: ${result.error.message}`
+ : describeFfmpegFailure(result, timeoutMs)),
),
};
}
@@ -793,14 +798,25 @@ async function generateSilence(
};
}
-async function mixAudioTracks(
+/** A mix graph's work grows with the film length (about 11 ms per film second at 96 inputs); so does its deadline. */
+const MIX_TIMEOUT_MS_PER_FILM_SECOND = 100;
+
+function mixProcessTimeout(
+ config: Partial<Pick<EngineConfig, "ffmpegProcessTimeout">> | undefined,
+ totalDuration: number,
+): number {
+ const configured = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout;
+ return Math.max(configured, Math.ceil(totalDuration * MIX_TIMEOUT_MS_PER_FILM_SECOND));
+}
+
+async function mixAudioTracksFlat(
tracks: AudioTrack[],
outputPath: string,
totalDuration: number,
signal?: AbortSignal,
config?: Partial<Pick<EngineConfig, "ffmpegProcessTimeout" | "audioGain">>,
): Promise<MixResult> {
- const ffmpegProcessTimeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout;
+ const ffmpegProcessTimeout = mixProcessTimeout(config, totalDuration);
const masterOutputGain = config?.audioGain ?? DEFAULT_CONFIG.audioGain;
if (tracks.length === 0) {
@@ -989,7 +1005,7 @@ function groupNormalizeOptionUnsupported(stderr: string): boolean {
* compensation at all and is preferred; the manual per-own-count fallback
* only runs when this ffmpeg build's `amix` rejects the option.
*/
-async function mixGroupMembers(
+async function mixGroupMembersFlat(
memberTracks: AudioTrack[],
outputPath: string,
totalDuration: number,
@@ -999,7 +1015,7 @@ async function mixGroupMembers(
| { success: true; degradedAutomation?: boolean }
| { success: false; error: string; failure: AudioProcessingFailure }
> {
- const ffmpegProcessTimeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout;
+ const ffmpegProcessTimeout = mixProcessTimeout(config, totalDuration);
const outputDir = dirname(outputPath);
if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true });
@@ -1099,6 +1115,135 @@ async function mixGroupMembers(
return { success: true, degradedAutomation };
}
+/** Tracks per ffmpeg graph; a larger mix is summed in stages, so there is no cap on the total. */
+export const MAX_MIX_INPUTS = 96;
+
+type GroupMixResult =
+ | { success: true; degradedAutomation?: boolean }
+ | { success: false; error: string; failure: AudioProcessingFailure };
+
+/** Unity-sum `tracks` (the group gain law) slice by slice into one float WAV, with few full-length files at once. */
+async function sumInSlices(
+ tracks: AudioTrack[],
+ scratchDir: string,
+ totalDuration: number,
+ signal?: AbortSignal,
+ config?: Partial<Pick<EngineConfig, "ffmpegProcessTimeout">>,
+): Promise<
+ | { success: true; track: AudioTrack; degradedAutomation: boolean }
+ | Extract<GroupMixResult, { success: false }>
+> {
+ const wavTrack = (id: string, srcPath: string): AudioTrack => ({
+ id,
+ srcPath,
+ start: 0,
+ end: totalDuration,
+ mediaStart: 0,
+ duration: totalDuration,
+ volume: 1,
+ });
+ let running: AudioTrack | null = null;
+ let degradedAutomation = false;
+ for (let at = 0; at < tracks.length; at += MAX_MIX_INPUTS) {
+ const id = `sum-${at / MAX_MIX_INPUTS}`;
+ const slicePath = join(scratchDir, `${id}-slice.wav`);
+ const slice = await mixGroupMembersFlat(
+ tracks.slice(at, at + MAX_MIX_INPUTS),
+ slicePath,
+ totalDuration,
+ signal,
+ config,
+ );
+ if (!slice.success) return slice;
+ degradedAutomation ||= slice.degradedAutomation === true;
+ if (!running) {
+ running = wavTrack(id, slicePath);
+ continue;
+ }
+ const foldedPath = join(scratchDir, `${id}.wav`);
+ const folded = await mixGroupMembersFlat(
+ [running, wavTrack(`${id}-slice`, slicePath)],
+ foldedPath,
+ totalDuration,
+ signal,
+ config,
+ );
+ rmSync(running.srcPath, { force: true });
+ rmSync(slicePath, { force: true });
+ if (!folded.success) return folded;
+ running = wavTrack(id, foldedPath);
+ }
+ if (!running) throw new Error("[audioMixer] sumInSlices needs at least one track");
+ return { success: true, track: running, degradedAutomation };
+}
+
+async function mixGroupMembers(
+ memberTracks: AudioTrack[],
+ outputPath: string,
+ totalDuration: number,
+ signal?: AbortSignal,
+ config?: Partial<Pick<EngineConfig, "ffmpegProcessTimeout">>,
+): Promise<GroupMixResult> {
+ if (memberTracks.length <= MAX_MIX_INPUTS) {
+ return mixGroupMembersFlat(memberTracks, outputPath, totalDuration, signal, config);
+ }
+ mkdirSync(dirname(outputPath), { recursive: true });
+ const scratchDir = mkdtempSync(join(dirname(outputPath), ".mix-batches-"));
+ try {
+ const summed = await sumInSlices(memberTracks, scratchDir, totalDuration, signal, config);
+ if (!summed.success) return summed;
+ renameSync(summed.track.srcPath, outputPath);
+ return { success: true, degradedAutomation: summed.degradedAutomation };
+ } finally {
+ rmSync(scratchDir, { recursive: true, force: true });
+ }
+}
+
+async function mixAudioTracks(
+ tracks: AudioTrack[],
+ outputPath: string,
+ totalDuration: number,
+ signal?: AbortSignal,
+ config?: Partial<Pick<EngineConfig, "ffmpegProcessTimeout" | "audioGain">>,
+): Promise<MixResult> {
+ if (tracks.length <= MAX_MIX_INPUTS) {
+ return mixAudioTracksFlat(tracks, outputPath, totalDuration, signal, config);
+ }
+ mkdirSync(dirname(outputPath), { recursive: true });
+ const scratchDir = mkdtempSync(join(dirname(outputPath), ".mix-batches-"));
+ try {
+ const summed = await sumInSlices(tracks, scratchDir, totalDuration, signal, config);
+ if (!summed.success) {
+ return {
+ success: false,
+ outputPath,
+ durationMs: 0,
+ tracksProcessed: 0,
+ error: summed.error,
+ failures: [summed.failure],
+ };
+ }
+ const mixed = await mixAudioTracksFlat(
+ [summed.track],
+ outputPath,
+ totalDuration,
+ signal,
+ config,
+ );
+ return mixed.success
+ ? {
+ ...mixed,
+ tracksProcessed: tracks.length,
+ error: summed.degradedAutomation
+ ? "Volume automation exceeded this ffmpeg build's expression limits; rendered at base volume"
+ : mixed.error,
+ }
+ : mixed;
+ } finally {
+ rmSync(scratchDir, { recursive: true, force: true });
+ }
+}
+
/**
* Sum the tracks to a float WAV, run the master chain (if any) over the sum, then AAC-encode.
* Groups are already folded into `tracks`, and the sum stays float until the encoder.