Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
fix(engine): mix any number of audio clips by summing them in stages (#6120)
Miguel Ángel3cdae3804e5f
diff --git a/packages/engine/src/services/audioMixer.manyTracks.test.ts b/packages/engine/src/services/audioMixer.manyTracks.test.tsnew file mode 100644index 0000000000..3c91a5d14a--- /dev/null+++ b/packages/engine/src/services/audioMixer.manyTracks.test.ts@@ -0,0 +1,135 @@+import { spawnSync } from "node:child_process";+import { copyFileSync, mkdtempSync, readdirSync, rmSync } from "node:fs";+import { tmpdir } from "node:os";+import { dirname, join } from "node:path";+import { afterEach, describe, expect, it } from "vitest";+import { getFfmpegBinary } from "../utils/ffmpegBinaries.js";+import { MAX_MIX_INPUTS, MIXED_AUDIO_FILENAME, processCompositionAudio } from "./audioMixer.js";++const HAS_FFMPEG = spawnSync(getFfmpegBinary(), ["-version"], { encoding: "utf-8" }).status === 0;+const tempDirs: string[] = [];++function meanVolumeDb(path: string): number {+ const result = spawnSync(+ getFfmpegBinary(),+ ["-nostdin", "-hide_banner", "-i", path, "-af", "volumedetect", "-f", "null", "-"],+ { encoding: "utf-8" },+ );+ const match = result.stderr.match(/mean_volume:\s*(-?[\d.]+) dB/);+ if (result.status !== 0 || !match?.[1]) throw new Error(`Could not measure: ${result.stderr}`);+ return Number(match[1]);+}++function peakDb(path: string): number {+ const result = spawnSync(+ getFfmpegBinary(),+ ["-nostdin", "-hide_banner", "-i", path, "-af", "astats", "-f", "null", "-"],+ { encoding: "utf-8" },+ );+ const match = result.stderr.match(/Peak level dB:\s*(-?[\d.]+)/);+ if (result.status !== 0 || !match?.[1]) throw new Error(`Could not measure: ${result.stderr}`);+ return Number(match[1]);+}++function writeTone(path: string, seconds: number, gain: number): void {+ const result = spawnSync(+ getFfmpegBinary(),+ [+ "-nostdin",+ "-v",+ "error",+ "-f",+ "lavfi",+ "-i",+ `sine=frequency=440:duration=${seconds}:sample_rate=48000`,+ "-af",+ `volume=${gain}`,+ "-c:a",+ "pcm_s16le",+ path,+ ],+ { encoding: "utf-8" },+ );+ if (result.status !== 0) throw new Error(`Could not write tone: ${result.stderr}`);+}++const clip = (i: number, start: number, end: number, groupId?: string) => ({+ id: `c${i}`,+ src: `a${i}.wav`,+ start,+ end,+ mediaStart: 0,+ layer: i,+ volume: 1,+ type: "audio" as const,+ ...(groupId ? { groupId } : {}),+});++describe.skipIf(!HAS_FFMPEG)("a mix with more clips than one ffmpeg graph can hold", () => {+ afterEach(() => {+ for (const dir of tempDirs.splice(0)) rmSync(dir, { recursive: true, force: true });+ });++ async function mix(+ count: number,+ groupId: string | undefined,+ shape: { stackGain: number } | "sequence",+ ) {+ const projectDir = mkdtempSync(join(tmpdir(), "hf-many-"));+ const workDir = mkdtempSync(join(tmpdir(), "hf-many-work-"));+ tempDirs.push(projectDir, workDir);+ const stacked = shape !== "sequence";+ writeTone(join(projectDir, "a.wav"), stacked ? 2 : 0.2, stacked ? shape.stackGain : 1);+ for (let i = 0; i < count; i++)+ copyFileSync(join(projectDir, "a.wav"), join(projectDir, `a${i}.wav`));+ const clips = Array.from({ length: count }, (_, i) =>+ stacked ? clip(i, 0, 2, groupId) : clip(i, i * 0.05, (i + 1) * 0.05, groupId),+ );+ const out = join(projectDir, MIXED_AUDIO_FILENAME);+ const sum = join(projectDir, "sum.wav");+ const result = await processCompositionAudio(+ clips,+ projectDir,+ workDir,+ out,+ stacked ? 2 : count * 0.05,+ undefined,+ { keepEncoderInputAt: sum },+ );+ return { result, out, sum };+ }++ it("mixes 200 clips in one group at the level of a short group", async () => {+ const small = await mix(8, "sfx", "sequence");+ const many = await mix(200, "sfx", "sequence");+ expect(many.result.failures ?? []).toEqual([]);+ expect(many.result.success).toBe(true);+ expect(Math.abs(meanVolumeDb(many.out) - meanVolumeDb(small.out))).toBeLessThan(1);+ }, 120_000);++ it("mixes 200 ungrouped clips at the level of a short mix", async () => {+ const small = await mix(8, undefined, "sequence");+ const many = await mix(200, undefined, "sequence");+ expect(many.result.failures ?? []).toEqual([]);+ expect(many.result.success).toBe(true);+ expect(Math.abs(meanVolumeDb(many.out) - meanVolumeDb(small.out))).toBeLessThan(1);+ expect(readdirSync(dirname(many.out)).filter((f) => f.startsWith(".mix-batches-"))).toEqual([]);+ }, 120_000);++ it.each([+ ["ungrouped", undefined],+ ["grouped", "sfx"],+ ])(+ "sums %s clips past the slice size at the gain the flat mix gives",+ async (_name, groupId) => {+ // Stacked coherent clips add, so 120 at gain g and 60 at gain 2g have the same sum.+ // 60 is one flat graph; 120 is staged. The float sums must peak alike.+ const flat = await mix(60, groupId, { stackGain: 0.1 });+ const staged = await mix(MAX_MIX_INPUTS + 24, groupId, { stackGain: 0.05 });+ expect(flat.result.success).toBe(true);+ expect(staged.result.success).toBe(true);+ expect(Math.abs(peakDb(staged.sum) - peakDb(flat.sum))).toBeLessThan(0.1);+ },+ 120_000,+ );+});diff --git a/packages/engine/src/services/audioMixer.test.ts b/packages/engine/src/services/audioMixer.test.tsindex de45b80fcd..e2bcdfbbc8 100644--- a/packages/engine/src/services/audioMixer.test.ts+++ b/packages/engine/src/services/audioMixer.test.ts@@ -83,7 +83,7 @@ vi.mock("../utils/ffprobe.js", async (importOriginal) => {import { RATE_RANGE } from "@hyperframes/core/audio-automation";import { AudioFxRenderError } from "./audioFxRender.js";-import { parseAudioElements, processCompositionAudio } from "./audioMixer.js";+import { MAX_MIX_INPUTS, parseAudioElements, processCompositionAudio } from "./audioMixer.js";import { encodeAac } from "./aacEncode.js";describe("parseAudioElements strict literal timing", () => {@@ -1093,6 +1093,140 @@ describe("processCompositionAudio", () => {expect(runFfmpegMock).toHaveBeenCalledTimes(2);});+ it("keeps how long a killed group mix ran and ffmpeg's last words", async () => {+ const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));+ const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));+ tempDirs.push(baseDir, workDir);+ writeFileSync(join(baseDir, "voice.wav"), "stub");++ const streamLines = Array.from(+ { length: 400 },+ (_, i) =>+ ` Stream #${i}:0 -> #0:${i} (pcm_s16le (native) -> atrim (graph 0 input from stream ${i}:0))`,+ );+ runFfmpegMock+ .mockResolvedValueOnce({+ success: true,+ durationMs: 1,+ stderr: "",+ exitCode: 0,+ terminationReason: "exit",+ })+ .mockResolvedValueOnce({+ success: false,+ durationMs: 530_000,+ stderr: [...streamLines, "Exiting normally, received signal 15."].join("\n"),+ exitCode: 255,+ signal: null,+ terminationReason: "exit",+ failureReason: "external_interruption",+ });++ const result = await processCompositionAudio(+ [+ {+ id: "voice",+ src: "voice.wav",+ start: 0,+ end: 2,+ mediaStart: 0,+ layer: 0,+ volume: 1,+ type: "audio",+ groupId: "sfx",+ },+ ],+ baseDir,+ workDir,+ join(baseDir, "out.m4a"),+ 2,+ );++ const failure = result.failures?.find((f) => f.elementId === "sfx");+ expect(failure?.reason).toBe("external_interruption");+ expect(failure?.subprocess).toMatchObject({ exitCode: 255 });+ expect(failure?.detail).toContain("ran 530000 ms");+ expect(failure?.detail).toContain("Exiting normally, received signal 15.");+ });++ it.each([+ ["ungrouped", undefined],+ ["grouped", "sfx"],+ ])(+ "never hands one ffmpeg more inputs than a graph holds, %s",+ async (_name, groupId) => {+ const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));+ const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));+ tempDirs.push(baseDir, workDir);+ const trackCount = 2_000;+ const elements = Array.from({ length: trackCount }, (_, i) => {+ writeFileSync(join(baseDir, `clip-${i}.wav`), "stub");+ return {+ id: `clip-${i}`,+ src: `clip-${i}.wav`,+ start: i * 0.1,+ end: i * 0.1 + 0.5,+ mediaStart: 0,+ layer: i,+ volume: 1,+ type: "audio" as const,+ ...(groupId ? { groupId } : {}),+ };+ });+ runFfmpegMock.mockClear();++ const result = await processCompositionAudio(+ elements,+ baseDir,+ workDir,+ join(baseDir, "out.m4a"),+ trackCount * 0.1 + 0.5,+ );++ // The mocked ffmpeg writes no files, so only the ungrouped mix runs to the end.+ if (!groupId) expect(result.success).toBe(true);+ const inputsPerCall = runFfmpegMock.mock.calls.map(+ (call) => (call[0] as string[]).filter((arg) => arg === "-i").length,+ );+ expect(Math.max(...inputsPerCall)).toBeLessThanOrEqual(MAX_MIX_INPUTS);+ },+ 60_000,+ );++ it("lets a mix's ffmpeg deadline grow with the film length", async () => {+ const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));+ const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));+ tempDirs.push(baseDir, workDir);+ writeFileSync(join(baseDir, "voice.wav"), "stub");+ const voice = {+ id: "voice",+ src: "voice.wav",+ start: 0,+ end: 2,+ mediaStart: 0,+ layer: 0,+ volume: 1,+ type: "audio" as const,+ };+ const mixTimeout = async (totalDuration: number) => {+ runFfmpegMock.mockClear();+ await processCompositionAudio(+ [voice],+ baseDir,+ workDir,+ join(baseDir, "out.m4a"),+ totalDuration,+ );+ const mixCall = runFfmpegMock.mock.calls.find((call) =>+ (call[0] as string[]).includes("-filter_complex_script"),+ );+ return (mixCall?.[1] as { timeout?: number } | undefined)?.timeout;+ };++ expect(await mixTimeout(60)).toBe(300_000);+ expect(await mixTimeout(36_000)).toBe(3_600_000);+ });+it("preserves and classifies unsupported FFmpeg filter failures", async () => {const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));@@ -1466,21 +1600,22 @@ describe("processCompositionAudio", () => {expect(result.success).toBe(true);expect(result.tracksProcessed).toBe(trackCount);- const mixArgs = runFfmpegMock.mock.calls.at(-1)?.[0] as string[];- expect(mixArgs).toContain("-filter_complex_script");- expect(mixArgs).not.toContain("-filter_complex");-- // The only things that scale with track count are the -i pairs (short,- // fixed-size each) and the filter SCRIPT FILE's content (off the command- // line entirely) — not the args array's own total character length.- const argsLength = mixArgs.join(" ").length;- expect(argsLength).toBeLessThan(20_000);+ // 150 tracks exceed one graph: two slice sums, the fold of those two, then the final mix.+ const mixCalls = runFfmpegMock.mock.calls.slice(-4).map((call) => call[0] as string[]);+ for (const mixArgs of mixCalls) {+ expect(mixArgs).toContain("-filter_complex_script");+ expect(mixArgs).not.toContain("-filter_complex");+ // Only the -i pairs and the filter script's content scale with track count.+ expect(mixArgs.join(" ").length).toBeLessThan(20_000);+ }- const filter = capturedFilterScripts.at(-1);- expect(filter).toContain(`amix=inputs=${trackCount}`);+ const [first, second, folded, last] = capturedFilterScripts.slice(-4);+ expect(first).toContain(`amix=inputs=${MAX_MIX_INPUTS}`);+ expect(second).toContain(`amix=inputs=${trackCount - MAX_MIX_INPUTS}`);+ expect(folded).toContain("amix=inputs=2");+ expect(last).toContain("amix=inputs=1");// Each track is trimmed to its authored clip; only the mixed output is padded.- expect((filter?.match(/atrim=/g) ?? []).length).toBe(trackCount + 1);- expect((filter?.match(/apad,/g) ?? []).length).toBe(1);+ expect((last?.match(/apad,/g) ?? []).length).toBe(1);// 150 real files + 150 existence checks: ~60ms on Linux, but well past the 5s// default on the Windows lane, where each file create is orders slower.}, 30_000);diff --git a/packages/engine/src/services/audioMixer.ts b/packages/engine/src/services/audioMixer.tsindex 908b381486..7b0c233910 100644--- a/packages/engine/src/services/audioMixer.ts+++ b/packages/engine/src/services/audioMixer.ts@@ -392,7 +392,11 @@ function boundedDetail(message: string, maxLength = 2_000): string {/(^|[\s"'(])\/.+?(?=:\s[A-Z]|\s(?:ENOENT|EACCES|EPERM)\b|\r?$)/gm,"$1<redacted-path>",);- return redacted.length <= maxLength ? redacted : `${redacted.slice(0, maxLength - 1)}…`;+ // Keep both ends: the head names the failure, the tail holds ffmpeg's last words.+ const head = Math.floor(maxLength / 4);+ return redacted.length <= maxLength+ ? redacted+ : `${redacted.slice(0, head)}…${redacted.slice(-(maxLength - head - 1))}`;}function probeFailure(message: string, elementId: string): AudioProcessingFailure {@@ -486,9 +490,10 @@ function ffmpegFailure(elementId,subprocess: audioSubprocessEvidence(result),detail: boundedDetail(- result.error?.message- ? `${describeFfmpegFailure(result, timeoutMs)}: ${result.error.message}`- : describeFfmpegFailure(result, timeoutMs),+ `(ran ${result.durationMs} ms, limit ${timeoutMs} ms) ` ++ (result.error?.message+ ? `${describeFfmpegFailure(result, timeoutMs)}: ${result.error.message}`+ : describeFfmpegFailure(result, timeoutMs)),),};}@@ -793,14 +798,25 @@ async function generateSilence(};}-async function mixAudioTracks(+/** A mix graph's work grows with the film length (about 11 ms per film second at 96 inputs); so does its deadline. */+const MIX_TIMEOUT_MS_PER_FILM_SECOND = 100;++function mixProcessTimeout(+ config: Partial<Pick<EngineConfig, "ffmpegProcessTimeout">> | undefined,+ totalDuration: number,+): number {+ const configured = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout;+ return Math.max(configured, Math.ceil(totalDuration * MIX_TIMEOUT_MS_PER_FILM_SECOND));+}++async function mixAudioTracksFlat(tracks: AudioTrack[],outputPath: string,totalDuration: number,signal?: AbortSignal,config?: Partial<Pick<EngineConfig, "ffmpegProcessTimeout" | "audioGain">>,): Promise<MixResult> {- const ffmpegProcessTimeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout;+ const ffmpegProcessTimeout = mixProcessTimeout(config, totalDuration);const masterOutputGain = config?.audioGain ?? DEFAULT_CONFIG.audioGain;if (tracks.length === 0) {@@ -989,7 +1005,7 @@ function groupNormalizeOptionUnsupported(stderr: string): boolean {* compensation at all and is preferred; the manual per-own-count fallback* only runs when this ffmpeg build's `amix` rejects the option.*/-async function mixGroupMembers(+async function mixGroupMembersFlat(memberTracks: AudioTrack[],outputPath: string,totalDuration: number,@@ -999,7 +1015,7 @@ async function mixGroupMembers(| { success: true; degradedAutomation?: boolean }| { success: false; error: string; failure: AudioProcessingFailure }> {- const ffmpegProcessTimeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout;+ const ffmpegProcessTimeout = mixProcessTimeout(config, totalDuration);const outputDir = dirname(outputPath);if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true });@@ -1099,6 +1115,135 @@ async function mixGroupMembers(return { success: true, degradedAutomation };}+/** Tracks per ffmpeg graph; a larger mix is summed in stages, so there is no cap on the total. */+export const MAX_MIX_INPUTS = 96;++type GroupMixResult =+ | { success: true; degradedAutomation?: boolean }+ | { success: false; error: string; failure: AudioProcessingFailure };++/** Unity-sum `tracks` (the group gain law) slice by slice into one float WAV, with few full-length files at once. */+async function sumInSlices(+ tracks: AudioTrack[],+ scratchDir: string,+ totalDuration: number,+ signal?: AbortSignal,+ config?: Partial<Pick<EngineConfig, "ffmpegProcessTimeout">>,+): Promise<+ | { success: true; track: AudioTrack; degradedAutomation: boolean }+ | Extract<GroupMixResult, { success: false }>+> {+ const wavTrack = (id: string, srcPath: string): AudioTrack => ({+ id,+ srcPath,+ start: 0,+ end: totalDuration,+ mediaStart: 0,+ duration: totalDuration,+ volume: 1,+ });+ let running: AudioTrack | null = null;+ let degradedAutomation = false;+ for (let at = 0; at < tracks.length; at += MAX_MIX_INPUTS) {+ const id = `sum-${at / MAX_MIX_INPUTS}`;+ const slicePath = join(scratchDir, `${id}-slice.wav`);+ const slice = await mixGroupMembersFlat(+ tracks.slice(at, at + MAX_MIX_INPUTS),+ slicePath,+ totalDuration,+ signal,+ config,+ );+ if (!slice.success) return slice;+ degradedAutomation ||= slice.degradedAutomation === true;+ if (!running) {+ running = wavTrack(id, slicePath);+ continue;+ }+ const foldedPath = join(scratchDir, `${id}.wav`);+ const folded = await mixGroupMembersFlat(+ [running, wavTrack(`${id}-slice`, slicePath)],+ foldedPath,+ totalDuration,+ signal,+ config,+ );+ rmSync(running.srcPath, { force: true });+ rmSync(slicePath, { force: true });+ if (!folded.success) return folded;+ running = wavTrack(id, foldedPath);+ }+ if (!running) throw new Error("[audioMixer] sumInSlices needs at least one track");+ return { success: true, track: running, degradedAutomation };+}++async function mixGroupMembers(+ memberTracks: AudioTrack[],+ outputPath: string,+ totalDuration: number,+ signal?: AbortSignal,+ config?: Partial<Pick<EngineConfig, "ffmpegProcessTimeout">>,+): Promise<GroupMixResult> {+ if (memberTracks.length <= MAX_MIX_INPUTS) {+ return mixGroupMembersFlat(memberTracks, outputPath, totalDuration, signal, config);+ }+ mkdirSync(dirname(outputPath), { recursive: true });+ const scratchDir = mkdtempSync(join(dirname(outputPath), ".mix-batches-"));+ try {+ const summed = await sumInSlices(memberTracks, scratchDir, totalDuration, signal, config);+ if (!summed.success) return summed;+ renameSync(summed.track.srcPath, outputPath);+ return { success: true, degradedAutomation: summed.degradedAutomation };+ } finally {+ rmSync(scratchDir, { recursive: true, force: true });+ }+}++async function mixAudioTracks(+ tracks: AudioTrack[],+ outputPath: string,+ totalDuration: number,+ signal?: AbortSignal,+ config?: Partial<Pick<EngineConfig, "ffmpegProcessTimeout" | "audioGain">>,+): Promise<MixResult> {+ if (tracks.length <= MAX_MIX_INPUTS) {+ return mixAudioTracksFlat(tracks, outputPath, totalDuration, signal, config);+ }+ mkdirSync(dirname(outputPath), { recursive: true });+ const scratchDir = mkdtempSync(join(dirname(outputPath), ".mix-batches-"));+ try {+ const summed = await sumInSlices(tracks, scratchDir, totalDuration, signal, config);+ if (!summed.success) {+ return {+ success: false,+ outputPath,+ durationMs: 0,+ tracksProcessed: 0,+ error: summed.error,+ failures: [summed.failure],+ };+ }+ const mixed = await mixAudioTracksFlat(+ [summed.track],+ outputPath,+ totalDuration,+ signal,+ config,+ );+ return mixed.success+ ? {+ ...mixed,+ tracksProcessed: tracks.length,+ error: summed.degradedAutomation+ ? "Volume automation exceeded this ffmpeg build's expression limits; rendered at base volume"+ : mixed.error,+ }+ : mixed;+ } finally {+ rmSync(scratchDir, { recursive: true, force: true });+ }+}+/*** Sum the tracks to a float WAV, run the master chain (if any) over the sum, then AAC-encode.* Groups are already folded into `tracks`, and the sum stays float until the encoder.