Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
fix(engine): avoid padding every audio mix input (#5563)
Ayodele Ayoola3de4763d53de
diff --git a/packages/engine/src/services/audioMixer.grouping.test.ts b/packages/engine/src/services/audioMixer.grouping.test.tsindex b8741ad644..3a66131bcd 100644--- a/packages/engine/src/services/audioMixer.grouping.test.ts+++ b/packages/engine/src/services/audioMixer.grouping.test.ts@@ -9,11 +9,10 @@ import { MIXED_AUDIO_FILENAME, parseAudioElements, processCompositionAudio } fro/*** Level arithmetic across the mix graph.*- * `mixAudioTracks` corrects for amix's own normalisation with a single global- * `masterOutputGain * tracks.length`. That correction is exact for one flat- * amix and wrong for any other shape — and it fails SILENTLY, in the export- * rather than in preview, because preview mixes through Web Audio and never- * runs this graph at all.+ * `mixAudioTracks` uses `amix=normalize=0` where supported. Older FFmpeg+ * builds fall back to padding every input and correcting by the input count.+ * The correction must use each mix's own count, or it fails silently in the+ * export while preview continues to mix through Web Audio.** Measured on a four-track mix (spike, 2026-08-14): keeping the global track* count on an outer amix that only has three inputs lands the whole mix@@ -145,15 +144,12 @@ describe.skipIf(!HAS_FFMPEG)(});it("does not lift the survivors when a shorter track ends", async () => {- // amix with normalize=true rescales by the number of CURRENTLY ACTIVE- // inputs, so a track ending mid-composition would hand the remaining ones- // a level jump. `apad` to the full duration is what holds every input- // active for the whole graph and neutralises that — an invariant the- // filter string relies on without saying so.+ // A track ending mid-composition must not change the survivors' level.+ // The main mix disables amix normalization; the legacy fallback pads all+ // inputs so its default normalization keeps a constant divisor.//// Measured without apad (spike, 2026-08-14): the tail runs +1.94 dB hot.- // Any group work that builds its own amix has to keep the padding, or- // inherit that bug one level down.+ // Any grouped mix that uses default normalization still needs padding.const projectDir = mkdtempSync(join(tmpdir(), "hf-grp-drop-"));const workDir = mkdtempSync(join(tmpdir(), "hf-grp-drop-work-"));tempDirs.push(projectDir, workDir);diff --git a/packages/engine/src/services/audioMixer.test.ts b/packages/engine/src/services/audioMixer.test.tsindex 612e69ebe0..8858349eac 100644--- a/packages/engine/src/services/audioMixer.test.ts+++ b/packages/engine/src/services/audioMixer.test.ts@@ -1,6 +1,6 @@// fallow-ignore-file code-duplicationimport { afterEach, describe, expect, it, vi } from "vitest";-import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";import { join } from "node:path";import { tmpdir } from "node:os";@@ -366,10 +366,10 @@ describe("processCompositionAudio", () => {const filter = capturedFilterScripts[1];expect(filter).toContain("volume=0");- expect(filter).toContain("[mixed]volume=1[out]");+ expect(filter).toContain("[mixed]volume=1,apad,asetpts=N/SR/TB,atrim=0:2[out]");expect(filter).toContain("apad,asetpts=N/SR/TB,atrim=0:2");expect(filter).not.toContain("whole_dur");- expect(filter).not.toContain("normalize=");+ expect(filter).toContain("normalize=0");expect(filter).not.toContain("weights=");});@@ -752,7 +752,7 @@ describe("processCompositionAudio", () => {expect(filter).toContain("atrim=0:2,");});- it("compensates amix normalization so multi-track master gain equals track count", async () => {+ it("sums unpadded tracks at authored gain and pads the mixed output once", async () => {const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));tempDirs.push(baseDir, workDir);@@ -807,9 +807,66 @@ describe("processCompositionAudio", () => {const filter = capturedFilterScripts[3];expect(filter).toContain("amix=inputs=3");- expect(filter).not.toContain("normalize=");- // masterOutputGain(1) × tracks(3) = 3- expect(filter).toContain("[mixed]volume=3[out]");+ expect(filter).toContain("amix=inputs=3:duration=longest:dropout_transition=0:normalize=0");+ expect(filter).toContain("[mixed]volume=1,apad,asetpts=N/SR/TB,atrim=0:2[out]");+ expect((filter.match(/apad,/g) ?? []).length).toBe(1);+ });++ it("retries the padded graph when ffmpeg does not support amix normalize", async () => {+ const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));+ const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));+ tempDirs.push(baseDir, workDir);+ writeFileSync(join(baseDir, "voice.wav"), "stub");++ runFfmpegMock+ .mockImplementationOnce(async () => ({+ success: true,+ durationMs: 1,+ stderr: "",+ exitCode: 0,+ }))+ .mockImplementationOnce(async (args: string[]) => {+ capturedFilterScripts.push(+ readFileSync(args[args.indexOf("-filter_complex_script") + 1]!, "utf8"),+ );+ return {+ success: false,+ durationMs: 1,+ stderr: "Error applying option 'normalize' to filter 'amix': Option not found",+ exitCode: 1,+ };+ })+ .mockImplementationOnce(async (args: string[]) => {+ capturedFilterScripts.push(+ readFileSync(args[args.indexOf("-filter_complex_script") + 1]!, "utf8"),+ );+ return { success: true, durationMs: 1, stderr: "", exitCode: 0 };+ });++ const result = await processCompositionAudio(+ [+ {+ id: "voice",+ src: "voice.wav",+ start: 0,+ end: 2,+ mediaStart: 0,+ layer: 0,+ volume: 1,+ type: "audio",+ },+ ],+ baseDir,+ workDir,+ join(baseDir, "out.m4a"),+ 2,+ );++ expect(result.success).toBe(true);+ expect(runFfmpegMock).toHaveBeenCalledTimes(3);+ expect(capturedFilterScripts[0]).toContain("normalize=0");+ expect(capturedFilterScripts[1]).toContain("apad,asetpts=N/SR/TB,atrim=0:2[a0]");+ expect(capturedFilterScripts[1]).not.toContain("normalize=");});it("fails the audio result instead of silently mixing after one track preparation fails", async () => {@@ -1268,22 +1325,17 @@ describe("processCompositionAudio", () => {const filter = capturedFilterScripts.at(-1);expect(filter).toContain(`amix=inputs=${trackCount}`);- // Each track is trimmed once to its authored clip and once after portable- // indefinite `apad` to cap the padded stream at composition duration.- expect((filter?.match(/atrim=/g) ?? []).length).toBe(trackCount * 2);- expect((filter?.match(/apad,/g) ?? []).length).toBe(trackCount);+ // Each track is trimmed to its authored clip; only the mixed output is padded.+ expect((filter?.match(/atrim=/g) ?? []).length).toBe(trackCount + 1);+ expect((filter?.match(/apad,/g) ?? []).length).toBe(1);// 150 real files + 150 existence checks: ~60ms on Linux, but well past the 5s// default on the Windows lane, where each file create is orders slower.}, 30_000);- it("renumbers timestamps between apad and atrim on every mixed branch", async () => {- // Regression: `apad` then `atrim` is the portable pad-to-length shape --+ it("renumbers timestamps between apad and atrim on the mixed output", async () => {// #2769 moved off `apad=whole_dur=` because some builds reject that option.- // But on FFmpeg 5.x-8.0.x the padded samples carry timestamps `atrim`- // misreads, so a delayed branch sounds at t=0 and, past three branches, the- // last one vanishes from the mix. `asetpts=N/SR/TB` between the two rebuilds- // the timestamps from the sample count and costs no portability, since all- // three filters exist in every build we support.+ // Keep the portable apad/asetpts/atrim shape on the mixed output so the+ // final stream has the requested duration and consistent timestamps.const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));tempDirs.push(baseDir, workDir);@@ -1322,10 +1374,8 @@ describe("processCompositionAudio", () => {const filter = capturedFilterScripts.at(-1) ?? "";const branches = filter.match(/apad[^;]*/g) ?? [];- expect(branches).toHaveLength(2);- for (const branch of branches) {- expect(branch).toMatch(/^apad,asetpts=N\/SR\/TB,atrim=0:/);- }+ expect(branches).toHaveLength(1);+ expect(branches[0]).toMatch(/^apad,asetpts=N\/SR\/TB,atrim=0:/);// The portability constraint #2769 established still holds.expect(filter).not.toContain("whole_dur");});diff --git a/packages/engine/src/services/audioMixer.ts b/packages/engine/src/services/audioMixer.tsindex 7937bf51c7..585c3f182d 100644--- a/packages/engine/src/services/audioMixer.ts+++ b/packages/engine/src/services/audioMixer.ts@@ -326,16 +326,18 @@ export function buildFadeFilters(return stages.length ? `,${stages.join(",")}` : "";}-/** One track: trim, volume, afade, delay/pad. asetpts after apad so FFmpeg 5-8 delayed branches do not land at t=0. */+/** One track: trim, volume, afade, delay. Legacy and group mixes also pad each branch. */export function buildTrackInputFilter(track: Pick<AudioTrack, "start" | "end" | "fadeIn" | "fadeOut" | "tailSeconds">,index: number,volumeFilter: string,totalDuration: number,+ padToDuration = true,): string {const delayMs = Math.round(track.start * 1000);const trimDuration = track.end - track.start + (track.tailSeconds ?? 0);- return `[${index}:a]atrim=0:${formatFilterNumber(trimDuration)},${volumeFilter}${buildFadeFilters(track)},adelay=${delayMs}|${delayMs},apad,asetpts=N/SR/TB,atrim=0:${formatFilterNumber(totalDuration)}[a${index}]`;+ const filters = `[${index}:a]atrim=0:${formatFilterNumber(trimDuration)},${volumeFilter}${buildFadeFilters(track)},adelay=${delayMs}|${delayMs}`;+ return `${filters}${padToDuration ? `,apad,asetpts=N/SR/TB,atrim=0:${formatFilterNumber(totalDuration)}` : ""}[a${index}]`;}interface ExtractResult {@@ -777,19 +779,21 @@ async function mixAudioTracks(const outputDir = dirname(outputPath);if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true });- const buildFilterComplex = (ignoreAutomation: boolean): string => {+ const buildFilterComplex = (ignoreAutomation: boolean, useNormalize: boolean): string => {const filterParts: string[] = [];tracks.forEach((track, i) => {const volumeFilter = buildVolumeExpression(track, ignoreAutomation);- filterParts.push(buildTrackInputFilter(track, i, volumeFilter, totalDuration));+ filterParts.push(buildTrackInputFilter(track, i, volumeFilter, totalDuration, !useNormalize));});const mixInputs = tracks.map((_, i) => `[a${i}]`).join("");- const mixFilter = `${mixInputs}amix=inputs=${tracks.length}:duration=longest:dropout_transition=0[mixed]`;- // amix divides output by inputs count (default normalize=true). Multiply master- // gain by track count so per-track volumes authored in data-volume are preserved.- const compensatedGain = masterOutputGain * tracks.length;- const postMixGainFilter = `[mixed]volume=${formatFilterNumber(compensatedGain)}[out]`;+ const mixFilter = `${mixInputs}amix=inputs=${tracks.length}:duration=longest:dropout_transition=0${useNormalize ? ":normalize=0" : ""}[mixed]`;+ // Older FFmpeg builds lack normalize=0. Their default amix divides by the+ // active input count, so keep all branches padded and compensate by N.+ const gain = masterOutputGain * (useNormalize ? 1 : tracks.length);+ const postMixGainFilter = useNormalize+ ? `[mixed]volume=${formatFilterNumber(gain)},apad,asetpts=N/SR/TB,atrim=0:${formatFilterNumber(totalDuration)}[out]`+ : `[mixed]volume=${formatFilterNumber(gain)}[out]`;return [...filterParts, mixFilter, postMixGainFilter].join(";");};@@ -804,14 +808,14 @@ async function mixAudioTracks(// alias from nightly builds; older stable builds do not understand the new// spelling. Prefer the legacy spelling for broad compatibility and retry// only when FFmpeg explicitly says that option is unavailable.- const runMix = async (ignoreAutomation: boolean) => {+ const runMix = async (ignoreAutomation: boolean, useNormalize: boolean) => {const inputs: string[] = [];tracks.forEach((track) => inputs.push("-i", track.srcPath));const scriptDir = mkdtempSync(join(outputDir, ".filter-complex-"));const scriptPath = join(scriptDir, "graph.txt");const fd = openSync(scriptPath, "wx", 0o600);try {- writeFileSync(fd, buildFilterComplex(ignoreAutomation));+ writeFileSync(fd, buildFilterComplex(ignoreAutomation, useNormalize));} finally {closeSync(fd);}@@ -840,7 +844,17 @@ async function mixAudioTracks(}};- let result = await runMix(false);+ let useNormalize = true;+ let result = await runMix(false, useNormalize);+ if (+ !result.success &&+ result.failureReason !== "external_interruption" &&+ !signal?.aborted &&+ groupNormalizeOptionUnsupported(result.stderr)+ ) {+ useNormalize = false;+ result = await runMix(false, useNormalize);+ }// Defense in depth: volume automation is folded into an FFmpeg `volume`// expression whose evaluator limits are build-dependent (see@@ -855,7 +869,7 @@ async function mixAudioTracks(!signal?.aborted &&hasAutomation) {- const retry = await runMix(true);+ const retry = await runMix(true, useNormalize);if (retry.success) {result = retry;degradedAutomation = true;@@ -919,8 +933,8 @@ function groupNormalizeOptionUnsupported(stderr: string): boolean {/*** Sub-mix one group's members into a single PCM WAV at full composition- * length — the same per-input treatment `mixAudioTracks` gives every track- * (atrim, volume, adelay, `apad` — RULES 6), writing PCM instead of AAC so the+ * length. Unlike the main mix, this still pads each member (atrim, volume,+ * adelay, `apad` — RULES 6), writing PCM instead of AAC so the* result can feed straight into `applyAudioFxChain` (which reads raw WAV* samples, not a decoded container).*