Skip to content

Write HTML. Render video. Built for agents.

fix(engine): avoid padding every audio mix input (#5563)

Ayodele Ayoola3de4763d53de
diff --git a/packages/engine/src/services/audioMixer.grouping.test.ts b/packages/engine/src/services/audioMixer.grouping.test.ts
index b8741ad644..3a66131bcd 100644
--- a/packages/engine/src/services/audioMixer.grouping.test.ts
+++ b/packages/engine/src/services/audioMixer.grouping.test.ts
@@ -9,11 +9,10 @@ import { MIXED_AUDIO_FILENAME, parseAudioElements, processCompositionAudio } fro
/**
* Level arithmetic across the mix graph.
*
- * `mixAudioTracks` corrects for amix's own normalisation with a single global
- * `masterOutputGain * tracks.length`. That correction is exact for one flat
- * amix and wrong for any other shape — and it fails SILENTLY, in the export
- * rather than in preview, because preview mixes through Web Audio and never
- * runs this graph at all.
+ * `mixAudioTracks` uses `amix=normalize=0` where supported. Older FFmpeg
+ * builds fall back to padding every input and correcting by the input count.
+ * The correction must use each mix's own count, or it fails silently in the
+ * export while preview continues to mix through Web Audio.
*
* Measured on a four-track mix (spike, 2026-08-14): keeping the global track
* count on an outer amix that only has three inputs lands the whole mix
@@ -145,15 +144,12 @@ describe.skipIf(!HAS_FFMPEG)(
});
it("does not lift the survivors when a shorter track ends", async () => {
- // amix with normalize=true rescales by the number of CURRENTLY ACTIVE
- // inputs, so a track ending mid-composition would hand the remaining ones
- // a level jump. `apad` to the full duration is what holds every input
- // active for the whole graph and neutralises that — an invariant the
- // filter string relies on without saying so.
+ // A track ending mid-composition must not change the survivors' level.
+ // The main mix disables amix normalization; the legacy fallback pads all
+ // inputs so its default normalization keeps a constant divisor.
//
// Measured without apad (spike, 2026-08-14): the tail runs +1.94 dB hot.
- // Any group work that builds its own amix has to keep the padding, or
- // inherit that bug one level down.
+ // Any grouped mix that uses default normalization still needs padding.
const projectDir = mkdtempSync(join(tmpdir(), "hf-grp-drop-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-grp-drop-work-"));
tempDirs.push(projectDir, workDir);
diff --git a/packages/engine/src/services/audioMixer.test.ts b/packages/engine/src/services/audioMixer.test.ts
index 612e69ebe0..8858349eac 100644
--- a/packages/engine/src/services/audioMixer.test.ts
+++ b/packages/engine/src/services/audioMixer.test.ts
@@ -1,6 +1,6 @@
// fallow-ignore-file code-duplication
import { afterEach, describe, expect, it, vi } from "vitest";
-import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { tmpdir } from "node:os";
@@ -366,10 +366,10 @@ describe("processCompositionAudio", () => {
const filter = capturedFilterScripts[1];
expect(filter).toContain("volume=0");
- expect(filter).toContain("[mixed]volume=1[out]");
+ expect(filter).toContain("[mixed]volume=1,apad,asetpts=N/SR/TB,atrim=0:2[out]");
expect(filter).toContain("apad,asetpts=N/SR/TB,atrim=0:2");
expect(filter).not.toContain("whole_dur");
- expect(filter).not.toContain("normalize=");
+ expect(filter).toContain("normalize=0");
expect(filter).not.toContain("weights=");
});
@@ -752,7 +752,7 @@ describe("processCompositionAudio", () => {
expect(filter).toContain("atrim=0:2,");
});
- it("compensates amix normalization so multi-track master gain equals track count", async () => {
+ it("sums unpadded tracks at authored gain and pads the mixed output once", async () => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
@@ -807,9 +807,66 @@ describe("processCompositionAudio", () => {
const filter = capturedFilterScripts[3];
expect(filter).toContain("amix=inputs=3");
- expect(filter).not.toContain("normalize=");
- // masterOutputGain(1) × tracks(3) = 3
- expect(filter).toContain("[mixed]volume=3[out]");
+ expect(filter).toContain("amix=inputs=3:duration=longest:dropout_transition=0:normalize=0");
+ expect(filter).toContain("[mixed]volume=1,apad,asetpts=N/SR/TB,atrim=0:2[out]");
+ expect((filter.match(/apad,/g) ?? []).length).toBe(1);
+ });
+
+ it("retries the padded graph when ffmpeg does not support amix normalize", async () => {
+ const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
+ const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
+ tempDirs.push(baseDir, workDir);
+ writeFileSync(join(baseDir, "voice.wav"), "stub");
+
+ runFfmpegMock
+ .mockImplementationOnce(async () => ({
+ success: true,
+ durationMs: 1,
+ stderr: "",
+ exitCode: 0,
+ }))
+ .mockImplementationOnce(async (args: string[]) => {
+ capturedFilterScripts.push(
+ readFileSync(args[args.indexOf("-filter_complex_script") + 1]!, "utf8"),
+ );
+ return {
+ success: false,
+ durationMs: 1,
+ stderr: "Error applying option 'normalize' to filter 'amix': Option not found",
+ exitCode: 1,
+ };
+ })
+ .mockImplementationOnce(async (args: string[]) => {
+ capturedFilterScripts.push(
+ readFileSync(args[args.indexOf("-filter_complex_script") + 1]!, "utf8"),
+ );
+ return { success: true, durationMs: 1, stderr: "", exitCode: 0 };
+ });
+
+ const result = await processCompositionAudio(
+ [
+ {
+ id: "voice",
+ src: "voice.wav",
+ start: 0,
+ end: 2,
+ mediaStart: 0,
+ layer: 0,
+ volume: 1,
+ type: "audio",
+ },
+ ],
+ baseDir,
+ workDir,
+ join(baseDir, "out.m4a"),
+ 2,
+ );
+
+ expect(result.success).toBe(true);
+ expect(runFfmpegMock).toHaveBeenCalledTimes(3);
+ expect(capturedFilterScripts[0]).toContain("normalize=0");
+ expect(capturedFilterScripts[1]).toContain("apad,asetpts=N/SR/TB,atrim=0:2[a0]");
+ expect(capturedFilterScripts[1]).not.toContain("normalize=");
});
it("fails the audio result instead of silently mixing after one track preparation fails", async () => {
@@ -1268,22 +1325,17 @@ describe("processCompositionAudio", () => {
const filter = capturedFilterScripts.at(-1);
expect(filter).toContain(`amix=inputs=${trackCount}`);
- // Each track is trimmed once to its authored clip and once after portable
- // indefinite `apad` to cap the padded stream at composition duration.
- expect((filter?.match(/atrim=/g) ?? []).length).toBe(trackCount * 2);
- expect((filter?.match(/apad,/g) ?? []).length).toBe(trackCount);
+ // Each track is trimmed to its authored clip; only the mixed output is padded.
+ expect((filter?.match(/atrim=/g) ?? []).length).toBe(trackCount + 1);
+ expect((filter?.match(/apad,/g) ?? []).length).toBe(1);
// 150 real files + 150 existence checks: ~60ms on Linux, but well past the 5s
// default on the Windows lane, where each file create is orders slower.
}, 30_000);
- it("renumbers timestamps between apad and atrim on every mixed branch", async () => {
- // Regression: `apad` then `atrim` is the portable pad-to-length shape --
+ it("renumbers timestamps between apad and atrim on the mixed output", async () => {
// #2769 moved off `apad=whole_dur=` because some builds reject that option.
- // But on FFmpeg 5.x-8.0.x the padded samples carry timestamps `atrim`
- // misreads, so a delayed branch sounds at t=0 and, past three branches, the
- // last one vanishes from the mix. `asetpts=N/SR/TB` between the two rebuilds
- // the timestamps from the sample count and costs no portability, since all
- // three filters exist in every build we support.
+ // Keep the portable apad/asetpts/atrim shape on the mixed output so the
+ // final stream has the requested duration and consistent timestamps.
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
@@ -1322,10 +1374,8 @@ describe("processCompositionAudio", () => {
const filter = capturedFilterScripts.at(-1) ?? "";
const branches = filter.match(/apad[^;]*/g) ?? [];
- expect(branches).toHaveLength(2);
- for (const branch of branches) {
- expect(branch).toMatch(/^apad,asetpts=N\/SR\/TB,atrim=0:/);
- }
+ expect(branches).toHaveLength(1);
+ expect(branches[0]).toMatch(/^apad,asetpts=N\/SR\/TB,atrim=0:/);
// The portability constraint #2769 established still holds.
expect(filter).not.toContain("whole_dur");
});
diff --git a/packages/engine/src/services/audioMixer.ts b/packages/engine/src/services/audioMixer.ts
index 7937bf51c7..585c3f182d 100644
--- a/packages/engine/src/services/audioMixer.ts
+++ b/packages/engine/src/services/audioMixer.ts
@@ -326,16 +326,18 @@ export function buildFadeFilters(
return stages.length ? `,${stages.join(",")}` : "";
}
-/** One track: trim, volume, afade, delay/pad. asetpts after apad so FFmpeg 5-8 delayed branches do not land at t=0. */
+/** One track: trim, volume, afade, delay. Legacy and group mixes also pad each branch. */
export function buildTrackInputFilter(
track: Pick<AudioTrack, "start" | "end" | "fadeIn" | "fadeOut" | "tailSeconds">,
index: number,
volumeFilter: string,
totalDuration: number,
+ padToDuration = true,
): string {
const delayMs = Math.round(track.start * 1000);
const trimDuration = track.end - track.start + (track.tailSeconds ?? 0);
- return `[${index}:a]atrim=0:${formatFilterNumber(trimDuration)},${volumeFilter}${buildFadeFilters(track)},adelay=${delayMs}|${delayMs},apad,asetpts=N/SR/TB,atrim=0:${formatFilterNumber(totalDuration)}[a${index}]`;
+ const filters = `[${index}:a]atrim=0:${formatFilterNumber(trimDuration)},${volumeFilter}${buildFadeFilters(track)},adelay=${delayMs}|${delayMs}`;
+ return `${filters}${padToDuration ? `,apad,asetpts=N/SR/TB,atrim=0:${formatFilterNumber(totalDuration)}` : ""}[a${index}]`;
}
interface ExtractResult {
@@ -777,19 +779,21 @@ async function mixAudioTracks(
const outputDir = dirname(outputPath);
if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true });
- const buildFilterComplex = (ignoreAutomation: boolean): string => {
+ const buildFilterComplex = (ignoreAutomation: boolean, useNormalize: boolean): string => {
const filterParts: string[] = [];
tracks.forEach((track, i) => {
const volumeFilter = buildVolumeExpression(track, ignoreAutomation);
- filterParts.push(buildTrackInputFilter(track, i, volumeFilter, totalDuration));
+ filterParts.push(buildTrackInputFilter(track, i, volumeFilter, totalDuration, !useNormalize));
});
const mixInputs = tracks.map((_, i) => `[a${i}]`).join("");
- const mixFilter = `${mixInputs}amix=inputs=${tracks.length}:duration=longest:dropout_transition=0[mixed]`;
- // amix divides output by inputs count (default normalize=true). Multiply master
- // gain by track count so per-track volumes authored in data-volume are preserved.
- const compensatedGain = masterOutputGain * tracks.length;
- const postMixGainFilter = `[mixed]volume=${formatFilterNumber(compensatedGain)}[out]`;
+ const mixFilter = `${mixInputs}amix=inputs=${tracks.length}:duration=longest:dropout_transition=0${useNormalize ? ":normalize=0" : ""}[mixed]`;
+ // Older FFmpeg builds lack normalize=0. Their default amix divides by the
+ // active input count, so keep all branches padded and compensate by N.
+ const gain = masterOutputGain * (useNormalize ? 1 : tracks.length);
+ const postMixGainFilter = useNormalize
+ ? `[mixed]volume=${formatFilterNumber(gain)},apad,asetpts=N/SR/TB,atrim=0:${formatFilterNumber(totalDuration)}[out]`
+ : `[mixed]volume=${formatFilterNumber(gain)}[out]`;
return [...filterParts, mixFilter, postMixGainFilter].join(";");
};
@@ -804,14 +808,14 @@ async function mixAudioTracks(
// alias from nightly builds; older stable builds do not understand the new
// spelling. Prefer the legacy spelling for broad compatibility and retry
// only when FFmpeg explicitly says that option is unavailable.
- const runMix = async (ignoreAutomation: boolean) => {
+ const runMix = async (ignoreAutomation: boolean, useNormalize: boolean) => {
const inputs: string[] = [];
tracks.forEach((track) => inputs.push("-i", track.srcPath));
const scriptDir = mkdtempSync(join(outputDir, ".filter-complex-"));
const scriptPath = join(scriptDir, "graph.txt");
const fd = openSync(scriptPath, "wx", 0o600);
try {
- writeFileSync(fd, buildFilterComplex(ignoreAutomation));
+ writeFileSync(fd, buildFilterComplex(ignoreAutomation, useNormalize));
} finally {
closeSync(fd);
}
@@ -840,7 +844,17 @@ async function mixAudioTracks(
}
};
- let result = await runMix(false);
+ let useNormalize = true;
+ let result = await runMix(false, useNormalize);
+ if (
+ !result.success &&
+ result.failureReason !== "external_interruption" &&
+ !signal?.aborted &&
+ groupNormalizeOptionUnsupported(result.stderr)
+ ) {
+ useNormalize = false;
+ result = await runMix(false, useNormalize);
+ }
// Defense in depth: volume automation is folded into an FFmpeg `volume`
// expression whose evaluator limits are build-dependent (see
@@ -855,7 +869,7 @@ async function mixAudioTracks(
!signal?.aborted &&
hasAutomation
) {
- const retry = await runMix(true);
+ const retry = await runMix(true, useNormalize);
if (retry.success) {
result = retry;
degradedAutomation = true;
@@ -919,8 +933,8 @@ function groupNormalizeOptionUnsupported(stderr: string): boolean {
/**
* Sub-mix one group's members into a single PCM WAV at full composition
- * length — the same per-input treatment `mixAudioTracks` gives every track
- * (atrim, volume, adelay, `apad` — RULES 6), writing PCM instead of AAC so the
+ * length. Unlike the main mix, this still pads each member (atrim, volume,
+ * adelay, `apad` — RULES 6), writing PCM instead of AAC so the
* result can feed straight into `applyAudioFxChain` (which reads raw WAV
* samples, not a decoded container).
*