Skip to content

Write HTML. Render video. Built for agents.

fix(engine): a sharp sound in a film's first moments no longer grows louder with each audio pass (#5956)

Miguel Ángel87c6e50f5ea9
diff --git a/packages/engine/src/index.ts b/packages/engine/src/index.ts
index 0550b6acec..c2a7e2f8cc 100644
--- a/packages/engine/src/index.ts
+++ b/packages/engine/src/index.ts
@@ -360,7 +360,7 @@ export {
type RunFfmpegOptions,
type RunFfmpegResult,
} from "./utils/runFfmpeg.js";
-export { aacSegmentCount } from "./services/aacEncode.js";
+export { aacSegmentCount, encodeAac } from "./services/aacEncode.js";
export {
ManagedChildProcess,
type ManagedChildProcessOptions,
diff --git a/packages/engine/src/services/aacEncode.test.ts b/packages/engine/src/services/aacEncode.test.ts
index abb13a5a1c..5964c68897 100644
--- a/packages/engine/src/services/aacEncode.test.ts
+++ b/packages/engine/src/services/aacEncode.test.ts
@@ -1,5 +1,5 @@
import { spawnSync } from "node:child_process";
-import { mkdtempSync, rmSync } from "node:fs";
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterAll, describe, expect, it } from "vitest";
@@ -55,6 +55,38 @@ function errorStats(source: Float32Array, coded: Float32Array, cutFrames: number
return { rms: Math.sqrt(sum / source.length), peakNearCut, peakElsewhere };
}
+/**
+ * Half a second of stereo: a quiet noise floor from the first sample, then a sharp two-tone
+ * onset. Shaped like a film whose sound opens softly; `seed` varies only the floor.
+ */
+function quietThenOnset(seed: number, f0: number): Float32Array {
+ const n = 24_000;
+ const onset = 2738;
+ const channel = (phase: [number, number], peak: number) => {
+ const tone = Array.from({ length: n }, (_, i) => {
+ const t = Math.max(0, (i - onset) / 48_000);
+ const envelope = Math.min(1, Math.max(0, (i - onset) / 27)) * Math.exp(-1.25 * t);
+ const f = 2 * Math.PI * f0 * t;
+ return envelope * (Math.sin(f + phase[0]) + 0.6 * Math.sin(2 * f + phase[1]));
+ });
+ const max = Math.max(...tone.map(Math.abs));
+ return tone.map((v) => (peak * v) / max);
+ };
+ const left = channel([1, 0], 0.35 * 0.25);
+ const right = channel([0.5, 2], 0.25);
+ const out = new Float32Array(n * 2);
+ let state = seed >>> 0;
+ for (let i = 0; i < 2 * n; i++) {
+ state = (Math.imul(state, 1664525) + 1013904223) >>> 0;
+ const floor = (state / 2 ** 31 - 1) * 10 ** (-70 / 20) * Math.min(1, (i >> 1) / 1500);
+ out[i] = (i % 2 ? right : left)[i >> 1]! + floor;
+ }
+ return out;
+}
+
+const peakOfFirst = (signal: Float32Array, frames: number) =>
+ signal.subarray(0, frames * 2).reduce((peak, v) => Math.max(peak, Math.abs(v)), 0);
+
describe("aacSegmentCount", () => {
it("keeps audio under two minutes in one encoder", () => {
expect(aacSegmentCount(119)).toBe(1);
@@ -115,6 +147,57 @@ describe.skipIf(!HAS_FFMPEG)("encodeAac", () => {
expect(streamEnd(split)).toBe(streamEnd(single));
}, 120_000);
+ it("keeps a sharp onset at its level when the audio opens quietly, encode after encode", async () => {
+ const growthDb: number[] = [];
+ for (const f0 of [250, 306, 360]) {
+ for (let seed = 1; seed <= 12; seed++) {
+ const raw = join(dir, "onset.f32");
+ writeFileSync(raw, quietThenOnset(seed, f0));
+ const source = join(dir, "onset.wav");
+ ffmpeg(["-f", "f32le", "-ar", "48000", "-ac", "2", "-i", raw, "-y", source]);
+ let input = source;
+ for (const generation of [1, 2]) {
+ const coded = join(dir, `onset-${generation}.m4a`);
+ expect((await encodeAac(input, coded, 0.5, { timeout: 60_000 })).success).toBe(true);
+ input = join(dir, `onset-${generation}.wav`);
+ ffmpeg(["-i", coded, "-c:a", "pcm_f32le", "-y", input]);
+ }
+ const before = peakOfFirst(decode(source), 4800);
+ growthDb.push(20 * Math.log10(peakOfFirst(decode(input), 4800) / before));
+ }
+ }
+ // Measured on ffmpeg 7.0.2 and 8.0.1: at most +0.1 dB. Opening cold, 3 of these grew 2.7-3.3 dB.
+ const report = growthDb.map((db) => db.toFixed(2)).join(" ");
+ expect(Math.max(...growthDb), `peak growth per signal, dB: ${report}`).toBeLessThan(1);
+ }, 120_000);
+
+ it("keeps a tempo-changed 44.1 kHz track at lag 0 against the same filter uncoded", async () => {
+ const source = join(dir, "rate.wav");
+ ffmpeg(["-f", "lavfi", "-i", "anoisesrc=d=6:c=pink:r=44100:a=0.3", "-ac", "2", "-y", source]);
+ const output = join(dir, "rate.m4a");
+ const encoded = await encodeAac(source, output, 4, { timeout: 60_000, filter: "atempo=1.5" });
+ expect(encoded.success).toBe(true);
+
+ const mono = (stereo: Float32Array) =>
+ Float32Array.from(
+ { length: stereo.length / 2 },
+ (_, i) => stereo[2 * i]! + stereo[2 * i + 1]!,
+ );
+ const reference = mono(decode(source, "atempo=1.5,aresample=48000"));
+ const coded = mono(decode(output));
+ const correlationAt = (lag: number) => {
+ let [dot, rr, cc] = [0, 0, 0];
+ for (let i = 72_000; i < 96_000; i++) {
+ const [r, c] = [reference[i]!, coded[i + lag] ?? 0];
+ [dot, rr, cc] = [dot + r * c, rr + r * r, cc + c * c];
+ }
+ return dot / Math.sqrt(rr * cc);
+ };
+ const scores = Array.from({ length: 4097 }, (_, k) => correlationAt(k - 2048));
+ expect(scores.indexOf(Math.max(...scores)) - 2048).toBe(0);
+ expect(correlationAt(0)).toBeGreaterThan(0.95);
+ }, 60_000);
+
it("leaves no click at the joins of a pure tone", async () => {
const source = join(dir, "tone.wav");
ffmpeg([
diff --git a/packages/engine/src/services/aacEncode.ts b/packages/engine/src/services/aacEncode.ts
index 12289330f9..898070ae37 100644
--- a/packages/engine/src/services/aacEncode.ts
+++ b/packages/engine/src/services/aacEncode.ts
@@ -12,11 +12,11 @@ const FRAME_SAMPLES = 1024;
const MIN_SEGMENT_SECONDS = 60;
// Fixed rather than the core count, so the bytes (and the planHash) do not depend on the machine.
const MAX_SEGMENTS = 8;
-// Encoded on each side of a segment and dropped. The encoder's rate control takes about
-// 10 s to settle (its error is higher until then), so a segment starts 10 s before its cut.
+// Encoded around each segment and dropped: rate control settles over ~10 s, and opening cold on
+// quiet sound codes the first loud frame up to 4.5 dB hot, so the first segment opens in silence.
const PRE_ROLL_FRAMES = 469;
const POST_ROLL_FRAMES = 8;
-export const AAC_ARGS = ["-acodec", "aac", "-b:a", "192k"];
+const AAC_ARGS = ["-acodec", "aac", "-b:a", "192k"];
export function aacSegmentCount(totalDuration: number): number {
return Math.max(1, Math.min(MAX_SEGMENTS, Math.floor(totalDuration / MIN_SEGMENT_SECONDS)));
@@ -45,26 +45,28 @@ function planSegments(totalDuration: number, count: number) {
const cuts = Array.from({ length: count + 1 }, (_, k) => Math.round((k * packets) / count));
return cuts.slice(0, -1).map((cut, k) => {
const next = cuts[k + 1]!;
- const firstFrame = Math.max(0, cut - PRE_ROLL_FRAMES);
- const end = k === count - 1 ? "" : `:end_sample=${(next + POST_ROLL_FRAMES) * FRAME_SAMPLES}`;
- const trim = `atrim=start_sample=${firstFrame * FRAME_SAMPLES}${end}`;
+ const firstFrame = cut - PRE_ROLL_FRAMES;
+ const end = k === count - 1 ? "" : `end_sample=${(next + POST_ROLL_FRAMES) * FRAME_SAMPLES}`;
+ const trim =
+ firstFrame < 0
+ ? `${end && `atrim=${end},`}adelay=delays=${-firstFrame * FRAME_SAMPLES}S:all=1`
+ : `atrim=start_sample=${firstFrame * FRAME_SAMPLES}${end && `:${end}`}`;
return { trim, keepFrom: cut - firstFrame, keepTo: next - firstFrame };
});
}
-/** `input` is a 48 kHz WAV of exactly `totalDuration`. */
+/** `input`, after `filter`, is audio of exactly `totalDuration`. */
export async function encodeAac(
input: string,
outputPath: string,
totalDuration: number,
- { signal, timeout }: { signal?: AbortSignal; timeout: number },
+ {
+ signal,
+ timeout,
+ filter,
+ segments: count = aacSegmentCount(totalDuration),
+ }: { signal?: AbortSignal; timeout: number; filter?: string; segments?: number },
): Promise<RunFfmpegResult> {
- const count = aacSegmentCount(totalDuration);
- if (count === 1) {
- const args = ["-i", input, "-t", String(totalDuration), ...AAC_ARGS, "-y", outputPath];
- return runFfmpeg(args, { signal, timeout });
- }
-
const dir = mkdtempSync(join(dirname(outputPath), ".aac-segments-"));
try {
const segments = planSegments(totalDuration, count).map((segment, k) => ({
@@ -73,10 +75,23 @@ export async function encodeAac(
}));
const results = await Promise.all(
segments.map(({ trim, path }) =>
- runFfmpeg(["-i", input, "-af", trim, ...AAC_ARGS, "-f", "adts", "-y", path], {
- signal,
- timeout,
- }),
+ runFfmpeg(
+ [
+ "-i",
+ input,
+ "-af",
+ [filter, `aresample=${SAMPLE_RATE}`, trim].filter(Boolean).join(","),
+ ...AAC_ARGS,
+ "-f",
+ "adts",
+ "-y",
+ path,
+ ],
+ {
+ signal,
+ timeout,
+ },
+ ),
),
);
const failed = results.find((result) => !result.success);
diff --git a/packages/engine/src/services/audioMixer.order.test.ts b/packages/engine/src/services/audioMixer.order.test.ts
index 88be132bbb..8aebe374a1 100644
--- a/packages/engine/src/services/audioMixer.order.test.ts
+++ b/packages/engine/src/services/audioMixer.order.test.ts
@@ -9,6 +9,13 @@ vi.mock("../utils/runFfmpeg.js", async (importOriginal) => ({
...(await importOriginal<typeof import("../utils/runFfmpeg.js")>()),
runFfmpeg: runFfmpegMock,
}));
+
+// The AAC encode is aacEncode.test.ts's subject; here the mix only has to hand it the float sum.
+vi.mock("./aacEncode.js", async (importOriginal) => ({
+ ...(await importOriginal<typeof import("./aacEncode.js")>()),
+ encodeAac: vi.fn(async () => ({ success: true, durationMs: 1, stderr: "", exitCode: 0 })),
+}));
+
vi.mock("../utils/ffprobe.js", () => ({
extractAudioMetadata: vi.fn(async () => ({ durationSeconds: 1, channels: 2 })),
}));
diff --git a/packages/engine/src/services/audioMixer.test.ts b/packages/engine/src/services/audioMixer.test.ts
index 8858349eac..624e3fdb67 100644
--- a/packages/engine/src/services/audioMixer.test.ts
+++ b/packages/engine/src/services/audioMixer.test.ts
@@ -41,6 +41,12 @@ vi.mock("../utils/runFfmpeg.js", async (importOriginal) => {
return { ...actual, runFfmpeg: runFfmpegMock };
});
+// The AAC encode is aacEncode.test.ts's subject; here the mix only has to hand it the float sum.
+vi.mock("./aacEncode.js", async (importOriginal) => ({
+ ...(await importOriginal<typeof import("./aacEncode.js")>()),
+ encodeAac: vi.fn(async () => ({ success: true, durationMs: 1, stderr: "", exitCode: 0 })),
+}));
+
// The FX render drives a headless browser; the mix only needs to know the
// processed file exists and how long a tail the chain asked for.
const { applyAudioFxChainMock } = vi.hoisted(() => ({
diff --git a/packages/engine/src/services/audioMixer.ts b/packages/engine/src/services/audioMixer.ts
index e33baaf377..1815c7bbdc 100644
--- a/packages/engine/src/services/audioMixer.ts
+++ b/packages/engine/src/services/audioMixer.ts
@@ -15,7 +15,7 @@ import {
} from "../utils/urlDownloader.js";
import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
import { formatFfmpegError, runFfmpeg, type RunFfmpegResult } from "../utils/runFfmpeg.js";
-import { AAC_ARGS, aacSegmentCount, encodeAac } from "./aacEncode.js";
+import { encodeAac } from "./aacEncode.js";
import { audioSubprocessEvidence } from "./audioFailureDiagnostic.js";
import { unwrapTemplate } from "../utils/htmlTemplate.js";
import { resolveMediaElementSrc, resolveProjectRelativeSrc } from "./videoFrameExtractor.js";
@@ -762,7 +762,6 @@ async function mixAudioTracks(
totalDuration: number,
signal?: AbortSignal,
config?: Partial<Pick<EngineConfig, "ffmpegProcessTimeout" | "audioGain">>,
- floatWav = false,
): Promise<MixResult> {
const ffmpegProcessTimeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout;
const masterOutputGain = config?.audioGain ?? DEFAULT_CONFIG.audioGain;
@@ -828,7 +827,10 @@ async function mixAudioTracks(
scriptPath,
"-map",
"[out]",
- ...(floatWav ? ["-acodec", "pcm_f32le", "-ar", "48000"] : AAC_ARGS),
+ "-acodec",
+ "pcm_f32le",
+ "-ar",
+ "48000",
"-t",
String(totalDuration),
"-y",
@@ -1073,7 +1075,7 @@ async function mixThroughFloatSum(
config?: Partial<Pick<EngineConfig, "ffmpegProcessTimeout" | "audioGain">>,
): Promise<MixResult> {
const sumPath = join(workDir, "master-sum.wav");
- const sum = await mixAudioTracks(tracks, sumPath, totalDuration, signal, config, true);
+ const sum = await mixAudioTracks(tracks, sumPath, totalDuration, signal, config);
if (!sum.success) return { ...sum, outputPath };
const encoderInput = masterChain
@@ -1609,7 +1611,7 @@ export async function processCompositionAudio(
let mixResult: MixResult;
try {
mixResult =
- (masterChain || aacSegmentCount(totalDuration) > 1) && tracks.length > 0
+ tracks.length > 0
? await mixThroughFloatSum(
tracks,
masterChain,
diff --git a/packages/producer/src/services/render/audioPadTrim.test.ts b/packages/producer/src/services/render/audioPadTrim.test.ts
index c5894f0c2e..bc4de97fa1 100644
--- a/packages/producer/src/services/render/audioPadTrim.test.ts
+++ b/packages/producer/src/services/render/audioPadTrim.test.ts
@@ -3,12 +3,12 @@
* `assemble()` step's audio duration correction.
*
* The helper is split into:
- * - `buildPadTrimAudioArgs(...)` — pure function, builds the ffmpeg argv
+ * - `buildPadTrimAudioPlan(...)` — pure function, builds the steps
* and labels the operation. Fully unit-testable.
* - `padOrTrimAudioToVideoFrameCount(...)` — wrapper that probes the
* video for frame count + fps, probes the audio for current duration,
- * and runs ffmpeg with the args from the pure helper. Tests inject
- * stubs for the probes and the ffmpeg runner.
+ * and runs the plan's steps. Tests inject stubs for the probes, the
+ * ffmpeg runner and the AAC encoder.
*
* No real ffmpeg/ffprobe runs in these tests.
*/
@@ -18,7 +18,6 @@ import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import {
- buildPadTrimAudioArgs,
buildPadTrimAudioPlan,
padOrTrimAudioToVideoFrameCount,
type AudioProbeInfo,
@@ -26,73 +25,55 @@ import {
type ProbeVideoFrameInfo,
} from "./audioPadTrim.js";
-describe("buildPadTrimAudioArgs", () => {
- it("emits a decode/filter/re-encode pad plan when audio is shorter than target", () => {
+describe("buildPadTrimAudioPlan", () => {
+ const filterOf = (plan: ReturnType<typeof buildPadTrimAudioPlan>) => {
+ const step = plan.steps[0]!;
+ return step.kind === "copy" ? undefined : step.filter;
+ };
+
+ it("pads to the exact target when audio is shorter", () => {
const plan = buildPadTrimAudioPlan("/tmp/in.aac", "/tmp/out.aac", 4.0, 5.0);
expect(plan.operation).toBe("pad");
expect(plan.steps).toHaveLength(1);
- const args = plan.steps[0]!.args;
- expect(args[args.indexOf("-i") + 1]).toBe("/tmp/in.aac");
- expect(args[args.indexOf("-af") + 1]).toBe("apad,atrim=0:5.000000");
- expect(args.join(" ")).not.toContain("whole_dur");
- expect(args[args.indexOf("-t") + 1]).toBe("5.000000");
- expect(args[args.indexOf("-c:a") + 1]).toBe("aac");
+ expect(plan.steps[0]!.kind).toBe("normalize");
+ expect(filterOf(plan)).toBe("apad,atrim=0:5.000000");
+ expect(filterOf(plan)).not.toContain("whole_dur");
// The single-step filter plan has no intermediate artifacts to clean up.
expect(plan.cleanupPaths).toEqual([]);
});
- it("keeps the legacy args helper on the first pad materialization step", () => {
- const { args, operation } = buildPadTrimAudioArgs("/tmp/in.aac", "/tmp/out.aac", 4.0, 5.0);
- expect(operation).toBe("pad");
- expect(args).toContain("/tmp/in.aac");
- expect(args[args.indexOf("-t") + 1]).toBe("5.000000");
- });
-
- it("filter-trims and re-encodes AAC packet padding beyond the target", () => {
- const { args, operation } = buildPadTrimAudioArgs(
- "/tmp/in.aac",
- "/tmp/out.m4a",
- 15.018667,
- 15.0,
- );
- expect(operation).toBe("trim");
- const filterIdx = args.indexOf("-af");
- expect(args[filterIdx + 1]).toBe("atrim=duration=15.000000,asetpts=PTS-STARTPTS");
- expect(args[args.indexOf("-t") + 1]).toBe("15.000000");
- const codecIdx = args.indexOf("-c:a");
- expect(args[codecIdx + 1]).toBe("aac");
- expect(args[args.indexOf("-b:a") + 1]).toBe("192k");
- expect(args.at(-1)).toBe("/tmp/out.m4a");
+ it("filter-trims AAC packet padding beyond the target", () => {
+ const plan = buildPadTrimAudioPlan("/tmp/in.aac", "/tmp/out.m4a", 15.018667, 15.0);
+ expect(plan.operation).toBe("trim");
+ expect(filterOf(plan)).toBe("atrim=duration=15.000000,asetpts=PTS-STARTPTS");
});
it("emits a plain copy when source duration matches target within ~1ms", () => {
- const { args, operation } = buildPadTrimAudioArgs("/tmp/in.aac", "/tmp/out.aac", 5.0, 5.0);
- expect(operation).toBe("copy");
- expect(args.indexOf("-af")).toBe(-1);
- expect(args.indexOf("-t")).toBe(-1);
- const codecIdx = args.indexOf("-c:a");
- expect(args[codecIdx + 1]).toBe("copy");
+ const plan = buildPadTrimAudioPlan("/tmp/in.aac", "/tmp/out.aac", 5.0, 5.0);
+ expect(plan.operation).toBe("copy");
+ expect(plan.steps).toEqual([
+ { kind: "copy", args: ["-i", "/tmp/in.aac", "-c:a", "copy", "-y", "/tmp/out.aac"] },
+ ]);
});
it("emits 6-decimal-place pad duration (no scientific notation)", () => {
// 1.23ms — just over the AUDIO_DURATION_TOLERANCE_SECONDS=1ms threshold,
// so we exercise the pad path with a tiny duration that would round to
// exponent notation if we used `toString()` instead of `toFixed(6)`.
- const { args, operation } = buildPadTrimAudioArgs("/tmp/in.aac", "/tmp/out.aac", 0.0, 0.00123);
- expect(operation).toBe("pad");
- const tIdx = args.indexOf("-t");
- expect(args[tIdx + 1]).toBe("0.001230");
+ const plan = buildPadTrimAudioPlan("/tmp/in.aac", "/tmp/out.aac", 0.0, 0.00123);
+ expect(plan.operation).toBe("pad");
+ expect(filterOf(plan)).toBe("apad,atrim=0:0.001230");
});
it("flags ~1ms drift as a copy (below the tolerance threshold)", () => {
- const close = buildPadTrimAudioArgs("/tmp/a.aac", "/tmp/o.aac", 5.0, 5.0005);
+ const close = buildPadTrimAudioPlan("/tmp/a.aac", "/tmp/o.aac", 5.0, 5.0005);
expect(close.operation).toBe("copy");
});
it("flags >1ms drift in either direction as pad/trim", () => {
- const padNeeded = buildPadTrimAudioArgs("/tmp/a.aac", "/tmp/o.aac", 5.0, 5.002);
+ const padNeeded = buildPadTrimAudioPlan("/tmp/a.aac", "/tmp/o.aac", 5.0, 5.002);
expect(padNeeded.operation).toBe("pad");
- const trimNeeded = buildPadTrimAudioArgs("/tmp/a.aac", "/tmp/o.aac", 5.002, 5.0);
+ const trimNeeded = buildPadTrimAudioPlan("/tmp/a.aac", "/tmp/o.aac", 5.002, 5.0);
expect(trimNeeded.operation).toBe("trim");
});
@@ -106,25 +87,21 @@ describe("buildPadTrimAudioArgs", () => {
5.0,
);
expect(winPlan.operation).toBe("pad");
- const args = winPlan.steps[0]!.args;
- expect(args).toContain("-af");
- expect(args[args.indexOf("-af") + 1]).toBe("apad,atrim=0:5.000000");
- expect(args.join(" ")).not.toContain("whole_dur");
+ expect(filterOf(winPlan)).toBe("apad,atrim=0:5.000000");
});
});
describe("padOrTrimAudioToVideoFrameCount", () => {
- // Build a minimal harness that stubs out the duration probes and ffmpeg runner.
+ type StepResult = { success: boolean; error?: string; failureReason?: "external_interruption" };
+ type Encode = { durationSeconds: number; filter: string };
+ // Build a minimal harness that stubs out the duration probes, the ffmpeg runner and the encoder.
function harness(opts: {
video: ProbeVideoFrameInfo | "throw";
audio: AudioProbeInfo | "throw";
- ffmpeg?: (args: string[]) => Promise<{
- success: boolean;
- error?: string;
- failureReason?: "external_interruption";
- }>;
- }): { input: PadTrimAudioInput; captured: { args: string[][] } } {
- const captured = { args: [] as string[][] };
+ step?: () => Promise<StepResult>;
+ }): { input: PadTrimAudioInput; captured: { args: string[][]; encodes: Encode[] } } {
+ const captured = { args: [] as string[][], encodes: [] as Encode[] };
+ const step = opts.step ?? (async () => ({ success: true }));
const input: PadTrimAudioInput = {
videoPath: "/tmp/v.mp4",
audioPath: "/tmp/a.aac",
@@ -137,12 +114,14 @@ describe("padOrTrimAudioToVideoFrameCount", () => {
if (opts.audio === "throw") throw new Error("audio probe boom");
return opts.audio;
},
- runFfmpeg:
- opts.ffmpeg ??
- (async (args) => {
- captured.args.push(args);
- return { success: true };
- }),
+ runFfmpeg: async (args) => {
+ captured.args.push(args);
+ return step();
+ },
+ encodeAac: async (_input, _output, durationSeconds, filter) => {
+ captured.encodes.push({ durationSeconds, filter });
+ return step();
+ },
};
return { input, captured };
}
@@ -175,10 +154,7 @@ describe("padOrTrimAudioToVideoFrameCount", () => {
expect(result.operation).toBe("pad");
expect(result.targetDurationSeconds).toBe(6);
expect(result.sourceDurationSeconds).toBe(5.5);
- expect(captured.args).toHaveLength(1);
- const tIdx = captured.args[0]!.indexOf("-t");
- expect(captured.args[0]![tIdx + 1]).toBe("6.000000");
- expect(captured.args[0]![captured.args[0]!.indexOf("-c:a") + 1]).toBe("aac");
+ expect(captured.encodes).toEqual([{ durationSeconds: 6, filter: "apad,atrim=0:6.000000" }]);
});
it("trims a video of N=120 frames at 30/1 fps with longer audio", async () => {
@@ -190,9 +166,9 @@ describe("padOrTrimAudioToVideoFrameCount", () => {
expect(result.success).toBe(true);
expect(result.operation).toBe("trim");
expect(result.targetDurationSeconds).toBe(4);
- expect(captured.args).toHaveLength(1);
- const filterIdx = captured.args[0]!.indexOf("-af");
- expect(captured.args[0]![filterIdx + 1]).toBe("atrim=duration=4.000000,asetpts=PTS-STARTPTS");
+ expect(captured.encodes).toEqual([
+ { durationSeconds: 4, filter: "atrim=duration=4.000000,asetpts=PTS-STARTPTS" },
+ ]);
});
it("emits a copy when audio duration already equals frameCount/fps", async () => {
@@ -206,6 +182,7 @@ describe("padOrTrimAudioToVideoFrameCount", () => {
expect(result.targetDurationSeconds).toBe(3);
expect(captured.args[0]!.includes("-af")).toBe(false);
expect(captured.args[0]!.includes("-t")).toBe(false);
+ expect(captured.encodes).toEqual([]);
});
it("handles NTSC fps (30000/1001) exactly", async () => {
@@ -218,8 +195,7 @@ describe("padOrTrimAudioToVideoFrameCount", () => {
expect(result.success).toBe(true);
expect(result.operation).toBe("pad");
expect(result.targetDurationSeconds).toBeCloseTo((120 * 1001) / 30000, 9);
- const tIdx = captured.args[0]!.indexOf("-t");
- expect(captured.args[0]![tIdx + 1]).toBe("4.004000");
+ expect(captured.encodes[0]!.filter).toBe("apad,atrim=0:4.004000");
});
it("propagates video probe failure as success=false", async () => {
@@ -268,7 +244,7 @@ describe("padOrTrimAudioToVideoFrameCount", () => {
const { input } = harness({
video: { frameCount: 180, fpsNum: 30, fpsDen: 1 },
audio: { durationSeconds: 5.0 },
- ffmpeg: async () => ({ success: false, error: "synthetic ffmpeg failure" }),
+ step: async () => ({ success: false, error: "synthetic ffmpeg failure" }),
});
const result = await padOrTrimAudioToVideoFrameCount(input);
expect(result.success).toBe(false);
@@ -281,7 +257,7 @@ describe("padOrTrimAudioToVideoFrameCount", () => {
const { input } = harness({
video: { frameCount: 180, fpsNum: 30, fpsDen: 1 },
audio: { durationSeconds: 5.0 },
- ffmpeg: async () => ({
+ step: async () => ({
success: false,
error: "synthetic interruption diagnostics",
failureReason: "external_interruption",
@@ -311,14 +287,14 @@ describe("padOrTrimAudioToVideoFrameCount", () => {
});
it("attenuates the duration-normalized artifact with AAC correction headroom", async () => {
- const calls: string[][] = [];
+ const calls: { durationSeconds: number; filter: string }[] = [];
const { input } = harness({
video: { frameCount: 90, fpsNum: 30, fpsDen: 1 },
audio: { durationSeconds: 3.029333 },
});
input.probeAudioTruePeakDbfs = async () => 1.5;
- input.runFfmpeg = async (args) => {
- calls.push(args);
+ input.encodeAac = async (_input, _output, durationSeconds, filter) => {
+ calls.push({ durationSeconds, filter });
return calls.length === 1
? { success: true }
: { success: false, error: "synthetic correction stop" };
@@ -329,9 +305,10 @@ describe("padOrTrimAudioToVideoFrameCount", () => {
expect(result.success).toBe(false);
expect(result.error).toBe("synthetic correction stop");
expect(calls).toHaveLength(2);
- const correctionArgs = calls[1]!;
- expect(correctionArgs[correctionArgs.indexOf("-af") + 1]).toBe("volume=-3.000dB");
- expect(correctionArgs[correctionArgs.indexOf("-t") + 1]).toBe("3.000000");
+ expect(calls[1]).toEqual({
+ durationSeconds: 3,
+ filter: "volume=-3.000dB,apad,atrim=0:3.000000",
+ });
});
it("keeps the end of a long correction failure, where ffmpeg says what went wrong", async () => {
@@ -343,7 +320,7 @@ describe("padOrTrimAudioToVideoFrameCount", () => {
const cause = "Error while filtering: the real ffmpeg cause";
let runs = 0;
input.probeAudioTruePeakDbfs = async () => 1.5;
- input.runFfmpeg = async () =>
+ input.encodeAac = async () =>
++runs === 1 ? { success: true } : { success: false, error: `[FFmpeg] ${banner}\n${cause}` };
const result = await padOrTrimAudioToVideoFrameCount(input);
@@ -365,13 +342,13 @@ describe("padOrTrimAudioToVideoFrameCount", () => {
input.audioPath = join(dir, "source.m4a");
input.outputPath = join(dir, "normalized.m4a");
input.probeAudioTruePeakDbfs = async () => attenuationDb * 0.4;
- input.runFfmpeg = async (args) => {
- const filter = args[args.indexOf("-af") + 1];
- if (filter?.startsWith("volume=")) {
- attenuationDb = Number(filter.slice("volume=".length, -2));
+ input.encodeAac = async (_input, output, _durationSeconds, filter) => {
+ const volume = /^volume=(-?[\d.]+)dB/.exec(filter);
+ if (volume) {
+ attenuationDb = Number(volume[1]);
corrections.push(attenuationDb);
}
- writeFileSync(args.at(-1)!, "");
+ writeFileSync(output, "");
return { success: true };
};
diff --git a/packages/producer/src/services/render/audioPadTrim.ts b/packages/producer/src/services/render/audioPadTrim.ts
index deda963538..59a5eb10fc 100644
--- a/packages/producer/src/services/render/audioPadTrim.ts
+++ b/packages/producer/src/services/render/audioPadTrim.ts
@@ -25,12 +25,14 @@ import { dirname, join } from "node:path";
import {
DEFAULT_CONFIG,
describeFfmpegFailure,
+ encodeAac,
extractAudioMetadata,
getFfprobeBinary,
ManagedChildProcess,
runFfmpeg,
trackChildProcess,
type AudioMetadata,
+ type RunFfmpegResult,
} from "@hyperframes/engine";
import { redactKnownPaths, redactTelemetryString } from "@hyperframes/core";
import { measureTruePeakDbfs } from "./audioTruePeak.js";
@@ -84,13 +86,24 @@ export interface PadTrimAudioInput {
probeVideoFrameInfo?: (videoPath: string) => Promise<ProbeVideoFrameInfo>;
probeAudioInfo?: (audioPath: string, signal?: AbortSignal) => Promise<AudioProbeInfo>;
probeAudioTruePeakDbfs?: (audioPath: string, signal?: AbortSignal) => Promise<number>;
- runFfmpeg?: (args: string[]) => Promise<{
- success: boolean;
- error?: string;
- failureReason?: "external_interruption";
- }>;
+ runFfmpeg?: (args: string[]) => Promise<StepResult>;
+ encodeAac?: AacEncoder;
}
+interface StepResult {
+ success: boolean;
+ error?: string;
+ failureReason?: "external_interruption";
+}
+
+/** Encodes `filter` applied to `inputPath` as AAC of exactly `durationSeconds`. */
+type AacEncoder = (
+ inputPath: string,
+ outputPath: string,
+ durationSeconds: number,
+ filter: string,
+) => Promise<StepResult>;
+
export type PadTrimOperation = "pad" | "trim" | "copy";
export interface PadTrimAudioResult {
@@ -110,12 +123,9 @@ export interface PadTrimAudioResult {
audioLoweredDb?: number;
}
-export type PadTrimAudioStepKind = "copy" | "trim" | "normalize";
-
-export interface PadTrimAudioStep {
- kind: PadTrimAudioStepKind;
- args: string[];
-}
+export type PadTrimAudioStep =
+ | { kind: "copy"; args: string[] }
+ | { kind: "trim" | "normalize"; filter: string };
export interface PadTrimAudioPlan {
operation: PadTrimOperation;
@@ -123,34 +133,13 @@ export interface PadTrimAudioPlan {
cleanupPaths: string[];
}
-function buildAacTruePeakCorrectionArgs(
- inputPath: string,
- outputPath: string,
- targetDurationSeconds: number,
- attenuationDb: number,
-): string[] {
- return [
- "-i",
- inputPath,
- "-map",
- "0:a:0",
- "-vn",
- "-af",
- `volume=${attenuationDb.toFixed(3)}dB`,
- "-t",
- formatSeconds(targetDurationSeconds),
- "-c:a",
- "aac",
- "-b:a",
- "192k",
- "-y",
- outputPath,
- ];
+function aacTruePeakCorrectionFilter(targetDurationSeconds: number, attenuationDb: number) {
+ return `volume=${attenuationDb.toFixed(3)}dB,apad,atrim=0:${formatSeconds(targetDurationSeconds)}`;
}
/**
- * Pure helper: decide the pad/trim operation and build the ffmpeg argv
- * sequence that materializes it. Exported separately so unit tests can pin
+ * Pure helper: decide the pad/trim operation and the steps that
+ * materialize it. Exported separately so unit tests can pin
* every branch without spawning ffmpeg.
*
* - `sourceDuration < targetDuration` → pad with `apad` to the exact target
@@ -180,25 +169,7 @@ export function buildPadTrimAudioPlan(
if (delta > 0) {
return {
operation: "pad",
- steps: [
- {
- kind: "normalize",
- args: [
- "-i",
- audioPath,
- "-af",
- `apad,atrim=0:${targetSec}`,
- "-t",
- targetSec,
- "-c:a",
- "aac",
- "-b:a",
- "192k",
- "-y",
- outputPath,
- ],
- },
- ],
+ steps: [{ kind: "normalize", filter: `apad,atrim=0:${targetSec}` }],
cleanupPaths: [],
};
}
@@ -207,48 +178,11 @@ export function buildPadTrimAudioPlan(
// into M4A so ffmpeg records the exact presentation duration.
return {
operation: "trim",
- steps: [
- {
- kind: "trim",
- args: [
- "-i",
- audioPath,
- "-af",
- `atrim=duration=${targetSec},asetpts=PTS-STARTPTS`,
- // `atrim` limits decoded samples but does not cap muxer timestamps
- // introduced by encoder delay/flush. The output duration contract
- // is enforced at the container boundary as well; without `-t`, a
- // long primed source can retain its original tail after re-encode.
- "-t",
- targetSec,
- "-c:a",
- "aac",
- "-b:a",
- "192k",
- "-y",
- outputPath,
- ],
- },
- ],
+ steps: [{ kind: "trim", filter: `atrim=duration=${targetSec},asetpts=PTS-STARTPTS` }],
cleanupPaths: [],
};
}
-export function buildPadTrimAudioArgs(
- audioPath: string,
- outputPath: string,
- sourceDurationSeconds: number,
- targetDurationSeconds: number,
-): { args: string[]; operation: PadTrimOperation } {
- const plan = buildPadTrimAudioPlan(
- audioPath,
- outputPath,
- sourceDurationSeconds,
- targetDurationSeconds,
- );
- return { operation: plan.operation, args: plan.steps[0]?.args ?? [] };
-}
-
/**
* Format a duration as a fixed-precision decimal string. ffmpeg parses
* scientific notation inconsistently across versions (some treat `1e-3` as
@@ -299,12 +233,13 @@ export async function padOrTrimAudioToVideoFrameCount(
const timeoutMs = input.timeoutMs ?? DEFAULT_CONFIG.ffmpegProcessTimeout;
const runner =
input.runFfmpeg ?? ((args: string[]) => defaultRunFfmpeg(args, timeoutMs, input.signal));
+ const encode = input.encodeAac ?? defaultAacEncoder(timeoutMs, input.signal);
// Injected runners usually do not materialize media. Tests that exercise
// peak correction opt in with a matching probe; production always uses the
// real post-codec measurement.
const probeTruePeak =
input.probeAudioTruePeakDbfs ??
- (input.runFfmpeg
+ (input.runFfmpeg || input.encodeAac
? undefined
: (audioPath: string, signal?: AbortSignal) =>
measureTruePeakDbfs(audioPath, { timeoutMs, signal }));
@@ -369,7 +304,10 @@ export async function padOrTrimAudioToVideoFrameCount(
try {
for (const step of plan.steps) {
- const ffmpegResult = await runner(step.args);
+ const ffmpegResult =
+ step.kind === "copy"
+ ? await runner(step.args)
+ : await encode(input.audioPath, input.outputPath, targetDurationSeconds, step.filter);
if (!ffmpegResult.success) {
return {
success: false,
@@ -404,7 +342,7 @@ export async function padOrTrimAudioToVideoFrameCount(
targetDurationSeconds,
signal: input.signal,
probeTruePeak,
- runner,
+ encode,
});
if (!correction.success) {
return {
@@ -450,7 +388,7 @@ export async function constrainAacTruePeak(input: {
targetDurationSeconds: input.targetDurationSeconds,
signal: input.signal,
probeTruePeak: (audioPath, signal) => measureTruePeakDbfs(audioPath, { timeoutMs, signal }),
- runner: (args) => defaultRunFfmpeg(args, timeoutMs, input.signal),
+ encode: defaultAacEncoder(timeoutMs, input.signal),
});
}
@@ -459,11 +397,7 @@ interface EnforceAacTruePeakInput {
targetDurationSeconds: number;
signal?: AbortSignal;
probeTruePeak: (audioPath: string, signal?: AbortSignal) => Promise<number>;
- runner: (args: string[]) => Promise<{
- success: boolean;
- error?: string;
- failureReason?: "external_interruption";
- }>;
+ encode: AacEncoder;
}
async function enforceAacTruePeak(input: EnforceAacTruePeakInput): Promise<{
@@ -498,13 +432,11 @@ async function enforceAacTruePeak(input: EnforceAacTruePeakInput): Promise<{
// Aim below the acceptance ceiling because this correction itself is
// another lossy AAC encode and can regenerate content-dependent peaks.
attenuationDb += correctionTargetDbfs - truePeakDbfs;
- const result = await input.runner(
- buildAacTruePeakCorrectionArgs(
- input.audioPath,
- correctedPath,
- input.targetDurationSeconds,
- attenuationDb,
- ),
+ const result = await input.encode(
+ input.audioPath,
+ correctedPath,
+ input.targetDurationSeconds,
+ aacTruePeakCorrectionFilter(input.targetDurationSeconds, attenuationDb),
);
if (!result.success) {
return {
@@ -645,12 +577,22 @@ async function defaultRunFfmpeg(
args: string[],
timeoutMs: number,
signal?: AbortSignal,
-): Promise<{
- success: boolean;
- error?: string;
- failureReason?: "external_interruption";
-}> {
- const result = await runFfmpeg(args, { signal, timeout: timeoutMs });
+): Promise<StepResult> {
+ return toStepResult(await runFfmpeg(args, { signal, timeout: timeoutMs }), timeoutMs);
+}
+
+function defaultAacEncoder(timeoutMs: number, signal?: AbortSignal): AacEncoder {
+ return async (inputPath, outputPath, durationSeconds, filter) => {
+ // One encoder, as before this went through encodeAac: splitting it is a separate change.
+ const options = { signal, timeout: timeoutMs, filter, segments: 1 };
+ return toStepResult(
+ await encodeAac(inputPath, outputPath, durationSeconds, options),
+ timeoutMs,
+ );
+ };
+}
+
+function toStepResult(result: RunFfmpegResult, timeoutMs: number): StepResult {
if (result.success) return { success: true };
return {
success: false,
diff --git a/packages/producer/tests/chat/output/output.mp4 b/packages/producer/tests/chat/output/output.mp4
index c858dd1a7f..c7c9186935 100644
--- a/packages/producer/tests/chat/output/output.mp4
+++ b/packages/producer/tests/chat/output/output.mp4
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:d002c0d9d9851ec6abce1c662ed7a0a7b78b993ec778650a9aca4a437edb4080
-size 887615
+oid sha256:ed03727bbc23262eeec9467f17fc7dff387f2c788e788329be63caffdb2961f6
+size 1315820
diff --git a/packages/producer/tests/missing-host-comp-id/output/output.mp4 b/packages/producer/tests/missing-host-comp-id/output/output.mp4
index 7af1b5459c..f9a44c8c94 100644
--- a/packages/producer/tests/missing-host-comp-id/output/output.mp4
+++ b/packages/producer/tests/missing-host-comp-id/output/output.mp4
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:af362a2663b53d50093bea4ea3a3fd8673c5cdbd8d2bb2be5e92e078b483420f
-size 118044
+oid sha256:3b825bd79b6e4547606d2c8ea4a6c101c84657196a6f29f208a13777493a5f84
+size 118358
diff --git a/packages/producer/tests/speed-ramp/output/output.mp4 b/packages/producer/tests/speed-ramp/output/output.mp4
index f0da3d4212..4beb857d08 100644
--- a/packages/producer/tests/speed-ramp/output/output.mp4
+++ b/packages/producer/tests/speed-ramp/output/output.mp4
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:345e129cfef4cea951ac3bbdddfaae3c8933f12e4a2c243a93ad70bfbca8db1d
-size 259520
+oid sha256:c74c635dae28d95137a446769a2c92359bf306ade885b8f705ad1610ba0f37fa
+size 260303
diff --git a/packages/producer/tests/style-17-prod/output/output.mp4 b/packages/producer/tests/style-17-prod/output/output.mp4
index 9eeed78c37..463dbbd18c 100644
--- a/packages/producer/tests/style-17-prod/output/output.mp4
+++ b/packages/producer/tests/style-17-prod/output/output.mp4
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:e43a77c57c8d14e0939688dfa1bb29becee026a6d9ed759e07c8b7ba26f5a1ef
-size 17110152
+oid sha256:1d7a2bebb2621a58f8df0d1f7c77e0eaf758d32bdd561deac84212a41768c72c
+size 17375894
diff --git a/packages/producer/tests/variables-prod/output/output.mp4 b/packages/producer/tests/variables-prod/output/output.mp4
index 209eec6f77..baa7607784 100644
--- a/packages/producer/tests/variables-prod/output/output.mp4
+++ b/packages/producer/tests/variables-prod/output/output.mp4
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:d13aa856b2eb5ca2d04cf7db8c9cf981bdea6bdd1d50485c07478d9e427bc255
-size 109724
+oid sha256:27b4c24b67034d86730b1b3ecf8afaf8599534fdd969117714b3f560db779364
+size 110038