Skip to content

Write HTML. Render video. Built for agents.

perf(engine): long renders encode their audio in parallel segments (#5788)

Miguel Ángel8e4ce7411825
diff --git a/packages/engine/src/services/aacEncode.test.ts b/packages/engine/src/services/aacEncode.test.ts
new file mode 100644
index 0000000000..abb13a5a1c
--- /dev/null
+++ b/packages/engine/src/services/aacEncode.test.ts
@@ -0,0 +1,155 @@
+import { spawnSync } from "node:child_process";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterAll, describe, expect, it } from "vitest";
+import { getFfmpegBinary, getFfprobeBinary } from "../utils/ffmpegBinaries.js";
+import { aacSegmentCount, encodeAac, splitAdtsFrames } from "./aacEncode.js";
+
+const HAS_FFMPEG = spawnSync(getFfmpegBinary(), ["-version"], { encoding: "utf-8" }).status === 0;
+
+function ffmpeg(args: string[]): Buffer {
+ const result = spawnSync(getFfmpegBinary(), ["-nostdin", "-loglevel", "error", ...args], {
+ maxBuffer: 1 << 28,
+ });
+ if (result.status !== 0) throw new Error(`ffmpeg failed: ${result.stderr?.toString()}`);
+ return result.stdout;
+}
+
+function decode(path: string, filter = "anull"): Float32Array {
+ const pcm = ffmpeg(["-i", path, "-af", filter, "-f", "f32le", "-ac", "2", "-"]);
+ return new Float32Array(pcm.buffer, pcm.byteOffset, pcm.length / 4);
+}
+
+/** Where the audio stream says it ends, in 1/48000 s ticks. */
+function streamEnd(path: string): string {
+ const args = ["-v", "error", "-select_streams", "a", "-show_entries", "stream=duration_ts"];
+ const result = spawnSync(getFfprobeBinary(), [...args, "-of", "csv=p=0", "--", path], {
+ encoding: "utf-8",
+ });
+ return result.stdout.trim();
+}
+
+/** Join k decodes from segment k-1's last packet and segment k's first: samples [(cut-1)*1024, cut*1024). */
+function joinCentres(seconds: number, segments: number): number[] {
+ const packets = Math.ceil(Math.round(seconds * 48_000) / 1024) + 1;
+ return Array.from({ length: segments - 1 }, (_, k) => {
+ const cut = Math.round(((k + 1) * packets) / segments);
+ return (cut - 1) * 1024 + 512;
+ });
+}
+
+/** Error against the source: overall RMS, and the peak within 4096 frames of each cut versus elsewhere. */
+function errorStats(source: Float32Array, coded: Float32Array, cutFrames: number[]) {
+ const nearCut = (i: number) =>
+ cutFrames.some((cut) => Math.abs(Math.floor(i / 2) - cut * 1024) < 4096);
+ let sum = 0;
+ let peakNearCut = 0;
+ let peakElsewhere = 0;
+ for (let i = 0; i < source.length; i++) {
+ const error = Math.abs(coded[i]! - source[i]!);
+ sum += error * error;
+ if (nearCut(i)) peakNearCut = Math.max(peakNearCut, error);
+ else peakElsewhere = Math.max(peakElsewhere, error);
+ }
+ return { rms: Math.sqrt(sum / source.length), peakNearCut, peakElsewhere };
+}
+
+describe("aacSegmentCount", () => {
+ it("keeps audio under two minutes in one encoder", () => {
+ expect(aacSegmentCount(119)).toBe(1);
+ });
+
+ it("gives each segment at least a minute, up to eight segments", () => {
+ expect(aacSegmentCount(180)).toBe(3);
+ expect(aacSegmentCount(1800)).toBe(8);
+ });
+});
+
+describe.skipIf(!HAS_FFMPEG)("encodeAac", () => {
+ const dir = mkdtempSync(join(tmpdir(), "hf-aac-encode-"));
+ afterAll(() => rmSync(dir, { recursive: true, force: true }));
+
+ it("encodes a long mix in parallel segments that decode like one encoder's output", async () => {
+ const source = join(dir, "mix.wav");
+ ffmpeg([
+ ...["-f", "lavfi", "-i", "anoisesrc=d=150.5:c=pink:r=48000:a=0.2"],
+ ...["-f", "lavfi", "-i", "sine=f=330:d=150.5:r=48000"],
+ ...["-filter_complex", "[0][1]amix=2:normalize=0,aformat=channel_layouts=stereo"],
+ ...["-c:a", "pcm_f32le", "-y", source],
+ ]);
+ const single = join(dir, "single.m4a");
+ const split = join(dir, "split.m4a");
+ ffmpeg(["-i", source, "-t", "150.5", "-acodec", "aac", "-b:a", "192k", "-y", single]);
+ expect((await encodeAac(source, split, 150.5, { timeout: 120_000 })).success).toBe(true);
+ // 150.5 s is not a whole number of AAC frames, so the last packet is short.
+ expect(streamEnd(split)).toBe(streamEnd(single));
+
+ const packets = (path: string) =>
+ splitAdtsFrames(ffmpeg(["-i", path, "-c", "copy", "-f", "adts", "-"]));
+ const singlePackets = packets(single);
+ const splitPackets = packets(split);
+ expect(splitPackets.length).toBe(singlePackets.length);
+ expect(Buffer.concat(splitPackets).equals(Buffer.concat(singlePackets))).toBe(false);
+
+ const reference = decode(source);
+ const singleDecoded = decode(single);
+ const splitDecoded = decode(split);
+ expect(splitDecoded.length).toBe(singleDecoded.length);
+ const cuts = [Math.round(singlePackets.length / 2)];
+ const singleError = errorStats(reference, singleDecoded, cuts);
+ const splitError = errorStats(reference, splitDecoded, cuts);
+ // Measured: under 1% more RMS error, and cut peaks at one encoder's own level.
+ expect(splitError.rms).toBeLessThan(singleError.rms * 1.02);
+ expect(splitError.peakNearCut).toBeLessThan(singleError.peakElsewhere);
+ }, 120_000);
+
+ it("encodes a frame-aligned duration whose sample count is not exact in floating point", async () => {
+ // 147.52 s is exactly 6915 AAC frames, but 147.52 * 48000 lands one ulp above it.
+ const source = join(dir, "aligned.wav");
+ ffmpeg(["-f", "lavfi", "-i", "sine=f=440:d=147.52:r=48000", "-ac", "2", "-y", source]);
+ const single = join(dir, "aligned-single.m4a");
+ const split = join(dir, "aligned-split.m4a");
+ ffmpeg(["-i", source, "-t", "147.52", "-acodec", "aac", "-b:a", "192k", "-y", single]);
+ expect((await encodeAac(source, split, 147.52, { timeout: 120_000 })).success).toBe(true);
+ expect(streamEnd(split)).toBe(streamEnd(single));
+ }, 120_000);
+
+ it("leaves no click at the joins of a pure tone", async () => {
+ const source = join(dir, "tone.wav");
+ ffmpeg([
+ ...["-f", "lavfi", "-i", "sine=f=440:d=150.5:r=48000"],
+ ...["-af", "volume=0.5,aformat=channel_layouts=stereo", "-c:a", "pcm_f32le", "-y", source],
+ ]);
+ const output = join(dir, "tone.m4a");
+ expect((await encodeAac(source, output, 150.5, { timeout: 120_000 })).success).toBe(true);
+
+ const joins = joinCentres(150.5, aacSegmentCount(150.5));
+ expect(joins).toHaveLength(1);
+ const window = 2 * 2400; // 50 ms of interleaved stereo
+ const peakAt = (signal: Float32Array, frame: number) => {
+ let peak = 0;
+ for (let i = Math.max(0, frame * 2 - window / 2); i < frame * 2 + window / 2; i++) {
+ peak = Math.max(peak, Math.abs(signal[i] ?? 0));
+ }
+ return peak;
+ };
+ const medianPeak = (signal: Float32Array) => {
+ const peaks = Array.from({ length: Math.floor(signal.length / window) }, (_, w) =>
+ peakAt(signal, w * (window / 2) + window / 4),
+ ).sort((a, b) => a - b);
+ return peaks[Math.floor(peaks.length / 2)]!;
+ };
+ // A 440 Hz tone has almost nothing above 2 kHz, so a click shows as a broadband spike there;
+ // a dropped or repeated frame shows as a phase jump against the source.
+ const highPassed = decode(output, Array(4).fill("highpass=f=2000").join(","));
+ const reference = decode(source);
+ const residual = decode(output).map((sample, i) => sample - (reference[i] ?? 0));
+ const [hpMedian, residualMedian] = [medianPeak(highPassed), medianPeak(residual)];
+ // Measured: joins sit at the median; the source's own stream-copy joins read about 40 dB above it.
+ for (const centre of joins) {
+ expect(peakAt(highPassed, centre)).toBeLessThan(hpMedian * 2);
+ expect(peakAt(residual, centre)).toBeLessThan(residualMedian * 2);
+ }
+ }, 120_000);
+});
diff --git a/packages/engine/src/services/aacEncode.ts b/packages/engine/src/services/aacEncode.ts
new file mode 100644
index 0000000000..12289330f9
--- /dev/null
+++ b/packages/engine/src/services/aacEncode.ts
@@ -0,0 +1,120 @@
+/**
+ * AAC for the mixed 48 kHz audio. One encoder is single-threaded (about 60x real time),
+ * so long audio is encoded as frame-aligned segments in parallel and their frames joined.
+ */
+
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { runFfmpeg, type RunFfmpegResult } from "../utils/runFfmpeg.js";
+
+const SAMPLE_RATE = 48_000;
+const FRAME_SAMPLES = 1024;
+const MIN_SEGMENT_SECONDS = 60;
+// Fixed rather than the core count, so the bytes (and the planHash) do not depend on the machine.
+const MAX_SEGMENTS = 8;
+// Encoded on each side of a segment and dropped. The encoder's rate control takes about
+// 10 s to settle (its error is higher until then), so a segment starts 10 s before its cut.
+const PRE_ROLL_FRAMES = 469;
+const POST_ROLL_FRAMES = 8;
+export const AAC_ARGS = ["-acodec", "aac", "-b:a", "192k"];
+
+export function aacSegmentCount(totalDuration: number): number {
+ return Math.max(1, Math.min(MAX_SEGMENTS, Math.floor(totalDuration / MIN_SEGMENT_SECONDS)));
+}
+
+export function splitAdtsFrames(bytes: Buffer): Buffer[] {
+ const frames: Buffer[] = [];
+ for (let at = 0; at < bytes.length; ) {
+ const length = ((bytes[at + 3]! & 0x03) << 11) | (bytes[at + 4]! << 3) | (bytes[at + 5]! >> 5);
+ if (bytes[at] !== 0xff || (bytes[at + 1]! & 0xf0) !== 0xf0 || length < 7) {
+ throw new Error(`Not an ADTS frame at byte ${at}`);
+ }
+ frames.push(bytes.subarray(at, at + length));
+ at += length;
+ }
+ return frames;
+}
+
+/**
+ * Packet g of the stream covers samples [(g-1)*1024, (g+1)*1024); packet 0 is the
+ * encoder delay. Segment k encodes from `firstFrame` and keeps its packets [keepFrom, keepTo).
+ */
+function planSegments(totalDuration: number, count: number) {
+ // Round to whole samples first: 147.52 * 48000 is 7080960.000000001 in floating point.
+ const packets = Math.ceil(Math.round(totalDuration * SAMPLE_RATE) / FRAME_SAMPLES) + 1;
+ const cuts = Array.from({ length: count + 1 }, (_, k) => Math.round((k * packets) / count));
+ return cuts.slice(0, -1).map((cut, k) => {
+ const next = cuts[k + 1]!;
+ const firstFrame = Math.max(0, cut - PRE_ROLL_FRAMES);
+ const end = k === count - 1 ? "" : `:end_sample=${(next + POST_ROLL_FRAMES) * FRAME_SAMPLES}`;
+ const trim = `atrim=start_sample=${firstFrame * FRAME_SAMPLES}${end}`;
+ return { trim, keepFrom: cut - firstFrame, keepTo: next - firstFrame };
+ });
+}
+
+/** `input` is a 48 kHz WAV of exactly `totalDuration`. */
+export async function encodeAac(
+ input: string,
+ outputPath: string,
+ totalDuration: number,
+ { signal, timeout }: { signal?: AbortSignal; timeout: number },
+): Promise<RunFfmpegResult> {
+ const count = aacSegmentCount(totalDuration);
+ if (count === 1) {
+ const args = ["-i", input, "-t", String(totalDuration), ...AAC_ARGS, "-y", outputPath];
+ return runFfmpeg(args, { signal, timeout });
+ }
+
+ const dir = mkdtempSync(join(dirname(outputPath), ".aac-segments-"));
+ try {
+ const segments = planSegments(totalDuration, count).map((segment, k) => ({
+ ...segment,
+ path: join(dir, `${k}.aac`),
+ }));
+ const results = await Promise.all(
+ segments.map(({ trim, path }) =>
+ runFfmpeg(["-i", input, "-af", trim, ...AAC_ARGS, "-f", "adts", "-y", path], {
+ signal,
+ timeout,
+ }),
+ ),
+ );
+ const failed = results.find((result) => !result.success);
+ if (failed) return failed;
+
+ const joined = join(dir, "joined.aac");
+ const frames = segments.flatMap(({ path, keepFrom, keepTo }) => {
+ const segmentFrames = splitAdtsFrames(readFileSync(path));
+ if (segmentFrames.length < keepTo) {
+ throw new Error(`AAC segment ${path} has ${segmentFrames.length} frames, needs ${keepTo}`);
+ }
+ return segmentFrames.slice(keepFrom, keepTo);
+ });
+ writeFileSync(joined, Buffer.concat(frames));
+ // ADTS carries neither the encoder delay nor the last packet's short duration: the offset
+ // restores the delay, and setts ends the stream at totalDuration as one encoder would.
+ const offset = String(-FRAME_SAMPLES / SAMPLE_RATE);
+ const end = `setts=duration='if(gte((PTS+DURATION)*TB,${totalDuration}),round(${totalDuration}/TB-PTS),DURATION)'`;
+ const muxed = await runFfmpeg(
+ [
+ "-itsoffset",
+ offset,
+ "-f",
+ "aac",
+ "-i",
+ joined,
+ "-c",
+ "copy",
+ "-bsf:a",
+ end,
+ "-y",
+ outputPath,
+ ],
+ { signal, timeout },
+ );
+ const slowest = Math.max(...results.map((result) => result.durationMs));
+ return { ...muxed, durationMs: slowest + muxed.durationMs };
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+}
diff --git a/packages/engine/src/services/audioMixer.ts b/packages/engine/src/services/audioMixer.ts
index 42c0b3cc82..e641fd6a6c 100644
--- a/packages/engine/src/services/audioMixer.ts
+++ b/packages/engine/src/services/audioMixer.ts
@@ -15,6 +15,7 @@ import {
} from "../utils/urlDownloader.js";
import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
import { formatFfmpegError, runFfmpeg, type RunFfmpegResult } from "../utils/runFfmpeg.js";
+import { AAC_ARGS, aacSegmentCount, encodeAac } from "./aacEncode.js";
import { audioSubprocessEvidence } from "./audioFailureDiagnostic.js";
import { unwrapTemplate } from "../utils/htmlTemplate.js";
import { resolveMediaElementSrc, resolveProjectRelativeSrc } from "./videoFrameExtractor.js";
@@ -827,7 +828,7 @@ async function mixAudioTracks(
scriptPath,
"-map",
"[out]",
- ...(floatWav ? ["-acodec", "pcm_f32le", "-ar", "48000"] : ["-acodec", "aac", "-b:a", "192k"]),
+ ...(floatWav ? ["-acodec", "pcm_f32le", "-ar", "48000"] : AAC_ARGS),
"-t",
String(totalDuration),
"-y",
@@ -1059,12 +1060,12 @@ async function mixGroupMembers(
}
/**
- * Sum the tracks to a float WAV, run the master chain over the sum, then AAC-encode.
+ * Sum the tracks to a float WAV, run the master chain (if any) over the sum, then AAC-encode.
* Groups are already folded into `tracks`, and the sum stays float until the encoder.
*/
-async function mixThroughMasterBus(
+async function mixThroughFloatSum(
tracks: AudioTrack[],
- masterChain: HfAudioFxChain,
+ masterChain: HfAudioFxChain | undefined,
outputPath: string,
workDir: string,
totalDuration: number,
@@ -1075,25 +1076,18 @@ async function mixThroughMasterBus(
const sum = await mixAudioTracks(tracks, sumPath, totalDuration, signal, config, true);
if (!sum.success) return { ...sum, outputPath };
- const fx = await applyAudioFxChain(sumPath, masterChain, join(workDir, "master-fx.wav"), {
- trackId: "master",
+ const encoderInput = masterChain
+ ? (
+ await applyAudioFxChain(sumPath, masterChain, join(workDir, "master-fx.wav"), {
+ trackId: "master",
+ signal,
+ })
+ ).path
+ : sumPath;
+ const encoded = await encodeAac(encoderInput, outputPath, totalDuration, {
signal,
+ timeout: config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout,
});
- const encoded = await runFfmpeg(
- [
- "-i",
- fx.path,
- "-t",
- String(totalDuration),
- "-acodec",
- "aac",
- "-b:a",
- "192k",
- "-y",
- outputPath,
- ],
- { signal, timeout: config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout },
- );
if (!encoded.success) {
const failure = ffmpegFailure("mix", encoded);
return {
@@ -1613,8 +1607,8 @@ export async function processCompositionAudio(
let mixResult: MixResult;
try {
mixResult =
- masterChain && tracks.length > 0
- ? await mixThroughMasterBus(
+ (masterChain || aacSegmentCount(totalDuration) > 1) && tracks.length > 0
+ ? await mixThroughFloatSum(
tracks,
masterChain,
outputPath,