Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
perf(engine): long renders encode their audio in parallel segments (#5788)
Miguel Ángel8e4ce7411825
diff --git a/packages/engine/src/services/aacEncode.test.ts b/packages/engine/src/services/aacEncode.test.tsnew file mode 100644index 0000000000..abb13a5a1c--- /dev/null+++ b/packages/engine/src/services/aacEncode.test.ts@@ -0,0 +1,155 @@+import { spawnSync } from "node:child_process";+import { mkdtempSync, rmSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { afterAll, describe, expect, it } from "vitest";+import { getFfmpegBinary, getFfprobeBinary } from "../utils/ffmpegBinaries.js";+import { aacSegmentCount, encodeAac, splitAdtsFrames } from "./aacEncode.js";++const HAS_FFMPEG = spawnSync(getFfmpegBinary(), ["-version"], { encoding: "utf-8" }).status === 0;++function ffmpeg(args: string[]): Buffer {+ const result = spawnSync(getFfmpegBinary(), ["-nostdin", "-loglevel", "error", ...args], {+ maxBuffer: 1 << 28,+ });+ if (result.status !== 0) throw new Error(`ffmpeg failed: ${result.stderr?.toString()}`);+ return result.stdout;+}++function decode(path: string, filter = "anull"): Float32Array {+ const pcm = ffmpeg(["-i", path, "-af", filter, "-f", "f32le", "-ac", "2", "-"]);+ return new Float32Array(pcm.buffer, pcm.byteOffset, pcm.length / 4);+}++/** Where the audio stream says it ends, in 1/48000 s ticks. */+function streamEnd(path: string): string {+ const args = ["-v", "error", "-select_streams", "a", "-show_entries", "stream=duration_ts"];+ const result = spawnSync(getFfprobeBinary(), [...args, "-of", "csv=p=0", "--", path], {+ encoding: "utf-8",+ });+ return result.stdout.trim();+}++/** Join k decodes from segment k-1's last packet and segment k's first: samples [(cut-1)*1024, cut*1024). */+function joinCentres(seconds: number, segments: number): number[] {+ const packets = Math.ceil(Math.round(seconds * 48_000) / 1024) + 1;+ return Array.from({ length: segments - 1 }, (_, k) => {+ const cut = Math.round(((k + 1) * packets) / segments);+ return (cut - 1) * 1024 + 512;+ });+}++/** Error against the source: overall RMS, and the peak within 4096 frames of each cut versus elsewhere. */+function errorStats(source: Float32Array, coded: Float32Array, cutFrames: number[]) {+ const nearCut = (i: number) =>+ cutFrames.some((cut) => Math.abs(Math.floor(i / 2) - cut * 1024) < 4096);+ let sum = 0;+ let peakNearCut = 0;+ let peakElsewhere = 0;+ for (let i = 0; i < source.length; i++) {+ const error = Math.abs(coded[i]! - source[i]!);+ sum += error * error;+ if (nearCut(i)) peakNearCut = Math.max(peakNearCut, error);+ else peakElsewhere = Math.max(peakElsewhere, error);+ }+ return { rms: Math.sqrt(sum / source.length), peakNearCut, peakElsewhere };+}++describe("aacSegmentCount", () => {+ it("keeps audio under two minutes in one encoder", () => {+ expect(aacSegmentCount(119)).toBe(1);+ });++ it("gives each segment at least a minute, up to eight segments", () => {+ expect(aacSegmentCount(180)).toBe(3);+ expect(aacSegmentCount(1800)).toBe(8);+ });+});++describe.skipIf(!HAS_FFMPEG)("encodeAac", () => {+ const dir = mkdtempSync(join(tmpdir(), "hf-aac-encode-"));+ afterAll(() => rmSync(dir, { recursive: true, force: true }));++ it("encodes a long mix in parallel segments that decode like one encoder's output", async () => {+ const source = join(dir, "mix.wav");+ ffmpeg([+ ...["-f", "lavfi", "-i", "anoisesrc=d=150.5:c=pink:r=48000:a=0.2"],+ ...["-f", "lavfi", "-i", "sine=f=330:d=150.5:r=48000"],+ ...["-filter_complex", "[0][1]amix=2:normalize=0,aformat=channel_layouts=stereo"],+ ...["-c:a", "pcm_f32le", "-y", source],+ ]);+ const single = join(dir, "single.m4a");+ const split = join(dir, "split.m4a");+ ffmpeg(["-i", source, "-t", "150.5", "-acodec", "aac", "-b:a", "192k", "-y", single]);+ expect((await encodeAac(source, split, 150.5, { timeout: 120_000 })).success).toBe(true);+ // 150.5 s is not a whole number of AAC frames, so the last packet is short.+ expect(streamEnd(split)).toBe(streamEnd(single));++ const packets = (path: string) =>+ splitAdtsFrames(ffmpeg(["-i", path, "-c", "copy", "-f", "adts", "-"]));+ const singlePackets = packets(single);+ const splitPackets = packets(split);+ expect(splitPackets.length).toBe(singlePackets.length);+ expect(Buffer.concat(splitPackets).equals(Buffer.concat(singlePackets))).toBe(false);++ const reference = decode(source);+ const singleDecoded = decode(single);+ const splitDecoded = decode(split);+ expect(splitDecoded.length).toBe(singleDecoded.length);+ const cuts = [Math.round(singlePackets.length / 2)];+ const singleError = errorStats(reference, singleDecoded, cuts);+ const splitError = errorStats(reference, splitDecoded, cuts);+ // Measured: under 1% more RMS error, and cut peaks at one encoder's own level.+ expect(splitError.rms).toBeLessThan(singleError.rms * 1.02);+ expect(splitError.peakNearCut).toBeLessThan(singleError.peakElsewhere);+ }, 120_000);++ it("encodes a frame-aligned duration whose sample count is not exact in floating point", async () => {+ // 147.52 s is exactly 6915 AAC frames, but 147.52 * 48000 lands one ulp above it.+ const source = join(dir, "aligned.wav");+ ffmpeg(["-f", "lavfi", "-i", "sine=f=440:d=147.52:r=48000", "-ac", "2", "-y", source]);+ const single = join(dir, "aligned-single.m4a");+ const split = join(dir, "aligned-split.m4a");+ ffmpeg(["-i", source, "-t", "147.52", "-acodec", "aac", "-b:a", "192k", "-y", single]);+ expect((await encodeAac(source, split, 147.52, { timeout: 120_000 })).success).toBe(true);+ expect(streamEnd(split)).toBe(streamEnd(single));+ }, 120_000);++ it("leaves no click at the joins of a pure tone", async () => {+ const source = join(dir, "tone.wav");+ ffmpeg([+ ...["-f", "lavfi", "-i", "sine=f=440:d=150.5:r=48000"],+ ...["-af", "volume=0.5,aformat=channel_layouts=stereo", "-c:a", "pcm_f32le", "-y", source],+ ]);+ const output = join(dir, "tone.m4a");+ expect((await encodeAac(source, output, 150.5, { timeout: 120_000 })).success).toBe(true);++ const joins = joinCentres(150.5, aacSegmentCount(150.5));+ expect(joins).toHaveLength(1);+ const window = 2 * 2400; // 50 ms of interleaved stereo+ const peakAt = (signal: Float32Array, frame: number) => {+ let peak = 0;+ for (let i = Math.max(0, frame * 2 - window / 2); i < frame * 2 + window / 2; i++) {+ peak = Math.max(peak, Math.abs(signal[i] ?? 0));+ }+ return peak;+ };+ const medianPeak = (signal: Float32Array) => {+ const peaks = Array.from({ length: Math.floor(signal.length / window) }, (_, w) =>+ peakAt(signal, w * (window / 2) + window / 4),+ ).sort((a, b) => a - b);+ return peaks[Math.floor(peaks.length / 2)]!;+ };+ // A 440 Hz tone has almost nothing above 2 kHz, so a click shows as a broadband spike there;+ // a dropped or repeated frame shows as a phase jump against the source.+ const highPassed = decode(output, Array(4).fill("highpass=f=2000").join(","));+ const reference = decode(source);+ const residual = decode(output).map((sample, i) => sample - (reference[i] ?? 0));+ const [hpMedian, residualMedian] = [medianPeak(highPassed), medianPeak(residual)];+ // Measured: joins sit at the median; the source's own stream-copy joins read about 40 dB above it.+ for (const centre of joins) {+ expect(peakAt(highPassed, centre)).toBeLessThan(hpMedian * 2);+ expect(peakAt(residual, centre)).toBeLessThan(residualMedian * 2);+ }+ }, 120_000);+});diff --git a/packages/engine/src/services/aacEncode.ts b/packages/engine/src/services/aacEncode.tsnew file mode 100644index 0000000000..12289330f9--- /dev/null+++ b/packages/engine/src/services/aacEncode.ts@@ -0,0 +1,120 @@+/**+ * AAC for the mixed 48 kHz audio. One encoder is single-threaded (about 60x real time),+ * so long audio is encoded as frame-aligned segments in parallel and their frames joined.+ */++import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";+import { dirname, join } from "node:path";+import { runFfmpeg, type RunFfmpegResult } from "../utils/runFfmpeg.js";++const SAMPLE_RATE = 48_000;+const FRAME_SAMPLES = 1024;+const MIN_SEGMENT_SECONDS = 60;+// Fixed rather than the core count, so the bytes (and the planHash) do not depend on the machine.+const MAX_SEGMENTS = 8;+// Encoded on each side of a segment and dropped. The encoder's rate control takes about+// 10 s to settle (its error is higher until then), so a segment starts 10 s before its cut.+const PRE_ROLL_FRAMES = 469;+const POST_ROLL_FRAMES = 8;+export const AAC_ARGS = ["-acodec", "aac", "-b:a", "192k"];++export function aacSegmentCount(totalDuration: number): number {+ return Math.max(1, Math.min(MAX_SEGMENTS, Math.floor(totalDuration / MIN_SEGMENT_SECONDS)));+}++export function splitAdtsFrames(bytes: Buffer): Buffer[] {+ const frames: Buffer[] = [];+ for (let at = 0; at < bytes.length; ) {+ const length = ((bytes[at + 3]! & 0x03) << 11) | (bytes[at + 4]! << 3) | (bytes[at + 5]! >> 5);+ if (bytes[at] !== 0xff || (bytes[at + 1]! & 0xf0) !== 0xf0 || length < 7) {+ throw new Error(`Not an ADTS frame at byte ${at}`);+ }+ frames.push(bytes.subarray(at, at + length));+ at += length;+ }+ return frames;+}++/**+ * Packet g of the stream covers samples [(g-1)*1024, (g+1)*1024); packet 0 is the+ * encoder delay. Segment k encodes from `firstFrame` and keeps its packets [keepFrom, keepTo).+ */+function planSegments(totalDuration: number, count: number) {+ // Round to whole samples first: 147.52 * 48000 is 7080960.000000001 in floating point.+ const packets = Math.ceil(Math.round(totalDuration * SAMPLE_RATE) / FRAME_SAMPLES) + 1;+ const cuts = Array.from({ length: count + 1 }, (_, k) => Math.round((k * packets) / count));+ return cuts.slice(0, -1).map((cut, k) => {+ const next = cuts[k + 1]!;+ const firstFrame = Math.max(0, cut - PRE_ROLL_FRAMES);+ const end = k === count - 1 ? "" : `:end_sample=${(next + POST_ROLL_FRAMES) * FRAME_SAMPLES}`;+ const trim = `atrim=start_sample=${firstFrame * FRAME_SAMPLES}${end}`;+ return { trim, keepFrom: cut - firstFrame, keepTo: next - firstFrame };+ });+}++/** `input` is a 48 kHz WAV of exactly `totalDuration`. */+export async function encodeAac(+ input: string,+ outputPath: string,+ totalDuration: number,+ { signal, timeout }: { signal?: AbortSignal; timeout: number },+): Promise<RunFfmpegResult> {+ const count = aacSegmentCount(totalDuration);+ if (count === 1) {+ const args = ["-i", input, "-t", String(totalDuration), ...AAC_ARGS, "-y", outputPath];+ return runFfmpeg(args, { signal, timeout });+ }++ const dir = mkdtempSync(join(dirname(outputPath), ".aac-segments-"));+ try {+ const segments = planSegments(totalDuration, count).map((segment, k) => ({+ ...segment,+ path: join(dir, `${k}.aac`),+ }));+ const results = await Promise.all(+ segments.map(({ trim, path }) =>+ runFfmpeg(["-i", input, "-af", trim, ...AAC_ARGS, "-f", "adts", "-y", path], {+ signal,+ timeout,+ }),+ ),+ );+ const failed = results.find((result) => !result.success);+ if (failed) return failed;++ const joined = join(dir, "joined.aac");+ const frames = segments.flatMap(({ path, keepFrom, keepTo }) => {+ const segmentFrames = splitAdtsFrames(readFileSync(path));+ if (segmentFrames.length < keepTo) {+ throw new Error(`AAC segment ${path} has ${segmentFrames.length} frames, needs ${keepTo}`);+ }+ return segmentFrames.slice(keepFrom, keepTo);+ });+ writeFileSync(joined, Buffer.concat(frames));+ // ADTS carries neither the encoder delay nor the last packet's short duration: the offset+ // restores the delay, and setts ends the stream at totalDuration as one encoder would.+ const offset = String(-FRAME_SAMPLES / SAMPLE_RATE);+ const end = `setts=duration='if(gte((PTS+DURATION)*TB,${totalDuration}),round(${totalDuration}/TB-PTS),DURATION)'`;+ const muxed = await runFfmpeg(+ [+ "-itsoffset",+ offset,+ "-f",+ "aac",+ "-i",+ joined,+ "-c",+ "copy",+ "-bsf:a",+ end,+ "-y",+ outputPath,+ ],+ { signal, timeout },+ );+ const slowest = Math.max(...results.map((result) => result.durationMs));+ return { ...muxed, durationMs: slowest + muxed.durationMs };+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+}diff --git a/packages/engine/src/services/audioMixer.ts b/packages/engine/src/services/audioMixer.tsindex 42c0b3cc82..e641fd6a6c 100644--- a/packages/engine/src/services/audioMixer.ts+++ b/packages/engine/src/services/audioMixer.ts@@ -15,6 +15,7 @@ import {} from "../utils/urlDownloader.js";import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";import { formatFfmpegError, runFfmpeg, type RunFfmpegResult } from "../utils/runFfmpeg.js";+import { AAC_ARGS, aacSegmentCount, encodeAac } from "./aacEncode.js";import { audioSubprocessEvidence } from "./audioFailureDiagnostic.js";import { unwrapTemplate } from "../utils/htmlTemplate.js";import { resolveMediaElementSrc, resolveProjectRelativeSrc } from "./videoFrameExtractor.js";@@ -827,7 +828,7 @@ async function mixAudioTracks(scriptPath,"-map","[out]",- ...(floatWav ? ["-acodec", "pcm_f32le", "-ar", "48000"] : ["-acodec", "aac", "-b:a", "192k"]),+ ...(floatWav ? ["-acodec", "pcm_f32le", "-ar", "48000"] : AAC_ARGS),"-t",String(totalDuration),"-y",@@ -1059,12 +1060,12 @@ async function mixGroupMembers(}/**- * Sum the tracks to a float WAV, run the master chain over the sum, then AAC-encode.+ * Sum the tracks to a float WAV, run the master chain (if any) over the sum, then AAC-encode.* Groups are already folded into `tracks`, and the sum stays float until the encoder.*/-async function mixThroughMasterBus(+async function mixThroughFloatSum(tracks: AudioTrack[],- masterChain: HfAudioFxChain,+ masterChain: HfAudioFxChain | undefined,outputPath: string,workDir: string,totalDuration: number,@@ -1075,25 +1076,18 @@ async function mixThroughMasterBus(const sum = await mixAudioTracks(tracks, sumPath, totalDuration, signal, config, true);if (!sum.success) return { ...sum, outputPath };- const fx = await applyAudioFxChain(sumPath, masterChain, join(workDir, "master-fx.wav"), {- trackId: "master",+ const encoderInput = masterChain+ ? (+ await applyAudioFxChain(sumPath, masterChain, join(workDir, "master-fx.wav"), {+ trackId: "master",+ signal,+ })+ ).path+ : sumPath;+ const encoded = await encodeAac(encoderInput, outputPath, totalDuration, {signal,+ timeout: config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout,});- const encoded = await runFfmpeg(- [- "-i",- fx.path,- "-t",- String(totalDuration),- "-acodec",- "aac",- "-b:a",- "192k",- "-y",- outputPath,- ],- { signal, timeout: config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout },- );if (!encoded.success) {const failure = ffmpegFailure("mix", encoded);return {@@ -1613,8 +1607,8 @@ export async function processCompositionAudio(let mixResult: MixResult;try {mixResult =- masterChain && tracks.length > 0- ? await mixThroughMasterBus(+ (masterChain || aacSegmentCount(totalDuration) > 1) && tracks.length > 0+ ? await mixThroughFloatSum(tracks,masterChain,outputPath,