Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
perf(producer): a cloud plan corrects its audio peak from the mix itself, in parallel (#5981)
Miguel Ángela125dbc19f36
diff --git a/packages/engine/src/services/audioMixer.test.ts b/packages/engine/src/services/audioMixer.test.tsindex 7b8194ccdb..fa615a1a94 100644--- a/packages/engine/src/services/audioMixer.test.ts+++ b/packages/engine/src/services/audioMixer.test.ts@@ -84,6 +84,7 @@ vi.mock("../utils/ffprobe.js", async (importOriginal) => {import { RATE_RANGE } from "@hyperframes/core/audio-automation";import { AudioFxRenderError } from "./audioFxRender.js";import { parseAudioElements, processCompositionAudio } from "./audioMixer.js";+import { encodeAac } from "./aacEncode.js";describe("parseAudioElements strict literal timing", () => {it.each(["", " ", "0s", "0abc", "0px", "-1s", "Infinity", "NaN", "0x10"])(@@ -1485,6 +1486,43 @@ describe("processCompositionAudio", () => {expect(prepareArgs).toContain(join(baseDir, "assets", filename));});+ it("keeps the float mix the AAC was encoded from when asked to", async () => {+ const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));+ const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));+ tempDirs.push(baseDir, workDir);+ writeFileSync(join(baseDir, "a.wav"), "stub");+ writeFileSync(join(baseDir, "b.wav"), "stub");+ runFfmpegMock.mockImplementation(async (args: string[]) => {+ if (args.at(-1)?.endsWith("master-sum.wav")) writeFileSync(args.at(-1)!, "float mix");+ return { success: true, durationMs: 1, stderr: "", exitCode: 0 };+ });+ const track = (id: string, layer: number) => ({+ id,+ src: `${id}.wav`,+ start: 0,+ end: 2,+ mediaStart: 0,+ layer,+ volume: 1,+ type: "audio" as const,+ });+ const kept = join(baseDir, "kept.wav");++ const result = await processCompositionAudio(+ [track("a", 0), track("b", 1)],+ baseDir,+ workDir,+ join(baseDir, "out.m4a"),+ 2,+ undefined,+ { keepEncoderInputAt: kept },+ );++ expect(result.success).toBe(true);+ expect(vi.mocked(encodeAac).mock.calls.at(-1)?.[0]).toBe(join(workDir, "master-sum.wav"));+ expect(readFileSync(kept, "utf8")).toBe("float mix");+ });+it("prepares browser root-absolute audio srcs from the project root", async () => {const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));diff --git a/packages/engine/src/services/audioMixer.ts b/packages/engine/src/services/audioMixer.tsindex 673c668287..c76bcd4088 100644--- a/packages/engine/src/services/audioMixer.ts+++ b/packages/engine/src/services/audioMixer.ts@@ -1,6 +1,15 @@// fallow-ignore-file complexity code-duplicationimport { isSelfOrAncestorHidden, memberGroupKey, isMemberGroupHidden } from "./mediaHidden.js";-import { closeSync, existsSync, mkdirSync, mkdtempSync, openSync, rmSync, writeFileSync } from "fs";+import {+ closeSync,+ existsSync,+ mkdirSync,+ mkdtempSync,+ openSync,+ renameSync,+ rmSync,+ writeFileSync,+} from "fs";import { join, dirname, isAbsolute, relative } from "path";import { parseHTML } from "linkedom";import { extractAudioMetadata, type AudioMetadata } from "../utils/ffprobe.js";@@ -1082,7 +1091,9 @@ async function mixThroughFloatSum(workDir: string,totalDuration: number,signal?: AbortSignal,- config?: Partial<Pick<EngineConfig, "ffmpegProcessTimeout" | "audioGain">>,+ config?: Partial<Pick<EngineConfig, "ffmpegProcessTimeout" | "audioGain">> & {+ keepEncoderInputAt?: string;+ },): Promise<MixResult> {const sumPath = join(workDir, "master-sum.wav");const sum = await mixAudioTracks(tracks, sumPath, totalDuration, signal, config);@@ -1111,6 +1122,7 @@ async function mixThroughFloatSum(failures: [failure],};}+ if (config?.keepEncoderInputAt) renameSync(encoderInput, config.keepEncoderInputAt);return { ...sum, outputPath, durationMs: sum.durationMs + encoded.durationMs };}@@ -1125,6 +1137,7 @@ export async function processCompositionAudio(/** Serialised master bus chain, from `parseMasterFxChain`. */masterFxChain?: string;urlByAbsentLocalPath?: ReadonlyMap<string, string>;+ keepEncoderInputAt?: string;},compiledDir?: string,): Promise<MixResult> {diff --git a/packages/producer/src/services/distributed/plan.ts b/packages/producer/src/services/distributed/plan.tsindex 3fce21da22..6cf90254e7 100644--- a/packages/producer/src/services/distributed/plan.ts+++ b/packages/producer/src/services/distributed/plan.ts@@ -1419,6 +1419,7 @@ export async function buildLocalExecutionPlan(// Chunks cut their audio from this track, so it is mixed to the frames' exact length in its one// encode; the assembler pads or trims the track of any other plan.const audioSeconds = chunksCarryAudio ? totalFrames / config.fps : probeResult.duration;+ const planMixPcmPath = join(workDir, "plan-mix.wav");stageStarted = Date.now();const [audioOutcome, overlapOutcome, frameCostOutcome] = await Promise.allSettled([runAudioStage({@@ -1431,6 +1432,7 @@ export async function buildLocalExecutionPlan(audioGain: cfg.audioGain,audios: composition.audios,masterFxChain: composition.masterFxChain,+ ...(chunksCarryAudio ? { keepEncoderInputAt: planMixPcmPath } : {}),abortSignal,assertNotAborted,}).finally(() => logPlanStage(log, "audio", stageStarted)),@@ -1497,10 +1499,11 @@ export async function buildLocalExecutionPlan(stageStarted = Date.now();const peak = await constrainAacTruePeak({audioPath: planAudioPath,+ ...(existsSync(planMixPcmPath) ? { sourcePath: planMixPcmPath } : {}),targetDurationSeconds: audioSeconds,signal: abortSignal,timeoutMs: cfg.ffmpegProcessTimeout,- });+ }).finally(() => rmSync(planMixPcmPath, { force: true }));if (!peak.success) throw encoderFailureError("[plan] audio true-peak check failed", peak);logPlanStage(log, "audio-peak", stageStarted);if (options.sliceChunkAudio) {diff --git a/packages/producer/src/services/render/audioPadTrim.ts b/packages/producer/src/services/render/audioPadTrim.tsindex 59a5eb10fc..426df25855 100644--- a/packages/producer/src/services/render/audioPadTrim.ts+++ b/packages/producer/src/services/render/audioPadTrim.ts@@ -373,6 +373,8 @@ export async function padOrTrimAudioToVideoFrameCount(*/export async function constrainAacTruePeak(input: {audioPath: string;+ /** The PCM `audioPath` was encoded from: a correction re-encodes it instead of decoding the AAC. */+ sourcePath?: string;targetDurationSeconds: number;signal?: AbortSignal;timeoutMs?: number;@@ -388,7 +390,7 @@ export async function constrainAacTruePeak(input: {targetDurationSeconds: input.targetDurationSeconds,signal: input.signal,probeTruePeak: (audioPath, signal) => measureTruePeakDbfs(audioPath, { timeoutMs, signal }),- encode: defaultAacEncoder(timeoutMs, input.signal),+ encode: defaultAacEncoder(timeoutMs, input.signal, input.sourcePath),});}@@ -581,12 +583,18 @@ async function defaultRunFfmpeg(return toStepResult(await runFfmpeg(args, { signal, timeout: timeoutMs }), timeoutMs);}-function defaultAacEncoder(timeoutMs: number, signal?: AbortSignal): AacEncoder {+function defaultAacEncoder(+ timeoutMs: number,+ signal?: AbortSignal,+ pcmSourcePath?: string,+): AacEncoder {return async (inputPath, outputPath, durationSeconds, filter) => {- // One encoder, as before this went through encodeAac: splitting it is a separate change.- const options = { signal, timeout: timeoutMs, filter, segments: 1 };+ // Each segment decodes its input from the start: cheap from PCM, a second lossy pass from AAC.+ const options = pcmSourcePath+ ? { signal, timeout: timeoutMs, filter }+ : { signal, timeout: timeoutMs, filter, segments: 1 };return toStepResult(- await encodeAac(inputPath, outputPath, durationSeconds, options),+ await encodeAac(pcmSourcePath ?? inputPath, outputPath, durationSeconds, options),timeoutMs,);};diff --git a/packages/producer/src/services/render/audioTruePeak.fakeFfmpeg.test.ts b/packages/producer/src/services/render/audioTruePeak.fakeFfmpeg.test.tsindex be6bbbebe1..4d2ee4618c 100644--- a/packages/producer/src/services/render/audioTruePeak.fakeFfmpeg.test.ts+++ b/packages/producer/src/services/render/audioTruePeak.fakeFfmpeg.test.ts@@ -1,13 +1,19 @@-import { mkdtempSync, rmSync } from "node:fs";+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";import { tmpdir } from "node:os";import { join } from "node:path";import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from "vitest";-import { extractAudioMetadata, runFfmpeg, type RunFfmpegResult } from "@hyperframes/engine";-import { padOrTrimAudioToVideoFrameCount } from "./audioPadTrim.js";+import {+ encodeAac,+ extractAudioMetadata,+ runFfmpeg,+ type RunFfmpegResult,+} from "@hyperframes/engine";+import { constrainAacTruePeak, padOrTrimAudioToVideoFrameCount } from "./audioPadTrim.js";import { measureTruePeakDbfs } from "./audioTruePeak.js";vi.mock("@hyperframes/engine", async (importOriginal) => ({...(await importOriginal<typeof import("@hyperframes/engine")>()),+ encodeAac: vi.fn(),extractAudioMetadata: vi.fn(),runFfmpeg: vi.fn(),}));@@ -36,7 +42,10 @@ const scanFilters = () =>.filter(Boolean);beforeEach(() => probed(180, 48_000));-afterEach(() => vi.mocked(runFfmpeg).mockReset());+afterEach(() => {+ vi.mocked(runFfmpeg).mockReset();+ vi.mocked(encodeAac).mockReset();+});describe("measureTruePeakDbfs against ffmpeg's stderr", () => {it("throws when ffmpeg prints no true-peak summary", async () => {@@ -100,3 +109,50 @@ describe("a local render's audio pad/trim", () => {]);});});++describe("the true-peak correction", () => {+ const dir = mkdtempSync(join(tmpdir(), "hf-peak-correct-"));+ afterAll(() => rmSync(dir, { recursive: true, force: true }));++ async function correctHotAudio(sourcePath?: string) {+ const scan = (dbfs: string) =>+ ran(true, `${FRAME.replaceAll("-3.0", dbfs)}\n${SUMMARY.replace("-3.0", dbfs)}`);+ vi.mocked(runFfmpeg).mockImplementation(async (args, opts) => {+ const result = scan(args.some((arg) => arg.includes(".true-peak-")) ? "-1.6" : "-0.3");+ opts?.onStderr?.(result.stderr);+ return result;+ });+ vi.mocked(encodeAac).mockImplementation(async (_input, output) => {+ writeFileSync(output, "corrected");+ return ran(true, "");+ });+ const audioPath = join(dir, "audio.m4a");+ writeFileSync(audioPath, "mixed");+ const result = await constrainAacTruePeak({+ audioPath,+ ...(sourcePath ? { sourcePath } : {}),+ targetDurationSeconds: 180,+ timeoutMs: 1000,+ });+ expect(result).toEqual({ success: true, loweredDb: 1.2 });+ expect(vi.mocked(encodeAac)).toHaveBeenCalledTimes(1);+ return { audioPath, call: vi.mocked(encodeAac).mock.calls[0]! };+ }++ it("re-encodes the plan's PCM mix, not the AAC, in the encoder's parallel segments", async () => {+ const sourcePath = join(dir, "mix.wav");+ const { call } = await correctHotAudio(sourcePath);+ expect(call[0]).toBe(sourcePath);+ expect(call[3]).toEqual({+ signal: undefined,+ timeout: 1000,+ filter: expect.stringMatching(/^volume=-1\.200dB,/),+ });+ });++ it("re-encodes an AAC without its PCM in one encoder, not decoded again per segment", async () => {+ const { audioPath, call } = await correctHotAudio();+ expect(call[0]).toBe(audioPath);+ expect(call[3]).toMatchObject({ segments: 1 });+ });+});diff --git a/packages/producer/src/services/render/stages/audioStage.ts b/packages/producer/src/services/render/stages/audioStage.tsindex 9523d7d89a..3d0c7c488f 100644--- a/packages/producer/src/services/render/stages/audioStage.ts+++ b/packages/producer/src/services/render/stages/audioStage.ts@@ -43,6 +43,8 @@ export interface AudioStageInput {audios: CompositionMetadata["audios"];masterFxChain?: string;urlByAbsentLocalPath?: ReadonlyMap<string, string>;+ /** Keep the float mix the AAC was encoded from here, for a later re-encode from it. */+ keepEncoderInputAt?: string;abortSignal: AbortSignal | undefined;assertNotAborted: () => void;/** Where a per-track failure's detail goes. Optional so tests need not pass one. */@@ -84,6 +86,7 @@ export async function runAudioStage(input: AudioStageInput): Promise<AudioStageRaudios,masterFxChain,urlByAbsentLocalPath,+ keepEncoderInputAt,abortSignal,assertNotAborted,log,@@ -111,7 +114,13 @@ export async function runAudioStage(input: AudioStageInput): Promise<AudioStageRaudioOutputPath,duration,abortSignal,- { ffmpegProcessTimeout, audioGain, masterFxChain, urlByAbsentLocalPath },+ {+ ffmpegProcessTimeout,+ audioGain,+ masterFxChain,+ urlByAbsentLocalPath,+ keepEncoderInputAt,+ },compiledDir,);} catch (err) {