Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
feat(producer): a cloud plan can leave its audio to be mixed beside its chunks (#5988)
Miguel Ángel8b410f1cb1b0
diff --git a/packages/producer/src/distributed.ts b/packages/producer/src/distributed.tsindex ba74fa4f78..45ca2520ad 100644--- a/packages/producer/src/distributed.ts+++ b/packages/producer/src/distributed.ts@@ -38,6 +38,8 @@ export {plan,rejectUnsupportedDistributedFormat,resolveChunkPlan,+ buildPlanAudio,+ plansSeparateAudio,// Typestype DistributedRenderConfig,type PlanResult,diff --git a/packages/producer/src/services/distributed/assemble.ts b/packages/producer/src/services/distributed/assemble.tsindex bd856203cd..3cfc7b66fa 100644--- a/packages/producer/src/services/distributed/assemble.ts+++ b/packages/producer/src/services/distributed/assemble.ts@@ -147,7 +147,8 @@ export async function assemble(}if (chunksAreStitchedMp4(planDir, cfr)) {- await writeStitchedMp4(chunkPaths, outputPath);+ const separateAudio = lockedAudio(planDir).separateAudio === true && audioPath !== null;+ await writeStitchedMp4(chunkPaths, outputPath, separateAudio ? audioPath : undefined);return {outputPath,durationMs: Date.now() - start,@@ -399,22 +400,22 @@ async function runAssembleFfmpeg(}}-/** Chunks that carry their audio are stitched under one index, the same bytes as the S3 stitch. */+/** Chunks carrying audio, or with it as a separate file, stitch under one index like S3's. */export function chunksAreStitchedMp4(planDir: string, cfr: boolean): boolean {- if (!chunksCarryAudio(planDir)) return false;+ const { chunksCarryAudio, separateAudio } = lockedAudio(planDir);+ if (!chunksCarryAudio && !separateAudio) return false;if (cfr) {- throw new Error("[assemble] cfr=true cannot re-encode a plan whose mp4 chunks carry audio");+ throw new Error("[assemble] cfr=true cannot re-encode a plan whose mp4 chunks are stitched");}return true;}-function chunksCarryAudio(planDir: string): boolean {+function lockedAudio(+ planDir: string,+): Pick<LockedRenderConfig, "chunksCarryAudio" | "separateAudio"> {const encoderJsonPath = join(planDir, "meta", "encoder.json");- if (!existsSync(encoderJsonPath)) return false;- return (- (JSON.parse(readFileSync(encoderJsonPath, "utf-8")) as LockedRenderConfig).chunksCarryAudio ===- true- );+ if (!existsSync(encoderJsonPath)) return {};+ return JSON.parse(readFileSync(encoderJsonPath, "utf-8")) as LockedRenderConfig;}/**diff --git a/packages/producer/src/services/distributed/chunkAudio.test.ts b/packages/producer/src/services/distributed/chunkAudio.test.tsindex 8e76465711..708be98e3e 100644--- a/packages/producer/src/services/distributed/chunkAudio.test.ts+++ b/packages/producer/src/services/distributed/chunkAudio.test.ts@@ -236,6 +236,13 @@ describe.skipIf(!hasFfmpeg)("chunks that carry their audio", () => {expect(frames(withAudio)).toEqual(frames(videoOnly));expect(pcm(withAudio) === pcm(videoOnly)).toBe(true);expect(streams(withAudio)).toEqual(streams(videoOnly));++ // Video-only chunks stitched with the whole plan audio as a separate file: the same packets.+ const separate = join(dir, "separate-audio.mp4");+ writeFileSync(join(planDir, "meta", "encoder.json"), JSON.stringify({ separateAudio: true }));+ await assemble(planDir, videoPaths, audioPath, separate);+ expect(packets(separate)).toEqual(packets(withAudio));+ expect(streams(separate)).toEqual(streams(withAudio));},60_000,);diff --git a/packages/producer/src/services/distributed/mp4Stitch.test.ts b/packages/producer/src/services/distributed/mp4Stitch.test.tsindex 0f58da9ffa..63cd66c66e 100644--- a/packages/producer/src/services/distributed/mp4Stitch.test.ts+++ b/packages/producer/src/services/distributed/mp4Stitch.test.ts@@ -107,15 +107,20 @@ function samplesOf(file: Buffer, track: ReturnType<typeof readTracks>[number]):return read;}-/** The stitched file: the header, then each chunk's mdat payload in order. */-async function stitchInMemory(files: Buffer[]): Promise<Buffer> {- const layouts = await Promise.all(files.map((file) => readMp4Layout(memoryRange(file))));+/** The stitched file: the header, then `audio`'s and each chunk's mdat payload in order. */+async function stitchInMemory(files: Buffer[], audio?: Buffer): Promise<Buffer> {+ const sources = audio ? [audio, ...files] : files;+ const layouts = await Promise.all(sources.map((file) => readMp4Layout(memoryRange(file))));return Buffer.concat([- buildStitchedHeader(layouts),- ...layouts.map((l, i) => files[i]!.subarray(l.payloadStart, l.payloadEnd)),+ audio ? buildStitchedHeader(layouts.slice(1), layouts[0]) : buildStitchedHeader(layouts),+ ...layouts.map((l, i) => sources[i]!.subarray(l.payloadStart, l.payloadEnd)),]);}+/** A track's tkhd track_ID (version 0, as fakeChunk writes it). */+const trackIdOf = (moov: Buffer, track: ReturnType<typeof readTracks>[number]) =>+ moov.readUInt32BE(track.trak.bodyStart + 8 + 12);+describe("buildStitchedHeader", () => {const video = (sizes: number[], mediaTime = 0): FakeTrack => ({handler: "vide",@@ -199,6 +204,44 @@ describe("buildStitchedHeader", () => {expect(tracks[1]!.durations).toEqual([1024, 1024, 1024, 640]);});+ it("puts a separate audio file's track and payload before every chunk's", async () => {+ const chunks = [fakeChunk(0, [video([500, 120])]), fakeChunk(1, [video([90])])];+ const mix = fakeChunk(2, [audio([1024, 1024, 640], 1024)]);+ const file = await stitchInMemory(+ chunks.map((chunk) => chunk.file),+ mix.file,+ );+ const { moov } = await readMp4Layout(memoryRange(file));+ const tracks = readTracks(moov);+ expect(tracks.map((track) => [track.handler, track.mediaTime])).toEqual([+ ["vide", 0],+ ["soun", 1024],+ ]);+ expect(tracks.map((track) => trackIdOf(moov, track))).toEqual([1, 2]);+ expect(samplesOf(file, tracks[0]!)).toEqual([+ ...chunks[0]!.samples[0]!,+ ...chunks[1]!.samples[0]!,+ ]);+ expect(samplesOf(file, tracks[1]!)).toEqual(mix.samples[0]!);+ expect(tracks[1]!.durations).toEqual([1024, 1024, 640]);+ expect(Math.max(...tracks[1]!.chunkOffsets)).toBeLessThan(Math.min(...tracks[0]!.chunkOffsets));+ });++ it.each([0, 1])(+ "refuses chunk %i carrying audio when the audio is a separate file",+ async (carrier) => {+ const chunk = (i: number) =>+ fakeChunk(i, i === carrier ? [video([500]), audio([1024], 0)] : [video([500])]);+ const layouts = await Promise.all(+ [chunk(0), chunk(1)].map((c) => readMp4Layout(memoryRange(c.file))),+ );+ const mix = await readMp4Layout(memoryRange(fakeChunk(2, [audio([1024], 1024)]).file));+ expect(() => buildStitchedHeader(layouts, mix)).toThrow(+ `chunk ${carrier} carries its own audio`,+ );+ },+ );+it.each([["an audio codec chunk 0 does not use",@@ -246,6 +289,26 @@ describe("writeStitchedMp4", () => {expect(readdirSync(dir).sort()).toEqual(["a.mp4", "b.mp4", "out.mp4"]);});+ it("writes a separate audio file's payload right after the header", async () => {+ const files = [fakeChunk(0, [video([500, 120])]).file, fakeChunk(1, [video([480])]).file];+ const mix = fakeChunk(2, [+ {+ handler: "soun",+ timescale: 48000,+ mediaTime: 1024,+ durations: [1024, 640],+ sizes: [40, 41],+ },+ ]).file;+ const output = join(dir, "out.mp4");+ await writeStitchedMp4(+ [write("a.mp4", files[0]!), write("b.mp4", files[1]!)],+ output,+ write("audio.m4a", mix),+ );+ expect(readFileSync(output).equals(await stitchInMemory(files, mix))).toBe(true);+ });+it("opens no output for chunks it refuses", async () => {const paths = [write("a.mp4", fakeChunk(0, [video([500])]).file),diff --git a/packages/producer/src/services/distributed/mp4Stitch.ts b/packages/producer/src/services/distributed/mp4Stitch.tsindex 1b76643120..70e9179277 100644--- a/packages/producer/src/services/distributed/mp4Stitch.ts+++ b/packages/producer/src/services/distributed/mp4Stitch.ts@@ -1,6 +1,6 @@-// Assemble for chunks that carry their audio: one faststart header (ftyp, a moov indexing every-// chunk's samples, a 64-bit mdat header); the chunks' mdat payloads follow it unchanged and in-// order, so storage can copy them into place.+// Assemble for mp4 chunks: one faststart header (ftyp, a moov indexing every chunk's samples, a+// 64-bit mdat header); the payloads follow it unchanged and in order (a separate audio file's+// first, then the chunks'), so storage can copy them into place.import {createReadStream,@@ -270,6 +270,23 @@ function timescaleOf(buf: Buffer, header: Box): number {return buf.readUInt32BE(header.bodyStart + (buf[header.bodyStart] === 1 ? 20 : 12));}+const TRACK_ID_AT: Record<string, [number, number]> = { tkhd: [12, 20], mvhd: [96, 108] };++function idOf(buf: Buffer, target: Box): number {+ return buf.readUInt32BE(+ target.bodyStart + TRACK_ID_AT[target.type]![buf[target.bodyStart] === 1 ? 1 : 0],+ );+}++function withId(boxBytes: Buffer, type: string, id: number): Buffer {+ const header = boxAt(boxBytes, 0, 0, boxBytes.length);+ boxBytes.writeUInt32BE(+ id,+ header.bodyStart + TRACK_ID_AT[type]![boxBytes[header.bodyStart] === 1 ? 1 : 0],+ );+ return boxBytes;+}+interface MergedTrack {durations: number[];sizes: number[];@@ -330,28 +347,55 @@ function planTracksOf(tracks: readonly TrackTable[], plan: readonly TrackTable[]return matched;}-// ftyp + moov + mdat header for the chunks' payloads placed in order after it. The moov is chunk-// 0's with merged sample tables; chunk 0's audio priming skip stays in an edit list.-export function buildStitchedHeader(layouts: readonly Mp4Layout[]): Buffer {- const plan = readTracks(layouts[0]!.moov);- const chunks = layouts.map((layout, i) => planTracksOf(readTracks(layout.moov), plan, i));+function sourceTracks(layouts: readonly Mp4Layout[], audio: Mp4Layout | undefined) {+ const chunkPlan = readTracks(layouts[0]!.moov);+ const audioTracks = audio ? readTracks(audio.moov) : [];+ if (audio && (audioTracks.length !== 1 || audioTracks[0]!.handler !== "soun")) {+ throw new Error("[mp4Stitch] the separate audio file must hold one audio track");+ }+ const plan = [...chunkPlan, ...audioTracks];+ const chunks = layouts.map((layout, i) => {+ const tracks = readTracks(layout.moov);+ if (audio && tracks.some((track) => track.handler === "soun")) {+ throw new Error(`[mp4Stitch] chunk ${i} carries its own audio`);+ }+ return planTracksOf(tracks, plan, i);+ });+ return {+ plan,+ chunkTrackCount: chunkPlan.length,+ sources: audio ? [[...chunkPlan.map(() => null), audioTracks[0]!], ...chunks] : chunks,+ };+}++// ftyp + moov + mdat header for `audio`'s payload, then the chunks', after it. The moov is chunk 0's+// with merged sample tables and `audio`'s track (new id); the audio priming stays in an edit list.+export function buildStitchedHeader(layouts: readonly Mp4Layout[], audio?: Mp4Layout): Buffer {+ const { plan, chunkTrackCount, sources } = sourceTracks(layouts, audio);+ const sourceLayouts = audio ? [audio, ...layouts] : layouts;const payloadBases: number[] = [];let payloadBytes = 0;- for (const layout of layouts) {+ for (const layout of sourceLayouts) {payloadBases.push(payloadBytes);payloadBytes += layout.payloadEnd - layout.payloadStart;}const template = layouts[0]!.moov;const root = boxAt(template, 0, 0, template.length);- const movieTimescale = timescaleOf(template, child(template, root, "mvhd"));+ const mvhd = child(template, root, "mvhd");+ const movieTimescale = timescaleOf(template, mvhd);+ const chunkTrackIds = plan+ .slice(0, chunkTrackCount)+ .map((track) => idOf(template, child(template, track.trak, "tkhd")));+ const audioTrackId = Math.max(...chunkTrackIds) + 1;const buildMoov = (dataStart: number): Buffer => {const traks = plan.map((track, t) => {- // The chunks that carry this track, with their place in the file.- const parts = chunks.flatMap((tracks, chunk) => {+ // The sources that carry this track, with their place in the file.+ const parts = sources.flatMap((tracks, chunk) => {const part = tracks[t];return part ? [{ ...part, chunk }] : [];});+ const moov = t < chunkTrackCount ? template : audio!.moov;const merged: MergedTrack = {durations: parts.flatMap((part) => part.durations),sizes: parts.flatMap((part) => part.sizes),@@ -362,9 +406,9 @@ export function buildStitchedHeader(layouts: readonly Mp4Layout[]): Buffer {chunkOffsets: parts.flatMap((part) =>part.chunkOffsets.map((offset) => {const i = part.chunk;- const { payloadStart, payloadEnd } = layouts[i]!;+ const { payloadStart, payloadEnd } = sourceLayouts[i]!;if (offset < payloadStart || offset >= payloadEnd) {- throw new Error(`[mp4Stitch] chunk ${i} has samples outside its mdat`);+ throw new Error(`[mp4Stitch] source ${i} has samples outside its mdat`);}return offset - payloadStart + payloadBases[i]! + dataStart;}),@@ -382,21 +426,29 @@ export function buildStitchedHeader(layouts: readonly Mp4Layout[]): Buffer {// An edit list rounds the shown length to the movie timescale, so write one only to skip// leading samples (the AAC priming).const edts = track.mediaTime ? box("edts", fullBox("elst", 1, elst)) : Buffer.alloc(0);- const trak = rebuild(template, track.trak, {- tkhd: (tkhd) => Buffer.concat([withDuration(template, tkhd, shown), edts]),+ const tkhdOf = (tkhd: Box) => {+ const timed = withDuration(moov, tkhd, shown);+ return t < chunkTrackCount ? timed : withId(timed, "tkhd", audioTrackId);+ };+ const trak = rebuild(moov, track.trak, {+ tkhd: (tkhd) => Buffer.concat([tkhdOf(tkhd), edts]),edts: () => Buffer.alloc(0),mdia: (mdia) =>- rebuild(template, mdia, {- mdhd: (mdhd) => withDuration(template, mdhd, mediaDuration),- minf: (minf) => rebuild(template, minf, { stbl: () => buildStbl(track.stsd, merged) }),+ rebuild(moov, mdia, {+ mdhd: (mdhd) => withDuration(moov, mdhd, mediaDuration),+ minf: (minf) => rebuild(moov, minf, { stbl: () => buildStbl(track.stsd, merged) }),}),});return { trak, shown };});let next = 0;+ const movieHeader = withDuration(template, mvhd, Math.max(...traks.map((t) => t.shown)));return rebuild(template, root, {- mvhd: (mvhd) => withDuration(template, mvhd, Math.max(...traks.map((t) => t.shown))),- trak: () => traks[next++]!.trak,+ mvhd: () => (audio ? withId(movieHeader, "mvhd", audioTrackId + 1) : movieHeader),+ trak: () =>+ ++next < chunkTrackCount+ ? traks[next - 1]!.trak+ : Buffer.concat(traks.slice(next - 1).map((t) => t.trak)),});};@@ -435,20 +487,24 @@ async function copyPayload(path: string, layout: Mp4Layout, index: number, out:}}-/** Local join: the stitched header, then each chunk's mdat payload in order. Like the ffmpeg- * join, it writes in a work folder beside `outputPath` and renames only a finished file there. */+/** Local join: the header, then `audioPath`'s and each chunk's payload. Like the ffmpeg join, it+ * writes in a work folder beside `outputPath` and renames only a finished file there. */export async function writeStitchedMp4(chunkPaths: readonly string[],outputPath: string,+ audioPath?: string,): Promise<void> {- const layouts = await Promise.all(chunkPaths.map((path) => readMp4Layout(fileRange(path))));- const header = buildStitchedHeader(layouts);+ const sourcePaths = audioPath ? [audioPath, ...chunkPaths] : chunkPaths;+ const layouts = await Promise.all(sourcePaths.map((path) => readMp4Layout(fileRange(path))));+ const header = audioPath+ ? buildStitchedHeader(layouts.slice(1), layouts[0])+ : buildStitchedHeader(layouts);mkdirSync(dirname(outputPath), { recursive: true });const workDir = mkdtempSync(`${outputPath}.stitch-`);const out = createWriteStream(join(workDir, "out.mp4"));try {out.write(header);- for (const [i, path] of chunkPaths.entries()) await copyPayload(path, layouts[i]!, i, out);+ for (const [i, path] of sourcePaths.entries()) await copyPayload(path, layouts[i]!, i, out);out.end();await finished(out);renameSync(join(workDir, "out.mp4"), outputPath);diff --git a/packages/producer/src/services/distributed/plan.test.ts b/packages/producer/src/services/distributed/plan.test.tsindex cb4d7bd33f..c279428aab 100644--- a/packages/producer/src/services/distributed/plan.test.ts+++ b/packages/producer/src/services/distributed/plan.test.ts@@ -41,6 +41,7 @@ import { CURRENT_PLAN_PROTOCOL } from "./planProtocol.js";import {applyDistributedAudioWarningPolicy,buildLocalExecutionPlan,+ buildPlanAudio,buildChunkSlices,CHUNK_EXCEEDS_WORKER_TIMEOUT,ChunkExceedsWorkerTimeoutError,@@ -1040,6 +1041,78 @@ describe.skipIf(spawnSync("ffmpeg", ["-version"]).status !== 0)("stitched mp4 plexpect(cfrEncoder.chunksCarryAudio).toBeUndefined();}, 60_000);+ it("leaves the audio to buildPlanAudio, which makes the track the plan would", async () => {+ // A hot tone, so the true-peak hold re-encodes as well.+ const project = audioProject("project-separate-audio");+ const config = { fps: 30, width: 320, height: 240, format: "mp4" } as const;+ const carried = join(runRoot, "plan-carried-audio");+ await buildLocalExecutionPlan(project, config, carried, { chunksCarryAudio: true });++ const separate = join(runRoot, "plan-separate-audio");+ await buildLocalExecutionPlan(project, config, separate, {+ chunksCarryAudio: true,+ deferAudio: true,+ });+ const encoder = JSON.parse(readFileSync(join(separate, "meta", "encoder.json"), "utf-8"));+ expect([encoder.chunksCarryAudio, encoder.separateAudio]).toEqual([undefined, true]);+ expect(existsSync(join(separate, "audio.m4a"))).toBe(false);++ const work = join(runRoot, "separate-audio-work");+ mkdirSync(work, { recursive: true });+ const audio = join(work, "audio.m4a");+ expect(await buildPlanAudio(project, config, work, audio)).toEqual({ hasAudio: true });+ expect(readFileSync(audio).equals(readFileSync(join(carried, "audio.m4a")))).toBe(true);++ const withCfr = join(runRoot, "plan-separate-audio-cfr");+ await buildLocalExecutionPlan(project, { ...config, cfr: true }, withCfr, { deferAudio: true });+ const cfrEncoder = JSON.parse(readFileSync(join(withCfr, "meta", "encoder.json"), "utf-8"));+ expect(cfrEncoder.separateAudio).toBeUndefined();+ expect(existsSync(join(withCfr, "audio.m4a"))).toBe(true);+ }, 60_000);++ it("gives buildPlanAudio the sound of a video whose src the scene script sets", async () => {+ const project = join(runRoot, "project-script-src");+ mkdirSync(project, { recursive: true });+ const clip = spawnSync("ffmpeg", [+ "-nostdin",+ "-v",+ "error",+ "-f",+ "lavfi",+ "-i",+ "testsrc=s=320x240:r=30:d=1",+ "-f",+ "lavfi",+ "-i",+ "sine=frequency=440:sample_rate=48000:d=1",+ "-c:v",+ "libx264",+ "-pix_fmt",+ "yuv420p",+ "-c:a",+ "aac",+ "-shortest",+ join(project, "clip.mp4"),+ ]);+ if (clip.status !== 0) throw new Error(`ffmpeg clip failed: ${clip.stderr}`);+ writeFileSync(+ join(project, "index.html"),+ `<!doctype html><html><head><meta charset="utf-8"></head><body>+ <div data-composition-id="root" data-width="320" data-height="240" data-duration="1" data-no-timeline>+ <video id="v" data-has-audio="true" data-start="0" data-duration="1" playsinline></video>+ </div>+ <script>document.getElementById("v").src = "clip.mp4";</script></body></html>`,+ );+ const config = { fps: 30, width: 320, height: 240, format: "mp4" } as const;+ const carried = join(runRoot, "plan-script-src");+ await buildLocalExecutionPlan(project, config, carried, { chunksCarryAudio: true });+ const work = join(runRoot, "script-src-work");+ mkdirSync(work, { recursive: true });+ const audio = join(work, "audio.m4a");+ expect(await buildPlanAudio(project, config, work, audio)).toEqual({ hasAudio: true });+ expect(readFileSync(audio).equals(readFileSync(join(carried, "audio.m4a")))).toBe(true);+ }, 60_000);+it("encodes the audio once, at the frames' exact length", async () => {// 1.01 s at 30 fps is 31 frames: 49,600 samples, 48.4375 AAC frames of 1,024.const project = audioProject("project-one-encode", 0.3, 1.01);diff --git a/packages/producer/src/services/distributed/plan.ts b/packages/producer/src/services/distributed/plan.tsindex 6cf90254e7..7fb2775440 100644--- a/packages/producer/src/services/distributed/plan.ts+++ b/packages/producer/src/services/distributed/plan.ts@@ -321,6 +321,8 @@ export interface BuildLocalExecutionPlanOptions {readonly chunksCarryAudio?: boolean;/** With `chunksCarryAudio`, also write each chunk's audio packets to their own file. Plan v2 only. */readonly sliceChunkAudio?: boolean;+ /** Leave the audio to {@link buildPlanAudio}; chunks render video only. mp4 without `cfr` only. */+ readonly deferAudio?: boolean;}export interface RemoteProjectFile {@@ -882,6 +884,7 @@ function buildLockedRenderConfig(input: {chunkCount: number;runtimeEnv: Record<string, string>;chunksCarryAudio: boolean;+ separateAudio: boolean;}): LockedRenderConfig {const { config, forceScreenshot, deviceScaleFactor, ffmpegVersion } = input;const { encoder, pixelFormat, preset } = resolveEncoderTriple(config);@@ -917,6 +920,7 @@ function buildLockedRenderConfig(input: {runtimeEnv: input.runtimeEnv,variables: config.variables,...(input.chunksCarryAudio ? { chunksCarryAudio: true } : {}),+ ...(input.separateAudio ? { separateAudio: true } : {}),};}@@ -1082,13 +1086,23 @@ function createVideoSourceShipper(input: {* Build the shared local execution representation used by both transport* protocols. See the module docstring for the directory layout.*/-export async function buildLocalExecutionPlan(+/** Whether the plan leaves its audio to {@link buildPlanAudio}. */+export function plansSeparateAudio(+ config: Pick<DistributedRenderConfig, "format" | "cfr">,+ options: Pick<BuildLocalExecutionPlanOptions, "deferAudio">,+): boolean {+ return options.deferAudio === true && config.format === "mp4" && config.cfr !== true;+}++// The plan's copy, compile and probe, shared by the plan and by buildPlanAudio so both see the+// same composition.+// fallow-ignore-next-line complexity+async function compilePlanComposition(projectDir: string,config: DistributedRenderConfig,- executionPlanDir: string,- options: Readonly<BuildLocalExecutionPlanOptions> = {},-): Promise<LocalExecutionPlan> {- const planDir = executionPlanDir;+ planDir: string,+ options: Pick<BuildLocalExecutionPlanOptions, "executionPlanSizeLimitBytes" | "remoteFiles">,+) {// Plan-time validation. Rejections here surface as typed errors with// non-retryable codes so workflow adapters don't waste retry budget on// banned configs. Runs BEFORE any directory creation so a banned input@@ -1290,6 +1304,121 @@ export async function buildLocalExecutionPlan(});}}+ return {+ projectDir,+ log,+ sizeLimitBytes,+ abortSignal,+ assertNotAborted,+ cfg,+ job,+ workDir,+ compiledDir,+ finalCompiledDir,+ remoteFiles,+ urlByAbsentLocalPath,+ localPathByUrl,+ compileResult,+ compiled,+ composition,+ deviceScaleFactor,+ forceScreenshot,+ width,+ height,+ probeResult,+ totalFrames,+ };+}++type PlanComposition = Awaited<ReturnType<typeof compilePlanComposition>>;++/** The composition's mix at `seconds`, keeping the float mix at `pcmPath` when given. */+function mixPlanAudio(c: PlanComposition, compiledDir: string, seconds: number, pcmPath?: string) {+ return runAudioStage({+ projectDir: c.projectDir,+ workDir: c.workDir,+ compiledDir,+ urlByAbsentLocalPath: c.urlByAbsentLocalPath,+ duration: seconds,+ ffmpegProcessTimeout: c.cfg.ffmpegProcessTimeout,+ audioGain: c.cfg.audioGain,+ audios: c.composition.audios,+ masterFxChain: c.composition.masterFxChain,+ ...(pcmPath ? { keepEncoderInputAt: pcmPath } : {}),+ abortSignal: c.abortSignal,+ assertNotAborted: c.assertNotAborted,+ });+}++/** Extracts the composition's videos and adds the sound of each that carries one to its audios. */+async function extractPlanVideos(c: PlanComposition, deferRangeExtraction: boolean | undefined) {+ const extractResult = await runExtractVideosStage({+ projectDir: c.projectDir,+ compiledDir: c.compiledDir,+ job: c.job,+ cfg: c.cfg,+ composition: c.composition,+ abortSignal: c.abortSignal,+ assertNotAborted: c.assertNotAborted,+ materializeSymlinks: true,+ deferRangeExtraction,+ urlByAbsentLocalPath: c.urlByAbsentLocalPath,+ });+ if (extractResult.failureToEnforce) throw extractResult.failureToEnforce;+ if (extractResult.extractionResult) {+ // A chunk cannot fall back to decoding a source the planner held: a missing frame set+ // renders as a silent blank video, so every declared source must extract.+ assertVideoExtractionSucceeded(extractResult.extractionResult);+ }+ return extractResult;+}++/** Holds the mixed track at `audioPath` to the delivery true-peak ceiling, from its float mix. */+async function holdPlanAudioPeak(+ c: PlanComposition,+ audioPath: string,+ pcmPath: string,+ seconds: number,+): Promise<void> {+ const peak = await constrainAacTruePeak({+ audioPath,+ ...(existsSync(pcmPath) ? { sourcePath: pcmPath } : {}),+ targetDurationSeconds: seconds,+ signal: c.abortSignal,+ timeoutMs: c.cfg.ffmpegProcessTimeout,+ }).finally(() => rmSync(pcmPath, { force: true }));+ if (!peak.success) throw encoderFailureError("[plan] audio true-peak check failed", peak);+}++export async function buildLocalExecutionPlan(+ projectDir: string,+ config: DistributedRenderConfig,+ executionPlanDir: string,+ options: Readonly<BuildLocalExecutionPlanOptions> = {},+): Promise<LocalExecutionPlan> {+ const planDir = executionPlanDir;+ const c = await compilePlanComposition(projectDir, config, planDir, options);+ const {+ log,+ sizeLimitBytes,+ cfg,+ job,+ workDir,+ compiledDir,+ finalCompiledDir,+ remoteFiles,+ urlByAbsentLocalPath,+ localPathByUrl,+ compileResult,+ compiled,+ composition,+ deviceScaleFactor,+ forceScreenshot,+ width,+ height,+ probeResult,+ totalFrames,+ } = c;// ── Chunking decisions ── (before extraction so an impossible plan fails fast; graded// media is re-sized from its measured frame cost once its video frames exist)@@ -1312,29 +1441,9 @@ export async function buildLocalExecutionPlan(// ── Extract videos ──// `materializeSymlinks: true` recursively copies frames so the planDir is// self-contained (symlinks don't survive S3/GCS round-trips).- stageStarted = Date.now();- const extractResult = await runExtractVideosStage({- projectDir,- compiledDir,- job,- cfg,- composition,- abortSignal,- assertNotAborted,- materializeSymlinks: true,- deferRangeExtraction: options.deferVideoExtraction,- urlByAbsentLocalPath,- });+ let stageStarted = Date.now();+ const extractResult = await extractPlanVideos(c, options.deferVideoExtraction);logPlanStage(log, "extract", stageStarted);- if (extractResult.failureToEnforce) throw extractResult.failureToEnforce;- if (extractResult.extractionResult) {- // Distributed chunks cannot safely fall back to native remote decoding:- // the planner-local source may be unavailable on another worker, and a- // missing frame set otherwise renders as a silent blank video. Unlike the- // separately canaried in-process policy, distributed planning always- // requires every declared source to extract successfully.- assertVideoExtractionSucceeded(extractResult.extractionResult);- }// Skip `extractResult.frameLookup.cleanup()`: it would rm-rf each// video's outputDir, but in `plan()` those directories ARE the source// material the renames below move into `planDir/video-frames/`.@@ -1415,27 +1524,22 @@ export async function buildLocalExecutionPlan(// Only the assembler reads the mix, so hashing and publishing the final// compiled and video trees run beside it instead of after it.const digests: PlanFileDigests = options.digests ?? new Map();- const chunksCarryAudio = carriesChunkAudio(config, options);+ const separateAudio = plansSeparateAudio(config, options);+ const chunksCarryAudio = !separateAudio && carriesChunkAudio(config, options);// Chunks cut their audio from this track, so it is mixed to the frames' exact length in its one// encode; the assembler pads or trims the track of any other plan.const audioSeconds = chunksCarryAudio ? totalFrames / config.fps : probeResult.duration;const planMixPcmPath = join(workDir, "plan-mix.wav");stageStarted = Date.now();const [audioOutcome, overlapOutcome, frameCostOutcome] = await Promise.allSettled([- runAudioStage({- projectDir,- workDir,- compiledDir: finalCompiledDir,- urlByAbsentLocalPath,- duration: audioSeconds,- ffmpegProcessTimeout: cfg.ffmpegProcessTimeout,- audioGain: cfg.audioGain,- audios: composition.audios,- masterFxChain: composition.masterFxChain,- ...(chunksCarryAudio ? { keepEncoderInputAt: planMixPcmPath } : {}),- abortSignal,- assertNotAborted,- }).finally(() => logPlanStage(log, "audio", stageStarted)),+ separateAudio+ ? null+ : mixPlanAudio(+ c,+ finalCompiledDir,+ audioSeconds,+ chunksCarryAudio ? planMixPcmPath : undefined,+ ).finally(() => logPlanStage(log, "audio", stageStarted)),(async () => {for (const dir of [finalCompiledDir, videoFramesDst, videoSourcesDir]) {await prehashPlanFiles(dir, digests);@@ -1487,24 +1591,17 @@ export async function buildLocalExecutionPlan();const chunks = buildChunkSlices(totalFrames, chunkCount, effectiveChunkSize);const audioResult = audioOutcome.value;- if (audioResult.audioError) {+ if (audioResult?.audioError) {applyDistributedAudioWarningPolicy(job, audioResult.audioError, audioResult.audioFailures, log);}const planAudioPath = join(planDir, PLAN_AUDIO_RELATIVE_PATH);- if (audioResult.hasAudio && existsSync(audioResult.audioOutputPath)) {+ if (audioResult?.hasAudio && existsSync(audioResult.audioOutputPath)) {renameSync(audioResult.audioOutputPath, planAudioPath);}if (chunksCarryAudio && existsSync(planAudioPath)) {stageStarted = Date.now();- const peak = await constrainAacTruePeak({- audioPath: planAudioPath,- ...(existsSync(planMixPcmPath) ? { sourcePath: planMixPcmPath } : {}),- targetDurationSeconds: audioSeconds,- signal: abortSignal,- timeoutMs: cfg.ffmpegProcessTimeout,- }).finally(() => rmSync(planMixPcmPath, { force: true }));- if (!peak.success) throw encoderFailureError("[plan] audio true-peak check failed", peak);+ await holdPlanAudioPeak(c, planAudioPath, planMixPcmPath, audioSeconds);logPlanStage(log, "audio-peak", stageStarted);if (options.sliceChunkAudio) {stageStarted = Date.now();@@ -1533,6 +1630,7 @@ export async function buildLocalExecutionPlan(chunkCount,runtimeEnv,chunksCarryAudio,+ separateAudio,});// ── Freeze the plan ──@@ -1601,7 +1699,7 @@ export async function buildLocalExecutionPlan(fontSnapshotSha,durationSeconds: job.duration ?? 0,totalFrames,- hasAudio: audioResult.hasAudio,+ hasAudio: audioResult?.hasAudio ?? false,digests,});logPlanStage(log, "freeze", stageStarted);@@ -1633,6 +1731,44 @@ export async function buildLocalExecutionPlan(};}+/**+ * The audio of a plan made with `deferAudio`: the mix and true-peak hold that plan would have+ * made, written to `outputPath` (on `workRoot`'s filesystem) at exactly the video's length.+ */+export async function buildPlanAudio(+ projectDir: string,+ config: DistributedRenderConfig,+ workRoot: string,+ outputPath: string,+ options: Pick<BuildLocalExecutionPlanOptions, "remoteFiles"> = {},+): Promise<{ hasAudio: boolean }> {+ const c = await compilePlanComposition(projectDir, config, workRoot, {+ ...options,+ executionPlanSizeLimitBytes: Number.MAX_SAFE_INTEGER,+ });+ try {+ const seconds = c.totalFrames / config.fps;+ const pcmPath = join(c.workDir, "plan-mix.wav");+ let stageStarted = Date.now();+ await extractPlanVideos(c, true);+ logPlanStage(c.log, "extract", stageStarted);+ stageStarted = Date.now();+ const mixed = await mixPlanAudio(c, c.compiledDir, seconds, pcmPath);+ logPlanStage(c.log, "audio", stageStarted);+ if (mixed.audioError) {+ applyDistributedAudioWarningPolicy(c.job, mixed.audioError, mixed.audioFailures, c.log);+ }+ if (!mixed.hasAudio || !existsSync(mixed.audioOutputPath)) return { hasAudio: false };+ renameSync(mixed.audioOutputPath, outputPath);+ stageStarted = Date.now();+ await holdPlanAudioPeak(c, outputPath, pcmPath, seconds);+ logPlanStage(c.log, "audio-peak", stageStarted);+ return { hasAudio: true };+ } finally {+ rmSync(c.workDir, { recursive: true, force: true });+ }+}+/*** Legacy v1 transport wrapper around {@link buildLocalExecutionPlan}.*diff --git a/packages/producer/src/services/distributed/planV2.ts b/packages/producer/src/services/distributed/planV2.tsindex a287aa0e2f..45a67ece44 100644--- a/packages/producer/src/services/distributed/planV2.ts+++ b/packages/producer/src/services/distributed/planV2.ts@@ -150,6 +150,7 @@ export interface PlanV2WithPublisherOptions {/** See {@link BuildLocalExecutionPlanOptions.remoteFiles}; needs a publisher with `referenceStoredBlob`. */readonly remoteFiles?: BuildLocalExecutionPlanOptions["remoteFiles"];readonly chunksCarryAudio?: boolean;+ readonly deferAudio?: boolean;}export interface PlanV2MaterializationResult {@@ -772,6 +773,7 @@ export async function planV2WithPublisher(),chunksCarryAudio: options.chunksCarryAudio,sliceChunkAudio: true,+ deferAudio: options.deferAudio,});return await publishExecutionPlan(stagingRoot, publisher, { digests, log: config.logger });} catch (error) {diff --git a/packages/producer/src/services/render/stages/freezePlan.ts b/packages/producer/src/services/render/stages/freezePlan.tsindex ba054823d2..b155fb3b4d 100644--- a/packages/producer/src/services/render/stages/freezePlan.ts+++ b/packages/producer/src/services/render/stages/freezePlan.ts@@ -96,6 +96,7 @@ export interface LockedRenderConfig {variables?: Record<string, unknown>;chunksCarryAudio?: true;+ separateAudio?: true;}export interface CompositionMetadataJson {