Skip to content

Write HTML. Render video. Built for agents.

feat(producer): a cloud plan can leave its audio to be mixed beside its chunks (#5988)

Miguel Ángel8b410f1cb1b0
diff --git a/packages/producer/src/distributed.ts b/packages/producer/src/distributed.ts
index ba74fa4f78..45ca2520ad 100644
--- a/packages/producer/src/distributed.ts
+++ b/packages/producer/src/distributed.ts
@@ -38,6 +38,8 @@ export {
plan,
rejectUnsupportedDistributedFormat,
resolveChunkPlan,
+ buildPlanAudio,
+ plansSeparateAudio,
// Types
type DistributedRenderConfig,
type PlanResult,
diff --git a/packages/producer/src/services/distributed/assemble.ts b/packages/producer/src/services/distributed/assemble.ts
index bd856203cd..3cfc7b66fa 100644
--- a/packages/producer/src/services/distributed/assemble.ts
+++ b/packages/producer/src/services/distributed/assemble.ts
@@ -147,7 +147,8 @@ export async function assemble(
}
if (chunksAreStitchedMp4(planDir, cfr)) {
- await writeStitchedMp4(chunkPaths, outputPath);
+ const separateAudio = lockedAudio(planDir).separateAudio === true && audioPath !== null;
+ await writeStitchedMp4(chunkPaths, outputPath, separateAudio ? audioPath : undefined);
return {
outputPath,
durationMs: Date.now() - start,
@@ -399,22 +400,22 @@ async function runAssembleFfmpeg(
}
}
-/** Chunks that carry their audio are stitched under one index, the same bytes as the S3 stitch. */
+/** Chunks carrying audio, or with it as a separate file, stitch under one index like S3's. */
export function chunksAreStitchedMp4(planDir: string, cfr: boolean): boolean {
- if (!chunksCarryAudio(planDir)) return false;
+ const { chunksCarryAudio, separateAudio } = lockedAudio(planDir);
+ if (!chunksCarryAudio && !separateAudio) return false;
if (cfr) {
- throw new Error("[assemble] cfr=true cannot re-encode a plan whose mp4 chunks carry audio");
+ throw new Error("[assemble] cfr=true cannot re-encode a plan whose mp4 chunks are stitched");
}
return true;
}
-function chunksCarryAudio(planDir: string): boolean {
+function lockedAudio(
+ planDir: string,
+): Pick<LockedRenderConfig, "chunksCarryAudio" | "separateAudio"> {
const encoderJsonPath = join(planDir, "meta", "encoder.json");
- if (!existsSync(encoderJsonPath)) return false;
- return (
- (JSON.parse(readFileSync(encoderJsonPath, "utf-8")) as LockedRenderConfig).chunksCarryAudio ===
- true
- );
+ if (!existsSync(encoderJsonPath)) return {};
+ return JSON.parse(readFileSync(encoderJsonPath, "utf-8")) as LockedRenderConfig;
}
/**
diff --git a/packages/producer/src/services/distributed/chunkAudio.test.ts b/packages/producer/src/services/distributed/chunkAudio.test.ts
index 8e76465711..708be98e3e 100644
--- a/packages/producer/src/services/distributed/chunkAudio.test.ts
+++ b/packages/producer/src/services/distributed/chunkAudio.test.ts
@@ -236,6 +236,13 @@ describe.skipIf(!hasFfmpeg)("chunks that carry their audio", () => {
expect(frames(withAudio)).toEqual(frames(videoOnly));
expect(pcm(withAudio) === pcm(videoOnly)).toBe(true);
expect(streams(withAudio)).toEqual(streams(videoOnly));
+
+ // Video-only chunks stitched with the whole plan audio as a separate file: the same packets.
+ const separate = join(dir, "separate-audio.mp4");
+ writeFileSync(join(planDir, "meta", "encoder.json"), JSON.stringify({ separateAudio: true }));
+ await assemble(planDir, videoPaths, audioPath, separate);
+ expect(packets(separate)).toEqual(packets(withAudio));
+ expect(streams(separate)).toEqual(streams(withAudio));
},
60_000,
);
diff --git a/packages/producer/src/services/distributed/mp4Stitch.test.ts b/packages/producer/src/services/distributed/mp4Stitch.test.ts
index 0f58da9ffa..63cd66c66e 100644
--- a/packages/producer/src/services/distributed/mp4Stitch.test.ts
+++ b/packages/producer/src/services/distributed/mp4Stitch.test.ts
@@ -107,15 +107,20 @@ function samplesOf(file: Buffer, track: ReturnType<typeof readTracks>[number]):
return read;
}
-/** The stitched file: the header, then each chunk's mdat payload in order. */
-async function stitchInMemory(files: Buffer[]): Promise<Buffer> {
- const layouts = await Promise.all(files.map((file) => readMp4Layout(memoryRange(file))));
+/** The stitched file: the header, then `audio`'s and each chunk's mdat payload in order. */
+async function stitchInMemory(files: Buffer[], audio?: Buffer): Promise<Buffer> {
+ const sources = audio ? [audio, ...files] : files;
+ const layouts = await Promise.all(sources.map((file) => readMp4Layout(memoryRange(file))));
return Buffer.concat([
- buildStitchedHeader(layouts),
- ...layouts.map((l, i) => files[i]!.subarray(l.payloadStart, l.payloadEnd)),
+ audio ? buildStitchedHeader(layouts.slice(1), layouts[0]) : buildStitchedHeader(layouts),
+ ...layouts.map((l, i) => sources[i]!.subarray(l.payloadStart, l.payloadEnd)),
]);
}
+/** A track's tkhd track_ID (version 0, as fakeChunk writes it). */
+const trackIdOf = (moov: Buffer, track: ReturnType<typeof readTracks>[number]) =>
+ moov.readUInt32BE(track.trak.bodyStart + 8 + 12);
+
describe("buildStitchedHeader", () => {
const video = (sizes: number[], mediaTime = 0): FakeTrack => ({
handler: "vide",
@@ -199,6 +204,44 @@ describe("buildStitchedHeader", () => {
expect(tracks[1]!.durations).toEqual([1024, 1024, 1024, 640]);
});
+ it("puts a separate audio file's track and payload before every chunk's", async () => {
+ const chunks = [fakeChunk(0, [video([500, 120])]), fakeChunk(1, [video([90])])];
+ const mix = fakeChunk(2, [audio([1024, 1024, 640], 1024)]);
+ const file = await stitchInMemory(
+ chunks.map((chunk) => chunk.file),
+ mix.file,
+ );
+ const { moov } = await readMp4Layout(memoryRange(file));
+ const tracks = readTracks(moov);
+ expect(tracks.map((track) => [track.handler, track.mediaTime])).toEqual([
+ ["vide", 0],
+ ["soun", 1024],
+ ]);
+ expect(tracks.map((track) => trackIdOf(moov, track))).toEqual([1, 2]);
+ expect(samplesOf(file, tracks[0]!)).toEqual([
+ ...chunks[0]!.samples[0]!,
+ ...chunks[1]!.samples[0]!,
+ ]);
+ expect(samplesOf(file, tracks[1]!)).toEqual(mix.samples[0]!);
+ expect(tracks[1]!.durations).toEqual([1024, 1024, 640]);
+ expect(Math.max(...tracks[1]!.chunkOffsets)).toBeLessThan(Math.min(...tracks[0]!.chunkOffsets));
+ });
+
+ it.each([0, 1])(
+ "refuses chunk %i carrying audio when the audio is a separate file",
+ async (carrier) => {
+ const chunk = (i: number) =>
+ fakeChunk(i, i === carrier ? [video([500]), audio([1024], 0)] : [video([500])]);
+ const layouts = await Promise.all(
+ [chunk(0), chunk(1)].map((c) => readMp4Layout(memoryRange(c.file))),
+ );
+ const mix = await readMp4Layout(memoryRange(fakeChunk(2, [audio([1024], 1024)]).file));
+ expect(() => buildStitchedHeader(layouts, mix)).toThrow(
+ `chunk ${carrier} carries its own audio`,
+ );
+ },
+ );
+
it.each([
[
"an audio codec chunk 0 does not use",
@@ -246,6 +289,26 @@ describe("writeStitchedMp4", () => {
expect(readdirSync(dir).sort()).toEqual(["a.mp4", "b.mp4", "out.mp4"]);
});
+ it("writes a separate audio file's payload right after the header", async () => {
+ const files = [fakeChunk(0, [video([500, 120])]).file, fakeChunk(1, [video([480])]).file];
+ const mix = fakeChunk(2, [
+ {
+ handler: "soun",
+ timescale: 48000,
+ mediaTime: 1024,
+ durations: [1024, 640],
+ sizes: [40, 41],
+ },
+ ]).file;
+ const output = join(dir, "out.mp4");
+ await writeStitchedMp4(
+ [write("a.mp4", files[0]!), write("b.mp4", files[1]!)],
+ output,
+ write("audio.m4a", mix),
+ );
+ expect(readFileSync(output).equals(await stitchInMemory(files, mix))).toBe(true);
+ });
+
it("opens no output for chunks it refuses", async () => {
const paths = [
write("a.mp4", fakeChunk(0, [video([500])]).file),
diff --git a/packages/producer/src/services/distributed/mp4Stitch.ts b/packages/producer/src/services/distributed/mp4Stitch.ts
index 1b76643120..70e9179277 100644
--- a/packages/producer/src/services/distributed/mp4Stitch.ts
+++ b/packages/producer/src/services/distributed/mp4Stitch.ts
@@ -1,6 +1,6 @@
-// Assemble for chunks that carry their audio: one faststart header (ftyp, a moov indexing every
-// chunk's samples, a 64-bit mdat header); the chunks' mdat payloads follow it unchanged and in
-// order, so storage can copy them into place.
+// Assemble for mp4 chunks: one faststart header (ftyp, a moov indexing every chunk's samples, a
+// 64-bit mdat header); the payloads follow it unchanged and in order (a separate audio file's
+// first, then the chunks'), so storage can copy them into place.
import {
createReadStream,
@@ -270,6 +270,23 @@ function timescaleOf(buf: Buffer, header: Box): number {
return buf.readUInt32BE(header.bodyStart + (buf[header.bodyStart] === 1 ? 20 : 12));
}
+const TRACK_ID_AT: Record<string, [number, number]> = { tkhd: [12, 20], mvhd: [96, 108] };
+
+function idOf(buf: Buffer, target: Box): number {
+ return buf.readUInt32BE(
+ target.bodyStart + TRACK_ID_AT[target.type]![buf[target.bodyStart] === 1 ? 1 : 0],
+ );
+}
+
+function withId(boxBytes: Buffer, type: string, id: number): Buffer {
+ const header = boxAt(boxBytes, 0, 0, boxBytes.length);
+ boxBytes.writeUInt32BE(
+ id,
+ header.bodyStart + TRACK_ID_AT[type]![boxBytes[header.bodyStart] === 1 ? 1 : 0],
+ );
+ return boxBytes;
+}
+
interface MergedTrack {
durations: number[];
sizes: number[];
@@ -330,28 +347,55 @@ function planTracksOf(tracks: readonly TrackTable[], plan: readonly TrackTable[]
return matched;
}
-// ftyp + moov + mdat header for the chunks' payloads placed in order after it. The moov is chunk
-// 0's with merged sample tables; chunk 0's audio priming skip stays in an edit list.
-export function buildStitchedHeader(layouts: readonly Mp4Layout[]): Buffer {
- const plan = readTracks(layouts[0]!.moov);
- const chunks = layouts.map((layout, i) => planTracksOf(readTracks(layout.moov), plan, i));
+function sourceTracks(layouts: readonly Mp4Layout[], audio: Mp4Layout | undefined) {
+ const chunkPlan = readTracks(layouts[0]!.moov);
+ const audioTracks = audio ? readTracks(audio.moov) : [];
+ if (audio && (audioTracks.length !== 1 || audioTracks[0]!.handler !== "soun")) {
+ throw new Error("[mp4Stitch] the separate audio file must hold one audio track");
+ }
+ const plan = [...chunkPlan, ...audioTracks];
+ const chunks = layouts.map((layout, i) => {
+ const tracks = readTracks(layout.moov);
+ if (audio && tracks.some((track) => track.handler === "soun")) {
+ throw new Error(`[mp4Stitch] chunk ${i} carries its own audio`);
+ }
+ return planTracksOf(tracks, plan, i);
+ });
+ return {
+ plan,
+ chunkTrackCount: chunkPlan.length,
+ sources: audio ? [[...chunkPlan.map(() => null), audioTracks[0]!], ...chunks] : chunks,
+ };
+}
+
+// ftyp + moov + mdat header for `audio`'s payload, then the chunks', after it. The moov is chunk 0's
+// with merged sample tables and `audio`'s track (new id); the audio priming stays in an edit list.
+export function buildStitchedHeader(layouts: readonly Mp4Layout[], audio?: Mp4Layout): Buffer {
+ const { plan, chunkTrackCount, sources } = sourceTracks(layouts, audio);
+ const sourceLayouts = audio ? [audio, ...layouts] : layouts;
const payloadBases: number[] = [];
let payloadBytes = 0;
- for (const layout of layouts) {
+ for (const layout of sourceLayouts) {
payloadBases.push(payloadBytes);
payloadBytes += layout.payloadEnd - layout.payloadStart;
}
const template = layouts[0]!.moov;
const root = boxAt(template, 0, 0, template.length);
- const movieTimescale = timescaleOf(template, child(template, root, "mvhd"));
+ const mvhd = child(template, root, "mvhd");
+ const movieTimescale = timescaleOf(template, mvhd);
+ const chunkTrackIds = plan
+ .slice(0, chunkTrackCount)
+ .map((track) => idOf(template, child(template, track.trak, "tkhd")));
+ const audioTrackId = Math.max(...chunkTrackIds) + 1;
const buildMoov = (dataStart: number): Buffer => {
const traks = plan.map((track, t) => {
- // The chunks that carry this track, with their place in the file.
- const parts = chunks.flatMap((tracks, chunk) => {
+ // The sources that carry this track, with their place in the file.
+ const parts = sources.flatMap((tracks, chunk) => {
const part = tracks[t];
return part ? [{ ...part, chunk }] : [];
});
+ const moov = t < chunkTrackCount ? template : audio!.moov;
const merged: MergedTrack = {
durations: parts.flatMap((part) => part.durations),
sizes: parts.flatMap((part) => part.sizes),
@@ -362,9 +406,9 @@ export function buildStitchedHeader(layouts: readonly Mp4Layout[]): Buffer {
chunkOffsets: parts.flatMap((part) =>
part.chunkOffsets.map((offset) => {
const i = part.chunk;
- const { payloadStart, payloadEnd } = layouts[i]!;
+ const { payloadStart, payloadEnd } = sourceLayouts[i]!;
if (offset < payloadStart || offset >= payloadEnd) {
- throw new Error(`[mp4Stitch] chunk ${i} has samples outside its mdat`);
+ throw new Error(`[mp4Stitch] source ${i} has samples outside its mdat`);
}
return offset - payloadStart + payloadBases[i]! + dataStart;
}),
@@ -382,21 +426,29 @@ export function buildStitchedHeader(layouts: readonly Mp4Layout[]): Buffer {
// An edit list rounds the shown length to the movie timescale, so write one only to skip
// leading samples (the AAC priming).
const edts = track.mediaTime ? box("edts", fullBox("elst", 1, elst)) : Buffer.alloc(0);
- const trak = rebuild(template, track.trak, {
- tkhd: (tkhd) => Buffer.concat([withDuration(template, tkhd, shown), edts]),
+ const tkhdOf = (tkhd: Box) => {
+ const timed = withDuration(moov, tkhd, shown);
+ return t < chunkTrackCount ? timed : withId(timed, "tkhd", audioTrackId);
+ };
+ const trak = rebuild(moov, track.trak, {
+ tkhd: (tkhd) => Buffer.concat([tkhdOf(tkhd), edts]),
edts: () => Buffer.alloc(0),
mdia: (mdia) =>
- rebuild(template, mdia, {
- mdhd: (mdhd) => withDuration(template, mdhd, mediaDuration),
- minf: (minf) => rebuild(template, minf, { stbl: () => buildStbl(track.stsd, merged) }),
+ rebuild(moov, mdia, {
+ mdhd: (mdhd) => withDuration(moov, mdhd, mediaDuration),
+ minf: (minf) => rebuild(moov, minf, { stbl: () => buildStbl(track.stsd, merged) }),
}),
});
return { trak, shown };
});
let next = 0;
+ const movieHeader = withDuration(template, mvhd, Math.max(...traks.map((t) => t.shown)));
return rebuild(template, root, {
- mvhd: (mvhd) => withDuration(template, mvhd, Math.max(...traks.map((t) => t.shown))),
- trak: () => traks[next++]!.trak,
+ mvhd: () => (audio ? withId(movieHeader, "mvhd", audioTrackId + 1) : movieHeader),
+ trak: () =>
+ ++next < chunkTrackCount
+ ? traks[next - 1]!.trak
+ : Buffer.concat(traks.slice(next - 1).map((t) => t.trak)),
});
};
@@ -435,20 +487,24 @@ async function copyPayload(path: string, layout: Mp4Layout, index: number, out:
}
}
-/** Local join: the stitched header, then each chunk's mdat payload in order. Like the ffmpeg
- * join, it writes in a work folder beside `outputPath` and renames only a finished file there. */
+/** Local join: the header, then `audioPath`'s and each chunk's payload. Like the ffmpeg join, it
+ * writes in a work folder beside `outputPath` and renames only a finished file there. */
export async function writeStitchedMp4(
chunkPaths: readonly string[],
outputPath: string,
+ audioPath?: string,
): Promise<void> {
- const layouts = await Promise.all(chunkPaths.map((path) => readMp4Layout(fileRange(path))));
- const header = buildStitchedHeader(layouts);
+ const sourcePaths = audioPath ? [audioPath, ...chunkPaths] : chunkPaths;
+ const layouts = await Promise.all(sourcePaths.map((path) => readMp4Layout(fileRange(path))));
+ const header = audioPath
+ ? buildStitchedHeader(layouts.slice(1), layouts[0])
+ : buildStitchedHeader(layouts);
mkdirSync(dirname(outputPath), { recursive: true });
const workDir = mkdtempSync(`${outputPath}.stitch-`);
const out = createWriteStream(join(workDir, "out.mp4"));
try {
out.write(header);
- for (const [i, path] of chunkPaths.entries()) await copyPayload(path, layouts[i]!, i, out);
+ for (const [i, path] of sourcePaths.entries()) await copyPayload(path, layouts[i]!, i, out);
out.end();
await finished(out);
renameSync(join(workDir, "out.mp4"), outputPath);
diff --git a/packages/producer/src/services/distributed/plan.test.ts b/packages/producer/src/services/distributed/plan.test.ts
index cb4d7bd33f..c279428aab 100644
--- a/packages/producer/src/services/distributed/plan.test.ts
+++ b/packages/producer/src/services/distributed/plan.test.ts
@@ -41,6 +41,7 @@ import { CURRENT_PLAN_PROTOCOL } from "./planProtocol.js";
import {
applyDistributedAudioWarningPolicy,
buildLocalExecutionPlan,
+ buildPlanAudio,
buildChunkSlices,
CHUNK_EXCEEDS_WORKER_TIMEOUT,
ChunkExceedsWorkerTimeoutError,
@@ -1040,6 +1041,78 @@ describe.skipIf(spawnSync("ffmpeg", ["-version"]).status !== 0)("stitched mp4 pl
expect(cfrEncoder.chunksCarryAudio).toBeUndefined();
}, 60_000);
+ it("leaves the audio to buildPlanAudio, which makes the track the plan would", async () => {
+ // A hot tone, so the true-peak hold re-encodes as well.
+ const project = audioProject("project-separate-audio");
+ const config = { fps: 30, width: 320, height: 240, format: "mp4" } as const;
+ const carried = join(runRoot, "plan-carried-audio");
+ await buildLocalExecutionPlan(project, config, carried, { chunksCarryAudio: true });
+
+ const separate = join(runRoot, "plan-separate-audio");
+ await buildLocalExecutionPlan(project, config, separate, {
+ chunksCarryAudio: true,
+ deferAudio: true,
+ });
+ const encoder = JSON.parse(readFileSync(join(separate, "meta", "encoder.json"), "utf-8"));
+ expect([encoder.chunksCarryAudio, encoder.separateAudio]).toEqual([undefined, true]);
+ expect(existsSync(join(separate, "audio.m4a"))).toBe(false);
+
+ const work = join(runRoot, "separate-audio-work");
+ mkdirSync(work, { recursive: true });
+ const audio = join(work, "audio.m4a");
+ expect(await buildPlanAudio(project, config, work, audio)).toEqual({ hasAudio: true });
+ expect(readFileSync(audio).equals(readFileSync(join(carried, "audio.m4a")))).toBe(true);
+
+ const withCfr = join(runRoot, "plan-separate-audio-cfr");
+ await buildLocalExecutionPlan(project, { ...config, cfr: true }, withCfr, { deferAudio: true });
+ const cfrEncoder = JSON.parse(readFileSync(join(withCfr, "meta", "encoder.json"), "utf-8"));
+ expect(cfrEncoder.separateAudio).toBeUndefined();
+ expect(existsSync(join(withCfr, "audio.m4a"))).toBe(true);
+ }, 60_000);
+
+ it("gives buildPlanAudio the sound of a video whose src the scene script sets", async () => {
+ const project = join(runRoot, "project-script-src");
+ mkdirSync(project, { recursive: true });
+ const clip = spawnSync("ffmpeg", [
+ "-nostdin",
+ "-v",
+ "error",
+ "-f",
+ "lavfi",
+ "-i",
+ "testsrc=s=320x240:r=30:d=1",
+ "-f",
+ "lavfi",
+ "-i",
+ "sine=frequency=440:sample_rate=48000:d=1",
+ "-c:v",
+ "libx264",
+ "-pix_fmt",
+ "yuv420p",
+ "-c:a",
+ "aac",
+ "-shortest",
+ join(project, "clip.mp4"),
+ ]);
+ if (clip.status !== 0) throw new Error(`ffmpeg clip failed: ${clip.stderr}`);
+ writeFileSync(
+ join(project, "index.html"),
+ `<!doctype html><html><head><meta charset="utf-8"></head><body>
+ <div data-composition-id="root" data-width="320" data-height="240" data-duration="1" data-no-timeline>
+ <video id="v" data-has-audio="true" data-start="0" data-duration="1" playsinline></video>
+ </div>
+ <script>document.getElementById("v").src = "clip.mp4";</script></body></html>`,
+ );
+ const config = { fps: 30, width: 320, height: 240, format: "mp4" } as const;
+ const carried = join(runRoot, "plan-script-src");
+ await buildLocalExecutionPlan(project, config, carried, { chunksCarryAudio: true });
+ const work = join(runRoot, "script-src-work");
+ mkdirSync(work, { recursive: true });
+ const audio = join(work, "audio.m4a");
+ expect(await buildPlanAudio(project, config, work, audio)).toEqual({ hasAudio: true });
+ expect(readFileSync(audio).equals(readFileSync(join(carried, "audio.m4a")))).toBe(true);
+ }, 60_000);
+
it("encodes the audio once, at the frames' exact length", async () => {
// 1.01 s at 30 fps is 31 frames: 49,600 samples, 48.4375 AAC frames of 1,024.
const project = audioProject("project-one-encode", 0.3, 1.01);
diff --git a/packages/producer/src/services/distributed/plan.ts b/packages/producer/src/services/distributed/plan.ts
index 6cf90254e7..7fb2775440 100644
--- a/packages/producer/src/services/distributed/plan.ts
+++ b/packages/producer/src/services/distributed/plan.ts
@@ -321,6 +321,8 @@ export interface BuildLocalExecutionPlanOptions {
readonly chunksCarryAudio?: boolean;
/** With `chunksCarryAudio`, also write each chunk's audio packets to their own file. Plan v2 only. */
readonly sliceChunkAudio?: boolean;
+ /** Leave the audio to {@link buildPlanAudio}; chunks render video only. mp4 without `cfr` only. */
+ readonly deferAudio?: boolean;
}
export interface RemoteProjectFile {
@@ -882,6 +884,7 @@ function buildLockedRenderConfig(input: {
chunkCount: number;
runtimeEnv: Record<string, string>;
chunksCarryAudio: boolean;
+ separateAudio: boolean;
}): LockedRenderConfig {
const { config, forceScreenshot, deviceScaleFactor, ffmpegVersion } = input;
const { encoder, pixelFormat, preset } = resolveEncoderTriple(config);
@@ -917,6 +920,7 @@ function buildLockedRenderConfig(input: {
runtimeEnv: input.runtimeEnv,
variables: config.variables,
...(input.chunksCarryAudio ? { chunksCarryAudio: true } : {}),
+ ...(input.separateAudio ? { separateAudio: true } : {}),
};
}
@@ -1082,13 +1086,23 @@ function createVideoSourceShipper(input: {
* Build the shared local execution representation used by both transport
* protocols. See the module docstring for the directory layout.
*/
-export async function buildLocalExecutionPlan(
+/** Whether the plan leaves its audio to {@link buildPlanAudio}. */
+export function plansSeparateAudio(
+ config: Pick<DistributedRenderConfig, "format" | "cfr">,
+ options: Pick<BuildLocalExecutionPlanOptions, "deferAudio">,
+): boolean {
+ return options.deferAudio === true && config.format === "mp4" && config.cfr !== true;
+}
+
+// The plan's copy, compile and probe, shared by the plan and by buildPlanAudio so both see the
+// same composition.
+// fallow-ignore-next-line complexity
+async function compilePlanComposition(
projectDir: string,
config: DistributedRenderConfig,
- executionPlanDir: string,
- options: Readonly<BuildLocalExecutionPlanOptions> = {},
-): Promise<LocalExecutionPlan> {
- const planDir = executionPlanDir;
+ planDir: string,
+ options: Pick<BuildLocalExecutionPlanOptions, "executionPlanSizeLimitBytes" | "remoteFiles">,
+) {
// Plan-time validation. Rejections here surface as typed errors with
// non-retryable codes so workflow adapters don't waste retry budget on
// banned configs. Runs BEFORE any directory creation so a banned input
@@ -1290,6 +1304,121 @@ export async function buildLocalExecutionPlan(
});
}
}
+ return {
+ projectDir,
+ log,
+ sizeLimitBytes,
+ abortSignal,
+ assertNotAborted,
+ cfg,
+ job,
+ workDir,
+ compiledDir,
+ finalCompiledDir,
+ remoteFiles,
+ urlByAbsentLocalPath,
+ localPathByUrl,
+ compileResult,
+ compiled,
+ composition,
+ deviceScaleFactor,
+ forceScreenshot,
+ width,
+ height,
+ probeResult,
+ totalFrames,
+ };
+}
+
+type PlanComposition = Awaited<ReturnType<typeof compilePlanComposition>>;
+
+/** The composition's mix at `seconds`, keeping the float mix at `pcmPath` when given. */
+function mixPlanAudio(c: PlanComposition, compiledDir: string, seconds: number, pcmPath?: string) {
+ return runAudioStage({
+ projectDir: c.projectDir,
+ workDir: c.workDir,
+ compiledDir,
+ urlByAbsentLocalPath: c.urlByAbsentLocalPath,
+ duration: seconds,
+ ffmpegProcessTimeout: c.cfg.ffmpegProcessTimeout,
+ audioGain: c.cfg.audioGain,
+ audios: c.composition.audios,
+ masterFxChain: c.composition.masterFxChain,
+ ...(pcmPath ? { keepEncoderInputAt: pcmPath } : {}),
+ abortSignal: c.abortSignal,
+ assertNotAborted: c.assertNotAborted,
+ });
+}
+
+/** Extracts the composition's videos and adds the sound of each that carries one to its audios. */
+async function extractPlanVideos(c: PlanComposition, deferRangeExtraction: boolean | undefined) {
+ const extractResult = await runExtractVideosStage({
+ projectDir: c.projectDir,
+ compiledDir: c.compiledDir,
+ job: c.job,
+ cfg: c.cfg,
+ composition: c.composition,
+ abortSignal: c.abortSignal,
+ assertNotAborted: c.assertNotAborted,
+ materializeSymlinks: true,
+ deferRangeExtraction,
+ urlByAbsentLocalPath: c.urlByAbsentLocalPath,
+ });
+ if (extractResult.failureToEnforce) throw extractResult.failureToEnforce;
+ if (extractResult.extractionResult) {
+ // A chunk cannot fall back to decoding a source the planner held: a missing frame set
+ // renders as a silent blank video, so every declared source must extract.
+ assertVideoExtractionSucceeded(extractResult.extractionResult);
+ }
+ return extractResult;
+}
+
+/** Holds the mixed track at `audioPath` to the delivery true-peak ceiling, from its float mix. */
+async function holdPlanAudioPeak(
+ c: PlanComposition,
+ audioPath: string,
+ pcmPath: string,
+ seconds: number,
+): Promise<void> {
+ const peak = await constrainAacTruePeak({
+ audioPath,
+ ...(existsSync(pcmPath) ? { sourcePath: pcmPath } : {}),
+ targetDurationSeconds: seconds,
+ signal: c.abortSignal,
+ timeoutMs: c.cfg.ffmpegProcessTimeout,
+ }).finally(() => rmSync(pcmPath, { force: true }));
+ if (!peak.success) throw encoderFailureError("[plan] audio true-peak check failed", peak);
+}
+
+export async function buildLocalExecutionPlan(
+ projectDir: string,
+ config: DistributedRenderConfig,
+ executionPlanDir: string,
+ options: Readonly<BuildLocalExecutionPlanOptions> = {},
+): Promise<LocalExecutionPlan> {
+ const planDir = executionPlanDir;
+ const c = await compilePlanComposition(projectDir, config, planDir, options);
+ const {
+ log,
+ sizeLimitBytes,
+ cfg,
+ job,
+ workDir,
+ compiledDir,
+ finalCompiledDir,
+ remoteFiles,
+ urlByAbsentLocalPath,
+ localPathByUrl,
+ compileResult,
+ compiled,
+ composition,
+ deviceScaleFactor,
+ forceScreenshot,
+ width,
+ height,
+ probeResult,
+ totalFrames,
+ } = c;
// ── Chunking decisions ── (before extraction so an impossible plan fails fast; graded
// media is re-sized from its measured frame cost once its video frames exist)
@@ -1312,29 +1441,9 @@ export async function buildLocalExecutionPlan(
// ── Extract videos ──
// `materializeSymlinks: true` recursively copies frames so the planDir is
// self-contained (symlinks don't survive S3/GCS round-trips).
- stageStarted = Date.now();
- const extractResult = await runExtractVideosStage({
- projectDir,
- compiledDir,
- job,
- cfg,
- composition,
- abortSignal,
- assertNotAborted,
- materializeSymlinks: true,
- deferRangeExtraction: options.deferVideoExtraction,
- urlByAbsentLocalPath,
- });
+ let stageStarted = Date.now();
+ const extractResult = await extractPlanVideos(c, options.deferVideoExtraction);
logPlanStage(log, "extract", stageStarted);
- if (extractResult.failureToEnforce) throw extractResult.failureToEnforce;
- if (extractResult.extractionResult) {
- // Distributed chunks cannot safely fall back to native remote decoding:
- // the planner-local source may be unavailable on another worker, and a
- // missing frame set otherwise renders as a silent blank video. Unlike the
- // separately canaried in-process policy, distributed planning always
- // requires every declared source to extract successfully.
- assertVideoExtractionSucceeded(extractResult.extractionResult);
- }
// Skip `extractResult.frameLookup.cleanup()`: it would rm-rf each
// video's outputDir, but in `plan()` those directories ARE the source
// material the renames below move into `planDir/video-frames/`.
@@ -1415,27 +1524,22 @@ export async function buildLocalExecutionPlan(
// Only the assembler reads the mix, so hashing and publishing the final
// compiled and video trees run beside it instead of after it.
const digests: PlanFileDigests = options.digests ?? new Map();
- const chunksCarryAudio = carriesChunkAudio(config, options);
+ const separateAudio = plansSeparateAudio(config, options);
+ const chunksCarryAudio = !separateAudio && carriesChunkAudio(config, options);
// Chunks cut their audio from this track, so it is mixed to the frames' exact length in its one
// encode; the assembler pads or trims the track of any other plan.
const audioSeconds = chunksCarryAudio ? totalFrames / config.fps : probeResult.duration;
const planMixPcmPath = join(workDir, "plan-mix.wav");
stageStarted = Date.now();
const [audioOutcome, overlapOutcome, frameCostOutcome] = await Promise.allSettled([
- runAudioStage({
- projectDir,
- workDir,
- compiledDir: finalCompiledDir,
- urlByAbsentLocalPath,
- duration: audioSeconds,
- ffmpegProcessTimeout: cfg.ffmpegProcessTimeout,
- audioGain: cfg.audioGain,
- audios: composition.audios,
- masterFxChain: composition.masterFxChain,
- ...(chunksCarryAudio ? { keepEncoderInputAt: planMixPcmPath } : {}),
- abortSignal,
- assertNotAborted,
- }).finally(() => logPlanStage(log, "audio", stageStarted)),
+ separateAudio
+ ? null
+ : mixPlanAudio(
+ c,
+ finalCompiledDir,
+ audioSeconds,
+ chunksCarryAudio ? planMixPcmPath : undefined,
+ ).finally(() => logPlanStage(log, "audio", stageStarted)),
(async () => {
for (const dir of [finalCompiledDir, videoFramesDst, videoSourcesDir]) {
await prehashPlanFiles(dir, digests);
@@ -1487,24 +1591,17 @@ export async function buildLocalExecutionPlan(
);
const chunks = buildChunkSlices(totalFrames, chunkCount, effectiveChunkSize);
const audioResult = audioOutcome.value;
- if (audioResult.audioError) {
+ if (audioResult?.audioError) {
applyDistributedAudioWarningPolicy(job, audioResult.audioError, audioResult.audioFailures, log);
}
const planAudioPath = join(planDir, PLAN_AUDIO_RELATIVE_PATH);
- if (audioResult.hasAudio && existsSync(audioResult.audioOutputPath)) {
+ if (audioResult?.hasAudio && existsSync(audioResult.audioOutputPath)) {
renameSync(audioResult.audioOutputPath, planAudioPath);
}
if (chunksCarryAudio && existsSync(planAudioPath)) {
stageStarted = Date.now();
- const peak = await constrainAacTruePeak({
- audioPath: planAudioPath,
- ...(existsSync(planMixPcmPath) ? { sourcePath: planMixPcmPath } : {}),
- targetDurationSeconds: audioSeconds,
- signal: abortSignal,
- timeoutMs: cfg.ffmpegProcessTimeout,
- }).finally(() => rmSync(planMixPcmPath, { force: true }));
- if (!peak.success) throw encoderFailureError("[plan] audio true-peak check failed", peak);
+ await holdPlanAudioPeak(c, planAudioPath, planMixPcmPath, audioSeconds);
logPlanStage(log, "audio-peak", stageStarted);
if (options.sliceChunkAudio) {
stageStarted = Date.now();
@@ -1533,6 +1630,7 @@ export async function buildLocalExecutionPlan(
chunkCount,
runtimeEnv,
chunksCarryAudio,
+ separateAudio,
});
// ── Freeze the plan ──
@@ -1601,7 +1699,7 @@ export async function buildLocalExecutionPlan(
fontSnapshotSha,
durationSeconds: job.duration ?? 0,
totalFrames,
- hasAudio: audioResult.hasAudio,
+ hasAudio: audioResult?.hasAudio ?? false,
digests,
});
logPlanStage(log, "freeze", stageStarted);
@@ -1633,6 +1731,44 @@ export async function buildLocalExecutionPlan(
};
}
+/**
+ * The audio of a plan made with `deferAudio`: the mix and true-peak hold that plan would have
+ * made, written to `outputPath` (on `workRoot`'s filesystem) at exactly the video's length.
+ */
+export async function buildPlanAudio(
+ projectDir: string,
+ config: DistributedRenderConfig,
+ workRoot: string,
+ outputPath: string,
+ options: Pick<BuildLocalExecutionPlanOptions, "remoteFiles"> = {},
+): Promise<{ hasAudio: boolean }> {
+ const c = await compilePlanComposition(projectDir, config, workRoot, {
+ ...options,
+ executionPlanSizeLimitBytes: Number.MAX_SAFE_INTEGER,
+ });
+ try {
+ const seconds = c.totalFrames / config.fps;
+ const pcmPath = join(c.workDir, "plan-mix.wav");
+ let stageStarted = Date.now();
+ await extractPlanVideos(c, true);
+ logPlanStage(c.log, "extract", stageStarted);
+ stageStarted = Date.now();
+ const mixed = await mixPlanAudio(c, c.compiledDir, seconds, pcmPath);
+ logPlanStage(c.log, "audio", stageStarted);
+ if (mixed.audioError) {
+ applyDistributedAudioWarningPolicy(c.job, mixed.audioError, mixed.audioFailures, c.log);
+ }
+ if (!mixed.hasAudio || !existsSync(mixed.audioOutputPath)) return { hasAudio: false };
+ renameSync(mixed.audioOutputPath, outputPath);
+ stageStarted = Date.now();
+ await holdPlanAudioPeak(c, outputPath, pcmPath, seconds);
+ logPlanStage(c.log, "audio-peak", stageStarted);
+ return { hasAudio: true };
+ } finally {
+ rmSync(c.workDir, { recursive: true, force: true });
+ }
+}
+
/**
* Legacy v1 transport wrapper around {@link buildLocalExecutionPlan}.
*
diff --git a/packages/producer/src/services/distributed/planV2.ts b/packages/producer/src/services/distributed/planV2.ts
index a287aa0e2f..45a67ece44 100644
--- a/packages/producer/src/services/distributed/planV2.ts
+++ b/packages/producer/src/services/distributed/planV2.ts
@@ -150,6 +150,7 @@ export interface PlanV2WithPublisherOptions {
/** See {@link BuildLocalExecutionPlanOptions.remoteFiles}; needs a publisher with `referenceStoredBlob`. */
readonly remoteFiles?: BuildLocalExecutionPlanOptions["remoteFiles"];
readonly chunksCarryAudio?: boolean;
+ readonly deferAudio?: boolean;
}
export interface PlanV2MaterializationResult {
@@ -772,6 +773,7 @@ export async function planV2WithPublisher(
),
chunksCarryAudio: options.chunksCarryAudio,
sliceChunkAudio: true,
+ deferAudio: options.deferAudio,
});
return await publishExecutionPlan(stagingRoot, publisher, { digests, log: config.logger });
} catch (error) {
diff --git a/packages/producer/src/services/render/stages/freezePlan.ts b/packages/producer/src/services/render/stages/freezePlan.ts
index ba054823d2..b155fb3b4d 100644
--- a/packages/producer/src/services/render/stages/freezePlan.ts
+++ b/packages/producer/src/services/render/stages/freezePlan.ts
@@ -96,6 +96,7 @@ export interface LockedRenderConfig {
variables?: Record<string, unknown>;
chunksCarryAudio?: true;
+ separateAudio?: true;
}
export interface CompositionMetadataJson {