Skip to content

Write HTML. Render video. Built for agents.

perf(producer): distributed mp4 chunks can carry their slice of the plan audio (#5803)

Miguel Ángel1a2194c7b4f3
diff --git a/packages/producer/src/services/distributed/assemble.ts b/packages/producer/src/services/distributed/assemble.ts
index 5647c3eb55..09431e404d 100644
--- a/packages/producer/src/services/distributed/assemble.ts
+++ b/packages/producer/src/services/distributed/assemble.ts
@@ -155,6 +155,9 @@ export async function assemble(
format,
fps,
fpsArg: fpsToFfmpegArg(fps),
+ chunkSeconds: chunksCarryAudio(planDir)
+ ? plan.chunks.map((chunk) => ((chunk.endFrame - chunk.startFrame) * fps.den) / fps.num)
+ : null,
signal: options?.abortSignal,
timeout: resolveConfig().ffmpegProcessTimeout,
log,
@@ -197,7 +200,10 @@ export async function assemble(
};
}
-function readAssemblePlan(planDir: string, chunkPaths: readonly string[]): PlanJsonForAssemble {
+function readAssemblePlan(
+ planDir: string,
+ chunkPaths: readonly string[],
+): PlanJsonForAssemble & { chunks: ChunkSliceJson[] } {
// ── 1. Validate planDir manifest matches chunkPaths shape ──────────────
const planJsonPath = join(planDir, "plan.json");
const chunksJsonPath = join(planDir, "meta", "chunks.json");
@@ -224,7 +230,7 @@ function readAssemblePlan(planDir: string, chunkPaths: readonly string[]): PlanJ
}
}
- return plan;
+ return { ...plan, chunks };
}
interface JoinContext {
@@ -236,6 +242,7 @@ interface JoinContext {
timeout: number;
log: ProducerLogger;
timed: <T>(stage: string, run: () => Promise<T>) => Promise<T>;
+ chunkSeconds: number[] | null;
}
// A one-entry concat list ignores the input-side `-r` (observed `359/12` in the
@@ -243,7 +250,7 @@ interface JoinContext {
function videoInputArgs(ctx: JoinContext, videoSources: readonly string[]): string[] {
return videoSources.length === 1
? ["-i", videoSources[0]!]
- : concatDemuxerInput(videoSources, ctx.workDir, ctx.fpsArg);
+ : concatDemuxerInput(videoSources, ctx.workDir, ctx.fpsArg, ctx.chunkSeconds);
}
// ── 2c. Optional exact-CFR re-encode ──────────────────────────────────
@@ -340,11 +347,18 @@ function concatDemuxerInput(
chunkPaths: readonly string[],
workDir: string,
fpsArg: string,
+ chunkSeconds: readonly number[] | null,
): string[] {
// ffmpeg's concat list escapes a single quote as `'\''`.
const listPath = join(workDir, "concat-list.txt");
- const body = chunkPaths.map((path) => `file '${path.replace(/'/g, "'\\''")}'`).join("\n");
- writeFileSync(listPath, `${body}\n`, "utf-8");
+ // Chunks that carry audio: read only their video (track 1), and end each chunk at its last frame
+ // rather than after its audio, which starts with the AAC priming and can run one packet longer.
+ const head = chunkSeconds ? ["stream", "exact_stream_id 1"] : [];
+ const body = chunkPaths.map((path, i) => {
+ const line = `file '${path.replace(/'/g, "'\\''")}'`;
+ return chunkSeconds ? `${line}\nduration ${chunkSeconds[i]!.toFixed(6)}` : line;
+ });
+ writeFileSync(listPath, `${[...head, ...body].join("\n")}\n`, "utf-8");
return ["-r", fpsArg, "-f", "concat", "-safe", "0", "-i", listPath];
}
@@ -379,6 +393,15 @@ async function runAssembleFfmpeg(
}
}
+function chunksCarryAudio(planDir: string): boolean {
+ const encoderJsonPath = join(planDir, "meta", "encoder.json");
+ if (!existsSync(encoderJsonPath)) return false;
+ return (
+ (JSON.parse(readFileSync(encoderJsonPath, "utf-8")) as LockedRenderConfig).chunksCarryAudio ===
+ true
+ );
+}
+
/**
* The cfr re-encode hardcodes `-c:v libx264`: reject non-mp4 formats and h265
* chunks (which it would silently transcode) before any ffmpeg runs.
diff --git a/packages/producer/src/services/distributed/chunkAudio.test.ts b/packages/producer/src/services/distributed/chunkAudio.test.ts
new file mode 100644
index 0000000000..832e3a0906
--- /dev/null
+++ b/packages/producer/src/services/distributed/chunkAudio.test.ts
@@ -0,0 +1,153 @@
+import { spawnSync } from "node:child_process";
+import { afterAll, beforeAll, describe, expect, it } from "bun:test";
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { assemble } from "./assemble.js";
+import { audioCutForChunk, muxChunkWithAudio } from "./chunkAudio.js";
+
+const FPS = { num: 30, den: 1 };
+
+describe("audioCutForChunk", () => {
+ it("gives each chunk the AAC packets that start in its range, priming to chunk 0", () => {
+ const grid = { sampleRate: 48000, firstStart: -1024, packetSamples: 1024, packetCount: 564 };
+ const chunks = [0, 120, 240].map((startFrame, index) => ({
+ index,
+ startFrame,
+ endFrame: startFrame + 120,
+ }));
+ const cuts = chunks.map((_, index) => audioCutForChunk(grid, chunks, index, FPS));
+ expect(cuts).toEqual([
+ { seekSeconds: null, durationSeconds: 4, firstPacket: 0, packetCount: 189 },
+ {
+ seekSeconds: 4,
+ durationSeconds: (375 * 1024 - 512) / 48000 - 4,
+ firstPacket: 189,
+ packetCount: 187,
+ },
+ {
+ seekSeconds: (375 * 1024 - 512) / 48000,
+ durationSeconds: null,
+ firstPacket: 376,
+ packetCount: 188,
+ },
+ ]);
+ });
+});
+
+const hasFfmpeg = spawnSync("ffmpeg", ["-version"]).status === 0;
+
+describe.skipIf(!hasFfmpeg)("chunks that carry their audio", () => {
+ const TOTAL_FRAMES = 91;
+ let root: string;
+
+ beforeAll(() => {
+ root = mkdtempSync(join(tmpdir(), "hf-chunk-audio-"));
+ });
+ afterAll(() => {
+ rmSync(root, { recursive: true, force: true });
+ });
+
+ function run(cmd: string, args: string[]): string {
+ const result = spawnSync(cmd, args, { maxBuffer: 1 << 28, encoding: "latin1" });
+ if (result.status !== 0) throw new Error(`${cmd} ${args.join(" ")}: ${result.stderr}`);
+ return result.stdout;
+ }
+ const ffmpeg = (args: string[]) => run("ffmpeg", ["-nostdin", "-loglevel", "error", ...args]);
+ const frames = (path: string) =>
+ ffmpeg(["-i", path, "-map", "0:v", "-f", "framemd5", "-"])
+ .split("\n")
+ .filter((line) => line && !line.startsWith("#"));
+ const pcm = (path: string) => ffmpeg(["-i", path, "-map", "0:a", "-f", "s16le", "-"]);
+ const streams = (path: string) =>
+ JSON.parse(
+ run("ffprobe", [
+ "-v",
+ "error",
+ "-count_packets",
+ "-show_entries",
+ "stream=codec_type,nb_read_packets,duration,r_frame_rate,avg_frame_rate",
+ "-of",
+ "json",
+ "--",
+ path,
+ ]),
+ ).streams as { codec_type: string; nb_read_packets: string; duration: string }[];
+
+ // Uneven chunks with a 1-frame tail, and a single chunk (read without a concat list today).
+ it.each([
+ ["three", [0, 47, 90, 91]],
+ ["one", [0, 91]],
+ ] as const)(
+ "%s chunk(s) join through today's assemble like video-only chunks",
+ async (name, edges) => {
+ const CHUNKS = edges.slice(1).map((end, index) => ({
+ index,
+ startFrame: edges[index]!,
+ endFrame: end,
+ }));
+ const dir = join(root, name);
+ mkdirSync(dir, { recursive: true });
+ const audioPath = join(dir, "audio.m4a");
+ ffmpeg(
+ `-f lavfi -i aevalsrc=0.4*sin(2*PI*(220+40*t)*t):s=48000:d=${TOTAL_FRAMES / 30} -c:a aac -b:a 192k -y`
+ .split(" ")
+ .concat(audioPath),
+ );
+ const planDir = join(dir, "plan");
+ mkdirSync(join(planDir, "meta"), { recursive: true });
+ writeFileSync(
+ join(planDir, "plan.json"),
+ JSON.stringify({
+ planHash: "fake",
+ totalFrames: TOTAL_FRAMES,
+ hasAudio: true,
+ dimensions: { fpsNum: 30, fpsDen: 1, width: 160, height: 120, format: "mp4" },
+ }),
+ );
+ writeFileSync(join(planDir, "meta", "chunks.json"), JSON.stringify(CHUNKS));
+
+ const videoPaths: string[] = [];
+ const chunkPaths: string[] = [];
+ for (const slice of CHUNKS) {
+ const n = String(slice.endFrame - slice.startFrame);
+ videoPaths.push(join(dir, `video-${slice.index}.mp4`));
+ chunkPaths.push(join(dir, `chunk-${slice.index}.mp4`));
+ // Closed GOP, no B-frames, headers repeated: shaped like renderChunk's encode.
+ ffmpeg(
+ `-f lavfi -i testsrc2=size=160x120:rate=30,trim=start_frame=${slice.startFrame}:end_frame=${slice.endFrame} -c:v libx264 -preset veryfast -g ${n} -keyint_min ${n} -sc_threshold 0 -bf 0 -x264-params scenecut=0:open-gop=0:repeat-headers=1 -pix_fmt yuv420p -avoid_negative_ts make_zero -y`
+ .split(" ")
+ .concat(videoPaths[slice.index]!),
+ );
+ await muxChunkWithAudio({
+ videoPath: videoPaths[slice.index]!,
+ audioPath,
+ outputPath: chunkPaths[slice.index]!,
+ chunks: CHUNKS,
+ chunkIndex: slice.index,
+ fps: FPS,
+ timeoutMs: 60_000,
+ });
+ }
+ // Every plan audio packet lands in exactly one chunk.
+ const audioPackets = Number(streams(audioPath)[0]!.nb_read_packets);
+ const chunkPackets = chunkPaths.map((path) => Number(streams(path)[1]!.nb_read_packets));
+ expect(chunkPackets.reduce((a, b) => a + b, 0)).toBe(audioPackets);
+
+ const withAudio = join(dir, "with-audio.mp4");
+ const videoOnly = join(dir, "video-only.mp4");
+ writeFileSync(
+ join(planDir, "meta", "encoder.json"),
+ JSON.stringify({ chunksCarryAudio: true }),
+ );
+ await assemble(planDir, chunkPaths, audioPath, withAudio);
+ // Today's join of the video-only encodes, from a plan without the option.
+ writeFileSync(join(planDir, "meta", "encoder.json"), "{}");
+ await assemble(planDir, videoPaths, audioPath, videoOnly);
+ expect(frames(withAudio)).toEqual(frames(videoOnly));
+ expect(pcm(withAudio) === pcm(videoOnly)).toBe(true);
+ expect(streams(withAudio)).toEqual(streams(videoOnly));
+ },
+ 60_000,
+ );
+});
diff --git a/packages/producer/src/services/distributed/chunkAudio.ts b/packages/producer/src/services/distributed/chunkAudio.ts
new file mode 100644
index 0000000000..db71f999dd
--- /dev/null
+++ b/packages/producer/src/services/distributed/chunkAudio.ts
@@ -0,0 +1,152 @@
+// Distributed mp4 chunks can carry their own slice of the plan audio: the AAC packets of the one
+// plan-wide encode that start inside the chunk's frame range, so the chunks' audio has no seams.
+
+import type { Fps } from "@hyperframes/core";
+import { appendRenderProvenanceArgs, describeFfmpegFailure, runFfmpeg } from "@hyperframes/engine";
+import { runFfprobeJson } from "../render/audioPadTrim.js";
+import type { ChunkSliceJson } from "../render/stages/freezePlan.js";
+
+/** Where AAC packet `j` of the plan audio starts: `firstStart + j * packetSamples`, in samples. */
+interface AacPacketGrid {
+ sampleRate: number;
+ firstStart: number;
+ packetSamples: number;
+ packetCount: number;
+}
+
+/** Input-side `-ss` / `-t` that select exactly packets `[first, end)` for one chunk. */
+interface AudioCut {
+ seekSeconds: number | null;
+ durationSeconds: number | null;
+ firstPacket: number;
+ packetCount: number;
+}
+
+// A chunk owns the packets starting in its frame range (chunk 0 also the priming before 0).
+// Cuts sit half a packet before a packet start so ffmpeg's rounding cannot move them.
+export function audioCutForChunk(
+ grid: AacPacketGrid,
+ chunks: readonly ChunkSliceJson[],
+ index: number,
+ fps: Fps,
+): AudioCut | null {
+ const firstPacketAt = (frame: number): number =>
+ Math.min(
+ grid.packetCount,
+ Math.max(
+ 0,
+ Math.ceil(
+ (frame * fps.den * grid.sampleRate - grid.firstStart * fps.num) /
+ (grid.packetSamples * fps.num),
+ ),
+ ),
+ );
+ const first = index === 0 ? 0 : firstPacketAt(chunks[index]!.startFrame);
+ const end =
+ index === chunks.length - 1 ? grid.packetCount : firstPacketAt(chunks[index + 1]!.startFrame);
+ if (end <= first) return null;
+ const cutSeconds = (packet: number): number =>
+ (grid.firstStart + packet * grid.packetSamples - grid.packetSamples / 2) / grid.sampleRate;
+ const seekSeconds = index === 0 ? null : cutSeconds(first);
+ const durationSeconds = end === grid.packetCount ? null : cutSeconds(end) - (seekSeconds ?? 0);
+ return { seekSeconds, durationSeconds, firstPacket: first, packetCount: end - first };
+}
+
+interface FfprobeAudioGrid {
+ streams?: { sample_rate?: string; time_base?: string; nb_frames?: string }[];
+ packets?: { pts?: number; duration?: number }[];
+}
+
+async function probeAacPacketGrid(audioPath: string): Promise<AacPacketGrid> {
+ const probe = await runFfprobeJson<FfprobeAudioGrid>([
+ "-v",
+ "error",
+ "-select_streams",
+ "a:0",
+ "-show_entries",
+ "stream=sample_rate,time_base,nb_frames:packet=pts,duration",
+ "-read_intervals",
+ "%+#1",
+ "-of",
+ "json",
+ "--",
+ audioPath,
+ ]);
+ const stream = probe.streams?.[0] ?? {};
+ const packet = probe.packets?.[0] ?? {};
+ const sampleRate = Number(stream.sample_rate);
+ const [tbNum = NaN, tbDen = NaN] = String(stream.time_base).split("/").map(Number);
+ const toSamples = (ticks?: number): number => (Number(ticks) * sampleRate * tbNum) / tbDen;
+ const grid = {
+ sampleRate,
+ firstStart: toSamples(packet.pts),
+ packetSamples: toSamples(packet.duration),
+ packetCount: Number(stream.nb_frames),
+ };
+ const counts = [grid.sampleRate, grid.packetSamples, grid.packetCount];
+ if (!Number.isFinite(grid.firstStart) || !counts.every((n) => Number.isFinite(n) && n > 0)) {
+ throw new Error(`[chunkAudio] could not read the AAC packet grid: ${JSON.stringify(probe)}`);
+ }
+ return grid;
+}
+
+interface MuxInput {
+ videoPath: string;
+ audioPath: string | null;
+ outputPath: string;
+ chunks: readonly ChunkSliceJson[];
+ chunkIndex: number;
+ fps: Fps;
+ signal?: AbortSignal;
+ timeoutMs: number;
+}
+
+function chunkMuxArgs(input: MuxInput, cut: AudioCut | null): string[] {
+ if (!cut) return ["-i", input.videoPath, "-map", "0:v:0"];
+ const seek = cut.seekSeconds === null ? [] : ["-ss", cut.seekSeconds.toFixed(6)];
+ const length = cut.durationSeconds === null ? [] : ["-t", cut.durationSeconds.toFixed(6)];
+ // A seek keeps the packet before the cut unless told not to; chunk 0 never
+ // seeks, so its priming packet (and the edit list that hides it) survive.
+ const exact = seek.length ? ["-copypriorss", "0"] : [];
+ return [
+ ...["-i", input.videoPath, ...seek, ...length, "-i", input.audioPath!],
+ ...["-map", "0:v:0", "-map", "1:a:0", ...exact],
+ ];
+}
+
+/** Mux one encoded chunk with the plan audio packets it owns into a regular mp4 at `outputPath`. */
+export async function muxChunkWithAudio(input: MuxInput): Promise<void> {
+ const grid = input.audioPath ? await probeAacPacketGrid(input.audioPath) : null;
+ const cut = grid ? audioCutForChunk(grid, input.chunks, input.chunkIndex, input.fps) : null;
+ const args = [...chunkMuxArgs(input, cut), "-c", "copy"];
+ appendRenderProvenanceArgs(args, input.outputPath);
+ args.push("-y", input.outputPath);
+ const result = await runFfmpeg(args, { signal: input.signal, timeout: input.timeoutMs });
+ if (!result.success) {
+ throw new Error(
+ `[chunkAudio] chunk mux failed: ${describeFfmpegFailure(result, input.timeoutMs)}`,
+ );
+ }
+ const slice = input.chunks[input.chunkIndex]!;
+ const expected: Record<string, number> = { video: slice.endFrame - slice.startFrame };
+ if (cut) expected.audio = cut.packetCount;
+ const probe = await runFfprobeJson<{
+ streams?: { codec_type?: string; nb_read_packets?: string }[];
+ }>([
+ "-v",
+ "error",
+ "-count_packets",
+ "-show_entries",
+ "stream=codec_type,nb_read_packets",
+ "-of",
+ "json",
+ "--",
+ input.outputPath,
+ ]);
+ for (const [type, count] of Object.entries(expected)) {
+ const packets = Number(probe.streams?.find((s) => s.codec_type === type)?.nb_read_packets ?? 0);
+ if (packets !== count) {
+ throw new Error(`[chunkAudio] ${type} packets in chunk ${packets}, expected ${count}`);
+ }
+ }
+}
diff --git a/packages/producer/src/services/distributed/plan.test.ts b/packages/producer/src/services/distributed/plan.test.ts
index f67c58bc4f..14750e1040 100644
--- a/packages/producer/src/services/distributed/plan.test.ts
+++ b/packages/producer/src/services/distributed/plan.test.ts
@@ -15,6 +15,7 @@
* inside the regression harness (`bun run --cwd packages/producer docker:test`).
*/
+import { spawnSync } from "node:child_process";
import { afterAll, beforeAll, describe, expect, it } from "bun:test";
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
@@ -853,6 +854,70 @@ describe("plan() — golden planDir + planHash determinism", () => {
);
});
+describe.skipIf(spawnSync("ffmpeg", ["-version"]).status !== 0)("stitched mp4 plans", () => {
+ function audioProject(name: string): string {
+ const dir = join(runRoot, name);
+ mkdirSync(dir, { recursive: true });
+ const tone = spawnSync("ffmpeg", [
+ "-nostdin",
+ "-v",
+ "error",
+ "-f",
+ "lavfi",
+ "-i",
+ "aevalsrc=0.99*sin(2*PI*440*t):s=48000:d=0.6",
+ "-c:a",
+ "pcm_s16le",
+ join(dir, "tone.wav"),
+ ]);
+ if (tone.status !== 0) throw new Error(`ffmpeg tone failed: ${tone.stderr}`);
+ writeFileSync(
+ join(dir, "index.html"),
+ `<!doctype html><html><head><meta charset="utf-8"></head><body>
+ <div data-composition-id="root" data-width="320" data-height="240" data-duration="1">
+ <audio data-composition-audio src="tone.wav"></audio>
+ </div></body></html>`,
+ );
+ return dir;
+ }
+
+ it("fits the audio length and peak and records the layout only for mp4 without cfr", async () => {
+ const project = audioProject("project-stitch");
+ const config = { fps: 30, width: 320, height: 240, format: "mp4" } as const;
+ const stitched = join(runRoot, "plan-stitch");
+ await buildLocalExecutionPlan(project, config, stitched, { chunksCarryAudio: true });
+ const encoder = JSON.parse(readFileSync(join(stitched, "meta", "encoder.json"), "utf-8"));
+ expect(encoder.chunksCarryAudio).toBe(true);
+ const probe = spawnSync(
+ "ffprobe",
+ [
+ "-v",
+ "error",
+ "-show_entries",
+ "format=duration",
+ "-of",
+ "csv=p=0",
+ join(stitched, "audio.m4a"),
+ ],
+ { encoding: "utf-8" },
+ );
+ expect(Math.abs(Number(probe.stdout) - 1)).toBeLessThan(0.001);
+ const volume = spawnSync(
+ "ffmpeg",
+ ["-nostdin", "-i", join(stitched, "audio.m4a"), "-af", "volumedetect", "-f", "null", "-"],
+ { encoding: "utf-8" },
+ );
+ expect(Number(/max_volume: (-?[\d.]+) dB/.exec(volume.stderr)?.[1])).toBeLessThan(-1);
+
+ const withCfr = join(runRoot, "plan-stitch-cfr");
+ await buildLocalExecutionPlan(project, { ...config, cfr: true }, withCfr, {
+ chunksCarryAudio: true,
+ });
+ const cfrEncoder = JSON.parse(readFileSync(join(withCfr, "meta", "encoder.json"), "utf-8"));
+ expect(cfrEncoder.chunksCarryAudio).toBeUndefined();
+ }, 60_000);
+});
+
describe("plan() — codec knob", () => {
const TIMEOUT_MS = 30_000;
diff --git a/packages/producer/src/services/distributed/plan.ts b/packages/producer/src/services/distributed/plan.ts
index 2a6bedf1f8..1a9edb3982 100644
--- a/packages/producer/src/services/distributed/plan.ts
+++ b/packages/producer/src/services/distributed/plan.ts
@@ -54,6 +54,8 @@ import {
type RenderStrictness,
} from "../renderOrchestrator.js";
import { closeFileServerSafely } from "../fileServer.js";
+import { padOrTrimAudioToVideoFrameCount } from "../render/audioPadTrim.js";
+import { encoderFailureError } from "../render/encoderInterruption.js";
import { runAudioStage } from "../render/stages/audioStage.js";
import { runCompileStage } from "../render/stages/compileStage.js";
import {
@@ -305,6 +307,11 @@ export interface BuildLocalExecutionPlanOptions {
readonly digests?: PlanFileDigests;
/** Runs while the audio mixes, once `compiled/` and the video trees are final. */
readonly whileAudioMixes?: () => Promise<void>;
+ /**
+ * mp4 chunks carry their own slice of the plan audio (renderChunk muxes it). Ignored for other
+ * formats and with `cfr`, which re-encodes.
+ */
+ readonly chunksCarryAudio?: boolean;
}
export function logPlanStage(log: ProducerLogger, stage: string, startedAt: number): void {
@@ -826,6 +833,7 @@ function buildLockedRenderConfig(input: {
effectiveChunkSize: number;
chunkCount: number;
runtimeEnv: Record<string, string>;
+ chunksCarryAudio: boolean;
}): LockedRenderConfig {
const { config, forceScreenshot, deviceScaleFactor, ffmpegVersion } = input;
const { encoder, pixelFormat, preset } = resolveEncoderTriple(config);
@@ -860,9 +868,44 @@ function buildLockedRenderConfig(input: {
chunkCount: input.chunkCount,
runtimeEnv: input.runtimeEnv,
variables: config.variables,
+ ...(input.chunksCarryAudio ? { chunksCarryAudio: true } : {}),
};
}
+/** Only mp4 without `cfr` can skip the assemble-time join; `cfr` re-encodes the joined video. */
+function carriesChunkAudio(
+ config: Pick<DistributedRenderConfig, "format" | "cfr">,
+ options: Pick<BuildLocalExecutionPlanOptions, "chunksCarryAudio">,
+): boolean {
+ return options.chunksCarryAudio === true && config.format === "mp4" && config.cfr !== true;
+}
+
+/**
+ * Chunks that carry their own audio cut it from the plan track, so the length and true-peak fix
+ * the assembler applies to a joined file happens here, once, before any chunk reads it.
+ */
+async function fitPlanAudioToFrames(
+ audioPath: string,
+ workDir: string,
+ totalFrames: number,
+ fps: number,
+ run: { signal?: AbortSignal; timeoutMs: number },
+): Promise<void> {
+ const fittedPath = join(workDir, "audio-fitted.m4a");
+ const result = await padOrTrimAudioToVideoFrameCount({
+ videoPath: audioPath,
+ probeVideoFrameInfo: async () => ({ frameCount: totalFrames, fpsNum: fps, fpsDen: 1 }),
+ audioPath,
+ outputPath: fittedPath,
+ signal: run.signal,
+ timeoutMs: run.timeoutMs,
+ });
+ if (!result.success) {
+ throw encoderFailureError("[plan] audio pad/trim failed", result);
+ }
+ renameSync(fittedPath, audioPath);
+}
+
/**
* Resolve the encoder + pixel-format + preset triple for a distributed
* render. Distributed mode is SDR-only: H.264 or H.265 8-bit for mp4,
@@ -1349,6 +1392,13 @@ export async function buildLocalExecutionPlan(
if (audioResult.hasAudio && existsSync(audioResult.audioOutputPath)) {
renameSync(audioResult.audioOutputPath, planAudioPath);
}
+ const chunksCarryAudio = carriesChunkAudio(config, options);
+ if (chunksCarryAudio && existsSync(planAudioPath)) {
+ await fitPlanAudioToFrames(planAudioPath, workDir, totalFrames, config.fps, {
+ signal: abortSignal,
+ timeoutMs: cfg.ffmpegProcessTimeout,
+ });
+ }
// ── Locked config ──
const ffmpegVersion = await readFfmpegVersion();
@@ -1364,6 +1414,7 @@ export async function buildLocalExecutionPlan(
effectiveChunkSize,
chunkCount,
runtimeEnv,
+ chunksCarryAudio,
});
// ── Freeze the plan ──
diff --git a/packages/producer/src/services/distributed/planV2.test.ts b/packages/producer/src/services/distributed/planV2.test.ts
index f808a5724d..6e49af0f3f 100644
--- a/packages/producer/src/services/distributed/planV2.test.ts
+++ b/packages/producer/src/services/distributed/planV2.test.ts
@@ -58,6 +58,7 @@ function createV1Plan(
root: string,
options?: {
audio?: boolean;
+ chunksCarryAudio?: boolean;
video?: boolean;
omitVideoMetadata?: boolean;
fpsNum?: number;
@@ -75,6 +76,7 @@ function createV1Plan(
): string {
const {
audio = false,
+ chunksCarryAudio = false,
video = false,
omitVideoMetadata = false,
fpsNum = 30,
@@ -100,7 +102,10 @@ function createV1Plan(
{ index: 1, startFrame: 1, endFrame: 2 },
]),
);
- writeFileSync(join(planDir, "meta", "encoder.json"), "{}");
+ writeFileSync(
+ join(planDir, "meta", "encoder.json"),
+ chunksCarryAudio ? JSON.stringify({ chunksCarryAudio: true }) : "{}",
+ );
writeFileSync(join(planDir, "meta", "composition.json"), "{}");
if (video) {
const framesDir = join(planDir, "video-frames", "hero");
@@ -416,6 +421,18 @@ describe("Plan v2 manifest", () => {
expect(assembler.some((artifact) => artifact.path === "compiled/index.html")).toBe(false);
});
+ it("also sends audio to every chunk when mp4 chunks carry their audio", () => {
+ const root = tempPath("hf-plan-v2-chunk-audio-targets-");
+ const v1 = createV1Plan(root, { audio: true, chunksCarryAudio: true });
+ const manifest = readPlanV2Manifest(createPlanV2FromV1(v1, join(root, "v2")).planDir);
+ for (const chunkIndex of [0, 1]) {
+ const chunk = listPlanV2ArtifactsForTarget(manifest, { role: "chunk", chunkIndex });
+ expect(chunk.some((artifact) => artifact.path === "audio.m4a")).toBe(true);
+ }
+ const assembler = listPlanV2ArtifactsForTarget(manifest, { role: "assembler" });
+ expect(assembler.some((artifact) => artifact.path === "audio.m4a")).toBe(true);
+ });
+
it("uses the runtime frame lookup to select exact video frames per chunk", () => {
const root = tempPath("hf-plan-v2-video-reachability-");
const result = createPlanV2FromV1(createV1Plan(root, { video: true }), join(root, "v2"));
diff --git a/packages/producer/src/services/distributed/planV2.ts b/packages/producer/src/services/distributed/planV2.ts
index d5f24c110a..282b1d81d2 100644
--- a/packages/producer/src/services/distributed/planV2.ts
+++ b/packages/producer/src/services/distributed/planV2.ts
@@ -140,6 +140,7 @@ export interface PlanV2WithPublisherOptions {
* can avoid a second set of data blocks.
*/
readonly stagingParentDir?: string;
+ readonly chunksCarryAudio?: boolean;
}
export interface PlanV2MaterializationResult {
@@ -267,8 +268,11 @@ function resolveExtractedVideoOutputDir(planDir: string, videoId: string): strin
function artifactTargets(
path: string,
videoDependencies: ReadonlyMap<string, readonly number[]> | null,
+ chunksCarryAudio: boolean,
): Pick<PlanV2Artifact, "chunks" | "assembler"> {
- if (isPlanAudioArtifactPath(path)) return { chunks: [], assembler: true };
+ if (isPlanAudioArtifactPath(path)) {
+ return { chunks: chunksCarryAudio ? "all" : [], assembler: true };
+ }
if (path === "plan.json" || path === "meta/chunks.json" || path === "meta/encoder.json") {
return { chunks: "all", assembler: true };
}
@@ -548,9 +552,11 @@ function buildPlanV2Publication(
}
validateExtractionCacheCompleteSentinels(executionPlanDir);
const videoDependencyPlan = buildVideoChunkDependencies(executionPlanDir, dimensions);
+ const encoder = readJsonFile(join(executionPlanDir, "meta", "encoder.json"), "meta/encoder.json");
+ const chunksCarryAudio = isRecord(encoder) && encoder.chunksCarryAudio === true;
for (const file of listFiles(executionPlanDir)) {
if (isExtractionCacheCompleteSentinelPath(file.path)) continue;
- const targets = artifactTargets(file.path, videoDependencyPlan.dependencies);
+ const targets = artifactTargets(file.path, videoDependencyPlan.dependencies, chunksCarryAudio);
if (
(file.path.startsWith("video-frames/") || file.path.startsWith("video-sources/")) &&
targets.chunks !== "all" &&
@@ -704,6 +710,7 @@ export async function planV2WithPublisher(
publishBlobFor(file.absolutePath, digests),
),
),
+ chunksCarryAudio: options.chunksCarryAudio,
});
return await publishExecutionPlan(stagingRoot, publisher, { digests, log: config.logger });
} catch (error) {
diff --git a/packages/producer/src/services/distributed/renderChunk.ts b/packages/producer/src/services/distributed/renderChunk.ts
index 0a85a3006c..86d580d07d 100644
--- a/packages/producer/src/services/distributed/renderChunk.ts
+++ b/packages/producer/src/services/distributed/renderChunk.ts
@@ -93,7 +93,9 @@ import {
PLAN_VIDEOS_META_RELATIVE_PATH,
type PlanVideosJson,
readFfmpegVersion,
+ resolvePlanAudioPath,
} from "./shared.js";
+import { muxChunkWithAudio } from "./chunkAudio.js";
import { DISTRIBUTED_RENDER_CAPABILITIES, readPlanProtocolV1 } from "./planProtocol.js";
import { validatePlanV2MaterializedTarget } from "./planV2.js";
@@ -1055,9 +1057,11 @@ export async function renderChunk(
const effectiveQuality = encoder.crf ?? preset.quality;
const effectiveBitrate = encoder.crf != null ? undefined : encoder.bitrate;
// For non-pngseq, encodeStage writes to `outputPath` when `isPngSequence`
- // is false. `videoOnlyPath` is the encoder's direct output (no mux —
- // mux happens in assemble()).
- const videoOnlyPath = outputChunkPath;
+ // is false. `videoOnlyPath` is the encoder's direct output: assemble muxes
+ // the audio, or a chunk that carries audio muxes its own below.
+ const videoOnlyPath = encoder.chunksCarryAudio
+ ? `${outputChunkPath}.video.mp4`
+ : outputChunkPath;
if (isPngSequence) {
if (!existsSync(outputChunkPath)) mkdirSync(outputChunkPath, { recursive: true });
} else {
@@ -1069,16 +1073,16 @@ export async function renderChunk(
await runEncodeStage({
job,
log,
- outputPath: outputChunkPath,
+ outputPath: videoOnlyPath,
framesDir,
videoOnlyPath,
width: Math.round(plan.dimensions.width * encoder.deviceScaleFactor),
height: Math.round(plan.dimensions.height * encoder.deviceScaleFactor),
needsAlpha: plan.dimensions.format !== "mp4",
captureImageFormat: captureOptions.format ?? "jpeg",
- // Each chunk produces video only — audio is muxed once at assemble
- // time. Suppressing `hasAudio` skips the png-sequence audio sidecar
- // AND the mp4 audio mux.
+ // The encoder writes video only; audio joins at assemble time or in the
+ // chunk that carries audio. Suppressing `hasAudio` skips the png-sequence audio
+ // sidecar AND the mp4 audio mux.
hasAudio: false,
isPngSequence,
// `DistributedFormat` has no "gif" member — distributed chunks are
@@ -1104,6 +1108,18 @@ export async function renderChunk(
lockGopForChunkConcat: !isPngSequence,
gopSize: framesInChunk,
});
+ if (encoder.chunksCarryAudio) {
+ await muxChunkWithAudio({
+ videoPath: videoOnlyPath,
+ audioPath: resolvePlanAudioPath(planDir),
+ outputPath: outputChunkPath,
+ chunks,
+ chunkIndex,
+ fps: { num: plan.dimensions.fpsNum, den: plan.dimensions.fpsDen },
+ timeoutMs: cfg.ffmpegProcessTimeout,
+ });
+ rmSync(videoOnlyPath, { force: true });
+ }
encodeStageMs = Date.now() - encodeStarted;
} finally {
// Cleanest path: captureStage closed the session for us. The defensive
diff --git a/packages/producer/src/services/render/audioPadTrim.ts b/packages/producer/src/services/render/audioPadTrim.ts
index c897dd60bb..ef1bdd45d3 100644
--- a/packages/producer/src/services/render/audioPadTrim.ts
+++ b/packages/producer/src/services/render/audioPadTrim.ts
@@ -656,7 +656,7 @@ async function defaultRunFfmpeg(
// ── ffprobe JSON runner (shared between fast/slow video probe paths) ─────
-async function runFfprobeJson<T>(args: string[], signal?: AbortSignal): Promise<T> {
+export async function runFfprobeJson<T>(args: string[], signal?: AbortSignal): Promise<T> {
// Callers bake the input path into `args` (terminated with "--"), so this
// helper cannot add the terminator itself — assert they did rather than
// let a dash-prefixed path silently reach ffprobe as an option.
diff --git a/packages/producer/src/services/render/stages/freezePlan.ts b/packages/producer/src/services/render/stages/freezePlan.ts
index 4b5b40e5da..f69d017a82 100644
--- a/packages/producer/src/services/render/stages/freezePlan.ts
+++ b/packages/producer/src/services/render/stages/freezePlan.ts
@@ -94,6 +94,8 @@ export interface LockedRenderConfig {
* before this field existed.
*/
variables?: Record<string, unknown>;
+
+ chunksCarryAudio?: true;
}
export interface CompositionMetadataJson {