Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
perf(producer): distributed mp4 chunks can carry their slice of the plan audio (#5803)
Miguel Ángel1a2194c7b4f3
diff --git a/packages/producer/src/services/distributed/assemble.ts b/packages/producer/src/services/distributed/assemble.tsindex 5647c3eb55..09431e404d 100644--- a/packages/producer/src/services/distributed/assemble.ts+++ b/packages/producer/src/services/distributed/assemble.ts@@ -155,6 +155,9 @@ export async function assemble(format,fps,fpsArg: fpsToFfmpegArg(fps),+ chunkSeconds: chunksCarryAudio(planDir)+ ? plan.chunks.map((chunk) => ((chunk.endFrame - chunk.startFrame) * fps.den) / fps.num)+ : null,signal: options?.abortSignal,timeout: resolveConfig().ffmpegProcessTimeout,log,@@ -197,7 +200,10 @@ export async function assemble(};}-function readAssemblePlan(planDir: string, chunkPaths: readonly string[]): PlanJsonForAssemble {+function readAssemblePlan(+ planDir: string,+ chunkPaths: readonly string[],+): PlanJsonForAssemble & { chunks: ChunkSliceJson[] } {// ── 1. Validate planDir manifest matches chunkPaths shape ──────────────const planJsonPath = join(planDir, "plan.json");const chunksJsonPath = join(planDir, "meta", "chunks.json");@@ -224,7 +230,7 @@ function readAssemblePlan(planDir: string, chunkPaths: readonly string[]): PlanJ}}- return plan;+ return { ...plan, chunks };}interface JoinContext {@@ -236,6 +242,7 @@ interface JoinContext {timeout: number;log: ProducerLogger;timed: <T>(stage: string, run: () => Promise<T>) => Promise<T>;+ chunkSeconds: number[] | null;}// A one-entry concat list ignores the input-side `-r` (observed `359/12` in the@@ -243,7 +250,7 @@ interface JoinContext {function videoInputArgs(ctx: JoinContext, videoSources: readonly string[]): string[] {return videoSources.length === 1? ["-i", videoSources[0]!]- : concatDemuxerInput(videoSources, ctx.workDir, ctx.fpsArg);+ : concatDemuxerInput(videoSources, ctx.workDir, ctx.fpsArg, ctx.chunkSeconds);}// ── 2c. Optional exact-CFR re-encode ──────────────────────────────────@@ -340,11 +347,18 @@ function concatDemuxerInput(chunkPaths: readonly string[],workDir: string,fpsArg: string,+ chunkSeconds: readonly number[] | null,): string[] {// ffmpeg's concat list escapes a single quote as `'\''`.const listPath = join(workDir, "concat-list.txt");- const body = chunkPaths.map((path) => `file '${path.replace(/'/g, "'\\''")}'`).join("\n");- writeFileSync(listPath, `${body}\n`, "utf-8");+ // Chunks that carry audio: read only their video (track 1), and end each chunk at its last frame+ // rather than after its audio, which starts with the AAC priming and can run one packet longer.+ const head = chunkSeconds ? ["stream", "exact_stream_id 1"] : [];+ const body = chunkPaths.map((path, i) => {+ const line = `file '${path.replace(/'/g, "'\\''")}'`;+ return chunkSeconds ? `${line}\nduration ${chunkSeconds[i]!.toFixed(6)}` : line;+ });+ writeFileSync(listPath, `${[...head, ...body].join("\n")}\n`, "utf-8");return ["-r", fpsArg, "-f", "concat", "-safe", "0", "-i", listPath];}@@ -379,6 +393,15 @@ async function runAssembleFfmpeg(}}+function chunksCarryAudio(planDir: string): boolean {+ const encoderJsonPath = join(planDir, "meta", "encoder.json");+ if (!existsSync(encoderJsonPath)) return false;+ return (+ (JSON.parse(readFileSync(encoderJsonPath, "utf-8")) as LockedRenderConfig).chunksCarryAudio ===+ true+ );+}+/*** The cfr re-encode hardcodes `-c:v libx264`: reject non-mp4 formats and h265* chunks (which it would silently transcode) before any ffmpeg runs.diff --git a/packages/producer/src/services/distributed/chunkAudio.test.ts b/packages/producer/src/services/distributed/chunkAudio.test.tsnew file mode 100644index 0000000000..832e3a0906--- /dev/null+++ b/packages/producer/src/services/distributed/chunkAudio.test.ts@@ -0,0 +1,153 @@+import { spawnSync } from "node:child_process";+import { afterAll, beforeAll, describe, expect, it } from "bun:test";+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { assemble } from "./assemble.js";+import { audioCutForChunk, muxChunkWithAudio } from "./chunkAudio.js";++const FPS = { num: 30, den: 1 };++describe("audioCutForChunk", () => {+ it("gives each chunk the AAC packets that start in its range, priming to chunk 0", () => {+ const grid = { sampleRate: 48000, firstStart: -1024, packetSamples: 1024, packetCount: 564 };+ const chunks = [0, 120, 240].map((startFrame, index) => ({+ index,+ startFrame,+ endFrame: startFrame + 120,+ }));+ const cuts = chunks.map((_, index) => audioCutForChunk(grid, chunks, index, FPS));+ expect(cuts).toEqual([+ { seekSeconds: null, durationSeconds: 4, firstPacket: 0, packetCount: 189 },+ {+ seekSeconds: 4,+ durationSeconds: (375 * 1024 - 512) / 48000 - 4,+ firstPacket: 189,+ packetCount: 187,+ },+ {+ seekSeconds: (375 * 1024 - 512) / 48000,+ durationSeconds: null,+ firstPacket: 376,+ packetCount: 188,+ },+ ]);+ });+});++const hasFfmpeg = spawnSync("ffmpeg", ["-version"]).status === 0;++describe.skipIf(!hasFfmpeg)("chunks that carry their audio", () => {+ const TOTAL_FRAMES = 91;+ let root: string;++ beforeAll(() => {+ root = mkdtempSync(join(tmpdir(), "hf-chunk-audio-"));+ });+ afterAll(() => {+ rmSync(root, { recursive: true, force: true });+ });++ function run(cmd: string, args: string[]): string {+ const result = spawnSync(cmd, args, { maxBuffer: 1 << 28, encoding: "latin1" });+ if (result.status !== 0) throw new Error(`${cmd} ${args.join(" ")}: ${result.stderr}`);+ return result.stdout;+ }+ const ffmpeg = (args: string[]) => run("ffmpeg", ["-nostdin", "-loglevel", "error", ...args]);+ const frames = (path: string) =>+ ffmpeg(["-i", path, "-map", "0:v", "-f", "framemd5", "-"])+ .split("\n")+ .filter((line) => line && !line.startsWith("#"));+ const pcm = (path: string) => ffmpeg(["-i", path, "-map", "0:a", "-f", "s16le", "-"]);+ const streams = (path: string) =>+ JSON.parse(+ run("ffprobe", [+ "-v",+ "error",+ "-count_packets",+ "-show_entries",+ "stream=codec_type,nb_read_packets,duration,r_frame_rate,avg_frame_rate",+ "-of",+ "json",+ "--",+ path,+ ]),+ ).streams as { codec_type: string; nb_read_packets: string; duration: string }[];++ // Uneven chunks with a 1-frame tail, and a single chunk (read without a concat list today).+ it.each([+ ["three", [0, 47, 90, 91]],+ ["one", [0, 91]],+ ] as const)(+ "%s chunk(s) join through today's assemble like video-only chunks",+ async (name, edges) => {+ const CHUNKS = edges.slice(1).map((end, index) => ({+ index,+ startFrame: edges[index]!,+ endFrame: end,+ }));+ const dir = join(root, name);+ mkdirSync(dir, { recursive: true });+ const audioPath = join(dir, "audio.m4a");+ ffmpeg(+ `-f lavfi -i aevalsrc=0.4*sin(2*PI*(220+40*t)*t):s=48000:d=${TOTAL_FRAMES / 30} -c:a aac -b:a 192k -y`+ .split(" ")+ .concat(audioPath),+ );+ const planDir = join(dir, "plan");+ mkdirSync(join(planDir, "meta"), { recursive: true });+ writeFileSync(+ join(planDir, "plan.json"),+ JSON.stringify({+ planHash: "fake",+ totalFrames: TOTAL_FRAMES,+ hasAudio: true,+ dimensions: { fpsNum: 30, fpsDen: 1, width: 160, height: 120, format: "mp4" },+ }),+ );+ writeFileSync(join(planDir, "meta", "chunks.json"), JSON.stringify(CHUNKS));++ const videoPaths: string[] = [];+ const chunkPaths: string[] = [];+ for (const slice of CHUNKS) {+ const n = String(slice.endFrame - slice.startFrame);+ videoPaths.push(join(dir, `video-${slice.index}.mp4`));+ chunkPaths.push(join(dir, `chunk-${slice.index}.mp4`));+ // Closed GOP, no B-frames, headers repeated: shaped like renderChunk's encode.+ ffmpeg(+ `-f lavfi -i testsrc2=size=160x120:rate=30,trim=start_frame=${slice.startFrame}:end_frame=${slice.endFrame} -c:v libx264 -preset veryfast -g ${n} -keyint_min ${n} -sc_threshold 0 -bf 0 -x264-params scenecut=0:open-gop=0:repeat-headers=1 -pix_fmt yuv420p -avoid_negative_ts make_zero -y`+ .split(" ")+ .concat(videoPaths[slice.index]!),+ );+ await muxChunkWithAudio({+ videoPath: videoPaths[slice.index]!,+ audioPath,+ outputPath: chunkPaths[slice.index]!,+ chunks: CHUNKS,+ chunkIndex: slice.index,+ fps: FPS,+ timeoutMs: 60_000,+ });+ }+ // Every plan audio packet lands in exactly one chunk.+ const audioPackets = Number(streams(audioPath)[0]!.nb_read_packets);+ const chunkPackets = chunkPaths.map((path) => Number(streams(path)[1]!.nb_read_packets));+ expect(chunkPackets.reduce((a, b) => a + b, 0)).toBe(audioPackets);++ const withAudio = join(dir, "with-audio.mp4");+ const videoOnly = join(dir, "video-only.mp4");+ writeFileSync(+ join(planDir, "meta", "encoder.json"),+ JSON.stringify({ chunksCarryAudio: true }),+ );+ await assemble(planDir, chunkPaths, audioPath, withAudio);+ // Today's join of the video-only encodes, from a plan without the option.+ writeFileSync(join(planDir, "meta", "encoder.json"), "{}");+ await assemble(planDir, videoPaths, audioPath, videoOnly);+ expect(frames(withAudio)).toEqual(frames(videoOnly));+ expect(pcm(withAudio) === pcm(videoOnly)).toBe(true);+ expect(streams(withAudio)).toEqual(streams(videoOnly));+ },+ 60_000,+ );+});diff --git a/packages/producer/src/services/distributed/chunkAudio.ts b/packages/producer/src/services/distributed/chunkAudio.tsnew file mode 100644index 0000000000..db71f999dd--- /dev/null+++ b/packages/producer/src/services/distributed/chunkAudio.ts@@ -0,0 +1,152 @@+// Distributed mp4 chunks can carry their own slice of the plan audio: the AAC packets of the one+// plan-wide encode that start inside the chunk's frame range, so the chunks' audio has no seams.++import type { Fps } from "@hyperframes/core";+import { appendRenderProvenanceArgs, describeFfmpegFailure, runFfmpeg } from "@hyperframes/engine";+import { runFfprobeJson } from "../render/audioPadTrim.js";+import type { ChunkSliceJson } from "../render/stages/freezePlan.js";++/** Where AAC packet `j` of the plan audio starts: `firstStart + j * packetSamples`, in samples. */+interface AacPacketGrid {+ sampleRate: number;+ firstStart: number;+ packetSamples: number;+ packetCount: number;+}++/** Input-side `-ss` / `-t` that select exactly packets `[first, end)` for one chunk. */+interface AudioCut {+ seekSeconds: number | null;+ durationSeconds: number | null;+ firstPacket: number;+ packetCount: number;+}++// A chunk owns the packets starting in its frame range (chunk 0 also the priming before 0).+// Cuts sit half a packet before a packet start so ffmpeg's rounding cannot move them.+export function audioCutForChunk(+ grid: AacPacketGrid,+ chunks: readonly ChunkSliceJson[],+ index: number,+ fps: Fps,+): AudioCut | null {+ const firstPacketAt = (frame: number): number =>+ Math.min(+ grid.packetCount,+ Math.max(+ 0,+ Math.ceil(+ (frame * fps.den * grid.sampleRate - grid.firstStart * fps.num) /+ (grid.packetSamples * fps.num),+ ),+ ),+ );+ const first = index === 0 ? 0 : firstPacketAt(chunks[index]!.startFrame);+ const end =+ index === chunks.length - 1 ? grid.packetCount : firstPacketAt(chunks[index + 1]!.startFrame);+ if (end <= first) return null;+ const cutSeconds = (packet: number): number =>+ (grid.firstStart + packet * grid.packetSamples - grid.packetSamples / 2) / grid.sampleRate;+ const seekSeconds = index === 0 ? null : cutSeconds(first);+ const durationSeconds = end === grid.packetCount ? null : cutSeconds(end) - (seekSeconds ?? 0);+ return { seekSeconds, durationSeconds, firstPacket: first, packetCount: end - first };+}++interface FfprobeAudioGrid {+ streams?: { sample_rate?: string; time_base?: string; nb_frames?: string }[];+ packets?: { pts?: number; duration?: number }[];+}++async function probeAacPacketGrid(audioPath: string): Promise<AacPacketGrid> {+ const probe = await runFfprobeJson<FfprobeAudioGrid>([+ "-v",+ "error",+ "-select_streams",+ "a:0",+ "-show_entries",+ "stream=sample_rate,time_base,nb_frames:packet=pts,duration",+ "-read_intervals",+ "%+#1",+ "-of",+ "json",+ "--",+ audioPath,+ ]);+ const stream = probe.streams?.[0] ?? {};+ const packet = probe.packets?.[0] ?? {};+ const sampleRate = Number(stream.sample_rate);+ const [tbNum = NaN, tbDen = NaN] = String(stream.time_base).split("/").map(Number);+ const toSamples = (ticks?: number): number => (Number(ticks) * sampleRate * tbNum) / tbDen;+ const grid = {+ sampleRate,+ firstStart: toSamples(packet.pts),+ packetSamples: toSamples(packet.duration),+ packetCount: Number(stream.nb_frames),+ };+ const counts = [grid.sampleRate, grid.packetSamples, grid.packetCount];+ if (!Number.isFinite(grid.firstStart) || !counts.every((n) => Number.isFinite(n) && n > 0)) {+ throw new Error(`[chunkAudio] could not read the AAC packet grid: ${JSON.stringify(probe)}`);+ }+ return grid;+}++interface MuxInput {+ videoPath: string;+ audioPath: string | null;+ outputPath: string;+ chunks: readonly ChunkSliceJson[];+ chunkIndex: number;+ fps: Fps;+ signal?: AbortSignal;+ timeoutMs: number;+}++function chunkMuxArgs(input: MuxInput, cut: AudioCut | null): string[] {+ if (!cut) return ["-i", input.videoPath, "-map", "0:v:0"];+ const seek = cut.seekSeconds === null ? [] : ["-ss", cut.seekSeconds.toFixed(6)];+ const length = cut.durationSeconds === null ? [] : ["-t", cut.durationSeconds.toFixed(6)];+ // A seek keeps the packet before the cut unless told not to; chunk 0 never+ // seeks, so its priming packet (and the edit list that hides it) survive.+ const exact = seek.length ? ["-copypriorss", "0"] : [];+ return [+ ...["-i", input.videoPath, ...seek, ...length, "-i", input.audioPath!],+ ...["-map", "0:v:0", "-map", "1:a:0", ...exact],+ ];+}++/** Mux one encoded chunk with the plan audio packets it owns into a regular mp4 at `outputPath`. */+export async function muxChunkWithAudio(input: MuxInput): Promise<void> {+ const grid = input.audioPath ? await probeAacPacketGrid(input.audioPath) : null;+ const cut = grid ? audioCutForChunk(grid, input.chunks, input.chunkIndex, input.fps) : null;+ const args = [...chunkMuxArgs(input, cut), "-c", "copy"];+ appendRenderProvenanceArgs(args, input.outputPath);+ args.push("-y", input.outputPath);+ const result = await runFfmpeg(args, { signal: input.signal, timeout: input.timeoutMs });+ if (!result.success) {+ throw new Error(+ `[chunkAudio] chunk mux failed: ${describeFfmpegFailure(result, input.timeoutMs)}`,+ );+ }+ const slice = input.chunks[input.chunkIndex]!;+ const expected: Record<string, number> = { video: slice.endFrame - slice.startFrame };+ if (cut) expected.audio = cut.packetCount;+ const probe = await runFfprobeJson<{+ streams?: { codec_type?: string; nb_read_packets?: string }[];+ }>([+ "-v",+ "error",+ "-count_packets",+ "-show_entries",+ "stream=codec_type,nb_read_packets",+ "-of",+ "json",+ "--",+ input.outputPath,+ ]);+ for (const [type, count] of Object.entries(expected)) {+ const packets = Number(probe.streams?.find((s) => s.codec_type === type)?.nb_read_packets ?? 0);+ if (packets !== count) {+ throw new Error(`[chunkAudio] ${type} packets in chunk ${packets}, expected ${count}`);+ }+ }+}diff --git a/packages/producer/src/services/distributed/plan.test.ts b/packages/producer/src/services/distributed/plan.test.tsindex f67c58bc4f..14750e1040 100644--- a/packages/producer/src/services/distributed/plan.test.ts+++ b/packages/producer/src/services/distributed/plan.test.ts@@ -15,6 +15,7 @@* inside the regression harness (`bun run --cwd packages/producer docker:test`).*/+import { spawnSync } from "node:child_process";import { afterAll, beforeAll, describe, expect, it } from "bun:test";import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";import { tmpdir } from "node:os";@@ -853,6 +854,70 @@ describe("plan() — golden planDir + planHash determinism", () => {);});+describe.skipIf(spawnSync("ffmpeg", ["-version"]).status !== 0)("stitched mp4 plans", () => {+ function audioProject(name: string): string {+ const dir = join(runRoot, name);+ mkdirSync(dir, { recursive: true });+ const tone = spawnSync("ffmpeg", [+ "-nostdin",+ "-v",+ "error",+ "-f",+ "lavfi",+ "-i",+ "aevalsrc=0.99*sin(2*PI*440*t):s=48000:d=0.6",+ "-c:a",+ "pcm_s16le",+ join(dir, "tone.wav"),+ ]);+ if (tone.status !== 0) throw new Error(`ffmpeg tone failed: ${tone.stderr}`);+ writeFileSync(+ join(dir, "index.html"),+ `<!doctype html><html><head><meta charset="utf-8"></head><body>+ <div data-composition-id="root" data-width="320" data-height="240" data-duration="1">+ <audio data-composition-audio src="tone.wav"></audio>+ </div></body></html>`,+ );+ return dir;+ }++ it("fits the audio length and peak and records the layout only for mp4 without cfr", async () => {+ const project = audioProject("project-stitch");+ const config = { fps: 30, width: 320, height: 240, format: "mp4" } as const;+ const stitched = join(runRoot, "plan-stitch");+ await buildLocalExecutionPlan(project, config, stitched, { chunksCarryAudio: true });+ const encoder = JSON.parse(readFileSync(join(stitched, "meta", "encoder.json"), "utf-8"));+ expect(encoder.chunksCarryAudio).toBe(true);+ const probe = spawnSync(+ "ffprobe",+ [+ "-v",+ "error",+ "-show_entries",+ "format=duration",+ "-of",+ "csv=p=0",+ join(stitched, "audio.m4a"),+ ],+ { encoding: "utf-8" },+ );+ expect(Math.abs(Number(probe.stdout) - 1)).toBeLessThan(0.001);+ const volume = spawnSync(+ "ffmpeg",+ ["-nostdin", "-i", join(stitched, "audio.m4a"), "-af", "volumedetect", "-f", "null", "-"],+ { encoding: "utf-8" },+ );+ expect(Number(/max_volume: (-?[\d.]+) dB/.exec(volume.stderr)?.[1])).toBeLessThan(-1);++ const withCfr = join(runRoot, "plan-stitch-cfr");+ await buildLocalExecutionPlan(project, { ...config, cfr: true }, withCfr, {+ chunksCarryAudio: true,+ });+ const cfrEncoder = JSON.parse(readFileSync(join(withCfr, "meta", "encoder.json"), "utf-8"));+ expect(cfrEncoder.chunksCarryAudio).toBeUndefined();+ }, 60_000);+});+describe("plan() — codec knob", () => {const TIMEOUT_MS = 30_000;diff --git a/packages/producer/src/services/distributed/plan.ts b/packages/producer/src/services/distributed/plan.tsindex 2a6bedf1f8..1a9edb3982 100644--- a/packages/producer/src/services/distributed/plan.ts+++ b/packages/producer/src/services/distributed/plan.ts@@ -54,6 +54,8 @@ import {type RenderStrictness,} from "../renderOrchestrator.js";import { closeFileServerSafely } from "../fileServer.js";+import { padOrTrimAudioToVideoFrameCount } from "../render/audioPadTrim.js";+import { encoderFailureError } from "../render/encoderInterruption.js";import { runAudioStage } from "../render/stages/audioStage.js";import { runCompileStage } from "../render/stages/compileStage.js";import {@@ -305,6 +307,11 @@ export interface BuildLocalExecutionPlanOptions {readonly digests?: PlanFileDigests;/** Runs while the audio mixes, once `compiled/` and the video trees are final. */readonly whileAudioMixes?: () => Promise<void>;+ /**+ * mp4 chunks carry their own slice of the plan audio (renderChunk muxes it). Ignored for other+ * formats and with `cfr`, which re-encodes.+ */+ readonly chunksCarryAudio?: boolean;}export function logPlanStage(log: ProducerLogger, stage: string, startedAt: number): void {@@ -826,6 +833,7 @@ function buildLockedRenderConfig(input: {effectiveChunkSize: number;chunkCount: number;runtimeEnv: Record<string, string>;+ chunksCarryAudio: boolean;}): LockedRenderConfig {const { config, forceScreenshot, deviceScaleFactor, ffmpegVersion } = input;const { encoder, pixelFormat, preset } = resolveEncoderTriple(config);@@ -860,9 +868,44 @@ function buildLockedRenderConfig(input: {chunkCount: input.chunkCount,runtimeEnv: input.runtimeEnv,variables: config.variables,+ ...(input.chunksCarryAudio ? { chunksCarryAudio: true } : {}),};}+/** Only mp4 without `cfr` can skip the assemble-time join; `cfr` re-encodes the joined video. */+function carriesChunkAudio(+ config: Pick<DistributedRenderConfig, "format" | "cfr">,+ options: Pick<BuildLocalExecutionPlanOptions, "chunksCarryAudio">,+): boolean {+ return options.chunksCarryAudio === true && config.format === "mp4" && config.cfr !== true;+}++/**+ * Chunks that carry their own audio cut it from the plan track, so the length and true-peak fix+ * the assembler applies to a joined file happens here, once, before any chunk reads it.+ */+async function fitPlanAudioToFrames(+ audioPath: string,+ workDir: string,+ totalFrames: number,+ fps: number,+ run: { signal?: AbortSignal; timeoutMs: number },+): Promise<void> {+ const fittedPath = join(workDir, "audio-fitted.m4a");+ const result = await padOrTrimAudioToVideoFrameCount({+ videoPath: audioPath,+ probeVideoFrameInfo: async () => ({ frameCount: totalFrames, fpsNum: fps, fpsDen: 1 }),+ audioPath,+ outputPath: fittedPath,+ signal: run.signal,+ timeoutMs: run.timeoutMs,+ });+ if (!result.success) {+ throw encoderFailureError("[plan] audio pad/trim failed", result);+ }+ renameSync(fittedPath, audioPath);+}+/*** Resolve the encoder + pixel-format + preset triple for a distributed* render. Distributed mode is SDR-only: H.264 or H.265 8-bit for mp4,@@ -1349,6 +1392,13 @@ export async function buildLocalExecutionPlan(if (audioResult.hasAudio && existsSync(audioResult.audioOutputPath)) {renameSync(audioResult.audioOutputPath, planAudioPath);}+ const chunksCarryAudio = carriesChunkAudio(config, options);+ if (chunksCarryAudio && existsSync(planAudioPath)) {+ await fitPlanAudioToFrames(planAudioPath, workDir, totalFrames, config.fps, {+ signal: abortSignal,+ timeoutMs: cfg.ffmpegProcessTimeout,+ });+ }// ── Locked config ──const ffmpegVersion = await readFfmpegVersion();@@ -1364,6 +1414,7 @@ export async function buildLocalExecutionPlan(effectiveChunkSize,chunkCount,runtimeEnv,+ chunksCarryAudio,});// ── Freeze the plan ──diff --git a/packages/producer/src/services/distributed/planV2.test.ts b/packages/producer/src/services/distributed/planV2.test.tsindex f808a5724d..6e49af0f3f 100644--- a/packages/producer/src/services/distributed/planV2.test.ts+++ b/packages/producer/src/services/distributed/planV2.test.ts@@ -58,6 +58,7 @@ function createV1Plan(root: string,options?: {audio?: boolean;+ chunksCarryAudio?: boolean;video?: boolean;omitVideoMetadata?: boolean;fpsNum?: number;@@ -75,6 +76,7 @@ function createV1Plan(): string {const {audio = false,+ chunksCarryAudio = false,video = false,omitVideoMetadata = false,fpsNum = 30,@@ -100,7 +102,10 @@ function createV1Plan({ index: 1, startFrame: 1, endFrame: 2 },]),);- writeFileSync(join(planDir, "meta", "encoder.json"), "{}");+ writeFileSync(+ join(planDir, "meta", "encoder.json"),+ chunksCarryAudio ? JSON.stringify({ chunksCarryAudio: true }) : "{}",+ );writeFileSync(join(planDir, "meta", "composition.json"), "{}");if (video) {const framesDir = join(planDir, "video-frames", "hero");@@ -416,6 +421,18 @@ describe("Plan v2 manifest", () => {expect(assembler.some((artifact) => artifact.path === "compiled/index.html")).toBe(false);});+ it("also sends audio to every chunk when mp4 chunks carry their audio", () => {+ const root = tempPath("hf-plan-v2-chunk-audio-targets-");+ const v1 = createV1Plan(root, { audio: true, chunksCarryAudio: true });+ const manifest = readPlanV2Manifest(createPlanV2FromV1(v1, join(root, "v2")).planDir);+ for (const chunkIndex of [0, 1]) {+ const chunk = listPlanV2ArtifactsForTarget(manifest, { role: "chunk", chunkIndex });+ expect(chunk.some((artifact) => artifact.path === "audio.m4a")).toBe(true);+ }+ const assembler = listPlanV2ArtifactsForTarget(manifest, { role: "assembler" });+ expect(assembler.some((artifact) => artifact.path === "audio.m4a")).toBe(true);+ });+it("uses the runtime frame lookup to select exact video frames per chunk", () => {const root = tempPath("hf-plan-v2-video-reachability-");const result = createPlanV2FromV1(createV1Plan(root, { video: true }), join(root, "v2"));diff --git a/packages/producer/src/services/distributed/planV2.ts b/packages/producer/src/services/distributed/planV2.tsindex d5f24c110a..282b1d81d2 100644--- a/packages/producer/src/services/distributed/planV2.ts+++ b/packages/producer/src/services/distributed/planV2.ts@@ -140,6 +140,7 @@ export interface PlanV2WithPublisherOptions {* can avoid a second set of data blocks.*/readonly stagingParentDir?: string;+ readonly chunksCarryAudio?: boolean;}export interface PlanV2MaterializationResult {@@ -267,8 +268,11 @@ function resolveExtractedVideoOutputDir(planDir: string, videoId: string): strinfunction artifactTargets(path: string,videoDependencies: ReadonlyMap<string, readonly number[]> | null,+ chunksCarryAudio: boolean,): Pick<PlanV2Artifact, "chunks" | "assembler"> {- if (isPlanAudioArtifactPath(path)) return { chunks: [], assembler: true };+ if (isPlanAudioArtifactPath(path)) {+ return { chunks: chunksCarryAudio ? "all" : [], assembler: true };+ }if (path === "plan.json" || path === "meta/chunks.json" || path === "meta/encoder.json") {return { chunks: "all", assembler: true };}@@ -548,9 +552,11 @@ function buildPlanV2Publication(}validateExtractionCacheCompleteSentinels(executionPlanDir);const videoDependencyPlan = buildVideoChunkDependencies(executionPlanDir, dimensions);+ const encoder = readJsonFile(join(executionPlanDir, "meta", "encoder.json"), "meta/encoder.json");+ const chunksCarryAudio = isRecord(encoder) && encoder.chunksCarryAudio === true;for (const file of listFiles(executionPlanDir)) {if (isExtractionCacheCompleteSentinelPath(file.path)) continue;- const targets = artifactTargets(file.path, videoDependencyPlan.dependencies);+ const targets = artifactTargets(file.path, videoDependencyPlan.dependencies, chunksCarryAudio);if ((file.path.startsWith("video-frames/") || file.path.startsWith("video-sources/")) &&targets.chunks !== "all" &&@@ -704,6 +710,7 @@ export async function planV2WithPublisher(publishBlobFor(file.absolutePath, digests),),),+ chunksCarryAudio: options.chunksCarryAudio,});return await publishExecutionPlan(stagingRoot, publisher, { digests, log: config.logger });} catch (error) {diff --git a/packages/producer/src/services/distributed/renderChunk.ts b/packages/producer/src/services/distributed/renderChunk.tsindex 0a85a3006c..86d580d07d 100644--- a/packages/producer/src/services/distributed/renderChunk.ts+++ b/packages/producer/src/services/distributed/renderChunk.ts@@ -93,7 +93,9 @@ import {PLAN_VIDEOS_META_RELATIVE_PATH,type PlanVideosJson,readFfmpegVersion,+ resolvePlanAudioPath,} from "./shared.js";+import { muxChunkWithAudio } from "./chunkAudio.js";import { DISTRIBUTED_RENDER_CAPABILITIES, readPlanProtocolV1 } from "./planProtocol.js";import { validatePlanV2MaterializedTarget } from "./planV2.js";@@ -1055,9 +1057,11 @@ export async function renderChunk(const effectiveQuality = encoder.crf ?? preset.quality;const effectiveBitrate = encoder.crf != null ? undefined : encoder.bitrate;// For non-pngseq, encodeStage writes to `outputPath` when `isPngSequence`- // is false. `videoOnlyPath` is the encoder's direct output (no mux —- // mux happens in assemble()).- const videoOnlyPath = outputChunkPath;+ // is false. `videoOnlyPath` is the encoder's direct output: assemble muxes+ // the audio, or a chunk that carries audio muxes its own below.+ const videoOnlyPath = encoder.chunksCarryAudio+ ? `${outputChunkPath}.video.mp4`+ : outputChunkPath;if (isPngSequence) {if (!existsSync(outputChunkPath)) mkdirSync(outputChunkPath, { recursive: true });} else {@@ -1069,16 +1073,16 @@ export async function renderChunk(await runEncodeStage({job,log,- outputPath: outputChunkPath,+ outputPath: videoOnlyPath,framesDir,videoOnlyPath,width: Math.round(plan.dimensions.width * encoder.deviceScaleFactor),height: Math.round(plan.dimensions.height * encoder.deviceScaleFactor),needsAlpha: plan.dimensions.format !== "mp4",captureImageFormat: captureOptions.format ?? "jpeg",- // Each chunk produces video only — audio is muxed once at assemble- // time. Suppressing `hasAudio` skips the png-sequence audio sidecar- // AND the mp4 audio mux.+ // The encoder writes video only; audio joins at assemble time or in the+ // chunk that carries audio. Suppressing `hasAudio` skips the png-sequence audio+ // sidecar AND the mp4 audio mux.hasAudio: false,isPngSequence,// `DistributedFormat` has no "gif" member — distributed chunks are@@ -1104,6 +1108,18 @@ export async function renderChunk(lockGopForChunkConcat: !isPngSequence,gopSize: framesInChunk,});+ if (encoder.chunksCarryAudio) {+ await muxChunkWithAudio({+ videoPath: videoOnlyPath,+ audioPath: resolvePlanAudioPath(planDir),+ outputPath: outputChunkPath,+ chunks,+ chunkIndex,+ fps: { num: plan.dimensions.fpsNum, den: plan.dimensions.fpsDen },+ timeoutMs: cfg.ffmpegProcessTimeout,+ });+ rmSync(videoOnlyPath, { force: true });+ }encodeStageMs = Date.now() - encodeStarted;} finally {// Cleanest path: captureStage closed the session for us. The defensivediff --git a/packages/producer/src/services/render/audioPadTrim.ts b/packages/producer/src/services/render/audioPadTrim.tsindex c897dd60bb..ef1bdd45d3 100644--- a/packages/producer/src/services/render/audioPadTrim.ts+++ b/packages/producer/src/services/render/audioPadTrim.ts@@ -656,7 +656,7 @@ async function defaultRunFfmpeg(// ── ffprobe JSON runner (shared between fast/slow video probe paths) ─────-async function runFfprobeJson<T>(args: string[], signal?: AbortSignal): Promise<T> {+export async function runFfprobeJson<T>(args: string[], signal?: AbortSignal): Promise<T> {// Callers bake the input path into `args` (terminated with "--"), so this// helper cannot add the terminator itself — assert they did rather than// let a dash-prefixed path silently reach ffprobe as an option.diff --git a/packages/producer/src/services/render/stages/freezePlan.ts b/packages/producer/src/services/render/stages/freezePlan.tsindex 4b5b40e5da..f69d017a82 100644--- a/packages/producer/src/services/render/stages/freezePlan.ts+++ b/packages/producer/src/services/render/stages/freezePlan.ts@@ -94,6 +94,8 @@ export interface LockedRenderConfig {* before this field existed.*/variables?: Record<string, unknown>;++ chunksCarryAudio?: true;}export interface CompositionMetadataJson {