Skip to content

Write HTML. Render video. Built for agents.

perf(producer): each distributed chunk downloads only its own slice of the plan audio (#5881)

Miguel Ángel52c5fae9a760
diff --git a/packages/producer/src/services/distributed/chunkAudio.test.ts b/packages/producer/src/services/distributed/chunkAudio.test.ts
index ac263f0a8b..b8b1fdcb66 100644
--- a/packages/producer/src/services/distributed/chunkAudio.test.ts
+++ b/packages/producer/src/services/distributed/chunkAudio.test.ts
@@ -4,7 +4,12 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { assemble } from "./assemble.js";
-import { audioCutForChunk, muxChunkWithAudio } from "./chunkAudio.js";
+import {
+ audioCutForChunk,
+ chunkAudioSource,
+ muxChunkWithAudio,
+ writeChunkAudioSlices,
+} from "./chunkAudio.js";
const FPS = { num: 30, den: 1 };
@@ -72,6 +77,23 @@ describe.skipIf(!hasFfmpeg)("chunks that carry their audio", () => {
path,
]),
).streams as { codec_type: string; nb_read_packets: string; duration: string }[];
+ // Each stream's packets (timing, size, payload hash), independent of how the muxer interleaved them.
+ const packets = (path: string) =>
+ run("ffprobe", [
+ ...["-v", "error", "-show_entries", "stream=index,start_pts,duration_ts:packet=stream_index"],
+ ...[
+ "-show_entries",
+ "packet=pts,dts,duration,size,flags,data_hash",
+ "-show_data_hash",
+ "sha256",
+ ],
+ ...["-of", "compact", "--", path],
+ ])
+ .split("\n")
+ .sort(
+ (a, b) =>
+ Number(/stream_index=(\d)/.exec(a)?.[1]) - Number(/stream_index=(\d)/.exec(b)?.[1]),
+ );
// Uneven chunks with a 1-frame tail, and a single chunk (read without a concat list today). At
// 60 fps a 1-frame tail is shorter than an AAC packet: 51 frames are 40800 samples, 41 packets
@@ -133,6 +155,23 @@ describe.skipIf(!hasFfmpeg)("chunks that carry their audio", () => {
timeoutMs: 60_000,
});
}
+ // A chunk muxing from its slice of a sliced plan writes the same packets as from the whole file.
+ const slicedPlanDir = join(dir, "sliced-plan");
+ await writeChunkAudioSlices({ audioPath, planDir: slicedPlanDir, chunks: CHUNKS, fps });
+ for (const slice of CHUNKS) {
+ const slicedChunk = join(dir, `sliced-chunk-${slice.index}.mp4`);
+ await muxChunkWithAudio({
+ videoPath: videoPaths[slice.index]!,
+ audioPath: chunkAudioSource(slicedPlanDir, slice.index, join(dir, "rebuilt.m4a")),
+ outputPath: slicedChunk,
+ chunks: CHUNKS,
+ chunkIndex: slice.index,
+ fps,
+ timeoutMs: 60_000,
+ });
+ expect(packets(slicedChunk)).toEqual(packets(chunkPaths[slice.index]!));
+ }
+
// Every plan audio packet lands in exactly one chunk; a tail that owns none has no audio track.
const audioOf = (path: string) => streams(path).find((s) => s.codec_type === "audio");
const audioPackets = Number(audioOf(audioPath)!.nb_read_packets);
diff --git a/packages/producer/src/services/distributed/chunkAudio.ts b/packages/producer/src/services/distributed/chunkAudio.ts
index db71f999dd..834709eb7c 100644
--- a/packages/producer/src/services/distributed/chunkAudio.ts
+++ b/packages/producer/src/services/distributed/chunkAudio.ts
@@ -1,10 +1,15 @@
// Distributed mp4 chunks can carry their own slice of the plan audio: the AAC packets of the one
// plan-wide encode that start inside the chunk's frame range, so the chunks' audio has no seams.
+import { closeSync, existsSync, ftruncateSync, mkdirSync, openSync, readFileSync } from "node:fs";
+import { writeFileSync, writeSync } from "node:fs";
+import { join } from "node:path";
import type { Fps } from "@hyperframes/core";
import { appendRenderProvenanceArgs, describeFfmpegFailure, runFfmpeg } from "@hyperframes/engine";
import { runFfprobeJson } from "../render/audioPadTrim.js";
import type { ChunkSliceJson } from "../render/stages/freezePlan.js";
+import { fileRange, readMp4Layout, readTracks } from "./mp4Stitch.js";
+import { resolvePlanAudioPath } from "./shared.js";
/** Where AAC packet `j` of the plan audio starts: `firstStart + j * packetSamples`, in samples. */
interface AacPacketGrid {
@@ -90,6 +95,96 @@ async function probeAacPacketGrid(audioPath: string): Promise<AacPacketGrid> {
return grid;
}
+// A sliced plan keeps the audio's moov once, in `frame.m4a` (the file minus its mdat payload),
+// and each chunk's packet bytes in their own file. A chunk rebuilds a sparse copy of the whole
+// file from them, so it runs the same cut on the same moov and bytes in any ffmpeg version.
+const SLICE_DIR = "audio-chunks";
+const SLICE_FRAME = `${SLICE_DIR}/frame.m4a`;
+const SLICE_LAYOUT = `${SLICE_DIR}/layout.json`;
+
+interface SliceLayout {
+ sizeBytes: number;
+ payloadStart: number;
+ payloadEnd: number;
+ partStarts: Record<string, number>;
+}
+
+export function chunkAudioSlicePath(index: number): string {
+ return `${SLICE_DIR}/${String(index).padStart(4, "0")}.bin`;
+}
+
+/** Which chunks need a sliced-audio file: `"all"`, one index, or `null` if it is not one. */
+export function chunkAudioSliceTarget(path: string): "all" | number | null {
+ if (path === SLICE_FRAME || path === SLICE_LAYOUT) return "all";
+ const match = /^audio-chunks\/(\d{4,})\.bin$/.exec(path);
+ return match ? Number(match[1]) : null;
+}
+
+export async function writeChunkAudioSlices(input: {
+ audioPath: string;
+ planDir: string;
+ chunks: readonly ChunkSliceJson[];
+ fps: Fps;
+}): Promise<void> {
+ const grid = await probeAacPacketGrid(input.audioPath);
+ const { payloadStart, payloadEnd, moov } = await readMp4Layout(fileRange(input.audioPath));
+ const [track] = readTracks(moov);
+ const offsets: number[] = [];
+ track!.chunkOffsets.forEach((offset, chunk) => {
+ for (let i = 0; i < track!.samplesPerChunk[chunk]!; i++) {
+ offsets.push(offset);
+ offset += track!.sizes[offsets.length - 1]!;
+ }
+ });
+ const bytes = readFileSync(input.audioPath);
+ const layout: SliceLayout = { sizeBytes: bytes.length, payloadStart, payloadEnd, partStarts: {} };
+ mkdirSync(join(input.planDir, SLICE_DIR), { recursive: true });
+ input.chunks.forEach((_, index) => {
+ const cut = audioCutForChunk(grid, input.chunks, index, input.fps);
+ if (!cut) return;
+ const last = cut.firstPacket + cut.packetCount - 1;
+ const start = offsets[cut.firstPacket]!;
+ layout.partStarts[index] = start;
+ const end = offsets[last]! + track!.sizes[last]!;
+ writeFileSync(join(input.planDir, chunkAudioSlicePath(index)), bytes.subarray(start, end));
+ });
+ const frame = [bytes.subarray(0, payloadStart), bytes.subarray(payloadEnd)];
+ writeFileSync(join(input.planDir, SLICE_FRAME), Buffer.concat(frame));
+ writeFileSync(join(input.planDir, SLICE_LAYOUT), JSON.stringify(layout));
+}
+
+/** The plan audio for a chunk: the whole file, or one rebuilt at `scratchPath` from its slice. */
+export function chunkAudioSource(
+ planDir: string,
+ chunkIndex: number,
+ scratchPath: string,
+): string | null {
+ const layoutPath = join(planDir, SLICE_LAYOUT);
+ if (!existsSync(layoutPath)) return resolvePlanAudioPath(planDir);
+ const layout = JSON.parse(readFileSync(layoutPath, "utf-8")) as SliceLayout;
+ const frame = readFileSync(join(planDir, SLICE_FRAME));
+ const fd = openSync(scratchPath, "w");
+ try {
+ writeSync(fd, frame, 0, layout.payloadStart, 0);
+ writeSync(
+ fd,
+ frame,
+ layout.payloadStart,
+ frame.length - layout.payloadStart,
+ layout.payloadEnd,
+ );
+ const start = layout.partStarts[chunkIndex];
+ if (start !== undefined) {
+ const part = readFileSync(join(planDir, chunkAudioSlicePath(chunkIndex)));
+ writeSync(fd, part, 0, part.length, start);
+ }
+ ftruncateSync(fd, layout.sizeBytes);
+ } finally {
+ closeSync(fd);
+ }
+ return scratchPath;
+}
+
interface MuxInput {
videoPath: string;
audioPath: string | null;
diff --git a/packages/producer/src/services/distributed/mp4Stitch.ts b/packages/producer/src/services/distributed/mp4Stitch.ts
index e84533b61d..1b76643120 100644
--- a/packages/producer/src/services/distributed/mp4Stitch.ts
+++ b/packages/producer/src/services/distributed/mp4Stitch.ts
@@ -409,7 +409,7 @@ export function buildStitchedHeader(layouts: readonly Mp4Layout[]): Buffer {
return Buffer.concat([ftyp, buildMoov(dataStart), mdat]);
}
-function fileRange(path: string): ReadRange {
+export function fileRange(path: string): ReadRange {
return async (start, end) => {
// Read-only, so the mode never applies; an explicit one keeps CodeQL's temp-file rule quiet.
const file = await open(path, "r", 0o600);
diff --git a/packages/producer/src/services/distributed/plan.ts b/packages/producer/src/services/distributed/plan.ts
index ed124b04cb..3fce21da22 100644
--- a/packages/producer/src/services/distributed/plan.ts
+++ b/packages/producer/src/services/distributed/plan.ts
@@ -100,6 +100,7 @@ import {
readFfmpegVersion,
readProducerVersion,
} from "./shared.js";
+import { writeChunkAudioSlices } from "./chunkAudio.js";
import { PLAN_PROTOCOL_V1, type PlanProtocolV1Descriptor } from "./planProtocol.js";
import {
measurePlanSizeBreakdown,
@@ -318,6 +319,8 @@ export interface BuildLocalExecutionPlanOptions {
readonly whileAudioMixes?: () => Promise<void>;
/** mp4 chunks carry their plan-audio slice unless cfr re-encodes. */
readonly chunksCarryAudio?: boolean;
+ /** With `chunksCarryAudio`, also write each chunk's audio packets to their own file. Plan v2 only. */
+ readonly sliceChunkAudio?: boolean;
}
export interface RemoteProjectFile {
@@ -1500,6 +1503,16 @@ export async function buildLocalExecutionPlan(
});
if (!peak.success) throw encoderFailureError("[plan] audio true-peak check failed", peak);
logPlanStage(log, "audio-peak", stageStarted);
+ if (options.sliceChunkAudio) {
+ stageStarted = Date.now();
+ await writeChunkAudioSlices({
+ audioPath: planAudioPath,
+ planDir,
+ chunks,
+ fps: { num: config.fps, den: 1 },
+ });
+ logPlanStage(log, "audio-slices", stageStarted);
+ }
}
// ── Locked config ──
diff --git a/packages/producer/src/services/distributed/planV2.test.ts b/packages/producer/src/services/distributed/planV2.test.ts
index 305d184cde..bbb1434d71 100644
--- a/packages/producer/src/services/distributed/planV2.test.ts
+++ b/packages/producer/src/services/distributed/planV2.test.ts
@@ -15,6 +15,7 @@ import { tmpdir } from "node:os";
import { dirname, join } from "node:path";
import { planFileSha256, recomputePlanHashFromPlanDir } from "../render/stages/freezePlan.js";
import { canonicalJsonStringify, sha256Hex } from "../render/stages/planHash.js";
+import { chunkAudioSlicePath } from "./chunkAudio.js";
import { buildLocalExecutionPlan } from "./plan.js";
import { CURRENT_PLAN_PROTOCOL } from "./planProtocol.js";
import { FFMPEG_VERSION_MISMATCH, renderChunk, RenderChunkValidationError } from "./renderChunk.js";
@@ -28,6 +29,7 @@ import {
PlanV2IntegrityError,
publishPlanV2FromV1,
publishPlanV2FromExecutionPlan,
+ type PlanV2Manifest,
planV2WithPublisher,
readPlanV2Manifest,
readValidatedPlanV2Materialization,
@@ -196,6 +198,19 @@ function createV1Plan(
return planDir;
}
+const slicedAudioPaths = (chunkIndex: number) => [
+ chunkAudioSlicePath(chunkIndex),
+ "audio-chunks/frame.m4a",
+ "audio-chunks/layout.json",
+];
+
+function chunkAudioPaths(manifest: Readonly<PlanV2Manifest>, chunkIndex: number): string[] {
+ return listPlanV2ArtifactsForTarget(manifest, { role: "chunk", chunkIndex })
+ .map((artifact) => artifact.path)
+ .filter((path) => path.includes("audio"))
+ .sort();
+}
+
describe("Plan v2 manifest", () => {
it("is deterministic without changing the manifest or result wire shapes", () => {
const root = tempPath("hf-plan-v2-determinism-");
@@ -435,6 +450,24 @@ describe("Plan v2 manifest", () => {
expect(assembler.some((artifact) => artifact.path === "audio.m4a")).toBe(false);
});
+ it("sends each chunk only its own audio slice when the plan cut one per chunk", () => {
+ const root = tempPath("hf-plan-v2-chunk-audio-slices-");
+ const v1 = createV1Plan(root, { audio: true, chunksCarryAudio: true });
+ mkdirSync(join(v1, "audio-chunks"));
+ writeFileSync(join(v1, "audio-chunks", "frame.m4a"), "moov");
+ writeFileSync(join(v1, "audio-chunks", "layout.json"), "{}");
+ for (const chunkIndex of [0, 1]) {
+ writeFileSync(join(v1, chunkAudioSlicePath(chunkIndex)), `slice ${chunkIndex}`);
+ }
+ refreshV1PlanHash(v1);
+ const manifest = readPlanV2Manifest(createPlanV2FromV1(v1, join(root, "v2")).planDir);
+ for (const chunkIndex of [0, 1]) {
+ expect(chunkAudioPaths(manifest, chunkIndex)).toEqual(slicedAudioPaths(chunkIndex));
+ }
+ const assembler = listPlanV2ArtifactsForTarget(manifest, { role: "assembler" });
+ expect(assembler.some((artifact) => artifact.path.includes("audio"))).toBe(false);
+ });
+
it("uses the runtime frame lookup to select exact video frames per chunk", () => {
const root = tempPath("hf-plan-v2-video-reachability-");
const result = createPlanV2FromV1(createV1Plan(root, { video: true }), join(root, "v2"));
@@ -1171,6 +1204,39 @@ describe("Plan v2 planning", () => {
},
60_000,
);
+
+ it.skipIf(!HAS_FFMPEG)(
+ "cuts the audio into one slice per chunk when chunks carry their audio",
+ async () => {
+ const root = tempPath("hf-plan-v2-audio-slices-");
+ const projectDir = join(root, "project");
+ mkdirSync(projectDir);
+ // 2.1 s is not a whole number of 1024-sample AAC frames.
+ const tone = spawnSync("ffmpeg", [
+ ...["-nostdin", "-v", "error", "-f", "lavfi", "-i", "sine=frequency=440:sample_rate=48000"],
+ ...["-t", "2.1", "-c:a", "pcm_s16le", join(projectDir, "tone.wav")],
+ ]);
+ expect(tone.status).toBe(0);
+ writeFileSync(
+ join(projectDir, "index.html"),
+ `<!doctype html><html><head><meta charset="utf-8"></head><body>
+ <div data-composition-id="root" data-width="320" data-height="240" data-duration="2.1">
+ <audio id="tone" src="tone.wav" data-start="0" data-duration="2.1"></audio><p>slices</p>
+ </div></body></html>`,
+ );
+ const manifest = await planV2WithPublisher(
+ projectDir,
+ { fps: 30, width: 320, height: 240, format: "mp4", chunkSize: 30 },
+ new LocalPlanV2ArtifactPublisher(join(root, "v2")),
+ { stagingParentDir: root, chunksCarryAudio: true },
+ );
+ expect(manifest.chunkCount).toBe(3);
+ for (const chunkIndex of [0, 1, 2]) {
+ expect(chunkAudioPaths(manifest, chunkIndex)).toEqual(slicedAudioPaths(chunkIndex));
+ }
+ },
+ 60_000,
+ );
});
describe("Plan v2 hash schema", () => {
diff --git a/packages/producer/src/services/distributed/planV2.ts b/packages/producer/src/services/distributed/planV2.ts
index 319082e610..a287aa0e2f 100644
--- a/packages/producer/src/services/distributed/planV2.ts
+++ b/packages/producer/src/services/distributed/planV2.ts
@@ -55,6 +55,7 @@ import {
} from "./planV2Publisher.js";
import { PLAN_V2_INTEGRITY_UNRECOVERABLE, PlanV2IntegrityError } from "./planV2Errors.js";
import { planV2BlobPath } from "./planV2Layout.js";
+import { chunkAudioSliceTarget } from "./chunkAudio.js";
import { resolveRenderFpsConfig } from "../fileServer.js";
import {
isPlanAudioArtifactPath,
@@ -283,11 +284,14 @@ function resolveExtractedVideoOutputDir(planDir: string, videoId: string): strin
function artifactTargets(
path: string,
videoDependencies: ReadonlyMap<string, readonly number[]> | null,
- chunksCarryAudio: boolean,
+ chunkAudio: "none" | "whole" | "sliced",
): Pick<PlanV2Artifact, "chunks" | "assembler"> {
if (isPlanAudioArtifactPath(path)) {
- return chunksCarryAudio ? { chunks: "all", assembler: false } : { chunks: [], assembler: true };
+ if (chunkAudio === "none") return { chunks: [], assembler: true };
+ return { chunks: chunkAudio === "whole" ? "all" : [], assembler: false };
}
+ const slice = chunkAudioSliceTarget(path);
+ if (slice !== null) return { chunks: slice === "all" ? "all" : [slice], assembler: false };
if (path === "plan.json" || path === "meta/chunks.json" || path === "meta/encoder.json") {
return { chunks: "all", assembler: true };
}
@@ -583,10 +587,16 @@ function buildPlanV2Publication(
validateExtractionCacheCompleteSentinels(executionPlanDir);
const videoDependencyPlan = buildVideoChunkDependencies(executionPlanDir, dimensions);
const encoder = readJsonFile(join(executionPlanDir, "meta", "encoder.json"), "meta/encoder.json");
- const chunksCarryAudio = isRecord(encoder) && encoder.chunksCarryAudio === true;
- for (const file of listFiles(executionPlanDir)) {
+ const files = listFiles(executionPlanDir);
+ const chunkAudio =
+ !isRecord(encoder) || encoder.chunksCarryAudio !== true
+ ? "none"
+ : files.some((file) => chunkAudioSliceTarget(file.path) !== null)
+ ? "sliced"
+ : "whole";
+ for (const file of files) {
if (isExtractionCacheCompleteSentinelPath(file.path)) continue;
- const targets = artifactTargets(file.path, videoDependencyPlan.dependencies, chunksCarryAudio);
+ const targets = artifactTargets(file.path, videoDependencyPlan.dependencies, chunkAudio);
if (
(file.path.startsWith("video-frames/") || file.path.startsWith("video-sources/")) &&
targets.chunks !== "all" &&
@@ -610,7 +620,7 @@ function buildPlanV2Publication(
if (!blobs.has(sha256)) storedBlobs.set(sha256, { sha256, sizeBytes });
artifacts.push({
...remote,
- ...artifactTargets(remote.path, videoDependencyPlan.dependencies, chunksCarryAudio),
+ ...artifactTargets(remote.path, videoDependencyPlan.dependencies, chunkAudio),
});
}
@@ -761,6 +771,7 @@ export async function planV2WithPublisher(
),
),
chunksCarryAudio: options.chunksCarryAudio,
+ sliceChunkAudio: true,
});
return await publishExecutionPlan(stagingRoot, publisher, { digests, log: config.logger });
} catch (error) {
diff --git a/packages/producer/src/services/distributed/renderChunk.ts b/packages/producer/src/services/distributed/renderChunk.ts
index 789fe83323..e504ecbc80 100644
--- a/packages/producer/src/services/distributed/renderChunk.ts
+++ b/packages/producer/src/services/distributed/renderChunk.ts
@@ -84,9 +84,8 @@ import {
PLAN_VIDEOS_META_RELATIVE_PATH,
type PlanVideosJson,
readFfmpegVersion,
- resolvePlanAudioPath,
} from "./shared.js";
-import { muxChunkWithAudio } from "./chunkAudio.js";
+import { chunkAudioSource, muxChunkWithAudio } from "./chunkAudio.js";
import { DISTRIBUTED_RENDER_CAPABILITIES, readPlanProtocolV1 } from "./planProtocol.js";
import { readValidatedPlanV2Materialization } from "./planV2.js";
import {
@@ -907,9 +906,10 @@ export async function renderChunk(
gopSize: framesInChunk,
});
if (encoder.chunksCarryAudio) {
+ const audioScratchPath = `${outputChunkPath}.audio.m4a`;
await muxChunkWithAudio({
videoPath: videoOnlyPath,
- audioPath: resolvePlanAudioPath(planDir),
+ audioPath: chunkAudioSource(planDir, chunkIndex, audioScratchPath),
outputPath: outputChunkPath,
chunks,
chunkIndex,
@@ -917,6 +917,7 @@ export async function renderChunk(
timeoutMs: cfg.ffmpegProcessTimeout,
});
rmSync(videoOnlyPath, { force: true });
+ rmSync(audioScratchPath, { force: true });
}
encodeStageMs = Date.now() - encodeStarted;
} finally {