Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
perf(producer): each distributed chunk downloads only its own slice of the plan audio (#5881)
Miguel Ángel52c5fae9a760
diff --git a/packages/producer/src/services/distributed/chunkAudio.test.ts b/packages/producer/src/services/distributed/chunkAudio.test.tsindex ac263f0a8b..b8b1fdcb66 100644--- a/packages/producer/src/services/distributed/chunkAudio.test.ts+++ b/packages/producer/src/services/distributed/chunkAudio.test.ts@@ -4,7 +4,12 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";import { tmpdir } from "node:os";import { join } from "node:path";import { assemble } from "./assemble.js";-import { audioCutForChunk, muxChunkWithAudio } from "./chunkAudio.js";+import {+ audioCutForChunk,+ chunkAudioSource,+ muxChunkWithAudio,+ writeChunkAudioSlices,+} from "./chunkAudio.js";const FPS = { num: 30, den: 1 };@@ -72,6 +77,23 @@ describe.skipIf(!hasFfmpeg)("chunks that carry their audio", () => {path,]),).streams as { codec_type: string; nb_read_packets: string; duration: string }[];+ // Each stream's packets (timing, size, payload hash), independent of how the muxer interleaved them.+ const packets = (path: string) =>+ run("ffprobe", [+ ...["-v", "error", "-show_entries", "stream=index,start_pts,duration_ts:packet=stream_index"],+ ...[+ "-show_entries",+ "packet=pts,dts,duration,size,flags,data_hash",+ "-show_data_hash",+ "sha256",+ ],+ ...["-of", "compact", "--", path],+ ])+ .split("\n")+ .sort(+ (a, b) =>+ Number(/stream_index=(\d)/.exec(a)?.[1]) - Number(/stream_index=(\d)/.exec(b)?.[1]),+ );// Uneven chunks with a 1-frame tail, and a single chunk (read without a concat list today). At// 60 fps a 1-frame tail is shorter than an AAC packet: 51 frames are 40800 samples, 41 packets@@ -133,6 +155,23 @@ describe.skipIf(!hasFfmpeg)("chunks that carry their audio", () => {timeoutMs: 60_000,});}+ // A chunk muxing from its slice of a sliced plan writes the same packets as from the whole file.+ const slicedPlanDir = join(dir, "sliced-plan");+ await writeChunkAudioSlices({ audioPath, planDir: slicedPlanDir, chunks: CHUNKS, fps });+ for (const slice of CHUNKS) {+ const slicedChunk = join(dir, `sliced-chunk-${slice.index}.mp4`);+ await muxChunkWithAudio({+ videoPath: videoPaths[slice.index]!,+ audioPath: chunkAudioSource(slicedPlanDir, slice.index, join(dir, "rebuilt.m4a")),+ outputPath: slicedChunk,+ chunks: CHUNKS,+ chunkIndex: slice.index,+ fps,+ timeoutMs: 60_000,+ });+ expect(packets(slicedChunk)).toEqual(packets(chunkPaths[slice.index]!));+ }+// Every plan audio packet lands in exactly one chunk; a tail that owns none has no audio track.const audioOf = (path: string) => streams(path).find((s) => s.codec_type === "audio");const audioPackets = Number(audioOf(audioPath)!.nb_read_packets);diff --git a/packages/producer/src/services/distributed/chunkAudio.ts b/packages/producer/src/services/distributed/chunkAudio.tsindex db71f999dd..834709eb7c 100644--- a/packages/producer/src/services/distributed/chunkAudio.ts+++ b/packages/producer/src/services/distributed/chunkAudio.ts@@ -1,10 +1,15 @@// Distributed mp4 chunks can carry their own slice of the plan audio: the AAC packets of the one// plan-wide encode that start inside the chunk's frame range, so the chunks' audio has no seams.+import { closeSync, existsSync, ftruncateSync, mkdirSync, openSync, readFileSync } from "node:fs";+import { writeFileSync, writeSync } from "node:fs";+import { join } from "node:path";import type { Fps } from "@hyperframes/core";import { appendRenderProvenanceArgs, describeFfmpegFailure, runFfmpeg } from "@hyperframes/engine";import { runFfprobeJson } from "../render/audioPadTrim.js";import type { ChunkSliceJson } from "../render/stages/freezePlan.js";+import { fileRange, readMp4Layout, readTracks } from "./mp4Stitch.js";+import { resolvePlanAudioPath } from "./shared.js";/** Where AAC packet `j` of the plan audio starts: `firstStart + j * packetSamples`, in samples. */interface AacPacketGrid {@@ -90,6 +95,96 @@ async function probeAacPacketGrid(audioPath: string): Promise<AacPacketGrid> {return grid;}+// A sliced plan keeps the audio's moov once, in `frame.m4a` (the file minus its mdat payload),+// and each chunk's packet bytes in their own file. A chunk rebuilds a sparse copy of the whole+// file from them, so it runs the same cut on the same moov and bytes in any ffmpeg version.+const SLICE_DIR = "audio-chunks";+const SLICE_FRAME = `${SLICE_DIR}/frame.m4a`;+const SLICE_LAYOUT = `${SLICE_DIR}/layout.json`;++interface SliceLayout {+ sizeBytes: number;+ payloadStart: number;+ payloadEnd: number;+ partStarts: Record<string, number>;+}++export function chunkAudioSlicePath(index: number): string {+ return `${SLICE_DIR}/${String(index).padStart(4, "0")}.bin`;+}++/** Which chunks need a sliced-audio file: `"all"`, one index, or `null` if it is not one. */+export function chunkAudioSliceTarget(path: string): "all" | number | null {+ if (path === SLICE_FRAME || path === SLICE_LAYOUT) return "all";+ const match = /^audio-chunks\/(\d{4,})\.bin$/.exec(path);+ return match ? Number(match[1]) : null;+}++export async function writeChunkAudioSlices(input: {+ audioPath: string;+ planDir: string;+ chunks: readonly ChunkSliceJson[];+ fps: Fps;+}): Promise<void> {+ const grid = await probeAacPacketGrid(input.audioPath);+ const { payloadStart, payloadEnd, moov } = await readMp4Layout(fileRange(input.audioPath));+ const [track] = readTracks(moov);+ const offsets: number[] = [];+ track!.chunkOffsets.forEach((offset, chunk) => {+ for (let i = 0; i < track!.samplesPerChunk[chunk]!; i++) {+ offsets.push(offset);+ offset += track!.sizes[offsets.length - 1]!;+ }+ });+ const bytes = readFileSync(input.audioPath);+ const layout: SliceLayout = { sizeBytes: bytes.length, payloadStart, payloadEnd, partStarts: {} };+ mkdirSync(join(input.planDir, SLICE_DIR), { recursive: true });+ input.chunks.forEach((_, index) => {+ const cut = audioCutForChunk(grid, input.chunks, index, input.fps);+ if (!cut) return;+ const last = cut.firstPacket + cut.packetCount - 1;+ const start = offsets[cut.firstPacket]!;+ layout.partStarts[index] = start;+ const end = offsets[last]! + track!.sizes[last]!;+ writeFileSync(join(input.planDir, chunkAudioSlicePath(index)), bytes.subarray(start, end));+ });+ const frame = [bytes.subarray(0, payloadStart), bytes.subarray(payloadEnd)];+ writeFileSync(join(input.planDir, SLICE_FRAME), Buffer.concat(frame));+ writeFileSync(join(input.planDir, SLICE_LAYOUT), JSON.stringify(layout));+}++/** The plan audio for a chunk: the whole file, or one rebuilt at `scratchPath` from its slice. */+export function chunkAudioSource(+ planDir: string,+ chunkIndex: number,+ scratchPath: string,+): string | null {+ const layoutPath = join(planDir, SLICE_LAYOUT);+ if (!existsSync(layoutPath)) return resolvePlanAudioPath(planDir);+ const layout = JSON.parse(readFileSync(layoutPath, "utf-8")) as SliceLayout;+ const frame = readFileSync(join(planDir, SLICE_FRAME));+ const fd = openSync(scratchPath, "w");+ try {+ writeSync(fd, frame, 0, layout.payloadStart, 0);+ writeSync(+ fd,+ frame,+ layout.payloadStart,+ frame.length - layout.payloadStart,+ layout.payloadEnd,+ );+ const start = layout.partStarts[chunkIndex];+ if (start !== undefined) {+ const part = readFileSync(join(planDir, chunkAudioSlicePath(chunkIndex)));+ writeSync(fd, part, 0, part.length, start);+ }+ ftruncateSync(fd, layout.sizeBytes);+ } finally {+ closeSync(fd);+ }+ return scratchPath;+}+interface MuxInput {videoPath: string;audioPath: string | null;diff --git a/packages/producer/src/services/distributed/mp4Stitch.ts b/packages/producer/src/services/distributed/mp4Stitch.tsindex e84533b61d..1b76643120 100644--- a/packages/producer/src/services/distributed/mp4Stitch.ts+++ b/packages/producer/src/services/distributed/mp4Stitch.ts@@ -409,7 +409,7 @@ export function buildStitchedHeader(layouts: readonly Mp4Layout[]): Buffer {return Buffer.concat([ftyp, buildMoov(dataStart), mdat]);}-function fileRange(path: string): ReadRange {+export function fileRange(path: string): ReadRange {return async (start, end) => {// Read-only, so the mode never applies; an explicit one keeps CodeQL's temp-file rule quiet.const file = await open(path, "r", 0o600);diff --git a/packages/producer/src/services/distributed/plan.ts b/packages/producer/src/services/distributed/plan.tsindex ed124b04cb..3fce21da22 100644--- a/packages/producer/src/services/distributed/plan.ts+++ b/packages/producer/src/services/distributed/plan.ts@@ -100,6 +100,7 @@ import {readFfmpegVersion,readProducerVersion,} from "./shared.js";+import { writeChunkAudioSlices } from "./chunkAudio.js";import { PLAN_PROTOCOL_V1, type PlanProtocolV1Descriptor } from "./planProtocol.js";import {measurePlanSizeBreakdown,@@ -318,6 +319,8 @@ export interface BuildLocalExecutionPlanOptions {readonly whileAudioMixes?: () => Promise<void>;/** mp4 chunks carry their plan-audio slice unless cfr re-encodes. */readonly chunksCarryAudio?: boolean;+ /** With `chunksCarryAudio`, also write each chunk's audio packets to their own file. Plan v2 only. */+ readonly sliceChunkAudio?: boolean;}export interface RemoteProjectFile {@@ -1500,6 +1503,16 @@ export async function buildLocalExecutionPlan(});if (!peak.success) throw encoderFailureError("[plan] audio true-peak check failed", peak);logPlanStage(log, "audio-peak", stageStarted);+ if (options.sliceChunkAudio) {+ stageStarted = Date.now();+ await writeChunkAudioSlices({+ audioPath: planAudioPath,+ planDir,+ chunks,+ fps: { num: config.fps, den: 1 },+ });+ logPlanStage(log, "audio-slices", stageStarted);+ }}// ── Locked config ──diff --git a/packages/producer/src/services/distributed/planV2.test.ts b/packages/producer/src/services/distributed/planV2.test.tsindex 305d184cde..bbb1434d71 100644--- a/packages/producer/src/services/distributed/planV2.test.ts+++ b/packages/producer/src/services/distributed/planV2.test.ts@@ -15,6 +15,7 @@ import { tmpdir } from "node:os";import { dirname, join } from "node:path";import { planFileSha256, recomputePlanHashFromPlanDir } from "../render/stages/freezePlan.js";import { canonicalJsonStringify, sha256Hex } from "../render/stages/planHash.js";+import { chunkAudioSlicePath } from "./chunkAudio.js";import { buildLocalExecutionPlan } from "./plan.js";import { CURRENT_PLAN_PROTOCOL } from "./planProtocol.js";import { FFMPEG_VERSION_MISMATCH, renderChunk, RenderChunkValidationError } from "./renderChunk.js";@@ -28,6 +29,7 @@ import {PlanV2IntegrityError,publishPlanV2FromV1,publishPlanV2FromExecutionPlan,+ type PlanV2Manifest,planV2WithPublisher,readPlanV2Manifest,readValidatedPlanV2Materialization,@@ -196,6 +198,19 @@ function createV1Plan(return planDir;}+const slicedAudioPaths = (chunkIndex: number) => [+ chunkAudioSlicePath(chunkIndex),+ "audio-chunks/frame.m4a",+ "audio-chunks/layout.json",+];++function chunkAudioPaths(manifest: Readonly<PlanV2Manifest>, chunkIndex: number): string[] {+ return listPlanV2ArtifactsForTarget(manifest, { role: "chunk", chunkIndex })+ .map((artifact) => artifact.path)+ .filter((path) => path.includes("audio"))+ .sort();+}+describe("Plan v2 manifest", () => {it("is deterministic without changing the manifest or result wire shapes", () => {const root = tempPath("hf-plan-v2-determinism-");@@ -435,6 +450,24 @@ describe("Plan v2 manifest", () => {expect(assembler.some((artifact) => artifact.path === "audio.m4a")).toBe(false);});+ it("sends each chunk only its own audio slice when the plan cut one per chunk", () => {+ const root = tempPath("hf-plan-v2-chunk-audio-slices-");+ const v1 = createV1Plan(root, { audio: true, chunksCarryAudio: true });+ mkdirSync(join(v1, "audio-chunks"));+ writeFileSync(join(v1, "audio-chunks", "frame.m4a"), "moov");+ writeFileSync(join(v1, "audio-chunks", "layout.json"), "{}");+ for (const chunkIndex of [0, 1]) {+ writeFileSync(join(v1, chunkAudioSlicePath(chunkIndex)), `slice ${chunkIndex}`);+ }+ refreshV1PlanHash(v1);+ const manifest = readPlanV2Manifest(createPlanV2FromV1(v1, join(root, "v2")).planDir);+ for (const chunkIndex of [0, 1]) {+ expect(chunkAudioPaths(manifest, chunkIndex)).toEqual(slicedAudioPaths(chunkIndex));+ }+ const assembler = listPlanV2ArtifactsForTarget(manifest, { role: "assembler" });+ expect(assembler.some((artifact) => artifact.path.includes("audio"))).toBe(false);+ });+it("uses the runtime frame lookup to select exact video frames per chunk", () => {const root = tempPath("hf-plan-v2-video-reachability-");const result = createPlanV2FromV1(createV1Plan(root, { video: true }), join(root, "v2"));@@ -1171,6 +1204,39 @@ describe("Plan v2 planning", () => {},60_000,);++ it.skipIf(!HAS_FFMPEG)(+ "cuts the audio into one slice per chunk when chunks carry their audio",+ async () => {+ const root = tempPath("hf-plan-v2-audio-slices-");+ const projectDir = join(root, "project");+ mkdirSync(projectDir);+ // 2.1 s is not a whole number of 1024-sample AAC frames.+ const tone = spawnSync("ffmpeg", [+ ...["-nostdin", "-v", "error", "-f", "lavfi", "-i", "sine=frequency=440:sample_rate=48000"],+ ...["-t", "2.1", "-c:a", "pcm_s16le", join(projectDir, "tone.wav")],+ ]);+ expect(tone.status).toBe(0);+ writeFileSync(+ join(projectDir, "index.html"),+ `<!doctype html><html><head><meta charset="utf-8"></head><body>+ <div data-composition-id="root" data-width="320" data-height="240" data-duration="2.1">+ <audio id="tone" src="tone.wav" data-start="0" data-duration="2.1"></audio><p>slices</p>+ </div></body></html>`,+ );+ const manifest = await planV2WithPublisher(+ projectDir,+ { fps: 30, width: 320, height: 240, format: "mp4", chunkSize: 30 },+ new LocalPlanV2ArtifactPublisher(join(root, "v2")),+ { stagingParentDir: root, chunksCarryAudio: true },+ );+ expect(manifest.chunkCount).toBe(3);+ for (const chunkIndex of [0, 1, 2]) {+ expect(chunkAudioPaths(manifest, chunkIndex)).toEqual(slicedAudioPaths(chunkIndex));+ }+ },+ 60_000,+ );});describe("Plan v2 hash schema", () => {diff --git a/packages/producer/src/services/distributed/planV2.ts b/packages/producer/src/services/distributed/planV2.tsindex 319082e610..a287aa0e2f 100644--- a/packages/producer/src/services/distributed/planV2.ts+++ b/packages/producer/src/services/distributed/planV2.ts@@ -55,6 +55,7 @@ import {} from "./planV2Publisher.js";import { PLAN_V2_INTEGRITY_UNRECOVERABLE, PlanV2IntegrityError } from "./planV2Errors.js";import { planV2BlobPath } from "./planV2Layout.js";+import { chunkAudioSliceTarget } from "./chunkAudio.js";import { resolveRenderFpsConfig } from "../fileServer.js";import {isPlanAudioArtifactPath,@@ -283,11 +284,14 @@ function resolveExtractedVideoOutputDir(planDir: string, videoId: string): strinfunction artifactTargets(path: string,videoDependencies: ReadonlyMap<string, readonly number[]> | null,- chunksCarryAudio: boolean,+ chunkAudio: "none" | "whole" | "sliced",): Pick<PlanV2Artifact, "chunks" | "assembler"> {if (isPlanAudioArtifactPath(path)) {- return chunksCarryAudio ? { chunks: "all", assembler: false } : { chunks: [], assembler: true };+ if (chunkAudio === "none") return { chunks: [], assembler: true };+ return { chunks: chunkAudio === "whole" ? "all" : [], assembler: false };}+ const slice = chunkAudioSliceTarget(path);+ if (slice !== null) return { chunks: slice === "all" ? "all" : [slice], assembler: false };if (path === "plan.json" || path === "meta/chunks.json" || path === "meta/encoder.json") {return { chunks: "all", assembler: true };}@@ -583,10 +587,16 @@ function buildPlanV2Publication(validateExtractionCacheCompleteSentinels(executionPlanDir);const videoDependencyPlan = buildVideoChunkDependencies(executionPlanDir, dimensions);const encoder = readJsonFile(join(executionPlanDir, "meta", "encoder.json"), "meta/encoder.json");- const chunksCarryAudio = isRecord(encoder) && encoder.chunksCarryAudio === true;- for (const file of listFiles(executionPlanDir)) {+ const files = listFiles(executionPlanDir);+ const chunkAudio =+ !isRecord(encoder) || encoder.chunksCarryAudio !== true+ ? "none"+ : files.some((file) => chunkAudioSliceTarget(file.path) !== null)+ ? "sliced"+ : "whole";+ for (const file of files) {if (isExtractionCacheCompleteSentinelPath(file.path)) continue;- const targets = artifactTargets(file.path, videoDependencyPlan.dependencies, chunksCarryAudio);+ const targets = artifactTargets(file.path, videoDependencyPlan.dependencies, chunkAudio);if ((file.path.startsWith("video-frames/") || file.path.startsWith("video-sources/")) &&targets.chunks !== "all" &&@@ -610,7 +620,7 @@ function buildPlanV2Publication(if (!blobs.has(sha256)) storedBlobs.set(sha256, { sha256, sizeBytes });artifacts.push({...remote,- ...artifactTargets(remote.path, videoDependencyPlan.dependencies, chunksCarryAudio),+ ...artifactTargets(remote.path, videoDependencyPlan.dependencies, chunkAudio),});}@@ -761,6 +771,7 @@ export async function planV2WithPublisher(),),chunksCarryAudio: options.chunksCarryAudio,+ sliceChunkAudio: true,});return await publishExecutionPlan(stagingRoot, publisher, { digests, log: config.logger });} catch (error) {diff --git a/packages/producer/src/services/distributed/renderChunk.ts b/packages/producer/src/services/distributed/renderChunk.tsindex 789fe83323..e504ecbc80 100644--- a/packages/producer/src/services/distributed/renderChunk.ts+++ b/packages/producer/src/services/distributed/renderChunk.ts@@ -84,9 +84,8 @@ import {PLAN_VIDEOS_META_RELATIVE_PATH,type PlanVideosJson,readFfmpegVersion,- resolvePlanAudioPath,} from "./shared.js";-import { muxChunkWithAudio } from "./chunkAudio.js";+import { chunkAudioSource, muxChunkWithAudio } from "./chunkAudio.js";import { DISTRIBUTED_RENDER_CAPABILITIES, readPlanProtocolV1 } from "./planProtocol.js";import { readValidatedPlanV2Materialization } from "./planV2.js";import {@@ -907,9 +906,10 @@ export async function renderChunk(gopSize: framesInChunk,});if (encoder.chunksCarryAudio) {+ const audioScratchPath = `${outputChunkPath}.audio.m4a`;await muxChunkWithAudio({videoPath: videoOnlyPath,- audioPath: resolvePlanAudioPath(planDir),+ audioPath: chunkAudioSource(planDir, chunkIndex, audioScratchPath),outputPath: outputChunkPath,chunks,chunkIndex,@@ -917,6 +917,7 @@ export async function renderChunk(timeoutMs: cfg.ffmpegProcessTimeout,});rmSync(videoOnlyPath, { force: true });+ rmSync(audioScratchPath, { force: true });}encodeStageMs = Date.now() - encodeStarted;} finally {