Skip to content

Write HTML. Render video. Built for agents.

fix(aws-lambda): make webm and mov audio in the separate audio task like mp4 (#6098)

Miguel Ángelf5f884e33d44
diff --git a/packages/aws-lambda/src/handler.test.ts b/packages/aws-lambda/src/handler.test.ts
index c552d8e5ab..8f0d2a1d4a 100644
--- a/packages/aws-lambda/src/handler.test.ts
+++ b/packages/aws-lambda/src/handler.test.ts
@@ -1099,6 +1099,7 @@ describe("handler dispatch", () => {
chunkCount: number,
separateAudio = false,
workerConcurrency?: number,
+ format: "mp4" | "webm" | "mov" = "mp4",
) {
const tmpRoot = makeTmpRoot();
const s3 = new FakeS3Client();
@@ -1114,8 +1115,9 @@ describe("handler dispatch", () => {
const v1Dir = join(tmpRoot, "v1");
makeMinimalV1PlanDir(v1Dir, !separateAudio, {
chunkCount,
- chunksCarryAudio: !separateAudio,
+ chunksCarryAudio: !separateAudio && format === "mp4",
separateAudio,
+ format,
});
return publishPlanV2FromExecutionPlan(v1Dir, publisher);
},
@@ -1136,7 +1138,7 @@ describe("handler dispatch", () => {
Action: "plan",
ProjectS3Uri: "s3://bucket/project.tar.gz",
PlanOutputS3Prefix: "s3://bucket/renders/f/",
- Config: { fps: 30, width: 640, height: 360, format: "mp4", workerConcurrency },
+ Config: { fps: 30, width: 640, height: 360, format, workerConcurrency },
},
deps,
);
@@ -1151,8 +1153,8 @@ describe("handler dispatch", () => {
PlanHash: planned.PlanHash,
ChunkS3Uris: chunkS3Uris,
AudioS3Uri: audioS3Uri,
- OutputS3Uri: "s3://bucket/renders/f/output.mp4",
- Format: "mp4",
+ OutputS3Uri: `s3://bucket/renders/f/output.${format}`,
+ Format: format,
},
deps,
);
@@ -1256,6 +1258,30 @@ describe("handler dispatch", () => {
},
);
+ it.each(["webm", "mov"] as const)(
+ "joins %s chunks with the audio task's track when the plan left its audio out",
+ async (format) => {
+ const { s3, assembleMock, assemble, planned } = await planStitched(2, true, 4, format);
+ expect(planned.SeparateAudio).toBe(true);
+ const audioUri = "s3://bucket/renders/f/audio/audio.m4a";
+ s3.objects.set(audioUri, Buffer.from("AAC-FROM-AUDIO-TASK"));
+ let audio: string | null = null;
+ assembleMock.mockImplementationOnce(async (_p, _c, audioPath, outputPath) => {
+ audio = audioPath && readFileSync(audioPath, "utf-8");
+ writeFileSync(outputPath, "LOCAL-JOIN");
+ return { outputPath, durationMs: 1, framesEncoded: 60, fileSize: 10 };
+ });
+ const chunks = [0, 1].map((index) => {
+ const uri = `s3://bucket/renders/f/chunks/000${index}.${format}`;
+ s3.objects.set(uri, Buffer.from("CHUNK"));
+ return uri;
+ });
+ await assemble(chunks, audioUri);
+ expect(assembleMock).toHaveBeenCalledTimes(1);
+ expect(audio).toBe("AAC-FROM-AUDIO-TASK");
+ },
+ );
+
it("refuses an audio URI for a v2 plan that ships its own audio", async () => {
const { assemble } = await planStitched(1);
await expect(
@@ -1366,12 +1392,24 @@ describe("handler — audio", () => {
expect(buildPlanAudio).toHaveBeenCalledTimes(1);
});
- it("leaves the audio to the plan when the stitch cannot take it (cfr)", async () => {
- const { result, s3, buildPlanAudio } = await runAudio({ cfr: true });
- expect(result).toMatchObject({ Action: "audio", HasAudio: false, AudioS3Uri: null });
- expect(buildPlanAudio).not.toHaveBeenCalled();
- expect(s3.ops).toHaveLength(0);
- });
+ it.each(["webm", "mov"])(
+ "mixes %s audio too, so one function makes every audio byte",
+ async (format) => {
+ const { result, buildPlanAudio } = await runAudio({ format });
+ expect(result).toMatchObject({ Action: "audio", HasAudio: true });
+ expect(buildPlanAudio).toHaveBeenCalledTimes(1);
+ },
+ );
+
+ it.each([{ cfr: true }, { format: "png-sequence" }])(
+ "leaves the audio to the plan for %o",
+ async (config) => {
+ const { result, s3, buildPlanAudio } = await runAudio(config);
+ expect(result).toMatchObject({ Action: "audio", HasAudio: false, AudioS3Uri: null });
+ expect(buildPlanAudio).not.toHaveBeenCalled();
+ expect(s3.ops).toHaveLength(0);
+ },
+ );
});
/** A real client (so it can pre-sign offline) whose requests go to the fake. */
@@ -1619,6 +1657,7 @@ function makeMinimalV1PlanDir(
chunksCarryAudio?: boolean;
separateAudio?: boolean;
withDeferredVideo?: boolean;
+ format?: string;
} = {},
): void {
const chunkCount = options.chunkCount ?? 1;
@@ -1629,7 +1668,13 @@ function makeMinimalV1PlanDir(
planHash: "a".repeat(64),
chunkCount,
totalFrames: 30 * chunkCount,
- dimensions: { fpsNum: 30, fpsDen: 1, width: 640, height: 360, format: "mp4" },
+ dimensions: {
+ fpsNum: 30,
+ fpsDen: 1,
+ width: 640,
+ height: 360,
+ format: options.format ?? "mp4",
+ },
ffmpegVersion: "6.0",
producerVersion: "test",
fontSnapshotSha: "font-snapshot-test",
diff --git a/packages/aws-lambda/src/handler.ts b/packages/aws-lambda/src/handler.ts
index f9f2932d82..8bab85b1a8 100644
--- a/packages/aws-lambda/src/handler.ts
+++ b/packages/aws-lambda/src/handler.ts
@@ -566,7 +566,7 @@ async function handleAudio(event: AudioEvent, deps?: HandlerDeps): Promise<Audio
AudioS3Uri: null,
DurationMs: Date.now() - started,
});
- // The plan mixed it (non-mp4, cfr): the assembler reads it from the manifest.
+ // The plan mixed it (png-sequence, cfr): the assembler reads it from the manifest.
if (!plansSeparateAudio(event.Config, { deferAudio: true })) return none();
const s3 = deps?.s3 ?? getS3Client();
await ensureChromePath(deps);
@@ -835,7 +835,11 @@ async function handleAssembleV2(
}
// `downloadAndMaterializePlanV2` materializes atomically. Audio is
// assembler-only and lives at the familiar v1-compatible location.
- const audioPath = resolvePlanAudioPath(planDir);
+ let audioPath = resolvePlanAudioPath(planDir);
+ if (event.AudioS3Uri !== null) {
+ audioPath = join(planDir, PLAN_AUDIO_RELATIVE_PATH);
+ await downloadS3ObjectToFile(s3, event.AudioS3Uri, audioPath);
+ }
const chunksStarted = Date.now();
const chunkPaths = await downloadChunkObjects(s3, event.ChunkS3Uris, work, event.Format);
const chunksMs = Date.now() - chunksStarted;
diff --git a/packages/producer/src/services/distributed/assemble.ts b/packages/producer/src/services/distributed/assemble.ts
index df76814284..bda1533755 100644
--- a/packages/producer/src/services/distributed/assemble.ts
+++ b/packages/producer/src/services/distributed/assemble.ts
@@ -400,10 +400,11 @@ async function runAssembleFfmpeg(
}
}
-/** Chunks carrying audio, or with it as a separate file, stitch under one index like S3's. */
+/** Mp4 chunks carrying audio, or with it as a separate file, stitch under one index like S3's. */
export function chunksAreStitchedMp4(planDir: string, cfr: boolean): boolean {
const { chunksCarryAudio, separateAudio } = lockedAudio(planDir);
if (!chunksCarryAudio && !separateAudio) return false;
+ if (planFormat(planDir) !== "mp4") return false;
if (cfr) {
throw new Error("[assemble] cfr=true cannot re-encode a plan whose mp4 chunks are stitched");
}
@@ -414,6 +415,11 @@ export function stitchesSeparateAudio(planDir: string): boolean {
return lockedAudio(planDir).separateAudio === true;
}
+function planFormat(planDir: string): DistributedFormat {
+ const plan = JSON.parse(readFileSync(join(planDir, "plan.json"), "utf-8")) as PlanJsonForAssemble;
+ return plan.dimensions.format;
+}
+
function lockedAudio(
planDir: string,
): Pick<LockedRenderConfig, "chunksCarryAudio" | "separateAudio"> {
diff --git a/packages/producer/src/services/distributed/plan.test.ts b/packages/producer/src/services/distributed/plan.test.ts
index f277e903ce..7790853666 100644
--- a/packages/producer/src/services/distributed/plan.test.ts
+++ b/packages/producer/src/services/distributed/plan.test.ts
@@ -1112,6 +1112,25 @@ describe.skipIf(spawnSync(getFfmpegBinary(), ["-version"]).status !== 0)(
expect(existsSync(join(withCfr, "audio.m4a"))).toBe(true);
}, 60_000);
+ it("leaves webm and mov audio to buildPlanAudio too; png-sequence keeps its own", async () => {
+ const project = audioProject("project-separate-audio-formats");
+ const config = { fps: 30, width: 320, height: 240 } as const;
+ for (const [format, separate] of [
+ ["webm", true],
+ ["mov", true],
+ ["png-sequence", false],
+ ] as const) {
+ const dir = join(runRoot, `plan-separate-audio-${format}`);
+ await buildLocalExecutionPlan(project, { ...config, format }, dir, { deferAudio: true });
+ const encoder = JSON.parse(readFileSync(join(dir, "meta", "encoder.json"), "utf-8"));
+ expect({ format, separate: encoder.separateAudio === true }).toEqual({ format, separate });
+ expect({ format, mixed: existsSync(join(dir, "audio.m4a")) }).toEqual({
+ format,
+ mixed: !separate,
+ });
+ }
+ }, 60_000);
+
it("gives buildPlanAudio the sound of a video whose src the scene script sets", async () => {
const project = join(runRoot, "project-script-src");
mkdirSync(project, { recursive: true });
diff --git a/packages/producer/src/services/distributed/plan.ts b/packages/producer/src/services/distributed/plan.ts
index 69d2e0c1d9..1857d463d1 100644
--- a/packages/producer/src/services/distributed/plan.ts
+++ b/packages/producer/src/services/distributed/plan.ts
@@ -1089,7 +1089,7 @@ export function plansSeparateAudio(
config: Pick<DistributedRenderConfig, "format" | "cfr">,
options: Pick<BuildLocalExecutionPlanOptions, "deferAudio">,
): boolean {
- return options.deferAudio === true && config.format === "mp4" && config.cfr !== true;
+ return options.deferAudio === true && config.format !== "png-sequence" && config.cfr !== true;
}
// The plan's copy, compile and probe, shared by the plan and by buildPlanAudio so both see the