Skip to content

Write HTML. Render video. Built for agents.

fix(engine): cloud renders fail instead of going silent when an audio read is cut off (#5996)

Miguel Ángelb4e63cf19f52
diff --git a/packages/engine/src/services/audioMixer.test.ts b/packages/engine/src/services/audioMixer.test.ts
index fa615a1a94..c75267d821 100644
--- a/packages/engine/src/services/audioMixer.test.ts
+++ b/packages/engine/src/services/audioMixer.test.ts
@@ -1589,6 +1589,8 @@ describe("processCompositionAudio", () => {
const extractArgs = runFfmpegMock.mock.calls[0]?.[0] ?? [];
expect(extractArgs.indexOf("-tls_verify")).toBeGreaterThanOrEqual(0);
expect(extractArgs.indexOf("-tls_verify")).toBeLessThan(extractArgs.indexOf("-i"));
+ expect(extractArgs.indexOf("-xerror")).toBeGreaterThanOrEqual(0);
+ expect(extractArgs.indexOf("-xerror")).toBeLessThan(extractArgs.indexOf("-i"));
expect(extractArgs.slice(extractArgs.indexOf("-ss"), extractArgs.indexOf(url) + 1)).toEqual([
"-ss",
"600",
diff --git a/packages/engine/src/services/audioMixer.ts b/packages/engine/src/services/audioMixer.ts
index c76bcd4088..443c340941 100644
--- a/packages/engine/src/services/audioMixer.ts
+++ b/packages/engine/src/services/audioMixer.ts
@@ -649,6 +649,8 @@ export function parseAudioElements(html: string): AudioElement[] {
return elements;
}
+const FAIL_ON_READ_ERROR_INSTEAD_OF_ENDING_THERE = ["-xerror"];
+
/** Decode `duration` s of audio from `mediaStart` in Chrome's media time. Until every stream has
* started, an input seek snaps audio to the video's first keyframe, so read by timestamp instead. */
async function extractAudioSegment(
@@ -668,7 +670,8 @@ async function extractAudioSegment(
const sourceDuration = sourceTimeAt(normalizedPlaybackRate, duration);
// A failed local probe keeps FFmpeg's default channel conversion and the input seek; a URL probe
// is retried once, then fails: a blip must not change how the next, successful read is mixed.
- const metadata = isHttpUrl(srcPath)
+ const readsUrl = isHttpUrl(srcPath);
+ const metadata = readsUrl
? await extractAudioMetadata(srcPath, { signal }).catch((error: unknown) => {
if (signal?.aborted) throw error;
return extractAudioMetadata(srcPath, { signal });
@@ -687,6 +690,7 @@ async function extractAudioSegment(
);
const args = [
+ ...(readsUrl ? FAIL_ON_READ_ERROR_INSTEAD_OF_ENDING_THERE : []),
...urlInputArgs(srcPath),
...(inputSeek ? ["-ss", String(mediaStart), "-t", String(sourceDuration)] : []),
"-i",
@@ -714,7 +718,12 @@ async function extractAudioSegment(
failure,
};
}
- const failure = !result.success ? ffmpegFailure(stage, result) : undefined;
+ let failure = result.success ? undefined : ffmpegFailure(stage, result);
+ const probedLinkFailedToRead =
+ readsUrl && (failure?.reason === "invalid_media" || failure?.reason === "ffmpeg_failed");
+ if (failure && probedLinkFailedToRead) {
+ failure = { ...failure, reason: "download_failed", owner: "system", retryable: true };
+ }
return {
success: result.success,
outputPath,
diff --git a/packages/engine/src/services/audioMixer.urlRead.test.ts b/packages/engine/src/services/audioMixer.urlRead.test.ts
new file mode 100644
index 0000000000..f3265ee796
--- /dev/null
+++ b/packages/engine/src/services/audioMixer.urlRead.test.ts
@@ -0,0 +1,139 @@
+import { spawnSync } from "node:child_process";
+import { mkdtempSync, readFileSync, rmSync } from "node:fs";
+import { createServer, type Server } from "node:http";
+import type { AddressInfo } from "node:net";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { getFfmpegBinary } from "../utils/ffmpegBinaries.js";
+import { processCompositionAudio } from "./audioMixer.js";
+
+const HAS_FFMPEG = spawnSync(getFfmpegBinary(), ["-version"], { encoding: "utf-8" }).status === 0;
+const BYTES_PER_SECOND = 48_000 * 2 * 2;
+
+// A range server like S3's (ffmpeg asks for `bytes=<start>-`). With `cut`, the first read starting
+// at `cut.from` or later (past the probes' header reads) stops at `cut.at`, and every later request
+// is refused, as for a link that just expired.
+function serve(body: Buffer, cut?: { from: number; at: number }): Promise<Server> {
+ let refusing = false;
+ const server = createServer((req, res) => {
+ if (refusing) return void res.writeHead(403).end("Request has expired");
+ const start = Number(/bytes=(\d+)-/.exec(req.headers.range ?? "")?.[1] ?? 0);
+ res.writeHead(206, {
+ "Accept-Ranges": "bytes",
+ "Content-Length": body.length - start,
+ "Content-Range": `bytes ${start}-${body.length - 1}/${body.length}`,
+ "Content-Type": "audio/wav",
+ });
+ const cutHere = cut !== undefined && start >= cut.from && start < cut.at;
+ if (!cutHere) return void res.end(body.subarray(start));
+ refusing = true;
+ res.write(body.subarray(start, cut.at), () => res.destroy());
+ });
+ return new Promise((resolve) => server.listen(0, "127.0.0.1", () => resolve(server)));
+}
+
+describe.skipIf(!HAS_FFMPEG)("audio cut from a cloud-held file's URL", () => {
+ let dir: string;
+ let server: Server | undefined;
+
+ function makeMedia(name: string, args: string[]): Buffer {
+ const path = join(dir, name);
+ const made = spawnSync(getFfmpegBinary(), ["-v", "error", "-y", ...args, path]);
+ expect(made.status).toBe(0);
+ return readFileSync(path);
+ }
+
+ const makeVoice = () =>
+ makeMedia("voice.wav", [
+ ...["-f", "lavfi", "-i", "sine=frequency=440:sample_rate=48000:duration=20"],
+ ...["-ac", "2", "-c:a", "pcm_s16le"],
+ ]);
+
+ beforeEach(() => {
+ dir = mkdtempSync(join(tmpdir(), "hf-audio-url-read-"));
+ });
+
+ afterEach(async () => {
+ await new Promise((resolve) => (server ? server.close(resolve) : resolve(undefined)));
+ server = undefined;
+ rmSync(dir, { recursive: true, force: true });
+ });
+
+ async function mixFromUrl(
+ clip: { name: string; type: "audio" | "video"; mediaStart: number; end: number } = {
+ name: "voice.wav",
+ type: "audio",
+ mediaStart: 2,
+ end: 10,
+ },
+ ) {
+ const { port } = server!.address() as AddressInfo;
+ const baseDir = join(dir, "project");
+ return processCompositionAudio(
+ [
+ {
+ id: "clip",
+ src: `takes/${clip.name}`,
+ start: 0,
+ end: clip.end,
+ mediaStart: clip.mediaStart,
+ layer: 0,
+ volume: 1,
+ type: clip.type,
+ },
+ ],
+ baseDir,
+ join(dir, "work"),
+ join(dir, "out.m4a"),
+ clip.end,
+ undefined,
+ {
+ urlByAbsentLocalPath: new Map([
+ [join(baseDir, "takes", clip.name), `http://127.0.0.1:${port}/${clip.name}`],
+ ]),
+ },
+ );
+ }
+
+ it("mixes the whole span when the URL serves it", async () => {
+ server = await serve(makeVoice());
+
+ const result = await mixFromUrl();
+
+ expect(result.failures).toBeUndefined();
+ expect(result.success).toBe(true);
+ });
+
+ it("fails the track when the read is cut mid-span, instead of padding it with silence", async () => {
+ // The clip reads source seconds 2 to 12; the link fails at second 6.
+ server = await serve(makeVoice(), { from: BYTES_PER_SECOND, at: 6 * BYTES_PER_SECOND });
+
+ const result = await mixFromUrl();
+
+ expect(result.success).toBe(false);
+ expect(result.failures).toEqual([
+ expect.objectContaining({
+ elementId: "clip",
+ stage: "prepare",
+ reason: "download_failed",
+ owner: "system",
+ retryable: true,
+ }),
+ ]);
+ });
+
+ it("mixes a video whose audio track ends before its picture", async () => {
+ const video = makeMedia("take.mp4", [
+ ...["-f", "lavfi", "-i", "testsrc2=size=64x64:rate=25:duration=6"],
+ ...["-f", "lavfi", "-i", "sine=frequency=440:sample_rate=48000:duration=4"],
+ ...["-c:v", "mpeg4", "-c:a", "aac", "-movflags", "+faststart"],
+ ]);
+ server = await serve(video);
+
+ const result = await mixFromUrl({ name: "take.mp4", type: "video", mediaStart: 0, end: 6 });
+
+ expect(result.failures).toBeUndefined();
+ expect(result.success).toBe(true);
+ });
+});