Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
fix(engine): cloud renders fail instead of going silent when an audio read is cut off (#5996)
Miguel Ángelb4e63cf19f52
diff --git a/packages/engine/src/services/audioMixer.test.ts b/packages/engine/src/services/audioMixer.test.tsindex fa615a1a94..c75267d821 100644--- a/packages/engine/src/services/audioMixer.test.ts+++ b/packages/engine/src/services/audioMixer.test.ts@@ -1589,6 +1589,8 @@ describe("processCompositionAudio", () => {const extractArgs = runFfmpegMock.mock.calls[0]?.[0] ?? [];expect(extractArgs.indexOf("-tls_verify")).toBeGreaterThanOrEqual(0);expect(extractArgs.indexOf("-tls_verify")).toBeLessThan(extractArgs.indexOf("-i"));+ expect(extractArgs.indexOf("-xerror")).toBeGreaterThanOrEqual(0);+ expect(extractArgs.indexOf("-xerror")).toBeLessThan(extractArgs.indexOf("-i"));expect(extractArgs.slice(extractArgs.indexOf("-ss"), extractArgs.indexOf(url) + 1)).toEqual(["-ss","600",diff --git a/packages/engine/src/services/audioMixer.ts b/packages/engine/src/services/audioMixer.tsindex c76bcd4088..443c340941 100644--- a/packages/engine/src/services/audioMixer.ts+++ b/packages/engine/src/services/audioMixer.ts@@ -649,6 +649,8 @@ export function parseAudioElements(html: string): AudioElement[] {return elements;}+const FAIL_ON_READ_ERROR_INSTEAD_OF_ENDING_THERE = ["-xerror"];+/** Decode `duration` s of audio from `mediaStart` in Chrome's media time. Until every stream has* started, an input seek snaps audio to the video's first keyframe, so read by timestamp instead. */async function extractAudioSegment(@@ -668,7 +670,8 @@ async function extractAudioSegment(const sourceDuration = sourceTimeAt(normalizedPlaybackRate, duration);// A failed local probe keeps FFmpeg's default channel conversion and the input seek; a URL probe// is retried once, then fails: a blip must not change how the next, successful read is mixed.- const metadata = isHttpUrl(srcPath)+ const readsUrl = isHttpUrl(srcPath);+ const metadata = readsUrl? await extractAudioMetadata(srcPath, { signal }).catch((error: unknown) => {if (signal?.aborted) throw error;return extractAudioMetadata(srcPath, { signal });@@ -687,6 +690,7 @@ async function extractAudioSegment();const args = [+ ...(readsUrl ? FAIL_ON_READ_ERROR_INSTEAD_OF_ENDING_THERE : []),...urlInputArgs(srcPath),...(inputSeek ? ["-ss", String(mediaStart), "-t", String(sourceDuration)] : []),"-i",@@ -714,7 +718,12 @@ async function extractAudioSegment(failure,};}- const failure = !result.success ? ffmpegFailure(stage, result) : undefined;+ let failure = result.success ? undefined : ffmpegFailure(stage, result);+ const probedLinkFailedToRead =+ readsUrl && (failure?.reason === "invalid_media" || failure?.reason === "ffmpeg_failed");+ if (failure && probedLinkFailedToRead) {+ failure = { ...failure, reason: "download_failed", owner: "system", retryable: true };+ }return {success: result.success,outputPath,diff --git a/packages/engine/src/services/audioMixer.urlRead.test.ts b/packages/engine/src/services/audioMixer.urlRead.test.tsnew file mode 100644index 0000000000..f3265ee796--- /dev/null+++ b/packages/engine/src/services/audioMixer.urlRead.test.ts@@ -0,0 +1,139 @@+import { spawnSync } from "node:child_process";+import { mkdtempSync, readFileSync, rmSync } from "node:fs";+import { createServer, type Server } from "node:http";+import type { AddressInfo } from "node:net";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { afterEach, beforeEach, describe, expect, it } from "vitest";+import { getFfmpegBinary } from "../utils/ffmpegBinaries.js";+import { processCompositionAudio } from "./audioMixer.js";++const HAS_FFMPEG = spawnSync(getFfmpegBinary(), ["-version"], { encoding: "utf-8" }).status === 0;+const BYTES_PER_SECOND = 48_000 * 2 * 2;++// A range server like S3's (ffmpeg asks for `bytes=<start>-`). With `cut`, the first read starting+// at `cut.from` or later (past the probes' header reads) stops at `cut.at`, and every later request+// is refused, as for a link that just expired.+function serve(body: Buffer, cut?: { from: number; at: number }): Promise<Server> {+ let refusing = false;+ const server = createServer((req, res) => {+ if (refusing) return void res.writeHead(403).end("Request has expired");+ const start = Number(/bytes=(\d+)-/.exec(req.headers.range ?? "")?.[1] ?? 0);+ res.writeHead(206, {+ "Accept-Ranges": "bytes",+ "Content-Length": body.length - start,+ "Content-Range": `bytes ${start}-${body.length - 1}/${body.length}`,+ "Content-Type": "audio/wav",+ });+ const cutHere = cut !== undefined && start >= cut.from && start < cut.at;+ if (!cutHere) return void res.end(body.subarray(start));+ refusing = true;+ res.write(body.subarray(start, cut.at), () => res.destroy());+ });+ return new Promise((resolve) => server.listen(0, "127.0.0.1", () => resolve(server)));+}++describe.skipIf(!HAS_FFMPEG)("audio cut from a cloud-held file's URL", () => {+ let dir: string;+ let server: Server | undefined;++ function makeMedia(name: string, args: string[]): Buffer {+ const path = join(dir, name);+ const made = spawnSync(getFfmpegBinary(), ["-v", "error", "-y", ...args, path]);+ expect(made.status).toBe(0);+ return readFileSync(path);+ }++ const makeVoice = () =>+ makeMedia("voice.wav", [+ ...["-f", "lavfi", "-i", "sine=frequency=440:sample_rate=48000:duration=20"],+ ...["-ac", "2", "-c:a", "pcm_s16le"],+ ]);++ beforeEach(() => {+ dir = mkdtempSync(join(tmpdir(), "hf-audio-url-read-"));+ });++ afterEach(async () => {+ await new Promise((resolve) => (server ? server.close(resolve) : resolve(undefined)));+ server = undefined;+ rmSync(dir, { recursive: true, force: true });+ });++ async function mixFromUrl(+ clip: { name: string; type: "audio" | "video"; mediaStart: number; end: number } = {+ name: "voice.wav",+ type: "audio",+ mediaStart: 2,+ end: 10,+ },+ ) {+ const { port } = server!.address() as AddressInfo;+ const baseDir = join(dir, "project");+ return processCompositionAudio(+ [+ {+ id: "clip",+ src: `takes/${clip.name}`,+ start: 0,+ end: clip.end,+ mediaStart: clip.mediaStart,+ layer: 0,+ volume: 1,+ type: clip.type,+ },+ ],+ baseDir,+ join(dir, "work"),+ join(dir, "out.m4a"),+ clip.end,+ undefined,+ {+ urlByAbsentLocalPath: new Map([+ [join(baseDir, "takes", clip.name), `http://127.0.0.1:${port}/${clip.name}`],+ ]),+ },+ );+ }++ it("mixes the whole span when the URL serves it", async () => {+ server = await serve(makeVoice());++ const result = await mixFromUrl();++ expect(result.failures).toBeUndefined();+ expect(result.success).toBe(true);+ });++ it("fails the track when the read is cut mid-span, instead of padding it with silence", async () => {+ // The clip reads source seconds 2 to 12; the link fails at second 6.+ server = await serve(makeVoice(), { from: BYTES_PER_SECOND, at: 6 * BYTES_PER_SECOND });++ const result = await mixFromUrl();++ expect(result.success).toBe(false);+ expect(result.failures).toEqual([+ expect.objectContaining({+ elementId: "clip",+ stage: "prepare",+ reason: "download_failed",+ owner: "system",+ retryable: true,+ }),+ ]);+ });++ it("mixes a video whose audio track ends before its picture", async () => {+ const video = makeMedia("take.mp4", [+ ...["-f", "lavfi", "-i", "testsrc2=size=64x64:rate=25:duration=6"],+ ...["-f", "lavfi", "-i", "sine=frequency=440:sample_rate=48000:duration=4"],+ ...["-c:v", "mpeg4", "-c:a", "aac", "-movflags", "+faststart"],+ ]);+ server = await serve(video);++ const result = await mixFromUrl({ name: "take.mp4", type: "video", mediaStart: 0, end: 6 });++ expect(result.failures).toBeUndefined();+ expect(result.success).toBe(true);+ });+});