Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
perf(engine): cloud renders cut a video's audio from its URL instead of downloading the file (#5972)
Miguel Ángela8a80323f69f
diff --git a/packages/engine/src/services/aacEncode.test.ts b/packages/engine/src/services/aacEncode.test.tsindex 5964c68897..7a0ce0d416 100644--- a/packages/engine/src/services/aacEncode.test.ts+++ b/packages/engine/src/services/aacEncode.test.ts@@ -105,7 +105,7 @@ describe.skipIf(!HAS_FFMPEG)("encodeAac", () => {it("encodes a long mix in parallel segments that decode like one encoder's output", async () => {const source = join(dir, "mix.wav");ffmpeg([- ...["-f", "lavfi", "-i", "anoisesrc=d=150.5:c=pink:r=48000:a=0.2"],+ ...["-f", "lavfi", "-i", "anoisesrc=d=150.5:c=pink:r=48000:a=0.2:seed=1"],...["-f", "lavfi", "-i", "sine=f=330:d=150.5:r=48000"],...["-filter_complex", "[0][1]amix=2:normalize=0,aformat=channel_layouts=stereo"],...["-c:a", "pcm_f32le", "-y", source],@@ -173,7 +173,16 @@ describe.skipIf(!HAS_FFMPEG)("encodeAac", () => {it("keeps a tempo-changed 44.1 kHz track at lag 0 against the same filter uncoded", async () => {const source = join(dir, "rate.wav");- ffmpeg(["-f", "lavfi", "-i", "anoisesrc=d=6:c=pink:r=44100:a=0.3", "-ac", "2", "-y", source]);+ ffmpeg([+ "-f",+ "lavfi",+ "-i",+ "anoisesrc=d=6:c=pink:r=44100:a=0.3:seed=1",+ "-ac",+ "2",+ "-y",+ source,+ ]);const output = join(dir, "rate.m4a");const encoded = await encodeAac(source, output, 4, { timeout: 60_000, filter: "atempo=1.5" });expect(encoded.success).toBe(true);diff --git a/packages/engine/src/services/audioMixer.test.ts b/packages/engine/src/services/audioMixer.test.tsindex 624e3fdb67..7b8194ccdb 100644--- a/packages/engine/src/services/audioMixer.test.ts+++ b/packages/engine/src/services/audioMixer.test.ts@@ -66,6 +66,16 @@ vi.mock("./audioFxRender.js", async (importOriginal) => {return { ...actual, applyAudioFxChain: applyAudioFxChainMock };});+// The real default CA bundle paths do not exist on Windows runners.+vi.mock("../utils/urlInput.js", async (importOriginal) => {+ const actual = await importOriginal<typeof import("../utils/urlInput.js")>();+ return {+ ...actual,+ urlInputArgs: (input: string, options?: Parameters<typeof actual.urlInputArgs>[1]) =>+ actual.urlInputArgs(input, { ...options, caBundles: [process.execPath] }),+ };+});+vi.mock("../utils/ffprobe.js", async (importOriginal) => {const actual = await importOriginal<typeof import("../utils/ffprobe.js")>();return { ...actual, extractAudioMetadata: extractAudioMetadataMock };@@ -1507,6 +1517,86 @@ describe("processCompositionAudio", () => {expect(runFfmpegMock.mock.calls[0]?.[0]).toContain(join(baseDir, ".media", "tone.wav"));});+ it("cuts a cloud-held project file's audio from its URL without downloading the file", async () => {+ const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));+ const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));+ tempDirs.push(baseDir, workDir);+ const fetchMock = vi.fn().mockRejectedValue(new Error("downloaded"));+ vi.stubGlobal("fetch", fetchMock);+ const url = "https://bucket.s3.amazonaws.com/cas/take.mov?X-Amz-Signature=s";++ const result = await processCompositionAudio(+ [+ {+ id: "take-audio",+ src: "takes/take.mov",+ start: 0,+ end: 2,+ mediaStart: 600,+ layer: 0,+ volume: 1,+ type: "video",+ },+ ],+ baseDir,+ workDir,+ join(baseDir, "out.m4a"),+ 2,+ undefined,+ { urlByAbsentLocalPath: new Map([[join(baseDir, "takes", "take.mov"), url]]) },+ );++ expect(result.success).toBe(true);+ expect(fetchMock).not.toHaveBeenCalled();+ const extractArgs = runFfmpegMock.mock.calls[0]?.[0] ?? [];+ expect(extractArgs.indexOf("-tls_verify")).toBeGreaterThanOrEqual(0);+ expect(extractArgs.indexOf("-tls_verify")).toBeLessThan(extractArgs.indexOf("-i"));+ expect(extractArgs.slice(extractArgs.indexOf("-ss"), extractArgs.indexOf(url) + 1)).toEqual([+ "-ss",+ "600",+ "-t",+ "2",+ "-i",+ url,+ ]);+ });++ it("fails a cloud-held file's audio when its URL probe fails twice instead of mixing it unprobed", async () => {+ const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));+ const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));+ tempDirs.push(baseDir, workDir);+ extractAudioMetadataMock.mockRejectedValue(new Error("Server returned 503"));+ const url = "https://bucket.s3.amazonaws.com/cas/take.mov?X-Amz-Signature=s";++ const result = await processCompositionAudio(+ [+ {+ id: "take-audio",+ src: "takes/take.mov",+ start: 0,+ end: 2,+ mediaStart: 600,+ layer: 0,+ volume: 1,+ type: "video",+ },+ ],+ baseDir,+ workDir,+ join(baseDir, "out.m4a"),+ 2,+ undefined,+ { urlByAbsentLocalPath: new Map([[join(baseDir, "takes", "take.mov"), url]]) },+ );++ expect(extractAudioMetadataMock).toHaveBeenCalledTimes(2);+ expect(extractAudioMetadataMock).toHaveBeenCalledWith(url, {+ signal: expect.any(AbortSignal),+ });+ expect(result.success).toBe(false);+ expect(runFfmpegMock.mock.calls.some(([args]) => args.includes(url))).toBe(false);+ });+it("preserves authored clip gain above unity for quiet-source boosting", async () => {const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));diff --git a/packages/engine/src/services/audioMixer.ts b/packages/engine/src/services/audioMixer.tsindex 1815c7bbdc..673c668287 100644--- a/packages/engine/src/services/audioMixer.ts+++ b/packages/engine/src/services/audioMixer.ts@@ -5,6 +5,7 @@ import { join, dirname, isAbsolute, relative } from "path";import { parseHTML } from "linkedom";import { extractAudioMetadata, type AudioMetadata } from "../utils/ffprobe.js";import { isNotMediaPayload } from "../utils/notMediaPayload.js";+import { urlInputArgs } from "../utils/urlInput.js";import { clampAudioGain } from "@hyperframes/core/audio-gain";import { clampFadesToDuration, readElementFades } from "@hyperframes/core/audio-fade";import {@@ -656,8 +657,14 @@ async function extractAudioSegment(if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true });const normalizedPlaybackRate = normalizeRateSpec(playbackRate);const sourceDuration = sourceTimeAt(normalizedPlaybackRate, duration);- // A failed probe keeps FFmpeg's default channel conversion and the input seek.- const metadata = await extractAudioMetadata(srcPath).catch(() => null);+ // A failed local probe keeps FFmpeg's default channel conversion and the input seek; a URL probe+ // is retried once, then fails: a blip must not change how the next, successful read is mixed.+ const metadata = isHttpUrl(srcPath)+ ? await extractAudioMetadata(srcPath, { signal }).catch((error: unknown) => {+ if (signal?.aborted) throw error;+ return extractAudioMetadata(srcPath, { signal });+ })+ : await extractAudioMetadata(srcPath).catch(() => null);const inputSeek = mediaStart >= (metadata?.latestStreamLeadSeconds ?? 0);const start = formatFilterNumber(mediaStart);const timestampTrim = inputSeek@@ -670,9 +677,12 @@ async function extractAudioSegment(timestampTrim,);- const args = inputSeek- ? ["-ss", String(mediaStart), "-t", String(sourceDuration), "-i", srcPath]- : ["-i", srcPath];+ const args = [+ ...urlInputArgs(srcPath),+ ...(inputSeek ? ["-ss", String(mediaStart), "-t", String(sourceDuration)] : []),+ "-i",+ srcPath,+ ];args.push("-vn", "-acodec", "pcm_s16le", "-ar", "48000", ...outputArgs);if (normalizedPlaybackRate !== 1) args.push("-t", String(duration));args.push("-y", outputPath);@@ -1166,13 +1176,18 @@ export async function processCompositionAudio(}try {let srcPath = element.src;+ let readInPlace = false;if (!isHttpUrl(srcPath)) {// Same browser-URL path semantics as videos.srcPath = resolveProjectRelativeSrc(element.src, baseDir, compiledDir);- srcPath = config?.urlByAbsentLocalPath?.get(srcPath) ?? srcPath;+ const remoteUrl = config?.urlByAbsentLocalPath?.get(srcPath);+ if (remoteUrl) {+ srcPath = remoteUrl;+ readInPlace = true;+ }}- if (isHttpUrl(srcPath)) {+ if (isHttpUrl(srcPath) && !readInPlace) {try {srcPath = await downloadToTemp(srcPath,@@ -1190,7 +1205,7 @@ export async function processCompositionAudio(}}- if (!existsSync(srcPath)) {+ if (!readInPlace && !existsSync(srcPath)) {failures.push({stage: "source",reason: "source_not_found",@@ -1226,7 +1241,10 @@ export async function processCompositionAudio(if (element.end - element.start <= 0) {let metadata;try {- metadata = await extractAudioMetadata(srcPath);+ metadata = await extractAudioMetadata(+ srcPath,+ readInPlace ? { signal: effectiveSignal } : undefined,+ );} catch (err: unknown) {failures.push(probeFailure(err instanceof Error ? err.message : String(err), element.id),diff --git a/packages/engine/src/utils/notMediaPayload.ts b/packages/engine/src/utils/notMediaPayload.tsindex 446e37ad09..a449582a20 100644--- a/packages/engine/src/utils/notMediaPayload.ts+++ b/packages/engine/src/utils/notMediaPayload.ts@@ -19,8 +19,8 @@ const MAX_LISTED_FINGERPRINTS = 8;** Only a 2xx response reaches this classifier. `downloadToTemp` rejects a* 404/410 as `http_not_found` before writing a byte (urlDownloader.ts- * `classifyHttpFailure`), and every ffprobe input is local — an http src is- * downloaded first. So the shapes that land here are a soft-404 or+ * `classifyHttpFailure`), and an authored http src is downloaded first (a cloud-held+ * project file is read by URL, which the sniff cannot open). So the shapes that land here are a soft-404 or* interstitial HTML body, an S3/CloudFront error document, or a JSON API error* body, each served with a success status.*