Skip to content

Write HTML. Render video. Built for agents.

perf(engine): cloud renders cut a video's audio from its URL instead of downloading the file (#5972)

Miguel Ángela8a80323f69f
diff --git a/packages/engine/src/services/aacEncode.test.ts b/packages/engine/src/services/aacEncode.test.ts
index 5964c68897..7a0ce0d416 100644
--- a/packages/engine/src/services/aacEncode.test.ts
+++ b/packages/engine/src/services/aacEncode.test.ts
@@ -105,7 +105,7 @@ describe.skipIf(!HAS_FFMPEG)("encodeAac", () => {
it("encodes a long mix in parallel segments that decode like one encoder's output", async () => {
const source = join(dir, "mix.wav");
ffmpeg([
- ...["-f", "lavfi", "-i", "anoisesrc=d=150.5:c=pink:r=48000:a=0.2"],
+ ...["-f", "lavfi", "-i", "anoisesrc=d=150.5:c=pink:r=48000:a=0.2:seed=1"],
...["-f", "lavfi", "-i", "sine=f=330:d=150.5:r=48000"],
...["-filter_complex", "[0][1]amix=2:normalize=0,aformat=channel_layouts=stereo"],
...["-c:a", "pcm_f32le", "-y", source],
@@ -173,7 +173,16 @@ describe.skipIf(!HAS_FFMPEG)("encodeAac", () => {
it("keeps a tempo-changed 44.1 kHz track at lag 0 against the same filter uncoded", async () => {
const source = join(dir, "rate.wav");
- ffmpeg(["-f", "lavfi", "-i", "anoisesrc=d=6:c=pink:r=44100:a=0.3", "-ac", "2", "-y", source]);
+ ffmpeg([
+ "-f",
+ "lavfi",
+ "-i",
+ "anoisesrc=d=6:c=pink:r=44100:a=0.3:seed=1",
+ "-ac",
+ "2",
+ "-y",
+ source,
+ ]);
const output = join(dir, "rate.m4a");
const encoded = await encodeAac(source, output, 4, { timeout: 60_000, filter: "atempo=1.5" });
expect(encoded.success).toBe(true);
diff --git a/packages/engine/src/services/audioMixer.test.ts b/packages/engine/src/services/audioMixer.test.ts
index 624e3fdb67..7b8194ccdb 100644
--- a/packages/engine/src/services/audioMixer.test.ts
+++ b/packages/engine/src/services/audioMixer.test.ts
@@ -66,6 +66,16 @@ vi.mock("./audioFxRender.js", async (importOriginal) => {
return { ...actual, applyAudioFxChain: applyAudioFxChainMock };
});
+// The real default CA bundle paths do not exist on Windows runners.
+vi.mock("../utils/urlInput.js", async (importOriginal) => {
+ const actual = await importOriginal<typeof import("../utils/urlInput.js")>();
+ return {
+ ...actual,
+ urlInputArgs: (input: string, options?: Parameters<typeof actual.urlInputArgs>[1]) =>
+ actual.urlInputArgs(input, { ...options, caBundles: [process.execPath] }),
+ };
+});
+
vi.mock("../utils/ffprobe.js", async (importOriginal) => {
const actual = await importOriginal<typeof import("../utils/ffprobe.js")>();
return { ...actual, extractAudioMetadata: extractAudioMetadataMock };
@@ -1507,6 +1517,86 @@ describe("processCompositionAudio", () => {
expect(runFfmpegMock.mock.calls[0]?.[0]).toContain(join(baseDir, ".media", "tone.wav"));
});
+ it("cuts a cloud-held project file's audio from its URL without downloading the file", async () => {
+ const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
+ const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
+ tempDirs.push(baseDir, workDir);
+ const fetchMock = vi.fn().mockRejectedValue(new Error("downloaded"));
+ vi.stubGlobal("fetch", fetchMock);
+ const url = "https://bucket.s3.amazonaws.com/cas/take.mov?X-Amz-Signature=s";
+
+ const result = await processCompositionAudio(
+ [
+ {
+ id: "take-audio",
+ src: "takes/take.mov",
+ start: 0,
+ end: 2,
+ mediaStart: 600,
+ layer: 0,
+ volume: 1,
+ type: "video",
+ },
+ ],
+ baseDir,
+ workDir,
+ join(baseDir, "out.m4a"),
+ 2,
+ undefined,
+ { urlByAbsentLocalPath: new Map([[join(baseDir, "takes", "take.mov"), url]]) },
+ );
+
+ expect(result.success).toBe(true);
+ expect(fetchMock).not.toHaveBeenCalled();
+ const extractArgs = runFfmpegMock.mock.calls[0]?.[0] ?? [];
+ expect(extractArgs.indexOf("-tls_verify")).toBeGreaterThanOrEqual(0);
+ expect(extractArgs.indexOf("-tls_verify")).toBeLessThan(extractArgs.indexOf("-i"));
+ expect(extractArgs.slice(extractArgs.indexOf("-ss"), extractArgs.indexOf(url) + 1)).toEqual([
+ "-ss",
+ "600",
+ "-t",
+ "2",
+ "-i",
+ url,
+ ]);
+ });
+
+ it("fails a cloud-held file's audio when its URL probe fails twice instead of mixing it unprobed", async () => {
+ const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
+ const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
+ tempDirs.push(baseDir, workDir);
+ extractAudioMetadataMock.mockRejectedValue(new Error("Server returned 503"));
+ const url = "https://bucket.s3.amazonaws.com/cas/take.mov?X-Amz-Signature=s";
+
+ const result = await processCompositionAudio(
+ [
+ {
+ id: "take-audio",
+ src: "takes/take.mov",
+ start: 0,
+ end: 2,
+ mediaStart: 600,
+ layer: 0,
+ volume: 1,
+ type: "video",
+ },
+ ],
+ baseDir,
+ workDir,
+ join(baseDir, "out.m4a"),
+ 2,
+ undefined,
+ { urlByAbsentLocalPath: new Map([[join(baseDir, "takes", "take.mov"), url]]) },
+ );
+
+ expect(extractAudioMetadataMock).toHaveBeenCalledTimes(2);
+ expect(extractAudioMetadataMock).toHaveBeenCalledWith(url, {
+ signal: expect.any(AbortSignal),
+ });
+ expect(result.success).toBe(false);
+ expect(runFfmpegMock.mock.calls.some(([args]) => args.includes(url))).toBe(false);
+ });
+
it("preserves authored clip gain above unity for quiet-source boosting", async () => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
diff --git a/packages/engine/src/services/audioMixer.ts b/packages/engine/src/services/audioMixer.ts
index 1815c7bbdc..673c668287 100644
--- a/packages/engine/src/services/audioMixer.ts
+++ b/packages/engine/src/services/audioMixer.ts
@@ -5,6 +5,7 @@ import { join, dirname, isAbsolute, relative } from "path";
import { parseHTML } from "linkedom";
import { extractAudioMetadata, type AudioMetadata } from "../utils/ffprobe.js";
import { isNotMediaPayload } from "../utils/notMediaPayload.js";
+import { urlInputArgs } from "../utils/urlInput.js";
import { clampAudioGain } from "@hyperframes/core/audio-gain";
import { clampFadesToDuration, readElementFades } from "@hyperframes/core/audio-fade";
import {
@@ -656,8 +657,14 @@ async function extractAudioSegment(
if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true });
const normalizedPlaybackRate = normalizeRateSpec(playbackRate);
const sourceDuration = sourceTimeAt(normalizedPlaybackRate, duration);
- // A failed probe keeps FFmpeg's default channel conversion and the input seek.
- const metadata = await extractAudioMetadata(srcPath).catch(() => null);
+ // A failed local probe keeps FFmpeg's default channel conversion and the input seek; a URL probe
+ // is retried once, then fails: a blip must not change how the next, successful read is mixed.
+ const metadata = isHttpUrl(srcPath)
+ ? await extractAudioMetadata(srcPath, { signal }).catch((error: unknown) => {
+ if (signal?.aborted) throw error;
+ return extractAudioMetadata(srcPath, { signal });
+ })
+ : await extractAudioMetadata(srcPath).catch(() => null);
const inputSeek = mediaStart >= (metadata?.latestStreamLeadSeconds ?? 0);
const start = formatFilterNumber(mediaStart);
const timestampTrim = inputSeek
@@ -670,9 +677,12 @@ async function extractAudioSegment(
timestampTrim,
);
- const args = inputSeek
- ? ["-ss", String(mediaStart), "-t", String(sourceDuration), "-i", srcPath]
- : ["-i", srcPath];
+ const args = [
+ ...urlInputArgs(srcPath),
+ ...(inputSeek ? ["-ss", String(mediaStart), "-t", String(sourceDuration)] : []),
+ "-i",
+ srcPath,
+ ];
args.push("-vn", "-acodec", "pcm_s16le", "-ar", "48000", ...outputArgs);
if (normalizedPlaybackRate !== 1) args.push("-t", String(duration));
args.push("-y", outputPath);
@@ -1166,13 +1176,18 @@ export async function processCompositionAudio(
}
try {
let srcPath = element.src;
+ let readInPlace = false;
if (!isHttpUrl(srcPath)) {
// Same browser-URL path semantics as videos.
srcPath = resolveProjectRelativeSrc(element.src, baseDir, compiledDir);
- srcPath = config?.urlByAbsentLocalPath?.get(srcPath) ?? srcPath;
+ const remoteUrl = config?.urlByAbsentLocalPath?.get(srcPath);
+ if (remoteUrl) {
+ srcPath = remoteUrl;
+ readInPlace = true;
+ }
}
- if (isHttpUrl(srcPath)) {
+ if (isHttpUrl(srcPath) && !readInPlace) {
try {
srcPath = await downloadToTemp(
srcPath,
@@ -1190,7 +1205,7 @@ export async function processCompositionAudio(
}
}
- if (!existsSync(srcPath)) {
+ if (!readInPlace && !existsSync(srcPath)) {
failures.push({
stage: "source",
reason: "source_not_found",
@@ -1226,7 +1241,10 @@ export async function processCompositionAudio(
if (element.end - element.start <= 0) {
let metadata;
try {
- metadata = await extractAudioMetadata(srcPath);
+ metadata = await extractAudioMetadata(
+ srcPath,
+ readInPlace ? { signal: effectiveSignal } : undefined,
+ );
} catch (err: unknown) {
failures.push(
probeFailure(err instanceof Error ? err.message : String(err), element.id),
diff --git a/packages/engine/src/utils/notMediaPayload.ts b/packages/engine/src/utils/notMediaPayload.ts
index 446e37ad09..a449582a20 100644
--- a/packages/engine/src/utils/notMediaPayload.ts
+++ b/packages/engine/src/utils/notMediaPayload.ts
@@ -19,8 +19,8 @@ const MAX_LISTED_FINGERPRINTS = 8;
*
* Only a 2xx response reaches this classifier. `downloadToTemp` rejects a
* 404/410 as `http_not_found` before writing a byte (urlDownloader.ts
- * `classifyHttpFailure`), and every ffprobe input is local — an http src is
- * downloaded first. So the shapes that land here are a soft-404 or
+ * `classifyHttpFailure`), and an authored http src is downloaded first (a cloud-held
+ * project file is read by URL, which the sniff cannot open). So the shapes that land here are a soft-404 or
* interstitial HTML body, an S3/CloudFront error document, or a JSON API error
* body, each served with a success status.
*