Skip to content

Write HTML. Render video. Built for agents.

fix(engine): hundreds of effects in one audio group mix in time, and a mix timeout is named (#6101)

Miguel Ángel1accb709ff05
diff --git a/packages/engine/src/services/audioMixer.test.ts b/packages/engine/src/services/audioMixer.test.ts
index 32a54d7ee4..de45b80fcd 100644
--- a/packages/engine/src/services/audioMixer.test.ts
+++ b/packages/engine/src/services/audioMixer.test.ts
@@ -703,6 +703,77 @@ describe("processCompositionAudio", () => {
expect(existsSync(workDir)).toBe(false);
});
+ const groupedEffects = (baseDir: string) => {
+ const track = (id: string, start: number) => ({
+ id,
+ src: `${id}.wav`,
+ start,
+ end: start + 1,
+ mediaStart: 0,
+ layer: 0,
+ volume: 1,
+ type: "audio" as const,
+ groupId: "sfx",
+ });
+ for (const id of ["hit", "whoosh", "pop"]) writeFileSync(join(baseDir, `${id}.wav`), "stub");
+ return [track("hit", 0), track("whoosh", 1), track("pop", 8)];
+ };
+
+ it("pads a group's members once after summing them, not each member to the film length", async () => {
+ // Per-member padding made the sub-mix cost members x film length: ~680 effects in a
+ // 10-minute film ran past the ffmpeg deadline.
+ const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
+ const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
+ tempDirs.push(baseDir, workDir);
+
+ await processCompositionAudio(
+ groupedEffects(baseDir),
+ baseDir,
+ workDir,
+ join(baseDir, "out.m4a"),
+ 600,
+ );
+
+ const groupMix = capturedFilterScripts.find((script) => script.includes("normalize=0"));
+ expect(groupMix).toContain("amix=inputs=3:duration=longest:dropout_transition=0:normalize=0");
+ expect(groupMix).not.toMatch(/adelay=[^;]*,apad/);
+ expect((groupMix?.match(/apad/g) ?? []).length).toBe(1);
+ expect(groupMix).toContain("apad,asetpts=N/SR/TB,atrim=0:600[out]");
+ });
+
+ it("names the deadline when a group sub-mix is killed for taking too long", async () => {
+ const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
+ const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
+ tempDirs.push(baseDir, workDir);
+ runFfmpegMock.mockImplementation(async (args: string[]) =>
+ args.includes("-filter_complex_script")
+ ? {
+ success: false,
+ durationMs: 300_000,
+ stderr: "Stream mapping:\n Stream #0:0 -> #0:0 (pcm_s16le (native))",
+ exitCode: 255,
+ terminationReason: "deadline",
+ }
+ : { success: true, durationMs: 1, stderr: "", exitCode: 0 },
+ );
+
+ const result = await processCompositionAudio(
+ groupedEffects(baseDir),
+ baseDir,
+ workDir,
+ join(baseDir, "out.m4a"),
+ 600,
+ undefined,
+ { ffmpegProcessTimeout: 300_000 },
+ );
+
+ expect(result.success).toBe(false);
+ const failure = result.failures?.find((f) => f.elementId === "sfx");
+ expect(failure?.reason).toBe("ffmpeg_timeout");
+ expect(failure?.detail).toContain("FFmpeg timed out after 300000 ms");
+ expect(failure?.detail).toContain("FFMPEG_PROCESS_TIMEOUT_MS");
+ });
+
it("lets an FX tail run past the clip, still bounded by the composition", async () => {
// A reverb is still decaying when the clip's own audio stops. Trimming at
// the clip boundary is what cut every tail short in the render.
diff --git a/packages/engine/src/services/audioMixer.ts b/packages/engine/src/services/audioMixer.ts
index eeb2f68fe0..908b381486 100644
--- a/packages/engine/src/services/audioMixer.ts
+++ b/packages/engine/src/services/audioMixer.ts
@@ -24,7 +24,7 @@ import {
writeUrlDownloadTelemetry,
} from "../utils/urlDownloader.js";
import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
-import { formatFfmpegError, runFfmpeg, type RunFfmpegResult } from "../utils/runFfmpeg.js";
+import { describeFfmpegFailure, runFfmpeg, type RunFfmpegResult } from "../utils/runFfmpeg.js";
import { encodeAac } from "./aacEncode.js";
import { audioSubprocessEvidence } from "./audioFailureDiagnostic.js";
import { unwrapTemplate } from "../utils/htmlTemplate.js";
@@ -446,6 +446,7 @@ function downloadFailure(error: unknown, elementId: string): AudioProcessingFail
function ffmpegFailure(
stage: Extract<AudioFailureStage, "extract" | "prepare" | "mix" | "silence">,
result: RunFfmpegResult,
+ timeoutMs: number,
elementId?: string,
): AudioProcessingFailure {
const stderr = result.stderr ?? "";
@@ -486,8 +487,8 @@ function ffmpegFailure(
subprocess: audioSubprocessEvidence(result),
detail: boundedDetail(
result.error?.message
- ? `${formatFfmpegError(result.exitCode, stderr)}: ${result.error.message}`
- : formatFfmpegError(result.exitCode, stderr),
+ ? `${describeFfmpegFailure(result, timeoutMs)}: ${result.error.message}`
+ : describeFfmpegFailure(result, timeoutMs),
),
};
}
@@ -724,7 +725,7 @@ async function extractAudioSegment(
failure,
};
}
- let failure = result.success ? undefined : ffmpegFailure(stage, result);
+ let failure = result.success ? undefined : ffmpegFailure(stage, result, ffmpegProcessTimeout);
const probedLinkFailedToRead =
readsUrl && (failure?.reason === "invalid_media" || failure?.reason === "ffmpeg_failed");
if (failure && probedLinkFailedToRead) {
@@ -780,7 +781,9 @@ async function generateSilence(
failure,
};
}
- const failure = !result.success ? ffmpegFailure("silence", result) : undefined;
+ const failure = !result.success
+ ? ffmpegFailure("silence", result, ffmpegProcessTimeout)
+ : undefined;
return {
success: result.success,
outputPath,
@@ -934,7 +937,7 @@ async function mixAudioTracks(
};
}
if (!result.success) {
- const failure = ffmpegFailure("mix", result);
+ const failure = ffmpegFailure("mix", result, ffmpegProcessTimeout);
return {
success: false,
outputPath,
@@ -1000,10 +1003,10 @@ async function mixGroupMembers(
const outputDir = dirname(outputPath);
if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true });
- const buildInputFilters = (ignoreKeyframes: boolean) =>
+ const buildInputFilters = (ignoreKeyframes: boolean, padMembers: boolean) =>
memberTracks.map((track, i) => {
const volumeFilter = buildVolumeExpression(track, ignoreKeyframes);
- return buildTrackInputFilter(track, i, volumeFilter, totalDuration);
+ return buildTrackInputFilter(track, i, volumeFilter, totalDuration, padMembers);
});
const mixInputs = memberTracks.map((_, i) => `[a${i}]`).join("");
@@ -1011,9 +1014,10 @@ async function mixGroupMembers(
useNormalize: boolean,
ignoreKeyframes = false,
): Promise<RunFfmpegResult> => {
- const inputFilters = buildInputFilters(ignoreKeyframes);
+ const inputFilters = buildInputFilters(ignoreKeyframes, !useNormalize);
+ const padUnitySumOnceToFilm = `[mixed];[mixed]apad,asetpts=N/SR/TB,atrim=0:${formatFilterNumber(totalDuration)}`;
const mixFilter = useNormalize
- ? `${mixInputs}amix=inputs=${memberTracks.length}:duration=longest:dropout_transition=0:normalize=0[out]`
+ ? `${mixInputs}amix=inputs=${memberTracks.length}:duration=longest:dropout_transition=0:normalize=0${padUnitySumOnceToFilm}[out]`
: // Every member is padded and trimmed to totalDuration above, and
// dropout_transition=0 keeps all N inputs active for that whole span.
// amix's default normalize therefore divides by exactly N throughout;
@@ -1089,7 +1093,7 @@ async function mixGroupMembers(
}
if (signal?.aborted || !result.success) {
- const failure = ffmpegFailure("mix", result);
+ const failure = ffmpegFailure("mix", result, ffmpegProcessTimeout);
return { success: false, error: failure.detail, failure };
}
return { success: true, degradedAutomation };
@@ -1127,7 +1131,11 @@ async function mixThroughFloatSum(
timeout: config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout,
});
if (!encoded.success) {
- const failure = ffmpegFailure("mix", encoded);
+ const failure = ffmpegFailure(
+ "mix",
+ encoded,
+ config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout,
+ );
return {
success: false,
outputPath,