Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
fix(engine): hundreds of effects in one audio group mix in time, and a mix timeout is named (#6101)
Miguel Ángel1accb709ff05
diff --git a/packages/engine/src/services/audioMixer.test.ts b/packages/engine/src/services/audioMixer.test.tsindex 32a54d7ee4..de45b80fcd 100644--- a/packages/engine/src/services/audioMixer.test.ts+++ b/packages/engine/src/services/audioMixer.test.ts@@ -703,6 +703,77 @@ describe("processCompositionAudio", () => {expect(existsSync(workDir)).toBe(false);});+ const groupedEffects = (baseDir: string) => {+ const track = (id: string, start: number) => ({+ id,+ src: `${id}.wav`,+ start,+ end: start + 1,+ mediaStart: 0,+ layer: 0,+ volume: 1,+ type: "audio" as const,+ groupId: "sfx",+ });+ for (const id of ["hit", "whoosh", "pop"]) writeFileSync(join(baseDir, `${id}.wav`), "stub");+ return [track("hit", 0), track("whoosh", 1), track("pop", 8)];+ };++ it("pads a group's members once after summing them, not each member to the film length", async () => {+ // Per-member padding made the sub-mix cost members x film length: ~680 effects in a+ // 10-minute film ran past the ffmpeg deadline.+ const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));+ const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));+ tempDirs.push(baseDir, workDir);++ await processCompositionAudio(+ groupedEffects(baseDir),+ baseDir,+ workDir,+ join(baseDir, "out.m4a"),+ 600,+ );++ const groupMix = capturedFilterScripts.find((script) => script.includes("normalize=0"));+ expect(groupMix).toContain("amix=inputs=3:duration=longest:dropout_transition=0:normalize=0");+ expect(groupMix).not.toMatch(/adelay=[^;]*,apad/);+ expect((groupMix?.match(/apad/g) ?? []).length).toBe(1);+ expect(groupMix).toContain("apad,asetpts=N/SR/TB,atrim=0:600[out]");+ });++ it("names the deadline when a group sub-mix is killed for taking too long", async () => {+ const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));+ const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));+ tempDirs.push(baseDir, workDir);+ runFfmpegMock.mockImplementation(async (args: string[]) =>+ args.includes("-filter_complex_script")+ ? {+ success: false,+ durationMs: 300_000,+ stderr: "Stream mapping:\n Stream #0:0 -> #0:0 (pcm_s16le (native))",+ exitCode: 255,+ terminationReason: "deadline",+ }+ : { success: true, durationMs: 1, stderr: "", exitCode: 0 },+ );++ const result = await processCompositionAudio(+ groupedEffects(baseDir),+ baseDir,+ workDir,+ join(baseDir, "out.m4a"),+ 600,+ undefined,+ { ffmpegProcessTimeout: 300_000 },+ );++ expect(result.success).toBe(false);+ const failure = result.failures?.find((f) => f.elementId === "sfx");+ expect(failure?.reason).toBe("ffmpeg_timeout");+ expect(failure?.detail).toContain("FFmpeg timed out after 300000 ms");+ expect(failure?.detail).toContain("FFMPEG_PROCESS_TIMEOUT_MS");+ });+it("lets an FX tail run past the clip, still bounded by the composition", async () => {// A reverb is still decaying when the clip's own audio stops. Trimming at// the clip boundary is what cut every tail short in the render.diff --git a/packages/engine/src/services/audioMixer.ts b/packages/engine/src/services/audioMixer.tsindex eeb2f68fe0..908b381486 100644--- a/packages/engine/src/services/audioMixer.ts+++ b/packages/engine/src/services/audioMixer.ts@@ -24,7 +24,7 @@ import {writeUrlDownloadTelemetry,} from "../utils/urlDownloader.js";import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";-import { formatFfmpegError, runFfmpeg, type RunFfmpegResult } from "../utils/runFfmpeg.js";+import { describeFfmpegFailure, runFfmpeg, type RunFfmpegResult } from "../utils/runFfmpeg.js";import { encodeAac } from "./aacEncode.js";import { audioSubprocessEvidence } from "./audioFailureDiagnostic.js";import { unwrapTemplate } from "../utils/htmlTemplate.js";@@ -446,6 +446,7 @@ function downloadFailure(error: unknown, elementId: string): AudioProcessingFailfunction ffmpegFailure(stage: Extract<AudioFailureStage, "extract" | "prepare" | "mix" | "silence">,result: RunFfmpegResult,+ timeoutMs: number,elementId?: string,): AudioProcessingFailure {const stderr = result.stderr ?? "";@@ -486,8 +487,8 @@ function ffmpegFailure(subprocess: audioSubprocessEvidence(result),detail: boundedDetail(result.error?.message- ? `${formatFfmpegError(result.exitCode, stderr)}: ${result.error.message}`- : formatFfmpegError(result.exitCode, stderr),+ ? `${describeFfmpegFailure(result, timeoutMs)}: ${result.error.message}`+ : describeFfmpegFailure(result, timeoutMs),),};}@@ -724,7 +725,7 @@ async function extractAudioSegment(failure,};}- let failure = result.success ? undefined : ffmpegFailure(stage, result);+ let failure = result.success ? undefined : ffmpegFailure(stage, result, ffmpegProcessTimeout);const probedLinkFailedToRead =readsUrl && (failure?.reason === "invalid_media" || failure?.reason === "ffmpeg_failed");if (failure && probedLinkFailedToRead) {@@ -780,7 +781,9 @@ async function generateSilence(failure,};}- const failure = !result.success ? ffmpegFailure("silence", result) : undefined;+ const failure = !result.success+ ? ffmpegFailure("silence", result, ffmpegProcessTimeout)+ : undefined;return {success: result.success,outputPath,@@ -934,7 +937,7 @@ async function mixAudioTracks(};}if (!result.success) {- const failure = ffmpegFailure("mix", result);+ const failure = ffmpegFailure("mix", result, ffmpegProcessTimeout);return {success: false,outputPath,@@ -1000,10 +1003,10 @@ async function mixGroupMembers(const outputDir = dirname(outputPath);if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true });- const buildInputFilters = (ignoreKeyframes: boolean) =>+ const buildInputFilters = (ignoreKeyframes: boolean, padMembers: boolean) =>memberTracks.map((track, i) => {const volumeFilter = buildVolumeExpression(track, ignoreKeyframes);- return buildTrackInputFilter(track, i, volumeFilter, totalDuration);+ return buildTrackInputFilter(track, i, volumeFilter, totalDuration, padMembers);});const mixInputs = memberTracks.map((_, i) => `[a${i}]`).join("");@@ -1011,9 +1014,10 @@ async function mixGroupMembers(useNormalize: boolean,ignoreKeyframes = false,): Promise<RunFfmpegResult> => {- const inputFilters = buildInputFilters(ignoreKeyframes);+ const inputFilters = buildInputFilters(ignoreKeyframes, !useNormalize);+ const padUnitySumOnceToFilm = `[mixed];[mixed]apad,asetpts=N/SR/TB,atrim=0:${formatFilterNumber(totalDuration)}`;const mixFilter = useNormalize- ? `${mixInputs}amix=inputs=${memberTracks.length}:duration=longest:dropout_transition=0:normalize=0[out]`+ ? `${mixInputs}amix=inputs=${memberTracks.length}:duration=longest:dropout_transition=0:normalize=0${padUnitySumOnceToFilm}[out]`: // Every member is padded and trimmed to totalDuration above, and// dropout_transition=0 keeps all N inputs active for that whole span.// amix's default normalize therefore divides by exactly N throughout;@@ -1089,7 +1093,7 @@ async function mixGroupMembers(}if (signal?.aborted || !result.success) {- const failure = ffmpegFailure("mix", result);+ const failure = ffmpegFailure("mix", result, ffmpegProcessTimeout);return { success: false, error: failure.detail, failure };}return { success: true, degradedAutomation };@@ -1127,7 +1131,11 @@ async function mixThroughFloatSum(timeout: config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout,});if (!encoded.success) {- const failure = ffmpegFailure("mix", encoded);+ const failure = ffmpegFailure(+ "mix",+ encoded,+ config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout,+ );return {success: false,outputPath,