Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
// fallow-ignore-file code-duplication
import { afterEach, describe, expect, it, vi } from "vitest";
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { tmpdir } from "node:os";
// The mix filter graph is written to a temp file and passed via
// a file-valued filter option (not inlined via -filter_complex) so the command
// line doesn't scale with track count — production code deletes that file
// the moment the (real) ffmpeg process exits. The mock captures each call's
// filter content synchronously, while the file still exists, into an
// index-aligned side array (rather than re-reading it from disk after
// processCompositionAudio resolves, by which point it's already gone).
const { runFfmpegMock, capturedFilterScripts, extractAudioMetadataMock } = vi.hoisted(() => {
const capturedFilterScripts: string[] = [];
return {
capturedFilterScripts,
extractAudioMetadataMock: vi.fn(async () => ({
durationSeconds: 2,
sampleRate: 48_000,
channels: 2,
audioCodec: "aac",
})),
runFfmpegMock: vi.fn(async (args: string[]) => {
const legacyIdx = args.indexOf("-filter_complex_script");
const currentIdx = args.indexOf("-/filter_complex");
const idx = legacyIdx >= 0 ? legacyIdx : currentIdx;
if (idx >= 0) {
const { readFileSync } = await import("node:fs");
capturedFilterScripts.push(readFileSync(args[idx + 1], "utf8"));
} else {
capturedFilterScripts.push("");
}
return { success: true, durationMs: 1, stderr: "", exitCode: 0 };
}),
};
});
vi.mock("../utils/runFfmpeg.js", async (importOriginal) => {
const actual = await importOriginal<typeof import("../utils/runFfmpeg.js")>();
return { ...actual, runFfmpeg: runFfmpegMock };
});
// The AAC encode is aacEncode.test.ts's subject; here the mix only has to hand it the float sum.
vi.mock("./aacEncode.js", async (importOriginal) => ({
...(await importOriginal<typeof import("./aacEncode.js")>()),
encodeAac: vi.fn(async () => ({ success: true, durationMs: 1, stderr: "", exitCode: 0 })),
}));
// The FX render drives a headless browser; the mix only needs to know the
// processed file exists and how long a tail the chain asked for.
const { applyAudioFxChainMock } = vi.hoisted(() => ({
applyAudioFxChainMock: vi.fn(
async (_src: string, _chain: unknown, outPath: string, options?: { envelope?: unknown }) => {
const { writeFileSync } = await import("node:fs");
writeFileSync(outPath, "stub");
// The real one bakes the volume envelope into its float output, so the
// mixer must not run its own pass afterwards.
return { path: outPath, envelopeBaked: Boolean(options?.envelope) };
},
),
}));
vi.mock("./audioFxRender.js", async (importOriginal) => {
const actual = await importOriginal<typeof import("./audioFxRender.js")>();
return { ...actual, applyAudioFxChain: applyAudioFxChainMock };
});
// The real default CA bundle paths do not exist on Windows runners.
vi.mock("../utils/urlInput.js", async (importOriginal) => {
const actual = await importOriginal<typeof import("../utils/urlInput.js")>();
return {
...actual,
urlInputArgs: (input: string, options?: Parameters<typeof actual.urlInputArgs>[1]) =>
actual.urlInputArgs(input, { ...options, caBundles: [process.execPath] }),
};
});
vi.mock("../utils/ffprobe.js", async (importOriginal) => {
const actual = await importOriginal<typeof import("../utils/ffprobe.js")>();
return { ...actual, extractAudioMetadata: extractAudioMetadataMock };
});
import { RATE_RANGE } from "@hyperframes/core/audio-automation";
import { AudioFxRenderError } from "./audioFxRender.js";
import { MAX_MIX_INPUTS, parseAudioElements, processCompositionAudio } from "./audioMixer.js";
import { encodeAac } from "./aacEncode.js";
describe("parseAudioElements strict literal timing", () => {
it.each(["", " ", "0s", "0abc", "0px", "-1s", "Infinity", "NaN", "0x10"])(
"does not drop hand-authored data-duration=%j as an explicit zero window",
(duration) => {
expect(
parseAudioElements(
`<audio id="a" src="tone.wav" data-start="0" data-duration="${duration}"></audio>`,
),
).toHaveLength(1);
},
);
});
describe("parseAudioElements — <source> children", () => {
it("discovers audio and audible-video tracks that use <source> children", () => {
const tracks = parseAudioElements(`
<audio id="bgm" data-start="2" data-end="7">
<source src="https://cdn.example.com/bgm.mp3" type="audio/mpeg">
<source src="_remote_media/bgm.ogg" type="audio/ogg">
</audio>
<video id="rec" data-has-audio="true" data-start="4" data-end="10">
<source src="https://cdn.example.com/rec.mp4" type="video/mp4">
<source src="_remote_media/rec.webm" type="video/webm">
</video>
`);
expect(tracks).toEqual([
expect.objectContaining({ id: "bgm", src: "_remote_media/bgm.ogg", type: "audio" }),
expect.objectContaining({
id: "rec-audio",
src: "_remote_media/rec.webm",
type: "video",
}),
]);
});
});
describe("processCompositionAudio", () => {
const tempDirs: string[] = [];
const gainChain = JSON.stringify({
version: 1,
nodes: [{ type: "gain", id: "g", params: { gain: -3 } }],
});
afterEach(() => {
vi.unstubAllGlobals();
runFfmpegMock.mockReset();
extractAudioMetadataMock.mockReset();
extractAudioMetadataMock.mockResolvedValue({
durationSeconds: 2,
sampleRate: 48_000,
channels: 2,
audioCodec: "aac",
});
applyAudioFxChainMock.mockClear();
capturedFilterScripts.length = 0;
for (const dir of tempDirs.splice(0)) {
rmSync(dir, { recursive: true, force: true });
}
});
it.each([
["compiled EOF", 4, 2],
["compiled past EOF", 5, 3],
["ordinary explicit zero", 6, 0],
])(
"drops a known zero timeline window without audio prep or mix: %s",
async (_label, start, mediaStart) => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
writeFileSync(join(baseDir, "tone.wav"), Buffer.from("RIFF0000WAVEfmt "));
const elements = parseAudioElements(
`<div data-composition-id="root" data-start="0" data-duration="10"><audio id="zero-window" src="tone.wav" data-start="${start}" data-duration="0" data-end="${start}" data-media-start="${mediaStart}"></audio></div>`,
);
const result = await processCompositionAudio(
elements,
baseDir,
workDir,
join(baseDir, "out.m4a"),
10,
);
expect(extractAudioMetadataMock).not.toHaveBeenCalled();
expect(runFfmpegMock).toHaveBeenCalledTimes(1);
expect(runFfmpegMock.mock.calls[0]?.[0].join(" ")).toContain("anullsrc");
expect(runFfmpegMock.mock.calls[0]?.[0]).not.toContain(join(baseDir, "tone.wav"));
expect(elements).toEqual([]);
expect(result).toMatchObject({ success: true, tracksProcessed: 0 });
},
);
it("classifies an HTML-as-200 audio source as deterministic user input", async () => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
const fetchMock = vi
.fn()
.mockResolvedValue(new Response("<!doctype html><html><body>denied</body></html>"));
vi.stubGlobal("fetch", fetchMock);
const result = await processCompositionAudio(
[
{
id: "remote-voice",
src: "https://cdn.example/voice",
start: 0,
end: 2,
mediaStart: 0,
layer: 0,
volume: 1,
type: "audio",
},
],
baseDir,
workDir,
join(baseDir, "out.m4a"),
2,
);
expect(fetchMock).toHaveBeenCalledTimes(1);
expect(result.failures).toEqual([
expect.objectContaining({
stage: "download",
owner: "user",
retryable: false,
}),
]);
});
it.each([
{
src: "assets/music.mp3",
detail:
'Source not found for audio element bed: src="assets/music.mp3" resolved to assets/music.mp3',
},
{
src: "/assets/music.mp3",
detail: "Source not found for audio element bed: resolved to assets/music.mp3",
},
])("names where a missing local audio file was looked for ($src)", async ({ src, detail }) => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-missing-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
const result = await processCompositionAudio(
[
{
id: "bed",
src,
start: 0,
end: 2,
mediaStart: 0,
layer: 0,
volume: 1,
type: "audio",
},
],
baseDir,
workDir,
join(baseDir, "out.m4a"),
2,
);
expect(result.failures).toEqual([
expect.objectContaining({
reason: "source_not_found",
elementId: "bed",
detail,
}),
]);
});
it.each([
{
message: "AbortError: ffprobe operation aborted",
reason: "cancelled",
owner: "user",
retryable: false,
},
{
message: "ffprobe timed out after inactivity deadline",
reason: "ffmpeg_timeout",
owner: "system",
retryable: true,
},
] as const)("classifies probe failure '$reason' independently", async (expected) => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
writeFileSync(join(baseDir, "voice.wav"), "stub");
extractAudioMetadataMock.mockRejectedValueOnce(new Error(expected.message));
const result = await processCompositionAudio(
[
{
id: "voice",
src: "voice.wav",
start: 0,
end: 0,
mediaStart: 0,
layer: 0,
volume: 1,
type: "audio",
},
],
baseDir,
workDir,
join(baseDir, "out.m4a"),
2,
);
expect(result.failures).toEqual([
expect.objectContaining({
stage: "probe",
reason: expected.reason,
owner: expected.owner,
retryable: expected.retryable,
}),
]);
});
// STUDIO-5433: an audio src that resolved to an HTML/XML page (an unresolved
// nested-composition preview URL, or a 403/404 body served as a 200) skips the
// probe entirely when the element carries an authored duration, and used to
// surface as `prepare/ffmpeg_failed` with owner "system" — an authoring bug
// paged as a platform fault, after every frame had already been captured.
it("classifies a document audio source as a user-owned invalid media source", async () => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
writeFileSync(join(baseDir, "bgm.mp3"), "<!DOCTYPE html><html><body>not audio</body></html>");
const result = await processCompositionAudio(
[
{
id: "bgm",
src: "bgm.mp3",
// Authored duration + loop is the shape that bypasses every probe.
start: 0,
end: 30,
mediaStart: 0,
layer: 0,
volume: 1,
type: "audio",
},
],
baseDir,
workDir,
join(baseDir, "out.m4a"),
30,
);
expect(result.failures).toEqual([
expect.objectContaining({
stage: "source",
reason: "invalid_media",
owner: "user",
retryable: false,
elementId: "bgm",
}),
]);
// Never reached ffmpeg: the whole point is failing before the work.
expect(runFfmpegMock).not.toHaveBeenCalled();
});
it("preserves muted tracks and uses unity master gain by default", async () => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
writeFileSync(join(baseDir, "voice.wav"), "stub");
const result = await processCompositionAudio(
[
{
id: "voice",
src: "voice.wav",
start: 0,
end: 2,
mediaStart: 0,
layer: 0,
volume: 0,
type: "audio",
},
],
baseDir,
workDir,
join(baseDir, "out.m4a"),
2,
);
expect(result.success).toBe(true);
expect(runFfmpegMock).toHaveBeenCalledTimes(2);
const filter = capturedFilterScripts[1];
expect(filter).toContain("volume=0");
expect(filter).toContain("[mixed]volume=1,apad,asetpts=N/SR/TB,atrim=0:2[out]");
expect(filter).toContain("apad,asetpts=N/SR/TB,atrim=0:2");
expect(filter).not.toContain("whole_dur");
expect(filter).toContain("normalize=0");
expect(filter).not.toContain("weights=");
});
it("trims the consumed source span and applies pitch-preserving tempo at 2x", async () => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
writeFileSync(join(baseDir, "timecode.wav"), "stub");
const result = await processCompositionAudio(
[
{
id: "timecode",
src: "timecode.wav",
start: 0,
end: 2,
mediaStart: 1,
playbackRate: 2,
layer: 0,
volume: 1,
type: "audio",
},
],
baseDir,
workDir,
join(baseDir, "out.m4a"),
2,
);
expect(result.success).toBe(true);
const prepareArgs = runFfmpegMock.mock.calls[0]?.[0] ?? [];
expect(prepareArgs).toEqual(expect.arrayContaining(["-ss", "1", "-t", "4", "-af", "atempo=2"]));
expect(prepareArgs.filter((arg) => arg === "-t")).toHaveLength(2);
expect(prepareArgs.at(-3)).toBe("2");
});
it.each([
{ rate: 0.1, filter: "atempo=0.5,atempo=0.5,atempo=0.5,atempo=0.8" },
{ rate: 5, filter: "atempo=2,atempo=2,atempo=1.25" },
])("builds a bounded atempo chain for normalized rate $rate", async ({ rate, filter }) => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
writeFileSync(join(baseDir, "timecode.wav"), "stub");
await processCompositionAudio(
[
{
id: "timecode",
src: "timecode.wav",
start: 0,
end: 2,
mediaStart: 0,
playbackRate: rate,
layer: 0,
volume: 1,
type: "audio",
},
],
baseDir,
workDir,
join(baseDir, "out.m4a"),
2,
);
expect(runFfmpegMock.mock.calls[0]?.[0]).toEqual(expect.arrayContaining(["-af", filter]));
});
it("bakes a rate lane as concatenated source slices whose tempo is the mean rate over each slice", async () => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
writeFileSync(join(baseDir, "timecode.wav"), "stub");
const rate = {
target: "rate",
points: [
{ t: 0, v: 2 },
{ t: 1, v: 2 },
],
};
await processCompositionAudio(
[
{
id: "timecode",
src: "timecode.wav",
start: 0,
end: 1,
mediaStart: 0,
playbackRate: rate,
layer: 0,
volume: 1,
type: "audio",
},
],
baseDir,
workDir,
join(baseDir, "out.m4a"),
1,
);
const args: string[] = runFfmpegMock.mock.calls[0]?.[0] ?? [];
const graph = args[args.indexOf("-filter_complex") + 1] ?? "";
// a constant 2x lane: 1s of composition consumes 2s of source, in 4 quarter-second slices
expect(args).toEqual(expect.arrayContaining(["-t", "2", "-map", "[out]"]));
expect(graph).toContain("asplit=4");
expect(graph).toContain(
"[s0]atrim=start=0:end=0.5,asetpts=PTS-STARTPTS,atempo=2,apad,asetpts=N/SR/TB,atrim=0:0.25[t0]",
);
expect(graph).toContain(
"[s3]atrim=start=1.5:end=2,asetpts=PTS-STARTPTS,atempo=2,apad,asetpts=N/SR/TB,atrim=0:0.25[t3]",
);
expect(graph).toContain("concat=n=4:v=0:a=1");
});
it("stretches each slice of a non-constant lane by its mean rate, and trims a video's audio to the consumed span", async () => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
writeFileSync(join(baseDir, "clip.mp4"), "stub");
const rate = {
target: "rate",
points: [
{ t: 0, v: 1 },
{ t: 1, v: 3 },
],
};
await processCompositionAudio(
[
{
id: "clip",
src: "clip.mp4",
start: 0,
end: 1,
mediaStart: 0,
playbackRate: rate,
layer: 0,
volume: 1,
type: "video",
},
],
baseDir,
workDir,
join(baseDir, "out.m4a"),
1,
);
const args: string[] = runFfmpegMock.mock.calls[0]?.[0] ?? [];
const graph = args[args.indexOf("-filter_complex") + 1] ?? "";
// geometric 1x to 3x over 1s consumes (3-1)/ln 3 = 1.8205 source seconds; quarter-second 0 has mean tempo 1.1508
expect(Number(args[args.indexOf("-t") + 1])).toBeCloseTo(1.8205, 2);
expect(
Number(graph.match(/\[s0\]atrim=[^,]*,asetpts=PTS-STARTPTS,atempo=([\d.]+)/)?.[1]),
).toBeCloseTo(1.1508, 2);
});
it("keeps automation on authored timeline time after constant retiming", async () => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
writeFileSync(join(baseDir, "timecode.wav"), "stub");
const automation = JSON.stringify({
version: 1,
lanes: [
{
target: "volume",
points: [
{ t: 0, v: 0 },
{ t: 2, v: 1 },
],
},
],
});
const fxChain = JSON.stringify({
version: 1,
nodes: [{ type: "gain", id: "gain", params: { gain: 1 } }],
});
await processCompositionAudio(
[
{
id: "timecode",
src: "timecode.wav",
start: 0,
end: 2,
mediaStart: 0,
playbackRate: 2,
layer: 0,
volume: 1,
automation,
fxChain,
type: "audio",
},
],
baseDir,
workDir,
join(baseDir, "out.m4a"),
2,
);
expect(runFfmpegMock.mock.calls[0]?.[0]).toEqual(
expect.arrayContaining(["-t", "4", "-af", "atempo=2"]),
);
expect(applyAudioFxChainMock.mock.calls[0]?.[3]).toMatchObject({
envelope: {
keyframes: [
{ time: 0, volume: 0 },
{ time: 2, volume: 1 },
],
trackStart: 0,
},
});
});
it.each([
{
gain: 0.5,
level: 1,
points: [
{ time: 0, volume: 0 },
{ time: 2, volume: 1 },
],
},
{
gain: 2,
level: 3,
points: [
{ time: 0, volume: 0 },
{ time: expect.closeTo(1.3270239018, 8), volume: expect.closeTo(1.9905358528, 8) },
{ time: 2, volume: expect.closeTo(1.9905358528, 8) },
],
},
])(
"applies pre-FX clip gain $gain with a ceiling-aware lane",
async ({ gain, level, points }) => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
writeFileSync(join(baseDir, "vo.wav"), "stub");
const automation = JSON.stringify({
version: 1,
lanes: [
{
target: "volume",
points: [
{ t: 0, v: 0 },
{ t: 2, v: level },
],
},
],
});
await processCompositionAudio(
[
{
id: "vo",
src: "vo.wav",
start: 0,
end: 2,
mediaStart: 0,
layer: 0,
volume: gain,
automation,
fxChain: gainChain,
type: "audio",
},
],
baseDir,
workDir,
join(baseDir, "out.m4a"),
2,
);
const fxOptions = applyAudioFxChainMock.mock.calls[0]?.[3];
expect(fxOptions?.inputGain).toBe(gain);
expect(fxOptions?.envelope?.keyframes).toEqual(points);
},
);
it.each([
["the master pass", {}, { masterFxChain: gainChain }],
["a group bus", { groupId: "bus", groupFxChain: gainChain }, {}],
])("removes its work directory when %s fails fatally", async (_label, extra, config) => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
writeFileSync(join(baseDir, "bed.wav"), "stub");
applyAudioFxChainMock.mockRejectedValueOnce(new AudioFxRenderError("browser would not start"));
await expect(
processCompositionAudio(
[
{
id: "bed",
src: "bed.wav",
start: 0,
end: 2,
mediaStart: 0,
layer: 0,
volume: 1,
type: "audio",
...extra,
},
],
baseDir,
workDir,
join(baseDir, "out.m4a"),
2,
undefined,
config,
),
).rejects.toThrow(AudioFxRenderError);
expect(existsSync(workDir)).toBe(false);
});
const groupedEffects = (baseDir: string) => {
const track = (id: string, start: number) => ({
id,
src: `${id}.wav`,
start,
end: start + 1,
mediaStart: 0,
layer: 0,
volume: 1,
type: "audio" as const,
groupId: "sfx",
});
for (const id of ["hit", "whoosh", "pop"]) writeFileSync(join(baseDir, `${id}.wav`), "stub");
return [track("hit", 0), track("whoosh", 1), track("pop", 8)];
};
it("pads a group's members once after summing them, not each member to the film length", async () => {
// Per-member padding made the sub-mix cost members x film length: ~680 effects in a
// 10-minute film ran past the ffmpeg deadline.
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
await processCompositionAudio(
groupedEffects(baseDir),
baseDir,
workDir,
join(baseDir, "out.m4a"),
600,
);
const groupMix = capturedFilterScripts.find((script) => script.includes("normalize=0"));
expect(groupMix).toContain("amix=inputs=3:duration=longest:dropout_transition=0:normalize=0");
expect(groupMix).not.toMatch(/adelay=[^;]*,apad/);
expect((groupMix?.match(/apad/g) ?? []).length).toBe(1);
expect(groupMix).toContain("apad,asetpts=N/SR/TB,atrim=0:600[out]");
});
it("names the deadline when a group sub-mix is killed for taking too long", async () => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
runFfmpegMock.mockImplementation(async (args: string[]) =>
args.includes("-filter_complex_script")
? {
success: false,
durationMs: 300_000,
stderr: "Stream mapping:\n Stream #0:0 -> #0:0 (pcm_s16le (native))",
exitCode: 255,
terminationReason: "deadline",
}
: { success: true, durationMs: 1, stderr: "", exitCode: 0 },
);
const result = await processCompositionAudio(
groupedEffects(baseDir),
baseDir,
workDir,
join(baseDir, "out.m4a"),
600,
undefined,
{ ffmpegProcessTimeout: 300_000 },
);
expect(result.success).toBe(false);
const failure = result.failures?.find((f) => f.elementId === "sfx");
expect(failure?.reason).toBe("ffmpeg_timeout");
expect(failure?.detail).toContain("FFmpeg timed out after 300000 ms");
expect(failure?.detail).toContain("FFMPEG_PROCESS_TIMEOUT_MS");
});
it("lets an FX tail run past the clip, still bounded by the composition", async () => {
// A reverb is still decaying when the clip's own audio stops. Trimming at
// the clip boundary is what cut every tail short in the render.
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
writeFileSync(join(baseDir, "bed.wav"), "stub");
const result = await processCompositionAudio(
[
{
id: "bed",
src: "bed.wav",
start: 0,
end: 2,
mediaStart: 0,
layer: 0,
volume: 1,
type: "audio",
fxChain: JSON.stringify({
version: 1,
nodes: [
{ type: "reverb", id: "r", params: { size: 0.5, damping: 0.5, wet: 0.4, dry: 0.7 } },
],
}),
},
],
baseDir,
workDir,
join(baseDir, "out.m4a"),
8,
);
expect(result.success).toBe(true);
expect(applyAudioFxChainMock).toHaveBeenCalledTimes(1);
const filter = capturedFilterScripts[capturedFilterScripts.length - 1];
// 2 s clip + the 1.9 s tail 0.6 + size * 2.6 generates.
expect(filter).toContain("atrim=0:3.9,");
// And still cut at the composition's end, so a tail cannot extend the video.
expect(filter).toContain("apad,asetpts=N/SR/TB,atrim=0:8");
});
it("hands the volume envelope to the FX pass instead of ducking the file after it", async () => {
// The FX pass writes 16-bit PCM, so a chain that overshoots full scale is
// clipped there. Ducking afterwards bakes that distortion in even though
// the lane pulls the track well down; the envelope has to travel into the
// FX pass and land on its float output.
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
writeFileSync(join(baseDir, "voice.wav"), "stub");
const result = await processCompositionAudio(
[
{
id: "voice",
src: "voice.wav",
start: 2,
end: 5,
mediaStart: 0,
layer: 0,
volume: 0.4,
volumeKeyframes: [
{ time: 2, volume: 1 },
{ time: 5, volume: 0.25 },
],
type: "audio",
fxChain: JSON.stringify({
version: 1,
nodes: [{ type: "peaking", id: "p", params: { frequency: 440, gain: 12, q: 1 } }],
}),
},
],
baseDir,
workDir,
join(baseDir, "out.m4a"),
5,
);
expect(result.success).toBe(true);
expect(applyAudioFxChainMock).toHaveBeenCalledTimes(1);
expect(applyAudioFxChainMock.mock.calls[0]?.[3]).toMatchObject({
envelope: {
keyframes: [
{ time: 2, volume: 1 },
{ time: 5, volume: 0.25 },
],
trackStart: 2,
baseVolume: 0.4,
},
});
// And the mixer trusts that bake: unity gain, no second pass, no ffmpeg
// volume expression re-applying the same envelope on top of it.
const filter = capturedFilterScripts[capturedFilterScripts.length - 1];
expect(filter).not.toContain(":eval=frame");
});
it("cuts at the clip boundary when the chain has no tail", async () => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
writeFileSync(join(baseDir, "bed.wav"), "stub");
await processCompositionAudio(
[
{
id: "bed",
src: "bed.wav",
start: 0,
end: 2,
mediaStart: 0,
layer: 0,
volume: 1,
type: "audio",
fxChain: JSON.stringify({
version: 1,
nodes: [{ type: "peaking", id: "n1", params: { frequency: 900, gain: -6, q: 1 } }],
}),
},
],
baseDir,
workDir,
join(baseDir, "out.m4a"),
8,
);
const filter = capturedFilterScripts[capturedFilterScripts.length - 1];
expect(filter).toContain("atrim=0:2,");
});
it("sums unpadded tracks at authored gain and pads the mixed output once", async () => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
writeFileSync(join(baseDir, "a.wav"), "stub");
writeFileSync(join(baseDir, "b.wav"), "stub");
writeFileSync(join(baseDir, "c.wav"), "stub");
const result = await processCompositionAudio(
[
{
id: "a",
src: "a.wav",
start: 0,
end: 2,
mediaStart: 0,
layer: 0,
volume: 0.8,
type: "audio",
},
{
id: "b",
src: "b.wav",
start: 0,
end: 2,
mediaStart: 0,
layer: 1,
volume: 1,
type: "audio",
},
{
id: "c",
src: "c.wav",
start: 0,
end: 2,
mediaStart: 0,
layer: 2,
volume: 0.5,
type: "audio",
},
],
baseDir,
workDir,
join(baseDir, "out.m4a"),
2,
);
expect(result.success).toBe(true);
// 3 prepare calls (one per track via Promise.all) precede the mix call,
// so the mix is at index 3, not index 1.
expect(runFfmpegMock).toHaveBeenCalledTimes(4);
const filter = capturedFilterScripts[3];
expect(filter).toContain("amix=inputs=3");
expect(filter).toContain("amix=inputs=3:duration=longest:dropout_transition=0:normalize=0");
expect(filter).toContain("[mixed]volume=1,apad,asetpts=N/SR/TB,atrim=0:2[out]");
expect((filter.match(/apad,/g) ?? []).length).toBe(1);
});
it("retries the padded graph when ffmpeg does not support amix normalize", async () => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
writeFileSync(join(baseDir, "voice.wav"), "stub");
runFfmpegMock
.mockImplementationOnce(async () => ({
success: true,
durationMs: 1,
stderr: "",
exitCode: 0,
}))
.mockImplementationOnce(async (args: string[]) => {
capturedFilterScripts.push(
readFileSync(args[args.indexOf("-filter_complex_script") + 1]!, "utf8"),
);
return {
success: false,
durationMs: 1,
stderr: "Error applying option 'normalize' to filter 'amix': Option not found",
exitCode: 1,
};
})
.mockImplementationOnce(async (args: string[]) => {
capturedFilterScripts.push(
readFileSync(args[args.indexOf("-filter_complex_script") + 1]!, "utf8"),
);
return { success: true, durationMs: 1, stderr: "", exitCode: 0 };
});
const result = await processCompositionAudio(
[
{
id: "voice",
src: "voice.wav",
start: 0,
end: 2,
mediaStart: 0,
layer: 0,
volume: 1,
type: "audio",
},
],
baseDir,
workDir,
join(baseDir, "out.m4a"),
2,
);
expect(result.success).toBe(true);
expect(runFfmpegMock).toHaveBeenCalledTimes(3);
expect(capturedFilterScripts[0]).toContain("normalize=0");
expect(capturedFilterScripts[1]).toContain("apad,asetpts=N/SR/TB,atrim=0:2[a0]");
expect(capturedFilterScripts[1]).not.toContain("normalize=");
});
it("fails the audio result instead of silently mixing after one track preparation fails", async () => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
writeFileSync(join(baseDir, "working.wav"), "stub");
writeFileSync(join(baseDir, "missing-cue.wav"), "stub");
const defaultImplementation = runFfmpegMock.getMockImplementation()!;
runFfmpegMock.mockImplementation(async (args: string[]) => {
const isMissingCuePrepare = args.includes(join(baseDir, "missing-cue.wav"));
return {
success: !isMissingCuePrepare,
durationMs: 1,
stderr: isMissingCuePrepare
? "https://media.example.test/private.wav?token=secret /tmp/hf/private secret.wav: Invalid data found when processing input"
: "",
exitCode: isMissingCuePrepare ? 1 : 0,
};
});
const result = await processCompositionAudio(
[
{
id: "working",
src: "working.wav",
start: 0,
end: 0.5,
mediaStart: 0,
layer: 0,
volume: 1,
type: "audio",
},
{
id: "missing-cue",
src: "missing-cue.wav",
start: 3.859,
end: 4.359,
mediaStart: 0,
layer: 1,
volume: 1,
type: "audio",
},
],
baseDir,
workDir,
join(baseDir, "out.m4a"),
5,
);
runFfmpegMock.mockImplementation(defaultImplementation);
expect(result.success).toBe(false);
expect(result.tracksProcessed).toBe(1);
expect(result.error).toContain("Invalid data found when processing input");
expect(result.error).toContain("<redacted-url>");
expect(result.error).toContain("<redacted-path>");
expect(result.error).not.toContain("token=secret");
expect(result.error).not.toContain("/tmp/hf/private");
expect(result.error).not.toContain("secret.wav");
expect(result.failures).toEqual([
expect.objectContaining({
stage: "prepare",
reason: "invalid_media",
owner: "user",
retryable: false,
elementId: "missing-cue",
}),
]);
expect(runFfmpegMock).toHaveBeenCalledTimes(2);
});
it("keeps how long a killed group mix ran and ffmpeg's last words", async () => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
writeFileSync(join(baseDir, "voice.wav"), "stub");
const streamLines = Array.from(
{ length: 400 },
(_, i) =>
` Stream #${i}:0 -> #0:${i} (pcm_s16le (native) -> atrim (graph 0 input from stream ${i}:0))`,
);
runFfmpegMock
.mockResolvedValueOnce({
success: true,
durationMs: 1,
stderr: "",
exitCode: 0,
terminationReason: "exit",
})
.mockResolvedValueOnce({
success: false,
durationMs: 530_000,
stderr: [...streamLines, "Exiting normally, received signal 15."].join("\n"),
exitCode: 255,
signal: null,
terminationReason: "exit",
failureReason: "external_interruption",
});
const result = await processCompositionAudio(
[
{
id: "voice",
src: "voice.wav",
start: 0,
end: 2,
mediaStart: 0,
layer: 0,
volume: 1,
type: "audio",
groupId: "sfx",
},
],
baseDir,
workDir,
join(baseDir, "out.m4a"),
2,
);
const failure = result.failures?.find((f) => f.elementId === "sfx");
expect(failure?.reason).toBe("external_interruption");
expect(failure?.subprocess).toMatchObject({ exitCode: 255 });
expect(failure?.detail).toContain("ran 530000 ms");
expect(failure?.detail).toContain("Exiting normally, received signal 15.");
});
it.each([
["ungrouped", undefined],
["grouped", "sfx"],
])(
"never hands one ffmpeg more inputs than a graph holds, %s",
async (_name, groupId) => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
const trackCount = 2_000;
const elements = Array.from({ length: trackCount }, (_, i) => {
writeFileSync(join(baseDir, `clip-${i}.wav`), "stub");
return {
id: `clip-${i}`,
src: `clip-${i}.wav`,
start: i * 0.1,
end: i * 0.1 + 0.5,
mediaStart: 0,
layer: i,
volume: 1,
type: "audio" as const,
...(groupId ? { groupId } : {}),
};
});
runFfmpegMock.mockClear();
const result = await processCompositionAudio(
elements,
baseDir,
workDir,
join(baseDir, "out.m4a"),
trackCount * 0.1 + 0.5,
);
// The mocked ffmpeg writes no files, so only the ungrouped mix runs to the end.
if (!groupId) expect(result.success).toBe(true);
const inputsPerCall = runFfmpegMock.mock.calls.map(
(call) => (call[0] as string[]).filter((arg) => arg === "-i").length,
);
expect(Math.max(...inputsPerCall)).toBeLessThanOrEqual(MAX_MIX_INPUTS);
},
60_000,
);
it("lets a mix's ffmpeg deadline grow with the film length", async () => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
writeFileSync(join(baseDir, "voice.wav"), "stub");
const voice = {
id: "voice",
src: "voice.wav",
start: 0,
end: 2,
mediaStart: 0,
layer: 0,
volume: 1,
type: "audio" as const,
};
const mixTimeout = async (totalDuration: number) => {
runFfmpegMock.mockClear();
await processCompositionAudio(
[voice],
baseDir,
workDir,
join(baseDir, "out.m4a"),
totalDuration,
);
const mixCall = runFfmpegMock.mock.calls.find((call) =>
(call[0] as string[]).includes("-filter_complex_script"),
);
return (mixCall?.[1] as { timeout?: number } | undefined)?.timeout;
};
expect(await mixTimeout(60)).toBe(300_000);
expect(await mixTimeout(36_000)).toBe(3_600_000);
});
it("preserves and classifies unsupported FFmpeg filter failures", async () => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
writeFileSync(join(baseDir, "voice.wav"), "stub");
runFfmpegMock
.mockResolvedValueOnce({
success: true,
durationMs: 1,
stderr: "",
exitCode: 0,
terminationReason: "exit",
})
.mockResolvedValueOnce({
success: false,
durationMs: 1,
stderr: "Error applying option 'whole_dur': Option not found",
exitCode: 8,
terminationReason: "exit",
});
const result = await processCompositionAudio(
[
{
id: "voice",
src: "voice.wav",
start: 0,
end: 2,
mediaStart: 0,
layer: 0,
volume: 1,
type: "audio",
},
],
baseDir,
workDir,
join(baseDir, "out.m4a"),
2,
);
expect(result.success).toBe(false);
expect(result.error).toContain("Option not found");
expect(result.failures).toEqual([
expect.objectContaining({
stage: "mix",
reason: "ffmpeg_unsupported",
owner: "system",
retryable: false,
}),
]);
});
it("preserves a sanitized FFmpeg spawn failure cause", async () => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
writeFileSync(join(baseDir, "voice.wav"), "stub");
runFfmpegMock.mockResolvedValueOnce({
success: false,
durationMs: 1,
stderr: "",
exitCode: null,
terminationReason: "spawn_error",
error: new Error("spawn C:\\private\\ffmpeg.exe ENOENT"),
});
const result = await processCompositionAudio(
[
{
id: "voice",
src: "voice.wav",
start: 0,
end: 2,
mediaStart: 0,
layer: 0,
volume: 1,
type: "audio",
},
],
baseDir,
workDir,
join(baseDir, "out.m4a"),
2,
);
expect(result.success).toBe(false);
expect(result.error).toContain("ENOENT");
expect(result.error).toContain("<redacted-path>");
expect(result.error).not.toContain("C:\\private\\ffmpeg.exe");
expect(result.failures).toEqual([
expect.objectContaining({
stage: "prepare",
reason: "ffmpeg_unavailable",
owner: "system",
retryable: true,
}),
]);
});
it("keeps invalid data from producer-generated mix inputs system-owned", async () => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
writeFileSync(join(baseDir, "voice.wav"), "stub");
runFfmpegMock
.mockResolvedValueOnce({
success: true,
durationMs: 1,
stderr: "",
exitCode: 0,
terminationReason: "exit",
})
.mockResolvedValueOnce({
success: false,
durationMs: 1,
stderr: "Invalid data found when processing input",
exitCode: 1,
terminationReason: "exit",
});
const result = await processCompositionAudio(
[
{
id: "voice",
src: "voice.wav",
start: 0,
end: 2,
mediaStart: 0,
layer: 0,
volume: 1,
type: "audio",
},
],
baseDir,
workDir,
join(baseDir, "out.m4a"),
2,
);
expect(result.failures).toEqual([
expect.objectContaining({
stage: "mix",
reason: "ffmpeg_failed",
owner: "system",
retryable: false,
}),
]);
});
it("bounds per-cause details and the aggregate error across many authored IDs", async () => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
const oversizedId = "authored-id-".repeat(300);
const result = await processCompositionAudio(
Array.from({ length: 3 }, (_, index) => ({
id: `${oversizedId}-${index}`,
src: `missing-${index}.wav`,
start: 0,
end: 2,
mediaStart: 0,
layer: index,
volume: 1,
type: "audio" as const,
})),
baseDir,
workDir,
join(baseDir, "out.m4a"),
2,
);
expect(result.success).toBe(false);
expect(result.error?.length).toBeLessThanOrEqual(2_000);
expect(result.failures).toHaveLength(3);
expect(result.failures?.every((failure) => failure.detail.length <= 2_000)).toBe(true);
});
it("uses frame-evaluated volume automation when keyframes are present", async () => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
writeFileSync(join(baseDir, "voice.wav"), "stub");
const result = await processCompositionAudio(
[
{
id: "voice",
src: "voice.wav",
start: 2,
end: 5,
mediaStart: 0,
layer: 0,
volume: 0,
volumeKeyframes: [
{ time: 2, volume: 0 },
{ time: 3, volume: 1 },
{ time: 5, volume: 0.5 },
],
type: "audio",
},
],
baseDir,
workDir,
join(baseDir, "out.m4a"),
5,
);
expect(result.success).toBe(true);
const filter = capturedFilterScripts[1];
expect(filter).toContain("volume=");
expect(filter).toContain(":eval=frame");
expect(filter).toContain("lt(t\\,1)");
expect(filter).toContain("adelay=2000|2000");
});
it("bounds expression nesting for dense keyframe automation without dropping the envelope", async () => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
writeFileSync(join(baseDir, "bgm.wav"), "stub");
// Mirrors the 60 Hz timeline probe: a 10s eased fade emits hundreds of
// keyframes. The nested-if volume expression must not grow one level per
// keyframe — past ~95 levels FFmpeg fails filter-graph init and the audio
// track is dropped entirely (GH #1066 follow-up).
const keyframes = Array.from({ length: 300 }, (_, i) => {
const time = (i / 299) * 10;
const volume =
time < 3 ? 0.8 * (time / 3) ** 2 : time < 7 ? 0.8 : 0.8 * (1 - (time - 7) / 3) ** 2;
return { time, volume };
});
const result = await processCompositionAudio(
[
{
id: "bgm",
src: "bgm.wav",
start: 0,
end: 10,
mediaStart: 0,
layer: 0,
volume: 0,
volumeKeyframes: keyframes,
type: "audio",
},
],
baseDir,
workDir,
join(baseDir, "out.m4a"),
10,
);
expect(result.success).toBe(true);
const filter = capturedFilterScripts[1];
// One nested `if(lt(...))` is emitted per segment; cap it well under the
// FFmpeg evaluator's nesting limit (MAX_VOLUME_SEGMENTS = 32).
const nestingDepth = (filter.match(/if\(lt\(t/g) ?? []).length;
expect(nestingDepth).toBeGreaterThan(1);
expect(nestingDepth).toBeLessThan(32);
// The simplified envelope still spans the clip: silent start, audible peak.
expect(filter).toContain(":eval=frame");
expect(filter).toMatch(/volume=if\(lt\(t\\,[0-9.]+\)\\,0\+/);
});
it("falls back to a static-volume mix instead of dropping audio when the automated mix fails", async () => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
writeFileSync(join(baseDir, "bgm.wav"), "stub");
// Simulate an ffmpeg build that rejects the automation expression: the
// first mix attempt fails, the static-volume retry succeeds. (prepare =
// call 0, automated mix = call 1, fallback mix = call 2.) These two
// one-time overrides bypass the default mock's capturedFilterScripts
// push, so they push an empty placeholder themselves to keep the array
// index-aligned with call order for the fallback mix's assertion below.
runFfmpegMock
.mockImplementationOnce(async () => {
capturedFilterScripts.push("");
return { success: true, durationMs: 1, stderr: "", exitCode: 0 };
})
.mockImplementationOnce(async () => {
capturedFilterScripts.push("");
return {
success: false,
durationMs: 1,
stderr: "Error initializing filters",
exitCode: 234,
};
});
const result = await processCompositionAudio(
[
{
id: "bgm",
src: "bgm.wav",
start: 0,
end: 5,
mediaStart: 0,
layer: 0,
volume: 0.8,
volumeKeyframes: [
{ time: 0, volume: 0.8 },
{ time: 5, volume: 0 },
],
type: "audio",
},
],
baseDir,
workDir,
join(baseDir, "out.m4a"),
5,
);
expect(result.success).toBe(true);
expect(result.tracksProcessed).toBe(1);
expect(runFfmpegMock).toHaveBeenCalledTimes(3);
// Degradation is surfaced, not silent — the track rendered at base volume.
expect(result.error).toMatch(/base volume/i);
// The fallback mix omits the automation expression (base volume only).
const fallbackFilter = capturedFilterScripts[2];
expect(fallbackFilter).not.toContain(":eval=frame");
expect(fallbackFilter).toContain("volume=0.8");
});
it("keeps the ffmpeg command line short with a large track count (regression for spawn ENAMETOOLONG)", async () => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
// Reported in the wild at 146 timed audio clips: the old inline
// -filter_complex string scaled with track count and blew past the OS
// command-line length limit. 150 tracks reproduces the same shape.
const trackCount = 150;
const elements = Array.from({ length: trackCount }, (_, i) => {
const filename = `clip-${i}.wav`;
writeFileSync(join(baseDir, filename), "stub");
return {
id: `clip-${i}`,
src: filename,
start: i * 0.1,
end: i * 0.1 + 0.5,
mediaStart: 0,
layer: i,
volume: 1,
type: "audio" as const,
};
});
const result = await processCompositionAudio(
elements,
baseDir,
workDir,
join(baseDir, "out.m4a"),
trackCount * 0.1 + 0.5,
);
expect(result.success).toBe(true);
expect(result.tracksProcessed).toBe(trackCount);
// 150 tracks exceed one graph: two slice sums, the fold of those two, then the final mix.
const mixCalls = runFfmpegMock.mock.calls.slice(-4).map((call) => call[0] as string[]);
for (const mixArgs of mixCalls) {
expect(mixArgs).toContain("-filter_complex_script");
expect(mixArgs).not.toContain("-filter_complex");
// Only the -i pairs and the filter script's content scale with track count.
expect(mixArgs.join(" ").length).toBeLessThan(20_000);
}
const [first, second, folded, last] = capturedFilterScripts.slice(-4);
expect(first).toContain(`amix=inputs=${MAX_MIX_INPUTS}`);
expect(second).toContain(`amix=inputs=${trackCount - MAX_MIX_INPUTS}`);
expect(folded).toContain("amix=inputs=2");
expect(last).toContain("amix=inputs=1");
// Each track is trimmed to its authored clip; only the mixed output is padded.
expect((last?.match(/apad,/g) ?? []).length).toBe(1);
// 150 real files + 150 existence checks: ~60ms on Linux, but well past the 5s
// default on the Windows lane, where each file create is orders slower.
}, 30_000);
it("renumbers timestamps between apad and atrim on the mixed output", async () => {
// #2769 moved off `apad=whole_dur=` because some builds reject that option.
// Keep the portable apad/asetpts/atrim shape on the mixed output so the
// final stream has the requested duration and consistent timestamps.
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
writeFileSync(join(baseDir, "a.wav"), "stub");
writeFileSync(join(baseDir, "b.wav"), "stub");
// Two branches, the second delayed: the shape that misplaced audio.
await processCompositionAudio(
[
{
id: "a",
src: "a.wav",
start: 0,
end: 1,
mediaStart: 0,
layer: 0,
volume: 1,
type: "audio",
},
{
id: "b",
src: "b.wav",
start: 4,
end: 5,
mediaStart: 0,
layer: 1,
volume: 1,
type: "audio",
},
],
baseDir,
workDir,
join(baseDir, "out.m4a"),
8,
);
const filter = capturedFilterScripts.at(-1) ?? "";
const branches = filter.match(/apad[^;]*/g) ?? [];
expect(branches).toHaveLength(1);
expect(branches[0]).toMatch(/^apad,asetpts=N\/SR\/TB,atrim=0:/);
// The portability constraint #2769 established still holds.
expect(filter).not.toContain("whole_dur");
});
it("retries with the current file-valued filter option when a nightly removes the legacy alias", async () => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
writeFileSync(join(baseDir, "voice.wav"), "stub");
runFfmpegMock
.mockImplementationOnce(async () => {
capturedFilterScripts.push("");
return { success: true, durationMs: 1, stderr: "", exitCode: 0 };
})
.mockImplementationOnce(async () => {
capturedFilterScripts.push("");
return {
success: false,
durationMs: 1,
stderr: "Unrecognized option 'filter_complex_script'.\nError splitting the argument list",
exitCode: 8,
};
});
const result = await processCompositionAudio(
[
{
id: "voice",
src: "voice.wav",
start: 0,
end: 2,
mediaStart: 0,
layer: 0,
volume: 1,
type: "audio",
},
],
baseDir,
workDir,
join(baseDir, "out.m4a"),
2,
);
expect(result.success).toBe(true);
expect(runFfmpegMock).toHaveBeenCalledTimes(3);
const legacyArgs = runFfmpegMock.mock.calls[1]?.[0] as string[];
const currentArgs = runFfmpegMock.mock.calls[2]?.[0] as string[];
expect(legacyArgs).toContain("-filter_complex_script");
expect(currentArgs).toContain("-/filter_complex");
expect(currentArgs).not.toContain("-filter_complex_script");
expect(capturedFilterScripts[2]).toContain("amix=inputs=1");
});
it("prepares percent-encoded non-Latin audio srcs from decoded filesystem paths", async () => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
const encodedFilename =
"%D9%87%D9%86%D8%A7%20%D9%85%D8%B1%D9%88%D8%A7%20-%20%D9%85%D8%A8%D8%A7%D8%B1%D9%83.mp4";
const filename = decodeURIComponent(encodedFilename);
mkdirSync(join(baseDir, "assets"), { recursive: true });
writeFileSync(join(baseDir, "assets", filename), "stub");
const result = await processCompositionAudio(
[
{
id: "voice",
src: `assets/${encodedFilename}`,
start: 0,
end: 2,
mediaStart: 0,
layer: 0,
volume: 1,
type: "audio",
},
],
baseDir,
workDir,
join(baseDir, "out.m4a"),
2,
);
expect(result.success).toBe(true);
expect(result.error).toBeUndefined();
expect(runFfmpegMock).toHaveBeenCalledTimes(2);
const prepareArgs = runFfmpegMock.mock.calls[0]?.[0];
expect(prepareArgs).toContain(join(baseDir, "assets", filename));
});
it("keeps the float mix the AAC was encoded from when asked to", async () => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
writeFileSync(join(baseDir, "a.wav"), "stub");
writeFileSync(join(baseDir, "b.wav"), "stub");
runFfmpegMock.mockImplementation(async (args: string[]) => {
if (args.at(-1)?.endsWith("master-sum.wav")) writeFileSync(args.at(-1)!, "float mix");
return { success: true, durationMs: 1, stderr: "", exitCode: 0 };
});
const track = (id: string, layer: number) => ({
id,
src: `${id}.wav`,
start: 0,
end: 2,
mediaStart: 0,
layer,
volume: 1,
type: "audio" as const,
});
const kept = join(baseDir, "kept.wav");
const result = await processCompositionAudio(
[track("a", 0), track("b", 1)],
baseDir,
workDir,
join(baseDir, "out.m4a"),
2,
undefined,
{ keepEncoderInputAt: kept },
);
expect(result.success).toBe(true);
expect(vi.mocked(encodeAac).mock.calls.at(-1)?.[0]).toBe(join(workDir, "master-sum.wav"));
expect(readFileSync(kept, "utf8")).toBe("float mix");
});
it("prepares browser root-absolute audio srcs from the project root", async () => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
mkdirSync(join(baseDir, ".media"), { recursive: true });
writeFileSync(join(baseDir, ".media", "tone.wav"), "stub");
const result = await processCompositionAudio(
[
{
id: "tone",
src: "/.media/tone.wav",
start: 0,
end: 1,
mediaStart: 0,
layer: 0,
volume: 1,
type: "audio",
},
],
baseDir,
workDir,
join(baseDir, "out.m4a"),
1,
);
expect(result.success).toBe(true);
expect(result.error).toBeUndefined();
expect(runFfmpegMock.mock.calls[0]?.[0]).toContain(join(baseDir, ".media", "tone.wav"));
});
it("cuts a cloud-held project file's audio from its URL without downloading the file", async () => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
const fetchMock = vi.fn().mockRejectedValue(new Error("downloaded"));
vi.stubGlobal("fetch", fetchMock);
const url = "https://bucket.s3.amazonaws.com/cas/take.mov?X-Amz-Signature=s";
const result = await processCompositionAudio(
[
{
id: "take-audio",
src: "takes/take.mov",
start: 0,
end: 2,
mediaStart: 600,
layer: 0,
volume: 1,
type: "video",
},
],
baseDir,
workDir,
join(baseDir, "out.m4a"),
2,
undefined,
{ urlByAbsentLocalPath: new Map([[join(baseDir, "takes", "take.mov"), url]]) },
);
expect(result.success).toBe(true);
expect(fetchMock).not.toHaveBeenCalled();
const extractArgs = runFfmpegMock.mock.calls[0]?.[0] ?? [];
expect(extractArgs.indexOf("-tls_verify")).toBeGreaterThanOrEqual(0);
expect(extractArgs.indexOf("-tls_verify")).toBeLessThan(extractArgs.indexOf("-i"));
expect(extractArgs.indexOf("-xerror")).toBeGreaterThanOrEqual(0);
expect(extractArgs.indexOf("-xerror")).toBeLessThan(extractArgs.indexOf("-i"));
expect(extractArgs.slice(extractArgs.indexOf("-ss"), extractArgs.indexOf(url) + 1)).toEqual([
"-ss",
"600",
"-t",
"2",
"-i",
url,
]);
});
it("fails a cloud-held file's audio when its URL probe fails twice instead of mixing it unprobed", async () => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
extractAudioMetadataMock.mockRejectedValue(new Error("Server returned 503"));
const url = "https://bucket.s3.amazonaws.com/cas/take.mov?X-Amz-Signature=s";
const result = await processCompositionAudio(
[
{
id: "take-audio",
src: "takes/take.mov",
start: 0,
end: 2,
mediaStart: 600,
layer: 0,
volume: 1,
type: "video",
},
],
baseDir,
workDir,
join(baseDir, "out.m4a"),
2,
undefined,
{ urlByAbsentLocalPath: new Map([[join(baseDir, "takes", "take.mov"), url]]) },
);
expect(extractAudioMetadataMock).toHaveBeenCalledTimes(2);
expect(extractAudioMetadataMock).toHaveBeenCalledWith(url, {
signal: expect.any(AbortSignal),
});
expect(result.success).toBe(false);
expect(runFfmpegMock.mock.calls.some(([args]) => args.includes(url))).toBe(false);
});
it("preserves authored clip gain above unity for quiet-source boosting", async () => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
writeFileSync(join(baseDir, "quiet.wav"), "stub");
const result = await processCompositionAudio(
[
{
id: "quiet",
src: "quiet.wav",
start: 0,
end: 2,
mediaStart: 0,
layer: 0,
volume: 3.98,
type: "audio",
},
],
baseDir,
workDir,
join(baseDir, "out.m4a"),
2,
);
expect(result.success).toBe(true);
expect(capturedFilterScripts[1]).toContain("volume=3.98");
});
it("clamps an out-of-range gain to the shared authoring ceiling", async () => {
const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-"));
const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-"));
tempDirs.push(baseDir, workDir);
writeFileSync(join(baseDir, "quiet.wav"), "stub");
const result = await processCompositionAudio(
[
{
id: "quiet",
src: "quiet.wav",
start: 0,
end: 2,
mediaStart: 0,
layer: 0,
volume: 99,
type: "audio",
},
],
baseDir,
workDir,
join(baseDir, "out.m4a"),
2,
);
expect(result.success).toBe(true);
// Pins the UPPER bound: the 3.98 case above only proves the clamp is not
// min(1, ...). Without this, changing MAX_AUDIO_GAIN's effect in the mixer
// leaves this suite green.
expect(capturedFilterScripts[1]).toContain("volume=3.981072");
});
});
describe("parseAudioElements — relative data-start resolution", () => {
it.each([
{
label: "valueless playback-start",
attributes: 'data-playback-start data-media-start="1.5"',
expected: 1.5,
},
{
label: "empty playback-start",
attributes: 'data-playback-start="" data-media-start="1.5"',
expected: 1.5,
},
{
label: "whitespace playback-start",
attributes: 'data-playback-start=" " data-media-start="1.5"',
expected: 1.5,
},
{
label: "invalid playback-start",
attributes: 'data-playback-start="later" data-media-start="1.5"',
expected: 1.5,
},
{ label: "missing media-start", attributes: "", expected: 0 },
{ label: "invalid media-start", attributes: 'data-media-start="later"', expected: 0 },
{
label: "negative playback-start",
attributes: 'data-playback-start="-1" data-media-start="1.5"',
expected: 1.5,
},
{ label: "negative media-start", attributes: 'data-media-start="-1"', expected: 0 },
{
label: "finite playback-start",
attributes: 'data-playback-start="2.25" data-media-start="1.5"',
expected: 2.25,
},
{
label: "zero playback-start",
attributes: 'data-playback-start="0" data-media-start="1.5"',
expected: 0,
},
])("uses finite playback-start -> media-start -> 0 for $label", ({ attributes, expected }) => {
const html = wrap(`<audio id="a0" data-start="0" src="a.m4a" ${attributes}></audio>`);
expect(parseAudioElements(html)[0]?.mediaStart).toBe(expected);
});
it("parses and normalizes constant playback rate", () => {
const html = wrap(
'<audio id="fast" data-start="0" src="a.m4a" data-playback-rate="2"></audio>' +
'<audio id="low" data-start="0" src="a.m4a" data-playback-rate="0.01"></audio>' +
'<audio id="high" data-start="0" src="a.m4a" data-playback-rate="20"></audio>' +
'<audio id="invalid" data-start="0" src="a.m4a" data-playback-rate="bad"></audio>',
);
const tracks = parseAudioElements(html);
expect(tracks.find((track) => track.id === "fast")?.playbackRate).toBe(2);
expect(tracks.find((track) => track.id === "low")?.playbackRate).toBe(RATE_RANGE.min);
expect(tracks.find((track) => track.id === "high")?.playbackRate).toBe(RATE_RANGE.max);
expect(tracks.find((track) => track.id === "invalid")?.playbackRate).toBe(1);
});
const wrap = (body: string) =>
`<div id="root" class="composition" data-composition-id="c" data-start="0" data-duration="10">${body}</div>`;
it("resolves a relative data-start reference to the target clip's end (matches video)", () => {
// <audio data-start="v0"> means 'start when clip v0 ends' = v0.start + v0.duration.
const html = wrap(
`<video id="v0" class="clip" data-start="0" data-duration="3" src="a.mp4" muted></video>` +
`<audio id="a0" data-start="v0" data-duration="2" src="a.m4a"></audio>`,
);
const els = parseAudioElements(html);
const a0 = els.find((e) => e.id === "a0");
expect(a0).toBeDefined();
// Regression guard: the pre-fix parseFloat("v0") produced NaN, and the
// mixer silently dropped the track.
expect(Number.isNaN(a0!.start)).toBe(false);
expect(a0!.start).toBe(3);
});
it("chains references and never emits NaN start (falls back to 0 for an unknown target)", () => {
const html = wrap(
`<video id="v0" class="clip" data-start="0" data-duration="2" src="a.mp4" muted></video>` +
`<video id="v1" class="clip" data-start="v0" data-duration="2" src="b.mp4" muted></video>` +
`<audio id="a1" data-start="v1" src="a.m4a"></audio>` +
`<audio id="a2" data-start="does-not-exist" src="b.m4a"></audio>`,
);
const els = parseAudioElements(html);
expect(els.find((e) => e.id === "a1")!.start).toBe(4); // v1 ends at 2+2
expect(els.find((e) => e.id === "a2")!.start).toBe(0); // unknown ref → 0, not NaN
});
it("still reads a numeric data-start unchanged", () => {
const html = wrap(`<audio id="a0" data-start="2.5" data-duration="1" src="a.m4a"></audio>`);
expect(parseAudioElements(html).find((e) => e.id === "a0")!.start).toBe(2.5);
});
it("resolves the reference for a data-has-audio video's audio track too", () => {
const html = wrap(
`<video id="v0" class="clip" data-start="0" data-duration="4" src="a.mp4" muted></video>` +
`<video id="v1" class="clip" data-start="v0" data-duration="2" src="b.mp4" data-has-audio="true"></video>`,
);
const track = parseAudioElements(html).find((e) => e.id === "v1-audio");
expect(track).toBeDefined();
expect(track!.start).toBe(4);
});
});
describe("parseAudioElements — hidden tracks", () => {
it("excludes directly hidden audio and audible video from the render mix", () => {
const html =
`<div data-composition-id="main" data-start="0" data-duration="3">` +
`<audio id="master" src="master.wav" data-start="0" data-duration="3"></audio>` +
`<audio id="hidden" src="hidden.wav" data-start="0" data-duration="3" data-hidden></audio>` +
`<video id="hidden-video" src="hidden.mp4" data-has-audio="true" data-hidden></video>` +
`</div>`;
expect(parseAudioElements(html).map((track) => track.id)).toEqual(["master"]);
});
it("excludes audio and audible video beneath a hidden ancestor", () => {
const html =
`<div data-composition-id="main" data-start="0" data-duration="3">` +
`<audio id="master" src="master.wav" data-start="0" data-duration="3"></audio>` +
`<section data-hidden>` +
`<audio id="nested-audio" src="nested.wav"></audio>` +
`<video id="nested-video" src="nested.mp4" data-has-audio="true"></video>` +
`</section>` +
`<video id="visible-video" src="visible.mp4" data-has-audio="true"></video>` +
`</div>`;
expect(parseAudioElements(html).map((track) => track.id)).toEqual([
"master",
"visible-video-audio",
]);
});
it("excludes every member of a hidden group, even though the members carry no data-hidden of their own", () => {
const html =
`<div data-composition-id="main" data-start="0" data-duration="3">` +
`<hf-audio-group id="vo" data-hidden></hf-audio-group>` +
`<audio id="master" src="master.wav" data-start="0" data-duration="3"></audio>` +
`<audio id="a" src="a.wav" data-start="0" data-duration="3" data-audio-group="vo"></audio>` +
`<audio id="b" src="b.wav" data-start="0" data-duration="3" data-audio-group="vo"></audio>` +
`</div>`;
expect(parseAudioElements(html).map((track) => track.id)).toEqual(["master"]);
});
it("does not mix a muted video that still declares data-has-audio", () => {
const html = `<video id="v" src="v.mp4" muted data-has-audio="true" data-start="0" data-duration="3"></video>`;
expect(parseAudioElements(html)).toEqual([]);
});
});
describe("parseAudioElements data-fx-chain", () => {
it("captures the serialised chain when present", () => {
const chain = `{"version":1,"nodes":[{"type":"peaking"}]}`;
const html = `<audio id="music" src="bgm.mp3" data-start="0" data-end="10" data-fx-chain='${chain}'></audio>`;
const [el] = parseAudioElements(html);
expect(el!.fxChain).toBe(chain);
});
it("leaves fxChain undefined when the attribute is absent", () => {
const html = `<audio id="music" src="bgm.mp3" data-start="0" data-end="10"></audio>`;
const [el] = parseAudioElements(html);
expect(el!.fxChain).toBeUndefined();
});
});