World's first open-source, agentic video production system. 12 production pipelines, 100+ tools, 700+ agent skill and production-knowledge files. Turn your AI coding assistant into a full video production studio.
Python agentagentic-aiaiclaudecopilotcursorelevenlabsffmpegfluximage-generationopen-sourceopenaipythonremotionstable-diffusiontext-to-speechtext-to-videovideo-generationvideo-production
fix(audio_mixer): stop amix from scaling tracks by 1/N (#761)
evgeniytiukov-wqa903d77ef0e3
diff --git a/tests/tools/test_audio_mixer_amix_normalize.py b/tests/tools/test_audio_mixer_amix_normalize.pynew file mode 100644index 00000000..c03a06ff--- /dev/null+++ b/tests/tools/test_audio_mixer_amix_normalize.py@@ -0,0 +1,285 @@+"""Regression tests: amix must not scale tracks by 1/N.++ffmpeg's `amix` defaults to `normalize=1`, which divides every input by the+input count. Narration segments are scheduled with `adelay`, so all of them+count as active from t=0: N sequential lines came out 20*log10(N) dB quiet+(-15.6 dB for 6 lines) and the music buried the voice. `duck` halved the+speech the same way, and `mix` silently overrode the per-track `volume` the+caller set. Every amix in `mix`, `duck` and `full_mix` now carries+`normalize=0`; outputs that skip loudnorm end in an `alimiter` so tracks that+overlap cannot hard-clip.+"""++import re+import shutil+import subprocess+import sys+from pathlib import Path++import pytest++PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent+sys.path.insert(0, str(PROJECT_ROOT))++from tools.audio.audio_mixer import AudioMixer # noqa: E402++LIMITER = "alimiter=limit=1:level=0[out]"+++def _stub_tracks(tmp_path, roles):+ tracks = []+ for i, role in enumerate(roles):+ path = tmp_path / f"{role}_{i}.wav"+ path.write_bytes(b"stub")+ tracks.append({"path": str(path), "role": role, "start_seconds": float(i)})+ return tracks+++def _build(monkeypatch, inputs):+ """Run the tool with ffmpeg stubbed out; return (filter graph, -map label)."""+ captured = []++ def fake_run(self, cmd, **kwargs):+ captured.append(list(cmd))++ class _R:+ stdout = "10.0\n"+ stderr = ""++ return _R()++ monkeypatch.setattr(AudioMixer, "run_command", fake_run)+ result = AudioMixer().execute(inputs)+ assert result.success, result.error++ ffmpeg_cmds = [c for c in captured if c and c[0] == "ffmpeg"]+ assert ffmpeg_cmds, "no ffmpeg command was built"+ cmd = ffmpeg_cmds[0]+ return cmd[cmd.index("-filter_complex") + 1], cmd[cmd.index("-map") + 1]+++def _amix_nodes(graph):+ return [node for node in re.split(r"[;,]", graph) if "amix=" in node]+++def _assert_all_amix_unnormalized(graph, expected_count):+ nodes = _amix_nodes(graph)+ assert len(nodes) == expected_count, graph+ for node in nodes:+ assert "normalize=0" in node, f"amix must disable normalize; got: {node}"+++# --- Offline: filter graph shape (no ffmpeg needed) -------------------------+++def test_mix_amix_disables_normalize(tmp_path, monkeypatch):+ tracks = _stub_tracks(tmp_path, ["speech"] * 6 + ["music"])+ tracks[-1]["volume"] = 0.3+ graph, _ = _build(+ monkeypatch,+ {"operation": "mix", "tracks": tracks, "output_path": str(tmp_path / "out.wav")},+ )+ _assert_all_amix_unnormalized(graph, 1)+++def test_full_mix_without_ducking_amix_disables_normalize(tmp_path, monkeypatch):+ tracks = _stub_tracks(tmp_path, ["speech"] * 6 + ["music"])+ graph, _ = _build(+ monkeypatch,+ {+ "operation": "full_mix",+ "tracks": tracks,+ "ducking": {"enabled": False},+ "output_path": str(tmp_path / "out.wav"),+ },+ )+ _assert_all_amix_unnormalized(graph, 1)+++def test_full_mix_with_ducking_every_amix_disables_normalize(tmp_path, monkeypatch):+ # speech amix, music amix, speech+music amix, +SFX amix+ tracks = _stub_tracks(tmp_path, ["speech"] * 3 + ["music"] * 2 + ["sfx"])+ graph, _ = _build(+ monkeypatch,+ {+ "operation": "full_mix",+ "tracks": tracks,+ "ducking": {"enabled": True},+ "output_path": str(tmp_path / "out.wav"),+ },+ )+ _assert_all_amix_unnormalized(graph, 4)+++def test_duck_amix_disables_normalize(tmp_path, monkeypatch):+ speech, music = tmp_path / "speech.wav", tmp_path / "music.wav"+ speech.write_bytes(b"stub")+ music.write_bytes(b"stub")+ graph, mapped = _build(+ monkeypatch,+ {+ "operation": "duck",+ "primary_audio": str(speech),+ "secondary_audio": str(music),+ "output_path": str(tmp_path / "out.wav"),+ },+ )+ _assert_all_amix_unnormalized(graph, 1)+ # duck has no loudnorm stage, so the summed output must be clip-guarded.+ assert graph.endswith(LIMITER), graph+ assert mapped == "[out]"++[email protected]("operation", ["mix", "full_mix"])+def test_output_without_loudnorm_ends_in_limiter(tmp_path, monkeypatch, operation):+ tracks = _stub_tracks(tmp_path, ["speech"] * 3)+ graph, mapped = _build(+ monkeypatch,+ {+ "operation": operation,+ "tracks": tracks,+ "normalize": False,+ "target_duration": 5,+ "output_path": str(tmp_path / "out.wav"),+ },+ )+ assert graph.endswith(LIMITER), graph+ assert mapped == "[out]"++[email protected]("operation", ["mix", "full_mix"])+def test_loudnorm_output_has_no_extra_limiter(tmp_path, monkeypatch, operation):+ tracks = _stub_tracks(tmp_path, ["speech"] * 3)+ graph, mapped = _build(+ monkeypatch,+ {+ "operation": operation,+ "tracks": tracks,+ "normalize": True,+ "output_path": str(tmp_path / "out.wav"),+ },+ )+ # loudnorm's true-peak limiter already caps the summed mix.+ assert "loudnorm=" in graph.rsplit(";", 1)[-1], graph+ assert "alimiter" not in graph+ assert mapped == "[out]"+++# --- End-to-end: measured levels (ffmpeg required) --------------------------++needs_ffmpeg = pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg required")++STARTS = [0.0, 1.0, 2.0, 3.0, 4.0, 5.0]+++def _render(path, expr, duration):+ subprocess.run(+ ["ffmpeg", "-y", "-f", "lavfi", "-i", f"aevalsrc={expr}:s=48000:d={duration}", str(path)],+ capture_output=True, check=True, timeout=60,+ )+++def _tone(path, freq, duration=0.5):+ # 0.5 amplitude (-6 dBFS peak): about as hot as typical TTS output.+ _render(path, f"0.5*sin(2*PI*{freq}*t)", duration)+++def _levels(path, ss=None, t=None):+ """Return (max_volume, mean_volume) in dB, as reported by volumedetect."""+ cmd = ["ffmpeg", "-hide_banner"]+ if ss is not None:+ cmd += ["-ss", str(ss), "-t", str(t)]+ cmd += ["-i", str(path), "-af", "volumedetect", "-f", "null", "-"]+ # ffmpeg logs UTF-8; decode explicitly so non-ASCII temp paths don't break+ # on Windows locales (e.g. cp1251).+ out = subprocess.run(+ cmd, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=60+ )+ found = {}+ for line in out.stderr.splitlines():+ for key in ("max_volume", "mean_volume"):+ if f"{key}:" in line:+ found[key] = float(line.split(f"{key}:")[1].split("dB")[0])+ assert {"max_volume", "mean_volume"} <= found.keys(), out.stderr+ return found["max_volume"], found["mean_volume"]+++@needs_ffmpeg+ "operation, ducking",+ [("mix", None), ("full_mix", False), ("full_mix", True)],+ ids=["mix", "full_mix-no-ducking", "full_mix-ducking"],+)+def test_sequential_lines_keep_standalone_peak(tmp_path, operation, ducking):+ """Six narration lines: each line's peak in the mix equals its own peak."""+ lines = []+ for i, start in enumerate(STARTS):+ path = tmp_path / f"line{i}.wav"+ _tone(path, 300 + 100 * i)+ lines.append({"path": str(path), "role": "speech", "start_seconds": start})+ standalone = [_levels(line["path"])[0] for line in lines]++ out = tmp_path / "out.wav"+ inputs = {"operation": operation, "tracks": list(lines), "normalize": False,+ "output_path": str(out)}+ if operation == "full_mix":+ inputs["ducking"] = {"enabled": ducking}+ if ducking:+ # A silent bed drives the whole sidechain graph without adding level.+ bed = tmp_path / "bed.wav"+ _render(bed, "0", 7)+ inputs["tracks"].append({"path": str(bed), "role": "music"})++ result = AudioMixer().execute(inputs)+ assert result.success, result.error++ for start, expected in zip(STARTS, standalone):+ peak, _ = _levels(out, ss=start + 0.1, t=0.3)+ assert peak == pytest.approx(expected, abs=0.2), (+ f"line at {start}s: standalone {expected} dB, in mix {peak} dB"+ )+++@needs_ffmpeg+def test_duck_keeps_speech_at_unity(tmp_path):+ speech, music = tmp_path / "speech.wav", tmp_path / "music.wav"+ _tone(speech, 440, duration=2)+ _render(music, "0", 3)+ out = tmp_path / "out.wav"++ result = AudioMixer().execute(+ {+ "operation": "duck",+ "primary_audio": str(speech),+ "secondary_audio": str(music),+ "output_path": str(out),+ }+ )+ assert result.success, result.error++ peak, _ = _levels(out, ss=0.5, t=1)+ assert peak == pytest.approx(_levels(speech)[0], abs=0.2)+++@needs_ffmpeg[email protected]("operation", ["mix", "full_mix"])+def test_overlapping_tracks_are_limited_not_clipped(tmp_path, operation):+ """Six in-phase -6 dBFS tones at t=0 sum to +9.5 dBFS without normalize."""+ tracks = []+ for i in range(6):+ path = tmp_path / f"t{i}.wav"+ _tone(path, 440, duration=1)+ tracks.append({"path": str(path), "role": "speech"})++ out = tmp_path / "out.wav"+ inputs = {"operation": operation, "tracks": tracks, "normalize": False,+ "output_path": str(out)}+ if operation == "full_mix":+ inputs["ducking"] = {"enabled": False}+ result = AudioMixer().execute(inputs)+ assert result.success, result.error++ peak, mean = _levels(out, ss=0.2, t=0.6)+ # Hard clipping squares the sine off (peak-to-RMS near 0 dB); the limiter+ # keeps a sine's ~3 dB crest factor.+ assert peak - mean > 2.0, f"output clipped: peak {peak} dB, mean {mean} dB"diff --git a/tools/audio/audio_mixer.py b/tools/audio/audio_mixer.pyindex 48dfcb6a..83bdac02 100644--- a/tools/audio/audio_mixer.py+++ b/tools/audio/audio_mixer.py@@ -219,7 +219,24 @@ class AudioMixer(BaseTool):target = -16.0# Clamp to a sane loudness range to avoid malformed ffmpeg args.target = max(-40.0, min(0.0, target))- return f"[{in_label}]loudnorm=I={target}:LRA=11:TP=-1.5[{out_label}]"+ # loudnorm upsamples to 192 kHz internally; resample back so the mix+ # lands at a delivery rate instead of a 4x-oversized file.+ return f"[{in_label}]loudnorm=I={target}:LRA=11:TP=-1.5,aresample=48000[{out_label}]"++ @staticmethod+ def _limiter_filter(in_label: str, out_label: str) -> str:+ """Clip guard for output paths that skip loudnorm.++ Every amix here runs with normalize=0 so tracks keep the levels the+ caller set, which means overlapping tracks can sum past full scale.+ With normalize=true, loudnorm's true-peak limiter absorbs that; without+ it the overs would hard-clip when the float mix is written as integer+ PCM. limit=1 only engages above 0 dBFS, so in-range audio passes at+ its original level; level=0 stops alimiter from re-gaining the output.+ The 5 ms lookahead delay is left uncompensated because the ``latency``+ option is missing from older FFmpeg builds (e.g. 4.x).+ """+ return f"[{in_label}]alimiter=limit=1:level=0[{out_label}]"def _track_filters(self, track: dict[str, Any]) -> list[str]:"""Build per-track filters on the source timeline before scheduling it.@@ -305,24 +322,25 @@ class AudioMixer(BaseTool):else:filter_parts.append(f"[{i}:a]acopy[a{i}]")- # Amix all processed streams+ # Amix all processed streams. normalize=0: amix's default normalize=1+ # scales every input by 1/N (re-scaling again as inputs end), which+ # silently overrides the per-track `volume` the caller set.mix_inputs = "".join(f"[a{i}]" for i in range(len(tracks)))filter_parts.append(- f"{mix_inputs}amix=inputs={len(tracks)}:duration=longest:dropout_transition=2[mixed]"+ f"{mix_inputs}amix=inputs={len(tracks)}:duration=longest:dropout_transition=2:normalize=0[mixed]")if normalize:filter_parts.append(self._loudnorm_filter(inputs, "mixed", "out"))- out_label = "[out]"else:- out_label = "[mixed]"+ filter_parts.append(self._limiter_filter("mixed", "out"))filter_complex = ";".join(filter_parts)cmd = ["ffmpeg", "-y"]cmd.extend(input_args)cmd.extend(["-filter_complex", filter_complex])- cmd.extend(["-map", out_label, str(output_path)])+ cmd.extend(["-map", "[out]", str(output_path)])self.run_command(cmd)@@ -419,7 +437,10 @@ class AudioMixer(BaseTool):f"threshold=0.02:ratio=9:attack={attack}:release={release}:"f"level_sc=1:mix=0.9[ducked];"f"[ducked]volume={music_vol * 3}[music_out];" # compensate sidechain level- f"[0:a][music_out]amix=inputs=2:duration=longest[out]"+ # normalize=0: the default normalize=1 halves both inputs, so the+ # speech came out 6 dB down. No loudnorm follows, so guard clipping.+ f"[0:a][music_out]amix=inputs=2:duration=longest:normalize=0[mixed];"+ + self._limiter_filter("mixed", "out"))cmd = [@@ -558,9 +579,14 @@ class AudioMixer(BaseTool):speech_indices = list(range(len(speech_tracks)))speech_labels = "".join(f"[a{i}]" for i in speech_indices)+ # normalize=0 throughout the ducking graph: amix's default+ # normalize=1 divides every input by the input count, so N narration+ # segments came out 20*log10(N) dB quiet (-15.6 dB for 6 lines) and+ # the music buried the voice. Segments are sequential, so summing is+ # correct; music level is already set by volume + ducking.if len(speech_tracks) > 1:filter_parts.append(- f"{speech_labels}amix=inputs={len(speech_tracks)}:duration=longest[speech_all]"+ f"{speech_labels}amix=inputs={len(speech_tracks)}:duration=longest:normalize=0[speech_all]")else:filter_parts.append(f"[a{speech_indices[0]}]acopy[speech_all]")@@ -580,7 +606,7 @@ class AudioMixer(BaseTool):if len(music_tracks) > 1:filter_parts.append(- f"{music_labels}amix=inputs={len(music_tracks)}:duration=longest[music_mix]"+ f"{music_labels}amix=inputs={len(music_tracks)}:duration=longest:normalize=0[music_mix]")music_in = "[music_mix]"else:@@ -600,7 +626,7 @@ class AudioMixer(BaseTool):)# Final mix: the other speech branch + ducked music- mix_label = "[speech_out][music_out]amix=inputs=2:duration=longest[premix]"+ mix_label = "[speech_out][music_out]amix=inputs=2:duration=longest:normalize=0[premix]"# Add SFX if presentsfx_start = len(speech_tracks) + len(music_tracks)@@ -608,16 +634,17 @@ class AudioMixer(BaseTool):sfx_labels = "".join(f"[a{i}]" for i in range(sfx_start, sfx_start + len(sfx_tracks)))filter_parts.append(mix_label.replace("[premix]", "[pressfx]"))filter_parts.append(- f"[pressfx]{sfx_labels}amix=inputs={1 + len(sfx_tracks)}:duration=longest[premix]"+ f"[pressfx]{sfx_labels}amix=inputs={1 + len(sfx_tracks)}:duration=longest:normalize=0[premix]")else:filter_parts.append(mix_label)else:- # No ducking: simple amix of all tracks+ # No ducking: simple amix of all tracks. normalize=0 for the same+ # reason as above — each track stays at the level the caller set.all_labels = "".join(f"[a{i}]" for i in range(len(all_tracks)))filter_parts.append(- f"{all_labels}amix=inputs={len(all_tracks)}:duration=longest:dropout_transition=2[premix]"+ f"{all_labels}amix=inputs={len(all_tracks)}:duration=longest:dropout_transition=2:normalize=0[premix]")# A ducked music stream is gated by the speech sidechain, so its tail@@ -635,16 +662,15 @@ class AudioMixer(BaseTool):# Normalizeif normalize:filter_parts.append(self._loudnorm_filter(inputs, premix_label, "out"))- out_label = "[out]"else:- out_label = f"[{premix_label}]"+ filter_parts.append(self._limiter_filter(premix_label, "out"))filter_complex = ";".join(p for p in filter_parts if p)cmd = ["ffmpeg", "-y"]cmd.extend(input_args)cmd.extend(["-filter_complex", filter_complex])- cmd.extend(["-map", out_label])+ cmd.extend(["-map", "[out]"])if target is not None:cmd.extend(["-t", str(target)])cmd.append(str(output_path))