EffectCraft: An open-source motion graphics and compositing tool written by scratch in pure Rust
#903: downmix 5.1 and other multichannel audio to stereo
vangel7664fabbb2fcad
diff --git a/crates/media/src/downmix.rs b/crates/media/src/downmix.rsnew file mode 100644index 000000000..2ef417dd7--- /dev/null+++ b/crates/media/src/downmix.rs@@ -0,0 +1,133 @@+//! Stereo downmix of multichannel audio, shared by preview, waveform and export mixdown (they+//! all read footage audio through `MediaPool::audio_samples`).+//!+//! The matrix is the usual ITU-R BS.775 one: front L/R at unity, the centre and every surround+//! or back channel at −3 dB (1/√2) into both sides' own side (centre into both), the low+//! frequency effects channel left out (as ffmpeg's and swresample's default, `lfe_mix_level` 0;+//! not verified against Adobe's documentation, which does not state it). A back-centre channel+//! reaches both sides at −6 dB. The whole matrix is then scaled by one fixed gain, the reciprocal+//! of the largest row sum (ffmpeg's default normalisation), so that full-scale input can never+//! exceed full scale: 5.1 is scaled by 1/(1 + 0.707 + 0.707). Mono stays duplicated at unity and+//! stereo is untouched.+//!+//! Channel order follows the WAVE / SMPTE convention by channel count (what FilmCraft's AC-3, PCM,+//! FLAC and Opus readers deliver): 3 = L R C, 4 = quad L R Ls Rs, 5 = L R C Ls Rs,+//! 6 = L R C LFE Ls Rs, 7 = L R C LFE Cs Ls Rs, 8 = L R C LFE Lb Rb Ls Rs. FilmCraft's AAC-LC+//! decoder instead delivers AAC's own order (C L R first, LFE last), so AAC gets its own tables.+//! More than 8 channels carry no known layout: only the first two (front L/R) are used.++use std::f32::consts::FRAC_1_SQRT_2;++/// A source channel's contribution to the left and right outputs, before normalisation.+type Pair = [f32; 2];++const FL: Pair = [1.0, 0.0];+const FR: Pair = [0.0, 1.0];+const FC: Pair = [FRAC_1_SQRT_2, FRAC_1_SQRT_2];+const LFE: Pair = [0.0, 0.0];+const SL: Pair = [FRAC_1_SQRT_2, 0.0];+const SR: Pair = [0.0, FRAC_1_SQRT_2];+const CS: Pair = [0.5, 0.5];++/// The per-channel `[left, right]` gains for `count` channels in the given order, normalised.+/// `aac` selects AAC's native channel order. Empty for zero channels.+pub fn stereo_gains(count: usize, aac: bool) -> Vec<Pair> {+ let raw: &[Pair] = match (count, aac) {+ (0, _) => &[],+ (1, _) => &[[1.0, 1.0]],+ (2, _) => &[FL, FR],+ (3, false) => &[FL, FR, FC],+ (3, true) => &[FC, FL, FR],+ (4, false) => &[FL, FR, SL, SR],+ (4, true) => &[FC, FL, FR, CS],+ (5, false) => &[FL, FR, FC, SL, SR],+ (5, true) => &[FC, FL, FR, SL, SR],+ (6, false) => &[FL, FR, FC, LFE, SL, SR],+ (6, true) => &[FC, FL, FR, SL, SR, LFE],+ (7, false) => &[FL, FR, FC, LFE, CS, SL, SR],+ (7, true) => &[FC, FL, FR, SL, SR, CS, LFE],+ (8, false) => &[FL, FR, FC, LFE, SL, SR, SL, SR],+ (8, true) => &[FC, FL, FR, SL, SR, SL, SR, LFE],+ _ => &[FL, FR],+ };+ let mut out = raw.to_vec();+ // Channels beyond a known layout (more than 8) are silent.+ out.resize(count, [0.0, 0.0]);+ let peak = out.iter().map(|g| g[0]).sum::<f32>().max(out.iter().map(|g| g[1]).sum::<f32>());+ // Mono is deliberately left at unity on both sides; so is anything already within range.+ if count > 2 && peak > 1.0 {+ for g in &mut out {+ g[0] /= peak;+ g[1] /= peak;+ }+ }+ out+}++/// Downmix planar `channels` to interleaved stereo in `out` (`out.len() / 2` frames); frames a+/// channel is too short for count as silence, extra frames in `out` stay as they are.+pub fn downmix_into(channels: &[Vec<f32>], aac: bool, out: &mut [f32]) {+ let gains = stereo_gains(channels.len(), aac);+ for (ch, g) in channels.iter().zip(&gains) {+ if g[0] == 0.0 && g[1] == 0.0 {+ continue;+ }+ for (o, v) in out.as_chunks_mut::<2>().0.iter_mut().zip(ch) {+ o[0] += v * g[0];+ o[1] += v * g[1];+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn every_channel_count_has_a_bounded_defined_matrix() {+ for aac in [false, true] {+ for n in 0..40usize {+ let g = stereo_gains(n, aac);+ assert_eq!(g.len(), n);+ let (l, r) = (g.iter().map(|p| p[0]).sum::<f32>(), g.iter().map(|p| p[1]).sum::<f32>());+ if n > 2 {+ assert!(l <= 1.0001 && r <= 1.0001, "{n} channels (aac {aac}): {l} {r}");+ assert!(l > 0.0 && r > 0.0);+ }+ }+ }+ }++ #[test]+ fn five_one_matches_the_bs775_matrix() {+ let k = 1.0 / (1.0 + 2.0 * FRAC_1_SQRT_2);+ let g = stereo_gains(6, false);+ assert!((g[0][0] - k).abs() < 1e-6 && g[0][1] == 0.0);+ assert!((g[2][0] - k * FRAC_1_SQRT_2).abs() < 1e-6 && (g[2][1] - k * FRAC_1_SQRT_2).abs() < 1e-6);+ assert_eq!(g[3], [0.0, 0.0], "LFE is left out");+ assert!((g[4][0] - k * FRAC_1_SQRT_2).abs() < 1e-6 && g[4][1] == 0.0);+ assert!((g[5][1] - k * FRAC_1_SQRT_2).abs() < 1e-6 && g[5][0] == 0.0);+ // AAC order: the centre is first, LFE last+ let a = stereo_gains(6, true);+ assert_eq!((a[0], a[5]), (g[2], g[3]));+ assert_eq!((a[1], a[2]), (g[0], g[1]));+ }++ #[test]+ fn mono_and_stereo_are_untouched_and_wide_layouts_use_the_front_pair() {+ assert_eq!(stereo_gains(1, false), vec![[1.0, 1.0]]);+ assert_eq!(stereo_gains(2, true), vec![FL, FR]);+ let g = stereo_gains(12, false);+ assert_eq!((g[0], g[1], g[2]), (FL, FR, [0.0, 0.0]));+ }++ #[test]+ fn ragged_and_empty_input_never_panics() {+ let mut out = vec![0.0; 8];+ downmix_into(&[], false, &mut out);+ downmix_into(&[vec![1.0; 100], vec![1.0; 2], vec![], vec![1.0; 3], vec![1.0; 1], vec![1.0]], true, &mut out);+ assert!(out.iter().all(|v| v.is_finite()));+ let mut none: [f32; 0] = [];+ downmix_into(&vec![vec![1.0; 3]; 7], false, &mut none);+ }+}diff --git a/crates/media/src/lib.rs b/crates/media/src/lib.rsindex 000c16245..2964dda52 100644--- a/crates/media/src/lib.rs+++ b/crates/media/src/lib.rs@@ -16,6 +16,7 @@#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]mod convert;+pub mod downmix;pub mod exr_channels;mod gif_anim;mod layered;diff --git a/crates/media/src/pool.rs b/crates/media/src/pool.rsindex ce980c563..f604b90ba 100644--- a/crates/media/src/pool.rs+++ b/crates/media/src/pool.rs@@ -448,8 +448,8 @@ impl MediaPool {}/// `frames` stereo sample frames of `footage`'s audio starting at source time `start`, at- /// `rate` Hz, interleaved (L R L R …). Mono is duplicated to both channels; channels beyond- /// the first two are dropped. Silence where there is no audio (or before the media starts).+ /// `rate` Hz, interleaved (L R L R …). Mono is duplicated to both channels; multichannel+ /// sources are downmixed to stereo ([`crate::downmix`]). Silence where there is no audio (or before the media starts)./// With a conformed audio folder, reads come from the conformed file once it is written.pub fn audio_samples(&self, footage: &Footage, start: Tick, frames: usize, rate: u32) -> Vec<f32> {if !footage.has_audio || footage.missing || rate == 0 || frames == 0 {@@ -828,7 +828,7 @@ impl FootageSource for MediaPool {/// `frames` stereo sample frames (interleaved) of `src` from sample `s0` at `rate` Hz, which/// FilmCraft serves as decoded when it is the source's own rate. Mono is duplicated to both-/// channels; silence before the start. An error where decoding fails.+/// channels, multichannel sources are downmixed ([`crate::downmix`]); silence before the start. An error where decoding fails.fn read_stereo(src: &SharedSource, s0: i64, frames: usize, rate: u32) -> std::result::Result<Vec<f32>, String> {let mut out = vec![0.0; frames * 2];let skip = usize::try_from(s0.saturating_neg()).unwrap_or(0);@@ -836,12 +836,9 @@ fn read_stereo(src: &SharedSource, s0: i64, frames: usize, rate: u32) -> std::rereturn Ok(out);}let buf = src.audio(s0.max(0), frames - skip, rate).map_err(|e| e.to_string())?;- let (Some(l), Some(r)) = (buf.channels.first(), buf.channels.get(1).or(buf.channels.first())) else { return Ok(out) };let Some(dst) = skip.checked_mul(2).and_then(|i| out.get_mut(i..)) else { return Ok(out) };- for ([o0, o1], (a, b)) in dst.as_chunks_mut::<2>().0.iter_mut().zip(l.iter().zip(r)) {- *o0 = *a;- *o1 = *b;- }+ let aac = src.info().audio.as_ref().is_some_and(|a| a.codec.starts_with("AAC"));+ crate::downmix::downmix_into(&buf.channels, aac, dst);Ok(out)}diff --git a/crates/media/tests/downmix.rs b/crates/media/tests/downmix.rsnew file mode 100644index 000000000..e0e7d2252--- /dev/null+++ b/crates/media/tests/downmix.rs@@ -0,0 +1,226 @@+//! Multichannel audio downmix (#903) against ffmpeg as an external oracle. Fixtures: ffmpeg with+//! one sine per channel, written to `target/fixtures/media`; tests skip when ffmpeg is missing.++use std::f64::consts::PI;+use std::path::{Path, PathBuf};+use std::process::Command;++use effectcraft_media::{MediaPool, probe};+use effectcraft_time::Tick;++const RATE: u32 = 48_000;++fn fixtures() -> PathBuf {+ Path::new(env!("CARGO_MANIFEST_DIR")).join("../../target/fixtures/media")+}++/// A fixture with `layout`'s channels `names` (join order) each carrying its own sine, encoded+/// with `codec_args`. `None` when ffmpeg is missing or cannot make it.+fn fixture(name: &str, layout: &str, names: &[&str], freqs: &[u32], codec_args: &[&str]) -> Option<PathBuf> {+ if !Command::new("ffmpeg").arg("-version").output().is_ok_and(|o| o.status.success()) {+ eprintln!("ffmpeg not found: skipping");+ return None;+ }+ let out = fixtures().join(name);+ if out.exists() {+ return Some(out);+ }+ std::fs::create_dir_all(fixtures()).ok()?;+ let ext = out.extension()?.to_string_lossy().to_string();+ let tmp = out.with_file_name(format!("tmp-{}-{name}", std::process::id())).with_extension(ext);+ let mut cmd = Command::new("ffmpeg");+ cmd.args(["-hide_banner", "-loglevel", "error", "-y"]);+ let mut map = String::new();+ for (i, (n, f)) in names.iter().zip(freqs).enumerate() {+ cmd.args(["-f", "lavfi", "-i", &format!("sine=frequency={f}:sample_rate={RATE}:duration=2")]);+ map.push_str(&format!("[{i}]"));+ let _ = n;+ }+ let post = if name.contains("loud") { ",volume=8:precision=float,alimiter=limit=1:level=disabled" } else { "" };+ let joins: Vec<String> = names.iter().enumerate().map(|(i, n)| format!("{i}.0-{n}")).collect();+ cmd.args(["-filter_complex", &format!("{map}join=inputs={}:channel_layout={layout}:map={}{}[a]", names.len(), joins.join("|"), post), "-map", "[a]"]);+ cmd.args(codec_args).arg(&tmp);+ let ok = cmd.status().ok()?.success();+ if !ok {+ eprintln!("ffmpeg could not make {name}");+ return None;+ }+ std::fs::rename(&tmp, &out).ok()?;+ Some(out)+}++/// Amplitude (peak) of the sine at `freq` in one channel of interleaved stereo.+fn amp(s: &[f32], ch: usize, freq: u32) -> f64 {+ let (mut re, mut im, mut n) = (0.0, 0.0, 0usize);+ for (i, f) in s.as_chunks::<2>().0.iter().enumerate() {+ let w = 2.0 * PI * f64::from(freq) * i as f64 / f64::from(RATE);+ re += f64::from(f[ch]) * w.cos();+ im += f64::from(f[ch]) * w.sin();+ n += 1;+ }+ 2.0 * (re * re + im * im).sqrt() / n.max(1) as f64+}++const K: f64 = std::f64::consts::FRAC_1_SQRT_2;++struct Case {+ file: &'static str,+ layout: &'static str,+ names: &'static [&'static str],+ codec: &'static [&'static str],+ /// Expected `[left, right]` gain of each channel (the same order as `names`).+ gains: Vec<[f64; 2]>,+ /// Allowed deviation of a channel's amplitude (lossy codecs).+ tol: f64,+}++const FREQS: [u32; 8] = [300, 500, 700, 900, 1100, 1300, 1500, 1700];++fn five_one() -> (&'static [&'static str], Vec<[f64; 2]>) {+ let k = 1.0 / (1.0 + 2.0 * K);+ (&["FL", "FR", "FC", "LFE", "BL", "BR"], vec![[k, 0.0], [0.0, k], [k * K, k * K], [0.0, 0.0], [k * K, 0.0], [0.0, k * K]])+}++fn seven_one() -> (&'static [&'static str], Vec<[f64; 2]>) {+ let k = 1.0 / (1.0 + 3.0 * K);+ (+ &["FL", "FR", "FC", "LFE", "BL", "BR", "SL", "SR"],+ vec![[k, 0.0], [0.0, k], [k * K, k * K], [0.0, 0.0], [k * K, 0.0], [0.0, k * K], [k * K, 0.0], [0.0, k * K]],+ )+}++fn cases() -> Vec<Case> {+ let (n51, g51) = five_one();+ let (n71, g71) = seven_one();+ let k5 = 1.0 / (1.0 + 2.0 * K);+ let kq = 1.0 / (1.0 + K);+ vec![+ Case { file: "dm51.m4a", layout: "5.1", names: n51, codec: &["-c:a", "aac", "-b:a", "384k"], gains: g51.clone(), tol: 0.15 },+ Case { file: "dm51.ac3.mp4", layout: "5.1", names: n51, codec: &["-c:a", "ac3", "-b:a", "640k"], gains: g51.clone(), tol: 0.15 },+ Case { file: "dm51.wav", layout: "5.1", names: n51, codec: &["-c:a", "pcm_s16le"], gains: g51.clone(), tol: 0.02 },+ Case { file: "dm51.flac", layout: "5.1", names: n51, codec: &["-c:a", "flac"], gains: g51, tol: 0.02 },+ Case { file: "dm71.wav", layout: "7.1", names: n71, codec: &["-c:a", "pcm_s16le"], gains: g71.clone(), tol: 0.02 },+ Case { file: "dm71.flac", layout: "7.1", names: n71, codec: &["-c:a", "flac"], gains: g71, tol: 0.02 },+ Case {+ file: "dm50.wav",+ layout: "5.0",+ names: &["FL", "FR", "FC", "BL", "BR"],+ codec: &["-c:a", "pcm_s16le"],+ gains: vec![[k5, 0.0], [0.0, k5], [k5 * K, k5 * K], [k5 * K, 0.0], [0.0, k5 * K]],+ tol: 0.02,+ },+ Case {+ file: "dmquad.wav",+ layout: "quad",+ names: &["FL", "FR", "BL", "BR"],+ codec: &["-c:a", "pcm_s16le"],+ gains: vec![[kq, 0.0], [0.0, kq], [kq * K, 0.0], [0.0, kq * K]],+ tol: 0.02,+ },+ ]+}++/// Our stereo mix of `p` over one second, from 0.2 s in.+fn ours(p: &Path) -> Vec<f32> {+ let f = probe(p).expect("probe");+ MediaPool::new().audio_samples(&f, Tick::from_seconds_f64(0.2), RATE as usize, RATE)+}++/// ffmpeg's own `-ac 2` downmix of the same second, interleaved. Written as 16-bit integers:+/// swresample only normalises the matrix (to keep full scale from clipping) for integer output.+fn ffmpeg_ac2(p: &Path) -> Vec<f32> {+ let out = Command::new("ffmpeg")+ .args(["-hide_banner", "-loglevel", "error", "-ss", "0.2", "-t", "1", "-i"])+ .arg(p)+ .args(["-ac", "2", "-ar", "48000", "-f", "s16le", "-"])+ .output()+ .expect("run ffmpeg");+ out.stdout.as_chunks::<2>().0.iter().map(|b| f32::from(i16::from_le_bytes(*b)) / 32768.0).collect()+}++fn rms(s: &[f32]) -> f64 {+ (s.iter().map(|v| f64::from(*v).powi(2)).sum::<f64>() / s.len().max(1) as f64).sqrt()+}++fn peak(s: &[f32]) -> f64 {+ s.iter().fold(0.0f64, |m, v| m.max(f64::from(v.abs())))+}++/// Each channel carries its own sine at amplitude 0.5·√... (ffmpeg's `sine` is −24 dBFS RMS,+/// peak ≈ 0.125). Every channel must reach each side at the BS.775 gain, and the overall level+/// must match ffmpeg's `-ac 2`.+#[test]+fn each_channel_reaches_stereo_at_its_bs775_gain() {+ for c in cases() {+ let freqs: Vec<u32> = FREQS.iter().copied().take(c.names.len()).collect();+ let Some(p) = fixture(c.file, c.layout, c.names, &freqs, c.codec) else { continue };+ let s = ours(&p);+ let sine = 0.125 * 1.0; // peak of ffmpeg's sine source (0.125 = 2^-3)+ for (i, (name, g)) in c.names.iter().zip(&c.gains).enumerate() {+ for (side, gain) in g.iter().enumerate() {+ let got = amp(&s, side, freqs[i]);+ let want = sine * gain;+ assert!((got - want).abs() <= sine * c.tol.max(0.02), "{}: {name} -> {} is {got:.4}, expected {want:.4}", c.file, ["L", "R"][side]);+ }+ }+ let theirs = ffmpeg_ac2(&p);+ let (a, b) = (rms(&s), rms(&theirs));+ let db = 20.0 * (a / b).log10();+ assert!(db.abs() < 1.0, "{}: level {db:.2} dB off ffmpeg -ac 2 (rms {a:.4} vs {b:.4})", c.file);+ let pdb = 20.0 * (peak(&s) / peak(&theirs)).log10();+ assert!(pdb.abs() < 1.5, "{}: peak {pdb:.2} dB off ffmpeg -ac 2", c.file);+ }+}++/// Full-scale in every channel must not clip.+#[test]+fn loud_surround_does_not_clip() {+ let Some(p) = fixture("dm51_loud.wav", "5.1", &["FL", "FR", "FC", "LFE", "BL", "BR"], &[300; 6], &["-c:a", "pcm_s16le"]) else { return };+ let s = ours(&p);+ assert!(peak(&s) <= 1.001, "peak {}", peak(&s));+ assert!(peak(&s) > 0.5, "audible, peak {}", peak(&s));+}++/// Mono and stereo footage keep their level (mono on both sides).+#[test]+fn mono_and_stereo_keep_their_level() {+ let Some(p) = fixture("dm10.wav", "mono", &["FC"], &[440], &["-c:a", "pcm_s16le"]) else { return };+ let s = ours(&p);+ assert!((amp(&s, 0, 440) - 0.125).abs() < 0.003 && (amp(&s, 1, 440) - 0.125).abs() < 0.003);+ let Some(p) = fixture("dm20.wav", "stereo", &["FL", "FR"], &[440, 660], &["-c:a", "pcm_s16le"]) else { return };+ let s = ours(&p);+ assert!((amp(&s, 0, 440) - 0.125).abs() < 0.003 && amp(&s, 1, 440) < 0.003);+ assert!((amp(&s, 1, 660) - 0.125).abs() < 0.003 && amp(&s, 0, 660) < 0.003);+}++/// Odd channel counts (3, 4, 7, 9, 12, 16) decode to finite stereo without panicking.+#[test]+fn hostile_channel_counts_do_not_panic() {+ for (n, layout) in [(3, "3.0"), (4, "quad"), (7, "6.1"), (9, "9 channels"), (12, "12 channels"), (16, "16 channels")] {+ let names: Vec<String> = (0..n).map(|i| format!("{i}")).collect();+ // channel names for unnamed layouts are DEFAULT-ordered by position+ let _ = names;+ let file = format!("dmn{n}.wav");+ let args = ["-f", "lavfi", "-i", &format!("sine=frequency=440:sample_rate={RATE}:duration=1")];+ if !Command::new("ffmpeg").arg("-version").output().is_ok_and(|o| o.status.success()) {+ return;+ }+ let out = fixtures().join(&file);+ std::fs::create_dir_all(fixtures()).expect("mkdir");+ if !out.exists() {+ let tmp = fixtures().join(format!("tmp-{}-{file}", std::process::id()));+ let af = format!("pan={layout}|{}", (0..n).map(|i| format!("c{i}=c0")).collect::<Vec<_>>().join("|"));+ let ok =+ Command::new("ffmpeg").args(["-hide_banner", "-loglevel", "error", "-y"]).args(args).args(["-af", &af, "-c:a", "pcm_s16le"]).arg(&tmp).status();+ if !ok.is_ok_and(|s| s.success()) {+ eprintln!("ffmpeg cannot make {n} channels: skipping");+ continue;+ }+ std::fs::rename(&tmp, &out).expect("rename");+ }+ let Ok(f) = probe(&out) else { continue };+ let s = MediaPool::new().audio_samples(&f, Tick::ZERO, 4800, RATE);+ assert_eq!(s.len(), 9600, "{n} channels");+ assert!(s.iter().all(|v| v.is_finite() && v.abs() <= 1.001), "{n} channels");+ }+}diff --git a/crates/media/tests/footage.rs b/crates/media/tests/footage.rsindex d26b1a1d1..82a6c8d0d 100644--- a/crates/media/tests/footage.rs+++ b/crates/media/tests/footage.rs@@ -615,9 +615,14 @@ fn flac_audio_of_any_shape_reads_in_device_blocks() {}at += 1024;}- // ffmpeg's sine source has an amplitude of 1/8, on both channels (mono is duplicated).+ // ffmpeg's sine source has an amplitude of 1/8 (mono is duplicated). The fixture carries+ // channels 0 and 1 at full level and the others at half, so a multichannel file sounds as+ // its stereo downmix (tests/downmix.rs checks that matrix channel by channel).+ let gains = effectcraft_media::downmix::stereo_gains(channels as usize, false);+ let side = |s: usize| gains.iter().enumerate().map(|(c, g)| f64::from(g[s]) * if c < 2 { 1.0 } else { 0.5 }).sum::<f64>();+ let want = 0.125 * ((side(0).powi(2) + side(1).powi(2)) / 2.0).sqrt() / 2f64.sqrt();let rms = (sum / n.max(1) as f64).sqrt();- assert!((rms - 0.125 / 2f64.sqrt()).abs() < 0.003, "{}: rms {rms}", p.display());+ assert!((rms - want).abs() < 0.003, "{}: rms {rms}, expected {want}", p.display());eprintln!("{}: read {secs} s in {:?}, slowest block after the first {worst:?}", p.display(), t0.elapsed());assert_eq!(pool.stats().opened, 1, "opened once, not per block");}