Skip to content

EffectCraft: An open-source motion graphics and compositing tool written by scratch in pure Rust

Media tests: every AAC layout mixes by speaker in MP4, Matroska and ADTS (#1701)

tylerjenningswec86f24afeaf
diff --git a/crates/media/tests/it/aac_layout.rs b/crates/media/tests/it/aac_layout.rs
new file mode 100644
index 000000000..9e5ddbb23
--- /dev/null
+++ b/crates/media/tests/it/aac_layout.rs
@@ -0,0 +1,224 @@
+//! Multichannel AAC reaches stereo by where each channel plays, whatever order its channel
+//! configuration decodes in (#1701). Fixtures are tones encoded with FilmCraft's AAC encoder (no
+//! ffmpeg needed, so nothing is skipped), with the configuration rewritten where the coded
+//! elements fit another: configurations 11 and 12 code the same elements as the encoder's 7- and
+//! 8-channel output, and a program config element can name any of them.
+
+use std::f64::consts::PI;
+use std::io::Cursor;
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use effectcraft_media::{MediaPool, probe, probe_bytes};
+use effectcraft_time::Tick;
+use filmcraft_aac::{AudioSpecificConfig, Encoder, EncoderConfig, ProgramConfig};
+
+const RATE: u32 = 48_000;
+/// Tone frequencies by channel; an LFE gets [`LFE_TONE`], which its element can carry.
+const FREQS: [u32; 8] = [300, 500, 700, 900, 1100, 1300, 1500, 1700];
+const LFE_TONE: u32 = 60;
+const AMP: f32 = 0.25;
+const K: f64 = std::f64::consts::FRAC_1_SQRT_2;
+
+/// A channel's content, by where it plays.
+#[derive(Clone, Copy, Debug, PartialEq)]
+enum Ch {
+ L,
+ R,
+ C,
+ Lfe,
+ Ls,
+ Rs,
+ Cs,
+}
+
+impl Ch {
+ /// ITU-R BS.775 gains into `[left, right]`, before normalisation.
+ fn gains(self) -> [f64; 2] {
+ match self {
+ Ch::L => [1.0, 0.0],
+ Ch::R => [0.0, 1.0],
+ Ch::C => [K, K],
+ Ch::Lfe => [0.0, 0.0],
+ Ch::Ls => [K, 0.0],
+ Ch::Rs => [0.0, K],
+ Ch::Cs => [0.5, 0.5],
+ }
+ }
+}
+
+struct Case {
+ name: &'static str,
+ /// What the encoder's input channels carry (its element order: C, L R, …, LFE last).
+ content: &'static [Ch],
+ /// The AudioSpecificConfig written to the container instead of the encoder's, if any.
+ asc: Option<AudioSpecificConfig>,
+}
+
+fn lc(config: u8) -> Option<AudioSpecificConfig> {
+ Some(AudioSpecificConfig::lc(3, config))
+}
+
+fn pce(front: Vec<(bool, u8)>, side: Vec<(bool, u8)>, back: Vec<(bool, u8)>, lfe: Vec<u8>, comment: &[u8]) -> Option<AudioSpecificConfig> {
+ let pce = ProgramConfig { object_type: 1, sf_index: 3, front, side, back, lfe, comment: comment.to_vec(), ..Default::default() };
+ Some(AudioSpecificConfig { pce: Some(pce), ..AudioSpecificConfig::lc(3, 0) })
+}
+
+use Ch::*;
+
+fn cases() -> Vec<Case> {
+ vec![
+ Case { name: "config3", content: &[C, L, R], asc: None },
+ Case { name: "config4", content: &[C, L, R, Cs], asc: None },
+ Case { name: "config5", content: &[C, L, R, Ls, Rs], asc: None },
+ Case { name: "config6", content: &[C, L, R, Ls, Rs, Lfe], asc: None },
+ // configuration 7 as ffmpeg writes 7.1: the second and third pairs are surrounds
+ Case { name: "config7", content: &[C, L, R, Ls, Rs, Ls, Rs, Lfe], asc: None },
+ Case { name: "config11", content: &[C, L, R, Ls, Rs, Cs, Lfe], asc: lc(11) },
+ Case { name: "config12", content: &[C, L, R, Ls, Rs, Ls, Rs, Lfe], asc: lc(12) },
+ // the encoder's own 6.1: a PCE listing the centre first
+ Case { name: "pce61-centre-first", content: &[C, L, R, Ls, Rs, Cs, Lfe], asc: None },
+ // ffmpeg's 6.1 PCE: front pair and centre, a side SCE, a back pair and SCE, no LFE
+ // element, its channels in ffmpeg's order L R C LFE Cs Ls Rs. The decoder places the
+ // encoder's SCE 0 in the front SCE, pair 0 front, pair 1 back, SCE 1 in the back SCE (tag
+ // 1) and the LFE in the side SCE left over (tag 5).
+ Case {
+ name: "pce61-ffmpeg",
+ content: &[C, L, R, Cs, Ls, Rs, Lfe],
+ asc: pce(vec![(true, 0), (false, 0)], vec![(false, 5)], vec![(true, 1), (false, 1)], vec![], b"Lavc62.28.102"),
+ },
+ ]
+}
+
+fn freq(content: &[Ch], i: usize) -> u32 {
+ if content.get(i) == Some(&Lfe) { LFE_TONE } else { FREQS[i] }
+}
+
+/// Two seconds of `content`'s tones encoded as AAC: the AudioSpecificConfig and access units.
+fn encode(content: &[Ch], adts: bool) -> (Vec<u8>, Vec<Vec<u8>>) {
+ let n = content.len();
+ let mut cfg = EncoderConfig::cbr(RATE, n, 96_000 * n as u32);
+ cfg.adts = adts;
+ let mut enc = Encoder::new(cfg).expect("encoder");
+ let planes: Vec<Vec<f32>> =
+ (0..n).map(|c| (0..2 * RATE).map(|i| AMP * (2.0 * PI * f64::from(freq(content, c)) * f64::from(i) / f64::from(RATE)).sin() as f32).collect()).collect();
+ let refs: Vec<&[f32]> = planes.iter().map(Vec::as_slice).collect();
+ let mut units = enc.encode(&refs);
+ units.extend(enc.flush());
+ (enc.audio_specific_config(), units)
+}
+
+fn mp4(asc: Vec<u8>, channels: u32, units: &[Vec<u8>]) -> Vec<u8> {
+ use filmcraft_isobmff::{Brand, Mp4Writer, SampleEntry, TrackConfig, WriteSample, WriterOptions};
+ let mut w = Mp4Writer::new(Cursor::new(Vec::new()), WriterOptions::new(Brand::Mp4)).expect("writer");
+ let t = w.add_track(TrackConfig::new(SampleEntry::aac(asc, channels, RATE), RATE)).expect("track");
+ for u in units {
+ w.write_sample(t, WriteSample { data: u, duration: 1024, composition_offset: 0, is_sync: true }).expect("sample");
+ }
+ w.finish().expect("finish").into_inner()
+}
+
+fn mkv(asc: Vec<u8>, channels: u32, units: &[Vec<u8>]) -> Vec<u8> {
+ use filmcraft_matroska::{MkvWriter, MuxOptions, TrackKind, TrackSpec};
+ let mut t = TrackSpec::new(TrackKind::Audio, "A_AAC");
+ t.codec_private = asc;
+ t.audio = Some((f64::from(RATE), u64::from(channels), None));
+ let mut w = MkvWriter::new(Cursor::new(Vec::new()), vec![t], MuxOptions::default()).expect("writer");
+ let frame_ns = 1_000_000_000 * 1024 / i64::from(RATE);
+ for (i, u) in units.iter().enumerate() {
+ w.write_frame(0, i as i64 * frame_ns, true, u, Some(frame_ns as u64)).expect("frame");
+ }
+ w.finish().expect("finish").into_inner()
+}
+
+/// The fixture written to disk as `name`.
+fn write(name: &str, bytes: &[u8]) -> PathBuf {
+ let dir = std::env::temp_dir().join(format!("effectcraft-aac-layout-{}", std::process::id()));
+ std::fs::create_dir_all(&dir).expect("mkdir");
+ let p = dir.join(name);
+ std::fs::write(&p, bytes).expect("write");
+ p
+}
+
+/// Amplitude of the tone at `freq` in one side of interleaved stereo.
+fn amp(s: &[f32], side: usize, freq: u32) -> f64 {
+ let (mut re, mut im, mut n) = (0.0, 0.0, 0usize);
+ for (i, f) in s.as_chunks::<2>().0.iter().enumerate() {
+ let w = 2.0 * PI * f64::from(freq) * i as f64 / f64::from(RATE);
+ re += f64::from(f[side]) * w.cos();
+ im += f64::from(f[side]) * w.sin();
+ n += 1;
+ }
+ 2.0 * (re * re + im * im).sqrt() / n.max(1) as f64
+}
+
+/// Stereo of one second from 0.5 s in, through the media pool as footage reads it.
+fn mix(pool: &MediaPool, footage: &effectcraft_project::Footage) -> Vec<f32> {
+ pool.audio_samples(footage, Tick::from_seconds_f64(0.5), RATE as usize, RATE)
+}
+
+/// Each channel of `content` reaches each side at its BS.775 gain, normalised by the larger
+/// side's sum (an LFE nowhere).
+fn check(label: &str, content: &[Ch], s: &[f32]) {
+ let raw: Vec<[f64; 2]> = content.iter().map(|c| c.gains()).collect();
+ let norm = raw.iter().map(|g| g[0]).sum::<f64>().max(raw.iter().map(|g| g[1]).sum::<f64>()).max(1.0);
+ for (i, (ch, g)) in content.iter().zip(&raw).enumerate() {
+ for side in 0..2 {
+ let (got, want) = (amp(s, side, freq(content, i)), f64::from(AMP) * g[side] / norm);
+ assert!(
+ (got - want).abs() <= f64::from(AMP) * 0.05,
+ "{label}: {ch:?} ({} Hz) -> {} is {got:.4}, expected {want:.4}",
+ freq(content, i),
+ ["L", "R"][side]
+ );
+ }
+ }
+}
+
+/// The LFE tone survives encoding, so its absence from the mix means it was left out.
+fn lfe_is_coded(case: &Case, bytes: Vec<u8>) {
+ if !case.content.contains(&Lfe) {
+ return;
+ }
+ let src = filmcraft_codecs::open_bytes("lfe.mp4", bytes.into()).expect("open");
+ let buf = src.audio(24_000, RATE as usize, RATE).expect("decode");
+ let best = buf.channels.iter().map(|ch| amp(&ch.iter().flat_map(|&v| [v, v]).collect::<Vec<_>>(), 0, LFE_TONE)).fold(0.0, f64::max);
+ assert!(best > 0.8 * f64::from(AMP), "{}: the LFE tone is coded ({best:.4})", case.name);
+}
+
+#[test]
+fn every_aac_layout_mixes_by_speaker_in_mp4() {
+ for case in cases() {
+ let (asc, units) = encode(case.content, false);
+ let asc = case.asc.as_ref().map_or(asc, AudioSpecificConfig::to_bytes);
+ let bytes = mp4(asc, case.content.len() as u32, &units);
+ lfe_is_coded(&case, bytes.clone());
+ let p = write(&format!("{}.mp4", case.name), &bytes);
+ check(case.name, case.content, &mix(&MediaPool::new(), &probe(&p).expect("probe")));
+ // in memory (`add_bytes`) as well as in place
+ let (path, bytes): (String, Arc<[u8]>) = (format!("/mem/{}.mp4", case.name), bytes.into());
+ let pool = MediaPool::new();
+ pool.add_bytes(&path, bytes.clone());
+ check(&format!("{} in memory", case.name), case.content, &mix(&pool, &probe_bytes(&path, bytes).expect("probe")));
+ }
+}
+
+#[test]
+fn matroska_aac_layouts_mix_by_speaker() {
+ for case in cases().into_iter().filter(|c| matches!(c.name, "config12" | "pce61-centre-first" | "pce61-ffmpeg")) {
+ let (asc, units) = encode(case.content, false);
+ let asc = case.asc.as_ref().map_or(asc, AudioSpecificConfig::to_bytes);
+ let p = write(&format!("{}.mkv", case.name), &mkv(asc, case.content.len() as u32, &units));
+ check(case.name, case.content, &mix(&MediaPool::new(), &probe(&p).expect("probe")));
+ }
+}
+
+/// ADTS signals configurations 1 to 7 only, which the channel count tells apart.
+#[test]
+fn adts_aac_layouts_mix_by_speaker() {
+ for case in cases().into_iter().filter(|c| c.asc.is_none() && c.content.len() != 7) {
+ let (_, units) = encode(case.content, true);
+ let p = write(&format!("{}.aac", case.name), &units.concat());
+ check(case.name, case.content, &mix(&MediaPool::new(), &probe(&p).expect("probe")));
+ }
+}
diff --git a/crates/media/tests/it/main.rs b/crates/media/tests/it/main.rs
index 5f7fb8037..69bed4a9f 100644
--- a/crates/media/tests/it/main.rs
+++ b/crates/media/tests/it/main.rs
@@ -1,5 +1,6 @@
//! The crate's integration tests, one binary (each module is a former `tests/<name>.rs`).
+mod aac_layout;
mod downmix;
mod exif_orientation;
mod footage;