EffectCraft: An open-source motion graphics and compositing tool written by scratch in pure Rust
Media tests: every AAC layout mixes by speaker in MP4, Matroska and ADTS (#1701)
tylerjenningswec86f24afeaf
diff --git a/crates/media/tests/it/aac_layout.rs b/crates/media/tests/it/aac_layout.rsnew file mode 100644index 000000000..9e5ddbb23--- /dev/null+++ b/crates/media/tests/it/aac_layout.rs@@ -0,0 +1,224 @@+//! Multichannel AAC reaches stereo by where each channel plays, whatever order its channel+//! configuration decodes in (#1701). Fixtures are tones encoded with FilmCraft's AAC encoder (no+//! ffmpeg needed, so nothing is skipped), with the configuration rewritten where the coded+//! elements fit another: configurations 11 and 12 code the same elements as the encoder's 7- and+//! 8-channel output, and a program config element can name any of them.++use std::f64::consts::PI;+use std::io::Cursor;+use std::path::PathBuf;+use std::sync::Arc;++use effectcraft_media::{MediaPool, probe, probe_bytes};+use effectcraft_time::Tick;+use filmcraft_aac::{AudioSpecificConfig, Encoder, EncoderConfig, ProgramConfig};++const RATE: u32 = 48_000;+/// Tone frequencies by channel; an LFE gets [`LFE_TONE`], which its element can carry.+const FREQS: [u32; 8] = [300, 500, 700, 900, 1100, 1300, 1500, 1700];+const LFE_TONE: u32 = 60;+const AMP: f32 = 0.25;+const K: f64 = std::f64::consts::FRAC_1_SQRT_2;++/// A channel's content, by where it plays.+#[derive(Clone, Copy, Debug, PartialEq)]+enum Ch {+ L,+ R,+ C,+ Lfe,+ Ls,+ Rs,+ Cs,+}++impl Ch {+ /// ITU-R BS.775 gains into `[left, right]`, before normalisation.+ fn gains(self) -> [f64; 2] {+ match self {+ Ch::L => [1.0, 0.0],+ Ch::R => [0.0, 1.0],+ Ch::C => [K, K],+ Ch::Lfe => [0.0, 0.0],+ Ch::Ls => [K, 0.0],+ Ch::Rs => [0.0, K],+ Ch::Cs => [0.5, 0.5],+ }+ }+}++struct Case {+ name: &'static str,+ /// What the encoder's input channels carry (its element order: C, L R, …, LFE last).+ content: &'static [Ch],+ /// The AudioSpecificConfig written to the container instead of the encoder's, if any.+ asc: Option<AudioSpecificConfig>,+}++fn lc(config: u8) -> Option<AudioSpecificConfig> {+ Some(AudioSpecificConfig::lc(3, config))+}++fn pce(front: Vec<(bool, u8)>, side: Vec<(bool, u8)>, back: Vec<(bool, u8)>, lfe: Vec<u8>, comment: &[u8]) -> Option<AudioSpecificConfig> {+ let pce = ProgramConfig { object_type: 1, sf_index: 3, front, side, back, lfe, comment: comment.to_vec(), ..Default::default() };+ Some(AudioSpecificConfig { pce: Some(pce), ..AudioSpecificConfig::lc(3, 0) })+}++use Ch::*;++fn cases() -> Vec<Case> {+ vec![+ Case { name: "config3", content: &[C, L, R], asc: None },+ Case { name: "config4", content: &[C, L, R, Cs], asc: None },+ Case { name: "config5", content: &[C, L, R, Ls, Rs], asc: None },+ Case { name: "config6", content: &[C, L, R, Ls, Rs, Lfe], asc: None },+ // configuration 7 as ffmpeg writes 7.1: the second and third pairs are surrounds+ Case { name: "config7", content: &[C, L, R, Ls, Rs, Ls, Rs, Lfe], asc: None },+ Case { name: "config11", content: &[C, L, R, Ls, Rs, Cs, Lfe], asc: lc(11) },+ Case { name: "config12", content: &[C, L, R, Ls, Rs, Ls, Rs, Lfe], asc: lc(12) },+ // the encoder's own 6.1: a PCE listing the centre first+ Case { name: "pce61-centre-first", content: &[C, L, R, Ls, Rs, Cs, Lfe], asc: None },+ // ffmpeg's 6.1 PCE: front pair and centre, a side SCE, a back pair and SCE, no LFE+ // element, its channels in ffmpeg's order L R C LFE Cs Ls Rs. The decoder places the+ // encoder's SCE 0 in the front SCE, pair 0 front, pair 1 back, SCE 1 in the back SCE (tag+ // 1) and the LFE in the side SCE left over (tag 5).+ Case {+ name: "pce61-ffmpeg",+ content: &[C, L, R, Cs, Ls, Rs, Lfe],+ asc: pce(vec![(true, 0), (false, 0)], vec![(false, 5)], vec![(true, 1), (false, 1)], vec![], b"Lavc62.28.102"),+ },+ ]+}++fn freq(content: &[Ch], i: usize) -> u32 {+ if content.get(i) == Some(&Lfe) { LFE_TONE } else { FREQS[i] }+}++/// Two seconds of `content`'s tones encoded as AAC: the AudioSpecificConfig and access units.+fn encode(content: &[Ch], adts: bool) -> (Vec<u8>, Vec<Vec<u8>>) {+ let n = content.len();+ let mut cfg = EncoderConfig::cbr(RATE, n, 96_000 * n as u32);+ cfg.adts = adts;+ let mut enc = Encoder::new(cfg).expect("encoder");+ let planes: Vec<Vec<f32>> =+ (0..n).map(|c| (0..2 * RATE).map(|i| AMP * (2.0 * PI * f64::from(freq(content, c)) * f64::from(i) / f64::from(RATE)).sin() as f32).collect()).collect();+ let refs: Vec<&[f32]> = planes.iter().map(Vec::as_slice).collect();+ let mut units = enc.encode(&refs);+ units.extend(enc.flush());+ (enc.audio_specific_config(), units)+}++fn mp4(asc: Vec<u8>, channels: u32, units: &[Vec<u8>]) -> Vec<u8> {+ use filmcraft_isobmff::{Brand, Mp4Writer, SampleEntry, TrackConfig, WriteSample, WriterOptions};+ let mut w = Mp4Writer::new(Cursor::new(Vec::new()), WriterOptions::new(Brand::Mp4)).expect("writer");+ let t = w.add_track(TrackConfig::new(SampleEntry::aac(asc, channels, RATE), RATE)).expect("track");+ for u in units {+ w.write_sample(t, WriteSample { data: u, duration: 1024, composition_offset: 0, is_sync: true }).expect("sample");+ }+ w.finish().expect("finish").into_inner()+}++fn mkv(asc: Vec<u8>, channels: u32, units: &[Vec<u8>]) -> Vec<u8> {+ use filmcraft_matroska::{MkvWriter, MuxOptions, TrackKind, TrackSpec};+ let mut t = TrackSpec::new(TrackKind::Audio, "A_AAC");+ t.codec_private = asc;+ t.audio = Some((f64::from(RATE), u64::from(channels), None));+ let mut w = MkvWriter::new(Cursor::new(Vec::new()), vec![t], MuxOptions::default()).expect("writer");+ let frame_ns = 1_000_000_000 * 1024 / i64::from(RATE);+ for (i, u) in units.iter().enumerate() {+ w.write_frame(0, i as i64 * frame_ns, true, u, Some(frame_ns as u64)).expect("frame");+ }+ w.finish().expect("finish").into_inner()+}++/// The fixture written to disk as `name`.+fn write(name: &str, bytes: &[u8]) -> PathBuf {+ let dir = std::env::temp_dir().join(format!("effectcraft-aac-layout-{}", std::process::id()));+ std::fs::create_dir_all(&dir).expect("mkdir");+ let p = dir.join(name);+ std::fs::write(&p, bytes).expect("write");+ p+}++/// Amplitude of the tone at `freq` in one side of interleaved stereo.+fn amp(s: &[f32], side: usize, freq: u32) -> f64 {+ let (mut re, mut im, mut n) = (0.0, 0.0, 0usize);+ for (i, f) in s.as_chunks::<2>().0.iter().enumerate() {+ let w = 2.0 * PI * f64::from(freq) * i as f64 / f64::from(RATE);+ re += f64::from(f[side]) * w.cos();+ im += f64::from(f[side]) * w.sin();+ n += 1;+ }+ 2.0 * (re * re + im * im).sqrt() / n.max(1) as f64+}++/// Stereo of one second from 0.5 s in, through the media pool as footage reads it.+fn mix(pool: &MediaPool, footage: &effectcraft_project::Footage) -> Vec<f32> {+ pool.audio_samples(footage, Tick::from_seconds_f64(0.5), RATE as usize, RATE)+}++/// Each channel of `content` reaches each side at its BS.775 gain, normalised by the larger+/// side's sum (an LFE nowhere).+fn check(label: &str, content: &[Ch], s: &[f32]) {+ let raw: Vec<[f64; 2]> = content.iter().map(|c| c.gains()).collect();+ let norm = raw.iter().map(|g| g[0]).sum::<f64>().max(raw.iter().map(|g| g[1]).sum::<f64>()).max(1.0);+ for (i, (ch, g)) in content.iter().zip(&raw).enumerate() {+ for side in 0..2 {+ let (got, want) = (amp(s, side, freq(content, i)), f64::from(AMP) * g[side] / norm);+ assert!(+ (got - want).abs() <= f64::from(AMP) * 0.05,+ "{label}: {ch:?} ({} Hz) -> {} is {got:.4}, expected {want:.4}",+ freq(content, i),+ ["L", "R"][side]+ );+ }+ }+}++/// The LFE tone survives encoding, so its absence from the mix means it was left out.+fn lfe_is_coded(case: &Case, bytes: Vec<u8>) {+ if !case.content.contains(&Lfe) {+ return;+ }+ let src = filmcraft_codecs::open_bytes("lfe.mp4", bytes.into()).expect("open");+ let buf = src.audio(24_000, RATE as usize, RATE).expect("decode");+ let best = buf.channels.iter().map(|ch| amp(&ch.iter().flat_map(|&v| [v, v]).collect::<Vec<_>>(), 0, LFE_TONE)).fold(0.0, f64::max);+ assert!(best > 0.8 * f64::from(AMP), "{}: the LFE tone is coded ({best:.4})", case.name);+}++#[test]+fn every_aac_layout_mixes_by_speaker_in_mp4() {+ for case in cases() {+ let (asc, units) = encode(case.content, false);+ let asc = case.asc.as_ref().map_or(asc, AudioSpecificConfig::to_bytes);+ let bytes = mp4(asc, case.content.len() as u32, &units);+ lfe_is_coded(&case, bytes.clone());+ let p = write(&format!("{}.mp4", case.name), &bytes);+ check(case.name, case.content, &mix(&MediaPool::new(), &probe(&p).expect("probe")));+ // in memory (`add_bytes`) as well as in place+ let (path, bytes): (String, Arc<[u8]>) = (format!("/mem/{}.mp4", case.name), bytes.into());+ let pool = MediaPool::new();+ pool.add_bytes(&path, bytes.clone());+ check(&format!("{} in memory", case.name), case.content, &mix(&pool, &probe_bytes(&path, bytes).expect("probe")));+ }+}++#[test]+fn matroska_aac_layouts_mix_by_speaker() {+ for case in cases().into_iter().filter(|c| matches!(c.name, "config12" | "pce61-centre-first" | "pce61-ffmpeg")) {+ let (asc, units) = encode(case.content, false);+ let asc = case.asc.as_ref().map_or(asc, AudioSpecificConfig::to_bytes);+ let p = write(&format!("{}.mkv", case.name), &mkv(asc, case.content.len() as u32, &units));+ check(case.name, case.content, &mix(&MediaPool::new(), &probe(&p).expect("probe")));+ }+}++/// ADTS signals configurations 1 to 7 only, which the channel count tells apart.+#[test]+fn adts_aac_layouts_mix_by_speaker() {+ for case in cases().into_iter().filter(|c| c.asc.is_none() && c.content.len() != 7) {+ let (_, units) = encode(case.content, true);+ let p = write(&format!("{}.aac", case.name), &units.concat());+ check(case.name, case.content, &mix(&MediaPool::new(), &probe(&p).expect("probe")));+ }+}diff --git a/crates/media/tests/it/main.rs b/crates/media/tests/it/main.rsindex 5f7fb8037..69bed4a9f 100644--- a/crates/media/tests/it/main.rs+++ b/crates/media/tests/it/main.rs@@ -1,5 +1,6 @@//! The crate's integration tests, one binary (each module is a former `tests/<name>.rs`).+mod aac_layout;mod downmix;mod exif_orientation;mod footage;