EffectCraft: An open-source motion graphics and compositing tool written by scratch in pure Rust
Media: read the AAC configuration of the MP4 or Matroska track FilmCraft plays, not an empty one before it (#1701)
tylerjenningsw800b555e514e
diff --git a/crates/media/src/aac.rs b/crates/media/src/aac.rsindex a2d7de37b..cd33eaf7b 100644--- a/crates/media/src/aac.rs+++ b/crates/media/src/aac.rs@@ -1,4 +1,4 @@-//! The AudioSpecificConfig a movie's first AAC track decodes with, from the containers that carry+//! The AudioSpecificConfig of the AAC track FilmCraft plays from a movie, from the containers that carry//! it outside the audio frames: MP4 / MOV (`esds`), Matroska (`CodecPrivate`) and LATM in MPEG//! TS / PS (its StreamMuxConfig). FilmCraft's other AAC is ADTS (in MPEG TS / PS or a raw `.aac`),//! whose channel configurations FilmCraft decodes (1 to 7) its channel count tells apart@@ -13,10 +13,11 @@ use filmcraft_media::ByteReader;/// as far as this, not through the whole file.const MPEG_HEAD: u64 = 8 << 20;-/// The configuration of the first audio track, if it is AAC: from an MP4 / MOV `header` (its-/// ftyp and moov, or the whole file), or else from `file`, read as Matroska or MPEG TS / PS.+/// The configuration of the audio track FilmCraft plays, if it is AAC: from an MP4 / MOV `header`+/// (its ftyp and moov, or the whole file) and `file`, or else from `file`, read as Matroska or+/// MPEG TS / PS.pub(crate) fn config(header: Option<&[u8]>, file: Option<&dyn ByteReader>) -> Option<AudioSpecificConfig> {- let asc = match header.and_then(mp4) {+ let asc = match header.and_then(|h| mp4(h, file)) {Some(asc) => asc,None => {let file = file?;@@ -28,19 +29,29 @@ pub(crate) fn config(header: Option<&[u8]>, file: Option<&dyn ByteReader>) -> OpAudioSpecificConfig::parse(&asc).ok()}-fn mp4(bytes: &[u8]) -> Option<Vec<u8>> {- let file = filmcraft_isobmff::open(bytes).ok()?;- let track = file.tracks.iter().find(|t| t.kind == filmcraft_isobmff::TrackKind::Audio)?;+/// The track FilmCraft's MP4 source plays: the first audio track with samples, a sample+/// description and a timescale, read from the whole file as it is (a fragmented movie's samples+/// are in its fragments, not the `moov`).+fn mp4(header: &[u8], file: Option<&dyn ByteReader>) -> Option<Vec<u8>> {+ let mp4 = match file {+ Some(f) => filmcraft_isobmff::open(Head(f, f.len())),+ None => filmcraft_isobmff::open(header),+ };+ let mp4 = mp4.ok()?;+ let playable =+ |t: &&filmcraft_isobmff::Track| t.kind == filmcraft_isobmff::TrackKind::Audio && !t.samples.is_empty() && !t.entries.is_empty() && t.timescale > 0;+ let track = mp4.tracks.iter().find(playable)?;match &track.entries.first()?.codec {filmcraft_isobmff::CodecConfig::Aac(a) => Some(a.asc.clone()),_ => None,}}+/// The track FilmCraft's Matroska source plays: the first audio track with frames, found by+/// indexing the clusters as that source does.fn matroska(file: &dyn ByteReader) -> Option<Vec<u8>> {- let opts = filmcraft_matroska::OpenOptions { index: false, ..Default::default() };- let mkv = filmcraft_matroska::open_with(Head(file, file.len()), &opts).ok()?;- match &mkv.tracks.iter().find(|t| t.kind == filmcraft_matroska::TrackKind::Audio)?.codec {+ let mkv = filmcraft_matroska::open(Head(file, file.len())).ok()?;+ match &mkv.tracks.iter().find(|t| t.kind == filmcraft_matroska::TrackKind::Audio && !t.samples.is_empty())?.codec {filmcraft_matroska::Codec::Aac { asc } => Some(asc.clone()),_ => None,}@@ -66,6 +77,15 @@ impl Head<'_> {}}+impl filmcraft_isobmff::ByteSource for Head<'_> {+ fn len(&self) -> u64 {+ self.1+ }+ fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {+ self.read(offset, buf)+ }+}+impl filmcraft_matroska::ByteSource for Head<'_> {fn len(&self) -> u64 {self.1diff --git a/crates/media/tests/it/aac_layout.rs b/crates/media/tests/it/aac_layout.rsindex 9e5ddbb23..c6403667b 100644--- a/crates/media/tests/it/aac_layout.rs+++ b/crates/media/tests/it/aac_layout.rs@@ -109,8 +109,17 @@ fn encode(content: &[Ch], adts: bool) -> (Vec<u8>, Vec<Vec<u8>>) {}fn mp4(asc: Vec<u8>, channels: u32, units: &[Vec<u8>]) -> Vec<u8> {+ mp4_after(None, asc, channels, units)+}++/// As [`mp4`], after an audio track with the configuration and channel count `empty` and no+/// samples, if any.+fn mp4_after(empty: Option<(Vec<u8>, u32)>, asc: Vec<u8>, channels: u32, units: &[Vec<u8>]) -> Vec<u8> {use filmcraft_isobmff::{Brand, Mp4Writer, SampleEntry, TrackConfig, WriteSample, WriterOptions};let mut w = Mp4Writer::new(Cursor::new(Vec::new()), WriterOptions::new(Brand::Mp4)).expect("writer");+ if let Some((asc, channels)) = empty {+ w.add_track(TrackConfig::new(SampleEntry::aac(asc, channels, RATE), RATE)).expect("empty track");+ }let t = w.add_track(TrackConfig::new(SampleEntry::aac(asc, channels, RATE), RATE)).expect("track");for u in units {w.write_sample(t, WriteSample { data: u, duration: 1024, composition_offset: 0, is_sync: true }).expect("sample");@@ -119,14 +128,25 @@ fn mp4(asc: Vec<u8>, channels: u32, units: &[Vec<u8>]) -> Vec<u8> {}fn mkv(asc: Vec<u8>, channels: u32, units: &[Vec<u8>]) -> Vec<u8> {+ mkv_after(None, asc, channels, units)+}++/// As [`mkv`], after an audio track with the configuration and channel count `empty` and no+/// frames, if any.+fn mkv_after(empty: Option<(Vec<u8>, u32)>, asc: Vec<u8>, channels: u32, units: &[Vec<u8>]) -> Vec<u8> {use filmcraft_matroska::{MkvWriter, MuxOptions, TrackKind, TrackSpec};- let mut t = TrackSpec::new(TrackKind::Audio, "A_AAC");- t.codec_private = asc;- t.audio = Some((f64::from(RATE), u64::from(channels), None));- let mut w = MkvWriter::new(Cursor::new(Vec::new()), vec![t], MuxOptions::default()).expect("writer");+ let track = |asc: Vec<u8>, channels: u32| {+ let mut t = TrackSpec::new(TrackKind::Audio, "A_AAC");+ t.codec_private = asc;+ t.audio = Some((f64::from(RATE), u64::from(channels), None));+ t+ };+ let tracks: Vec<TrackSpec> = empty.into_iter().map(|(asc, n)| track(asc, n)).chain([track(asc, channels)]).collect();+ let played = tracks.len() - 1;+ let mut w = MkvWriter::new(Cursor::new(Vec::new()), tracks, MuxOptions::default()).expect("writer");let frame_ns = 1_000_000_000 * 1024 / i64::from(RATE);for (i, u) in units.iter().enumerate() {- w.write_frame(0, i as i64 * frame_ns, true, u, Some(frame_ns as u64)).expect("frame");+ w.write_frame(played, i as i64 * frame_ns, true, u, Some(frame_ns as u64)).expect("frame");}w.finish().expect("finish").into_inner()}@@ -222,3 +242,21 @@ fn adts_aac_layouts_mix_by_speaker() {check(case.name, case.content, &mix(&MediaPool::new(), &probe(&p).expect("probe")));}}++/// Audio plays from the first track FilmCraft can play, so an audio track with no samples before+/// it lends it no configuration. Each case decodes in another order than the empty track's+/// configuration would (FilmCraft reorders 12 to WAVE / SMPTE but not 7, and 11 but not a PCE).+#[test]+fn an_empty_audio_track_lends_its_configuration_to_no_other() {+ for (name, empty) in [("config12", 7), ("pce61-centre-first", 11)] {+ let case = cases().into_iter().find(|c| c.name == name).expect("case");+ let n = case.content.len() as u32;+ let empty = Some((AudioSpecificConfig::lc(3, empty).to_bytes(), n));+ let (asc, units) = encode(case.content, false);+ let asc = case.asc.as_ref().map_or(asc, AudioSpecificConfig::to_bytes);+ let p = write(&format!("{name}-after-empty.mp4"), &mp4_after(empty.clone(), asc.clone(), n, &units));+ check(&format!("{name} MP4"), case.content, &mix(&MediaPool::new(), &probe(&p).expect("probe")));+ let p = write(&format!("{name}-after-empty.mkv"), &mkv_after(empty, asc, n, &units));+ check(&format!("{name} Matroska"), case.content, &mix(&MediaPool::new(), &probe(&p).expect("probe")));+ }+}