Skip to content

EffectCraft: An open-source motion graphics and compositing tool written by scratch in pure Rust

Media: read the AAC configuration of the MP4 or Matroska track FilmCraft plays, not an empty one before it (#1701)

tylerjenningsw800b555e514e
diff --git a/crates/media/src/aac.rs b/crates/media/src/aac.rs
index a2d7de37b..cd33eaf7b 100644
--- a/crates/media/src/aac.rs
+++ b/crates/media/src/aac.rs
@@ -1,4 +1,4 @@
-//! The AudioSpecificConfig a movie's first AAC track decodes with, from the containers that carry
+//! The AudioSpecificConfig of the AAC track FilmCraft plays from a movie, from the containers that carry
//! it outside the audio frames: MP4 / MOV (`esds`), Matroska (`CodecPrivate`) and LATM in MPEG
//! TS / PS (its StreamMuxConfig). FilmCraft's other AAC is ADTS (in MPEG TS / PS or a raw `.aac`),
//! whose channel configurations FilmCraft decodes (1 to 7) its channel count tells apart
@@ -13,10 +13,11 @@ use filmcraft_media::ByteReader;
/// as far as this, not through the whole file.
const MPEG_HEAD: u64 = 8 << 20;
-/// The configuration of the first audio track, if it is AAC: from an MP4 / MOV `header` (its
-/// ftyp and moov, or the whole file), or else from `file`, read as Matroska or MPEG TS / PS.
+/// The configuration of the audio track FilmCraft plays, if it is AAC: from an MP4 / MOV `header`
+/// (its ftyp and moov, or the whole file) and `file`, or else from `file`, read as Matroska or
+/// MPEG TS / PS.
pub(crate) fn config(header: Option<&[u8]>, file: Option<&dyn ByteReader>) -> Option<AudioSpecificConfig> {
- let asc = match header.and_then(mp4) {
+ let asc = match header.and_then(|h| mp4(h, file)) {
Some(asc) => asc,
None => {
let file = file?;
@@ -28,19 +29,29 @@ pub(crate) fn config(header: Option<&[u8]>, file: Option<&dyn ByteReader>) -> Op
AudioSpecificConfig::parse(&asc).ok()
}
-fn mp4(bytes: &[u8]) -> Option<Vec<u8>> {
- let file = filmcraft_isobmff::open(bytes).ok()?;
- let track = file.tracks.iter().find(|t| t.kind == filmcraft_isobmff::TrackKind::Audio)?;
+/// The track FilmCraft's MP4 source plays: the first audio track with samples, a sample
+/// description and a timescale, read from the whole file as it is (a fragmented movie's samples
+/// are in its fragments, not the `moov`).
+fn mp4(header: &[u8], file: Option<&dyn ByteReader>) -> Option<Vec<u8>> {
+ let mp4 = match file {
+ Some(f) => filmcraft_isobmff::open(Head(f, f.len())),
+ None => filmcraft_isobmff::open(header),
+ };
+ let mp4 = mp4.ok()?;
+ let playable =
+ |t: &&filmcraft_isobmff::Track| t.kind == filmcraft_isobmff::TrackKind::Audio && !t.samples.is_empty() && !t.entries.is_empty() && t.timescale > 0;
+ let track = mp4.tracks.iter().find(playable)?;
match &track.entries.first()?.codec {
filmcraft_isobmff::CodecConfig::Aac(a) => Some(a.asc.clone()),
_ => None,
}
}
+/// The track FilmCraft's Matroska source plays: the first audio track with frames, found by
+/// indexing the clusters as that source does.
fn matroska(file: &dyn ByteReader) -> Option<Vec<u8>> {
- let opts = filmcraft_matroska::OpenOptions { index: false, ..Default::default() };
- let mkv = filmcraft_matroska::open_with(Head(file, file.len()), &opts).ok()?;
- match &mkv.tracks.iter().find(|t| t.kind == filmcraft_matroska::TrackKind::Audio)?.codec {
+ let mkv = filmcraft_matroska::open(Head(file, file.len())).ok()?;
+ match &mkv.tracks.iter().find(|t| t.kind == filmcraft_matroska::TrackKind::Audio && !t.samples.is_empty())?.codec {
filmcraft_matroska::Codec::Aac { asc } => Some(asc.clone()),
_ => None,
}
@@ -66,6 +77,15 @@ impl Head<'_> {
}
}
+impl filmcraft_isobmff::ByteSource for Head<'_> {
+ fn len(&self) -> u64 {
+ self.1
+ }
+ fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {
+ self.read(offset, buf)
+ }
+}
+
impl filmcraft_matroska::ByteSource for Head<'_> {
fn len(&self) -> u64 {
self.1
diff --git a/crates/media/tests/it/aac_layout.rs b/crates/media/tests/it/aac_layout.rs
index 9e5ddbb23..c6403667b 100644
--- a/crates/media/tests/it/aac_layout.rs
+++ b/crates/media/tests/it/aac_layout.rs
@@ -109,8 +109,17 @@ fn encode(content: &[Ch], adts: bool) -> (Vec<u8>, Vec<Vec<u8>>) {
}
fn mp4(asc: Vec<u8>, channels: u32, units: &[Vec<u8>]) -> Vec<u8> {
+ mp4_after(None, asc, channels, units)
+}
+
+/// As [`mp4`], after an audio track with the configuration and channel count `empty` and no
+/// samples, if any.
+fn mp4_after(empty: Option<(Vec<u8>, u32)>, asc: Vec<u8>, channels: u32, units: &[Vec<u8>]) -> Vec<u8> {
use filmcraft_isobmff::{Brand, Mp4Writer, SampleEntry, TrackConfig, WriteSample, WriterOptions};
let mut w = Mp4Writer::new(Cursor::new(Vec::new()), WriterOptions::new(Brand::Mp4)).expect("writer");
+ if let Some((asc, channels)) = empty {
+ w.add_track(TrackConfig::new(SampleEntry::aac(asc, channels, RATE), RATE)).expect("empty track");
+ }
let t = w.add_track(TrackConfig::new(SampleEntry::aac(asc, channels, RATE), RATE)).expect("track");
for u in units {
w.write_sample(t, WriteSample { data: u, duration: 1024, composition_offset: 0, is_sync: true }).expect("sample");
@@ -119,14 +128,25 @@ fn mp4(asc: Vec<u8>, channels: u32, units: &[Vec<u8>]) -> Vec<u8> {
}
fn mkv(asc: Vec<u8>, channels: u32, units: &[Vec<u8>]) -> Vec<u8> {
+ mkv_after(None, asc, channels, units)
+}
+
+/// As [`mkv`], after an audio track with the configuration and channel count `empty` and no
+/// frames, if any.
+fn mkv_after(empty: Option<(Vec<u8>, u32)>, asc: Vec<u8>, channels: u32, units: &[Vec<u8>]) -> Vec<u8> {
use filmcraft_matroska::{MkvWriter, MuxOptions, TrackKind, TrackSpec};
- let mut t = TrackSpec::new(TrackKind::Audio, "A_AAC");
- t.codec_private = asc;
- t.audio = Some((f64::from(RATE), u64::from(channels), None));
- let mut w = MkvWriter::new(Cursor::new(Vec::new()), vec![t], MuxOptions::default()).expect("writer");
+ let track = |asc: Vec<u8>, channels: u32| {
+ let mut t = TrackSpec::new(TrackKind::Audio, "A_AAC");
+ t.codec_private = asc;
+ t.audio = Some((f64::from(RATE), u64::from(channels), None));
+ t
+ };
+ let tracks: Vec<TrackSpec> = empty.into_iter().map(|(asc, n)| track(asc, n)).chain([track(asc, channels)]).collect();
+ let played = tracks.len() - 1;
+ let mut w = MkvWriter::new(Cursor::new(Vec::new()), tracks, MuxOptions::default()).expect("writer");
let frame_ns = 1_000_000_000 * 1024 / i64::from(RATE);
for (i, u) in units.iter().enumerate() {
- w.write_frame(0, i as i64 * frame_ns, true, u, Some(frame_ns as u64)).expect("frame");
+ w.write_frame(played, i as i64 * frame_ns, true, u, Some(frame_ns as u64)).expect("frame");
}
w.finish().expect("finish").into_inner()
}
@@ -222,3 +242,21 @@ fn adts_aac_layouts_mix_by_speaker() {
check(case.name, case.content, &mix(&MediaPool::new(), &probe(&p).expect("probe")));
}
}
+
+/// Audio plays from the first track FilmCraft can play, so an audio track with no samples before
+/// it lends it no configuration. Each case decodes in another order than the empty track's
+/// configuration would (FilmCraft reorders 12 to WAVE / SMPTE but not 7, and 11 but not a PCE).
+#[test]
+fn an_empty_audio_track_lends_its_configuration_to_no_other() {
+ for (name, empty) in [("config12", 7), ("pce61-centre-first", 11)] {
+ let case = cases().into_iter().find(|c| c.name == name).expect("case");
+ let n = case.content.len() as u32;
+ let empty = Some((AudioSpecificConfig::lc(3, empty).to_bytes(), n));
+ let (asc, units) = encode(case.content, false);
+ let asc = case.asc.as_ref().map_or(asc, AudioSpecificConfig::to_bytes);
+ let p = write(&format!("{name}-after-empty.mp4"), &mp4_after(empty.clone(), asc.clone(), n, &units));
+ check(&format!("{name} MP4"), case.content, &mix(&MediaPool::new(), &probe(&p).expect("probe")));
+ let p = write(&format!("{name}-after-empty.mkv"), &mkv_after(empty, asc, n, &units));
+ check(&format!("{name} Matroska"), case.content, &mix(&MediaPool::new(), &probe(&p).expect("probe")));
+ }
+}