Skip to content

The animation engine for agents: blazingly fast math and 3D videos with Manim's Python API, on a Rust GPU core.

Python
22.9 KB
Raw Blame
//! MP4 (ISO/IEC 14496-12/14/15) holding an H.264 track and, if the film has sound, an AAC-LC track;
//! faststart.
//!
//! A video sample is a picture as x264 writes it (NAL units, each prefixed by its 4-byte size) and a
//! held picture is one sample that lasts longer — times are in frames (timescale = fps). With
//! B-frames (the slower presets), a picture can be decoded before the one it follows: the samples
//! are in decode order, each retaining its duration; signed composition offsets give its
//! presentation time. An audio
//! sample is an AAC access unit of 1024 samples; the encoder's priming before the sound's first
//! sample is skipped by an edit list, so the sound starts with the film, to the sample. Video
//! samples stream to a temporary file; `finish` writes the header (`moov`), then the samples,
//! a second of video and a second of sound at a time, so a player can start before the file has
//! arrived.

use std::fs::File;
use std::io::{self, BufReader, BufWriter, Read, Write};
use std::ops::Range;

pub struct Mp4 {
    path: String,
    part: String,
    data: Option<BufWriter<File>>,
    width: u32,
    height: u32,
    fps: u32,
    sps: Vec<u8>,
    pps: Vec<u8>,
    samples: Vec<Sample>,
    bytes: u64,
}

struct Sample {
    size: u32,
    pts: i64,
    duration: u32,
    key: bool,
}

/// A film's sound, encoded: AAC-LC access units of 1024 samples each, in order.
pub struct Audio {
    pub rate: u32,
    pub channels: u16,
    /// The AudioSpecificConfig (the decoder's configuration).
    pub config: Vec<u8>,
    /// Samples the encoder put before the sound's first (hidden by the edit list).
    pub priming: u32,
    /// The sound's own samples, after the priming: what plays.
    pub length: u64,
    pub units: Vec<Vec<u8>>,
}

/// Where each track's chunks go in `mdat`, in file order: (audio?, the samples it holds).
type Layout = Vec<(bool, Range<usize>)>;

impl Mp4 {
    /// `sps`, `pps`: the stream's parameter sets, without size prefixes.
    pub fn new(path: &str, width: u32, height: u32, fps: u32, sps: Vec<u8>, pps: Vec<u8>) -> io::Result<Self> {
        if sps.len() < 4 || pps.is_empty() {
            return Err(io::Error::new(io::ErrorKind::InvalidData, "H.264 parameter sets are missing"));
        }
        let part = format!("{path}.part");
        let data = BufWriter::with_capacity(1 << 20, File::create(&part)?);
        Ok(Self { path: path.to_string(), part, data: Some(data), width, height, fps, sps, pps, samples: Vec::new(), bytes: 0 })
    }

    pub fn sample(&mut self, data: &[u8], pts: i64, key: bool) -> io::Result<()> {
        if data.is_empty() {
            return Ok(());
        }
        self.data.as_mut().ok_or_else(|| io::Error::other("the video is finished"))?.write_all(data)?;
        self.bytes += data.len() as u64;
        self.samples.push(Sample { size: data.len() as u32, pts, duration: 0, key });
        Ok(())
    }

    /// Write the file: the video lasts until `end` (in frames — the last hold included), with
    /// `audio` beside it, if any. A file that could not be written whole is not left behind.
    pub fn finish(mut self, end: i64, audio: Option<&Audio>) -> io::Result<u64> {
        if let Some(mut data) = self.data.take() {
            data.flush()?;
        }
        // Recover each picture's authored duration in presentation order, retaining the
        // encoder's sample order. In particular, the final displayed picture owns the hold,
        // even when B-frames follow it in decode order.
        let mut order: Vec<_> = (0..self.samples.len()).collect();
        order.sort_unstable_by_key(|&i| self.samples[i].pts);
        let mut until = end;
        for i in order.into_iter().rev() {
            self.samples[i].duration = (until - self.samples[i].pts) as u32;
            until = self.samples[i].pts;
        }
        let written = self.write(end, audio);
        if written.is_err() {
            let _ = std::fs::remove_file(&self.path);
        }
        written
    }

    /// The chunks: the whole video in one without sound; with sound, a second of each in turn.
    fn layout(&self, audio: Option<&Audio>) -> Layout {
        let Some(audio) = audio else {
            return vec![(false, 0..self.samples.len())];
        };
        // A picture must arrive before its own presentation and every later picture in
        // decode order: any of them may need it as a reference. Signed composition offsets
        // make the sample-duration clock unsuitable as a progressive-download deadline.
        let mut deadlines: Vec<_> = self.samples.iter().map(|s| s.pts).collect();
        for i in (1..deadlines.len()).rev() {
            deadlines[i - 1] = deadlines[i - 1].min(deadlines[i]);
        }
        let (mut chunks, mut v, mut a, mut second) = (Vec::new(), 0, 0, 1i64);
        while v < self.samples.len() || a < audio.units.len() {
            let v0 = v;
            while v < self.samples.len() && deadlines[v] < second * self.fps as i64 {
                v += 1;
            }
            if v > v0 {
                chunks.push((false, v0..v));
            }
            let a0 = a;
            while a < audio.units.len() && (a as i64 * 1024) < second * audio.rate as i64 {
                a += 1;
            }
            if a > a0 {
                chunks.push((true, a0..a));
            }
            second += 1;
        }
        chunks
    }

    fn write(&self, end: i64, audio: Option<&Audio>) -> io::Result<u64> {
        let sound: u64 = audio.map_or(0, |a| a.units.iter().map(|u| u.len() as u64).sum());
        let body = self.bytes + sound;
        let wide = body + 16 > u32::MAX as u64;
        let header = if wide { 16 } else { 8 };
        let layout = self.layout(audio);
        let ftyp = ftyp();
        let probe = self.moov(end, audio, &layout, 0, wide)?;
        let start = (ftyp.len() + probe.len() + header) as u64;
        let moov = self.moov(end, audio, &layout, start, wide)?;
        let mut out = BufWriter::with_capacity(1 << 20, File::create(&self.path)?);
        out.write_all(&ftyp)?;
        out.write_all(&moov)?;
        if wide {
            out.write_all(&1u32.to_be_bytes())?;
            out.write_all(b"mdat")?;
            out.write_all(&(body + 16).to_be_bytes())?;
        } else {
            out.write_all(&((body + 8) as u32).to_be_bytes())?;
            out.write_all(b"mdat")?;
        }
        let mut video = BufReader::with_capacity(1 << 20, File::open(&self.part)?);
        for (is_audio, range) in &layout {
            if *is_audio {
                for unit in &audio.expect("audio chunks come with audio").units[range.clone()] {
                    out.write_all(unit)?;
                }
            } else {
                let bytes: u64 = self.samples[range.clone()].iter().map(|s| s.size as u64).sum();
                io::copy(&mut (&mut video).take(bytes), &mut out)?;
            }
        }
        out.flush()?;
        Ok((ftyp.len() + moov.len() + header) as u64 + body)
    }

    /// Each track's chunks: (their offsets in the file, the samples each holds).
    fn chunks(&self, audio: Option<&Audio>, layout: &Layout, offset: u64) -> [(Vec<u64>, Vec<u32>); 2] {
        let mut tracks: [(Vec<u64>, Vec<u32>); 2] = Default::default();
        let mut at = offset;
        for (is_audio, range) in layout {
            let size: u64 = if *is_audio {
                audio.map_or(0, |a| a.units[range.clone()].iter().map(|u| u.len() as u64).sum())
            } else {
                self.samples[range.clone()].iter().map(|s| s.size as u64).sum()
            };
            let track = &mut tracks[*is_audio as usize];
            track.0.push(at);
            track.1.push(range.len() as u32);
            at += size;
        }
        tracks
    }

    /// The header, with the samples starting at byte `offset` of the file.
    fn moov(&self, end: i64, audio: Option<&Audio>, layout: &Layout, offset: u64, wide: bool) -> io::Result<Vec<u8>> {
        // ISO/IEC 14496-12: decoding timestamps order samples; composition timestamps say
        // when they appear. Accumulate their own durations in decode order and preserve every
        // authored presentation time with signed ctts offsets. This also gives the last
        // displayed picture its proper duration, without a decode-preroll edit or extra frames.
        let duration = end.max(1) as u32;
        let deltas: Vec<_> = self.samples.iter().map(|s| s.duration).collect();
        let mut decode = 0;
        let offsets: Vec<_> = self.samples.iter().map(|s| {
            let offset = i32::try_from(s.pts - decode)
                .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "MP4 composition offset exceeds 32-bit range"))?;
            decode += s.duration as i64;
            Ok(offset as u32)
        }).collect::<io::Result<_>>()?;
        let [(video_at, video_counts), (audio_at, audio_counts)] = self.chunks(audio, layout, offset);
        // the sound's length in the movie's timescale (frames)
        let heard = audio.map_or(0, |a| (a.length * self.fps as u64).div_ceil(a.rate as u64) as u32);
        let mut w = Boxes::default();
        w.boxed(b"moov", |w| {
            w.full(b"mvhd", 0, 0, |w| {
                w.u32(0);
                w.u32(0);
                w.u32(self.fps);
                w.u32(duration.max(heard));
                w.u32(0x0001_0000);
                w.u16(0x0100);
                w.zeros(10);
                w.matrix();
                w.zeros(24);
                w.u32(if audio.is_some() { 3 } else { 2 });
            });
            w.boxed(b"trak", |w| {
                w.full(b"tkhd", 0, 3, |w| {
                    w.u32(0);
                    w.u32(0);
                    w.u32(1);
                    w.u32(0);
                    w.u32(duration);
                    w.zeros(8);
                    w.u16(0);
                    w.u16(0);
                    w.u16(0);
                    w.u16(0);
                    w.matrix();
                    w.u32(self.width << 16);
                    w.u32(self.height << 16);
                });
                w.boxed(b"mdia", |w| {
                    w.full(b"mdhd", 0, 0, |w| {
                        w.u32(0);
                        w.u32(0);
                        w.u32(self.fps);
                        w.u32(duration);
                        w.u16(0x55c4); // "und"
                        w.u16(0);
                    });
                    w.full(b"hdlr", 0, 0, |w| {
                        w.u32(0);
                        w.bytes(b"vide");
                        w.zeros(12);
                        w.bytes(b"VideoHandler\0");
                    });
                    w.boxed(b"minf", |w| {
                        w.full(b"vmhd", 0, 1, |w| w.zeros(8));
                        w.dinf();
                        w.boxed(b"stbl", |w| {
                            w.full(b"stsd", 0, 0, |w| {
                                w.u32(1);
                                self.avc1(w);
                            });
                            w.full(b"stts", 0, 0, |w| w.runs(&deltas));
                            if offsets.iter().any(|&o| o != 0) {
                                w.full(b"ctts", 1, 0, |w| w.runs(&offsets));
                            }
                            if self.samples.iter().any(|s| !s.key) {
                                let keys: Vec<u32> = (1..).zip(&self.samples).filter(|(_, s)| s.key).map(|(k, _)| k).collect();
                                w.full(b"stss", 0, 0, |w| {
                                    w.u32(keys.len() as u32);
                                    keys.iter().for_each(|&k| w.u32(k));
                                });
                            }
                            w.stsc(&video_counts);
                            w.full(b"stsz", 0, 0, |w| {
                                w.u32(0);
                                w.u32(self.samples.len() as u32);
                                self.samples.iter().for_each(|s| w.u32(s.size));
                            });
                            w.stco(&video_at, wide);
                        });
                    });
                });
            });
            if let Some(audio) = audio {
                self.sound(w, audio, heard, &audio_counts, &audio_at, wide);
            }
        });
        Ok(w.0)
    }

    /// The sound's track: AAC-LC, its priming cut by an edit list, a unit of pre-roll.
    fn sound(&self, w: &mut Boxes, audio: &Audio, heard: u32, counts: &[u32], at: &[u64], wide: bool) {
        let units = audio.units.len() as u32;
        let bitrate = (audio.units.iter().map(|u| u.len() as u64).sum::<u64>() * 8 * audio.rate as u64 / (units.max(1) as u64 * 1024)) as u32;
        w.boxed(b"trak", |w| {
            w.full(b"tkhd", 0, 3, |w| {
                w.u32(0);
                w.u32(0);
                w.u32(2);
                w.u32(0);
                w.u32(heard);
                w.zeros(8);
                w.u16(0);
                w.u16(1); // alternate group: sound
                w.u16(0x0100); // volume
                w.u16(0);
                w.matrix();
                w.u32(0);
                w.u32(0);
            });
            w.boxed(b"edts", |w| {
                w.full(b"elst", 0, 0, |w| {
                    w.u32(1);
                    w.u32(heard); // in the movie's timescale
                    w.u32(audio.priming); // in the track's: where the sound's first sample is
                    w.u16(1);
                    w.u16(0);
                });
            });
            w.boxed(b"mdia", |w| {
                w.full(b"mdhd", 0, 0, |w| {
                    w.u32(0);
                    w.u32(0);
                    w.u32(audio.rate);
                    w.u32(units * 1024);
                    w.u16(0x55c4); // "und"
                    w.u16(0);
                });
                w.full(b"hdlr", 0, 0, |w| {
                    w.u32(0);
                    w.bytes(b"soun");
                    w.zeros(12);
                    w.bytes(b"SoundHandler\0");
                });
                w.boxed(b"minf", |w| {
                    w.full(b"smhd", 0, 0, |w| w.u32(0));
                    w.dinf();
                    w.boxed(b"stbl", |w| {
                        w.full(b"stsd", 0, 0, |w| {
                            w.u32(1);
                            w.boxed(b"mp4a", |w| {
                                w.zeros(6);
                                w.u16(1); // data reference
                                w.zeros(8);
                                w.u16(audio.channels);
                                w.u16(16);
                                w.u32(0);
                                w.u32(audio.rate << 16);
                                w.full(b"esds", 0, 0, |w| {
                                    let config = &audio.config;
                                    // ES_Descriptor ⊃ DecoderConfigDescriptor ⊃ DecoderSpecificInfo;
                                    // SLConfigDescriptor
                                    w.u8(0x03);
                                    w.u8((3 + 2 + 13 + 2 + config.len() + 3) as u8);
                                    w.u16(0);
                                    w.u8(0);
                                    w.u8(0x04);
                                    w.u8((13 + 2 + config.len()) as u8);
                                    w.u8(0x40); // MPEG-4 audio
                                    w.u8(0x15); // audio stream
                                    w.bytes(&[0, 0x18, 0]); // buffer size: 6144 bytes
                                    w.u32(bitrate);
                                    w.u32(bitrate);
                                    w.u8(0x05);
                                    w.u8(config.len() as u8);
                                    w.bytes(config);
                                    w.u8(0x06);
                                    w.u8(1);
                                    w.u8(0x02);
                                });
                            });
                        });
                        w.full(b"stts", 0, 0, |w| {
                            w.u32(1);
                            w.u32(units);
                            w.u32(1024);
                        });
                        w.stsc(counts);
                        w.full(b"stsz", 0, 0, |w| {
                            w.u32(0);
                            w.u32(units);
                            audio.units.iter().for_each(|u| w.u32(u.len() as u32));
                        });
                        w.stco(at, wide);
                        // a decoder needs the unit before any it starts at
                        w.full(b"sgpd", 1, 0, |w| {
                            w.bytes(b"roll");
                            w.u32(2);
                            w.u32(1);
                            w.u16(0xffff); // -1
                        });
                        w.full(b"sbgp", 0, 0, |w| {
                            w.bytes(b"roll");
                            w.u32(1);
                            w.u32(units);
                            w.u32(1);
                        });
                    });
                });
            });
        });
    }

    fn avc1(&self, w: &mut Boxes) {
        w.boxed(b"avc1", |w| {
            w.zeros(6);
            w.u16(1); // data reference
            w.zeros(16);
            w.u16(self.width as u16);
            w.u16(self.height as u16);
            w.u32(0x0048_0000);
            w.u32(0x0048_0000);
            w.u32(0);
            w.u16(1); // frames per sample
            let name = b"manimgx";
            w.u8(name.len() as u8);
            w.bytes(name);
            w.zeros(31 - name.len());
            w.u16(0x0018);
            w.u16(0xffff);
            w.boxed(b"avcC", |w| {
                let (sps, pps) = (&self.sps, &self.pps);
                w.u8(1);
                w.u8(sps[1]); // profile
                w.u8(sps[2]); // constraints
                w.u8(sps[3]); // level
                w.u8(0xff); // 4-byte NAL sizes
                w.u8(0xe1); // one SPS
                w.u16(sps.len() as u16);
                w.bytes(sps);
                w.u8(1); // one PPS
                w.u16(pps.len() as u16);
                w.bytes(pps);
                if matches!(sps[1], 100 | 110 | 122 | 144) {
                    w.u8(0xfd); // 4:2:0
                    w.u8(0xf8); // 8-bit luma
                    w.u8(0xf8); // 8-bit chroma
                    w.u8(0);
                }
            });
        });
    }
}

impl Drop for Mp4 {
    /// The samples' file is temporary: finished or not, it goes (closed first, so that it can).
    fn drop(&mut self) {
        self.data = None;
        let _ = std::fs::remove_file(&self.part);
    }
}

fn ftyp() -> Vec<u8> {
    let mut w = Boxes::default();
    w.boxed(b"ftyp", |w| {
        w.bytes(b"iso4"); // signed composition offsets (ctts version 1)
        w.u32(0x200);
        for brand in [b"iso4", b"isom", b"iso2", b"avc1", b"mp41"] {
            w.bytes(brand);
        }
    });
    w.0
}

#[derive(Default)]
struct Boxes(Vec<u8>);

impl Boxes {
    fn u8(&mut self, v: u8) {
        self.0.push(v);
    }
    fn u16(&mut self, v: u16) {
        self.0.extend_from_slice(&v.to_be_bytes());
    }
    fn u32(&mut self, v: u32) {
        self.0.extend_from_slice(&v.to_be_bytes());
    }
    fn u64(&mut self, v: u64) {
        self.0.extend_from_slice(&v.to_be_bytes());
    }
    fn bytes(&mut self, b: &[u8]) {
        self.0.extend_from_slice(b);
    }
    fn zeros(&mut self, n: usize) {
        self.0.resize(self.0.len() + n, 0);
    }
    fn matrix(&mut self) {
        for v in [0x0001_0000, 0, 0, 0, 0x0001_0000, 0, 0, 0, 0x4000_0000u32] {
            self.u32(v);
        }
    }
    fn dinf(&mut self) {
        self.boxed(b"dinf", |w| {
            w.full(b"dref", 0, 0, |w| {
                w.u32(1);
                w.full(b"url ", 0, 1, |_| {});
            })
        });
    }
    /// How many samples a track's chunks hold, run-length: (first chunk, samples each).
    fn stsc(&mut self, counts: &[u32]) {
        self.full(b"stsc", 0, 0, |w| {
            let runs: Vec<(u32, u32)> = (1..).zip(counts).filter(|&(k, &n)| k == 1 || counts[k as usize - 2] != n).map(|(k, &n)| (k, n)).collect();
            w.u32(runs.len() as u32);
            for (first, n) in runs {
                w.u32(first);
                w.u32(n);
                w.u32(1);
            }
        });
    }
    /// Where a track's chunks start in the file.
    fn stco(&mut self, at: &[u64], wide: bool) {
        if wide {
            self.full(b"co64", 0, 0, |w| {
                w.u32(at.len() as u32);
                at.iter().for_each(|&o| w.u64(o));
            });
        } else {
            self.full(b"stco", 0, 0, |w| {
                w.u32(at.len() as u32);
                at.iter().for_each(|&o| w.u32(o as u32));
            });
        }
    }
    /// Run-length (count, value) entries.
    fn runs(&mut self, values: &[u32]) {
        let mut runs: Vec<(u32, u32)> = Vec::new();
        for &v in values {
            match runs.last_mut() {
                Some((n, last)) if *last == v => *n += 1,
                _ => runs.push((1, v)),
            }
        }
        self.u32(runs.len() as u32);
        for (n, v) in runs {
            self.u32(n);
            self.u32(v);
        }
    }
    fn boxed(&mut self, kind: &[u8; 4], body: impl FnOnce(&mut Self)) {
        let at = self.0.len();
        self.u32(0);
        self.bytes(kind);
        body(self);
        let size = (self.0.len() - at) as u32;
        self.0[at..at + 4].copy_from_slice(&size.to_be_bytes());
    }
    fn full(&mut self, kind: &[u8; 4], version: u8, flags: u32, body: impl FnOnce(&mut Self)) {
        self.boxed(kind, |w| {
            w.u32((version as u32) << 24 | flags);
            body(w);
        });
    }
}


#[cfg(test)]
mod tests {
    use super::*;

    fn movie(pictures: &[(i64, u32)]) -> Mp4 {
        Mp4 {
            path: String::new(), part: String::new(), data: None,
            width: 2, height: 2, fps: 1, sps: vec![0; 4], pps: vec![0], bytes: 0,
            samples: pictures.iter().map(|&(pts, duration)| Sample { size: 0, pts, duration, key: true }).collect(),
        }
    }

    #[test]
    fn progressive_chunks_deliver_references_before_their_earliest_picture() {
        // The P-picture shown at 10s is needed to decode the B-picture shown at 1s.
        // Its 20s hold must not put that B-picture behind 21s of audio.
        let movie = movie(&[(0, 1), (10, 20), (1, 9)]);
        let audio = Audio { rate: 1024, channels: 1, config: vec![], priming: 0, length: 30 * 1024, units: vec![vec![]; 30] };
        let chunks = movie.layout(Some(&audio));
        assert_eq!(&chunks[..4], &[(false, 0..1), (true, 0..1), (false, 1..3), (true, 1..2)]);
        assert!(chunks[4..].iter().all(|(sound, _)| *sound));
    }

    #[test]
    fn composition_offsets_cannot_wrap() {
        let movie = movie(&[(0, 1), (i32::MAX as i64 + 2, 1), (1, i32::MAX as u32 + 1)]);
        let error = movie.moov(i32::MAX as i64 + 3, None, &vec![(false, 0..3)], 0, false).unwrap_err();
        assert_eq!(error.kind(), io::ErrorKind::InvalidInput);
        assert!(error.to_string().contains("composition offset"));
    }
}