Skip to content

The animation engine for agents: blazingly fast math and 3D videos with Manim's Python API, on a Rust GPU core.

Python

Mux video and audio to MP4

apoogdk990c73de7dcb
diff --git a/rust/engine/src/export.rs b/rust/engine/src/export.rs
new file mode 100644
index 00000000..6d6f4ff0
--- /dev/null
+++ b/rust/engine/src/export.rs
@@ -0,0 +1,248 @@
+//! A film's video, natively: each frame drawn, converted to NV12 on the GPU where it changed,
+//! read back and handed to x264 on a thread of its own (see `encode`), muxed into an MP4.
+
+use std::collections::VecDeque;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::time::Instant;
+
+use crate::{encode, mp4};
+use wgpu::util::DeviceExt;
+
+use crate::render::{COUNT_BYTES, Camera, Frame, Gpu, Player, count};
+
+impl Player {
+ /// The macroblocks whose pixels can differ from the previous frame of the same view: the old
+ /// and new footprints of every record that changed. Anything else — a new camera, objects
+ /// added or removed, what cannot be bounded — is the whole frame.
+ fn damage(&self, previous: Option<&Frame>, frame: &Frame, columns: usize, rows: usize) -> Vec<u8> {
+ let everything = vec![1u8; columns * rows];
+ let Some(previous) = previous else { return everything };
+ if previous.view != frame.view || previous.records.len() != frame.records.len() {
+ return everything;
+ }
+ let camera = Camera::new(&frame.view);
+ let mut dirty = vec![0u8; columns * rows];
+ for (old, new) in previous.records.iter().zip(&frame.records) {
+ if bytemuck::bytes_of(old) == bytemuck::bytes_of(new) {
+ continue;
+ }
+ for r in [old, new] {
+ let Some([x0, y0, x1, y1]) = self.store.footprint(&camera, r) else { return everything };
+ if x1 < 0.0 || y1 < 0.0 || x0 >= camera.size[0] || y0 >= camera.size[1] {
+ continue; // off screen
+ }
+ let (c0, c1) = ((x0.max(0.0) / 16.0) as usize, ((x1 / 16.0) as usize).min(columns - 1));
+ let (r0, r1) = ((y0.max(0.0) / 16.0) as usize, ((y1 / 16.0) as usize).min(rows - 1));
+ for row in r0..=r1 {
+ dirty[row * columns + c0..=row * columns + c1].fill(1);
+ }
+ }
+ }
+ dirty
+ }
+
+ pub(crate) fn push_frames(&mut self, gpu: &mut Gpu, frames: Vec<Frame>, repeat: u32, key: bool) -> Result<(), String> {
+ let mut export = self.export.take().ok_or("no export is open")?;
+ let result = self.push_into(gpu, &mut export, frames, repeat, key);
+ self.export = Some(export);
+ result
+ }
+
+ /// Draw a frame into the export and convert only the macroblocks a change can touch — every
+ /// one when camera views are drawn too (their pictures change with no record changing).
+ fn push_into(&mut self, gpu: &mut Gpu, export: &mut Export, frames: Vec<Frame>, repeat: u32, key: bool) -> Result<(), String> {
+ let (columns, rows) = (export.width.div_ceil(16), export.height.div_ceil(16));
+ let total = columns * rows;
+ if export.pending.len() == RING {
+ self.retire(gpu, export)?;
+ }
+ let slot = export.drawn % RING;
+ let frame = frames.last().ok_or("no frame")?;
+ let (mut command, grouped, composited) = self.encode(gpu, &frames)?;
+ // a frame composited in groups settles the pixels two groups share at the seam between
+ // them, which a change anywhere can move
+ let whole = frames.len() > 1 || grouped || export.grouped;
+ export.grouped = grouped;
+ let dirty = if whole { vec![1u8; total] } else { self.damage(export.previous.as_ref(), frame, columns, rows) };
+ export.converted += dirty.iter().filter(|&&d| d != 0).count();
+ let listed: Vec<u32> = (0..total as u32).filter(|&k| dirty[k as usize] != 0).collect();
+ // a frame whose see-through fragments may overflow the lists is kept until its count is
+ // read (see `retire`)
+ let lists = self.lists.as_ref().filter(|_| composited && !listed.is_empty()).map(|l| l.capacity);
+ self.convert(gpu, export, &mut command, &listed, slot, lists.is_some());
+ gpu.queue.submit(Some(command.finish()));
+ export.map(slot, listed.len());
+ gpu.device.poll(wgpu::PollType::Poll).map_err(|e| e.to_string())?;
+ let first = export.previous.is_none();
+ let kept = lists.map(|capacity| (frames.clone(), capacity));
+ export.pending.push_back(Pending { slot, listed, repeat, first, key, kept });
+ export.previous = frames.into_iter().last();
+ export.drawn += 1;
+ Ok(())
+ }
+
+ /// Convert the listed macroblocks of the drawn frame into the export's `slot`, followed by
+ /// the lists' count when `counted`.
+ fn convert(&self, gpu: &Gpu, export: &Export, command: &mut wgpu::CommandEncoder, listed: &[u32], slot: usize, counted: bool) {
+ if listed.is_empty() {
+ return;
+ }
+ let columns = export.width.div_ceil(16);
+ let list = gpu.device.create_buffer_init(&wgpu::util::BufferInitDescriptor { label: Some("listed"), contents: bytemuck::cast_slice(listed), usage: wgpu::BufferUsages::STORAGE });
+ gpu.queue.write_buffer(&export.grid, 0, bytemuck::cast_slice(&[columns as u32, listed.len() as u32, 0, 0]));
+ let color = &self.targets.as_ref().expect("targets").frame.color;
+ let group = gpu.device.create_bind_group(&wgpu::BindGroupDescriptor {
+ label: Some("nv12"),
+ layout: &export.convert.get_bind_group_layout(0),
+ entries: &[
+ wgpu::BindGroupEntry { binding: 0, resource: wgpu::BindingResource::TextureView(color) },
+ wgpu::BindGroupEntry { binding: 1, resource: list.as_entire_binding() },
+ wgpu::BindGroupEntry { binding: 2, resource: export.packed.as_entire_binding() },
+ wgpu::BindGroupEntry { binding: 3, resource: export.grid.as_entire_binding() },
+ ],
+ });
+ {
+ let mut pass = command.begin_compute_pass(&Default::default());
+ pass.set_pipeline(&export.convert);
+ pass.set_bind_group(0, &group, &[]);
+ pass.dispatch_workgroups(listed.len() as u32, 1, 1);
+ }
+ let bytes = listed.len() as u64 * 384;
+ command.copy_buffer_to_buffer(&export.packed, 0, &export.ring[slot], 0, Some(bytes));
+ if let Some(lists) = self.lists.as_ref().filter(|_| counted) {
+ command.copy_buffer_to_buffer(&lists.appended, 0, &export.ring[slot], bytes, Some(COUNT_BYTES));
+ }
+ }
+
+ /// The oldest frame in flight: its changed macroblocks, read back, to the encoder — drawn
+ /// and converted again first if its see-through fragments overflowed the lists.
+ pub(crate) fn retire(&mut self, gpu: &mut Gpu, export: &mut Export) -> Result<(), String> {
+ let Some(p) = export.pending.pop_front() else { return Ok(()) };
+ let mut packets = Vec::new();
+ if !p.listed.is_empty() {
+ let bytes = p.listed.len() as u64 * 384;
+ export.wait(&gpu.device, p.slot)?;
+ if let Some((frames, capacity)) = &p.kept {
+ let mut capacity = *capacity;
+ loop {
+ let appended = count(&export.ring[p.slot].slice(bytes..bytes + COUNT_BYTES).get_mapped_range().map_err(|e| e.to_string())?);
+ if !self.overflowed(gpu, appended, capacity) {
+ break;
+ }
+ export.ring[p.slot].unmap();
+ let (mut command, ..) = self.encode(gpu, frames)?;
+ capacity = self.lists.as_ref().map_or(0, |l| l.capacity);
+ self.convert(gpu, export, &mut command, &p.listed, p.slot, true);
+ gpu.queue.submit(Some(command.finish()));
+ export.map(p.slot, p.listed.len());
+ export.wait(&gpu.device, p.slot)?;
+ }
+ }
+ packets = export.ring[p.slot].slice(..bytes).get_mapped_range().map_err(|e| e.to_string())?.to_vec();
+ export.ring[p.slot].unmap();
+ }
+ let change = encode::Change { listed: p.listed, packets, first: p.first, repeat: p.repeat, key: p.key };
+ export.worker.as_ref().ok_or("the video is finished")?.send(change)
+ }
+}
+
+/// Frames an export has in flight: drawn and converted, their pixels on the way back (deep
+/// enough that the drawing thread rarely waits for the GPU's completion).
+const RING: usize = 6;
+
+pub(crate) struct Pending {
+ slot: usize,
+ listed: Vec<u32>,
+ repeat: u32,
+ first: bool,
+ key: bool,
+ kept: Option<(Vec<Frame>, u64)>, // a frame composited through the lists, and their capacity
+}
+
+/// A video being written: each frame drawn, converted to NV12 where it changed, read back and
+/// handed — the changed macroblocks only — to the encoder's thread, which keeps the picture;
+/// `RING` frames in flight: the GPU draws the next frames while x264 encodes this one.
+pub(crate) struct Export {
+ worker: Option<encode::Worker>,
+ convert: wgpu::ComputePipeline, // RGBA → NV12, the listed macroblocks (`nv12.wgsl`)
+ packed: wgpu::Buffer,
+ ring: Vec<wgpu::Buffer>,
+ mapped: Vec<Arc<AtomicBool>>,
+ grid: wgpu::Buffer,
+ pub(crate) pending: VecDeque<Pending>,
+ previous: Option<Frame>,
+ grouped: bool, // the previous frame was composited in groups (see `push_into`)
+ width: usize,
+ height: usize,
+ drawn: usize,
+ converted: usize,
+ start: Instant,
+}
+
+impl Export {
+ #[allow(clippy::too_many_arguments)]
+ pub(crate) fn new(gpu: &Gpu, width: u32, height: u32, fps: u32, preset: &str, crf: f32, options: &[(String, String)], path: &str) -> Result<Self, String> {
+ let worker = encode::Worker::new(encode::Encoder::new(width, height, fps, preset, crf, options, path)?, width as usize, height as usize);
+ let size = (width.div_ceil(16) * height.div_ceil(16) * 384) as u64;
+ let buffer = |label, usage| gpu.device.create_buffer(&wgpu::BufferDescriptor { label: Some(label), size: size + COUNT_BYTES, usage, mapped_at_creation: false });
+ let convert = gpu.device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor {
+ label: Some("nv12"),
+ layout: None, // derived from the shader
+ module: &gpu.device.create_shader_module(wgpu::include_wgsl!("nv12.wgsl")),
+ entry_point: Some("cs_macroblocks"),
+ compilation_options: Default::default(),
+ cache: None,
+ });
+ Ok(Self {
+ worker: Some(worker),
+ convert,
+ packed: buffer("nv12", wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_SRC),
+ ring: (0..RING).map(|_| buffer("nv12 readback", wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::COPY_DST)).collect(),
+ mapped: (0..RING).map(|_| Default::default()).collect(),
+ grid: gpu.device.create_buffer(&wgpu::BufferDescriptor { label: Some("grid"), size: 16, usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, mapped_at_creation: false }),
+ pending: VecDeque::with_capacity(RING),
+ previous: None,
+ grouped: false,
+ width: width as usize,
+ height: height as usize,
+ drawn: 0,
+ converted: 0,
+ start: Instant::now(),
+ })
+ }
+
+ /// Read `slot` back once its frame is converted: its `listed` macroblocks, and the lists'
+ /// count after them.
+ fn map(&self, slot: usize, listed: usize) {
+ if listed > 0 {
+ let flag = self.mapped[slot].clone();
+ self.ring[slot].slice(..listed as u64 * 384 + COUNT_BYTES).map_async(wgpu::MapMode::Read, move |_| flag.store(true, Ordering::Release));
+ }
+ }
+
+ /// Wait until `slot` is read back.
+ fn wait(&self, device: &wgpu::Device, slot: usize) -> Result<(), String> {
+ while !self.mapped[slot].load(Ordering::Acquire) {
+ device.poll(wgpu::PollType::Wait { submission_index: None, timeout: None }).map_err(|e| e.to_string())?;
+ }
+ self.mapped[slot].store(false, Ordering::Release);
+ Ok(())
+ }
+
+ pub(crate) fn finish(mut self, audio: Option<mp4::Audio>) -> Result<(f64, f64, f64, u64), String> {
+ let total = self.width.div_ceil(16) * self.height.div_ceil(16);
+ let converted = self.converted as f64 / (self.drawn.max(1) * total) as f64;
+ let (bytes, x264) = self.worker.take().ok_or("the video is finished")?.join(audio)?;
+ Ok((self.start.elapsed().as_secs_f64(), x264, converted, bytes))
+ }
+}
+
+impl Drop for Export {
+ /// An export dropped before it finished (the scene failed) leaves no file.
+ fn drop(&mut self) {
+ if let Some(worker) = self.worker.take() {
+ worker.abort();
+ }
+ }
+}
diff --git a/rust/engine/src/mp4.rs b/rust/engine/src/mp4.rs
new file mode 100644
index 00000000..71104666
--- /dev/null
+++ b/rust/engine/src/mp4.rs
@@ -0,0 +1,527 @@
+//! MP4 (ISO/IEC 14496-12/14/15) holding an H.264 track and, if the film has sound, an AAC-LC track;
+//! faststart.
+//!
+//! A video sample is a picture as x264 writes it (NAL units, each prefixed by its 4-byte size) and a
+//! held picture is one sample that lasts longer — times are in frames (timescale = fps). An audio
+//! sample is an AAC access unit of 1024 samples; the encoder's priming before the sound's first
+//! sample is skipped by an edit list, so the sound starts with the film, to the sample. Video
+//! samples stream to a temporary file; `finish` writes the header (`moov`), then the samples,
+//! a second of video and a second of sound at a time, so a player can start before the file has
+//! arrived.
+
+use std::fs::File;
+use std::io::{self, BufReader, BufWriter, Read, Write};
+use std::ops::Range;
+
+pub struct Mp4 {
+ path: String,
+ part: String,
+ data: Option<BufWriter<File>>,
+ width: u32,
+ height: u32,
+ fps: u32,
+ sps: Vec<u8>,
+ pps: Vec<u8>,
+ samples: Vec<Sample>,
+ bytes: u64,
+}
+
+struct Sample {
+ size: u32,
+ pts: i64,
+ dts: i64,
+ key: bool,
+}
+
+/// A film's sound, encoded: AAC-LC access units of 1024 samples each, in order.
+pub struct Audio {
+ pub rate: u32,
+ pub channels: u16,
+ /// The AudioSpecificConfig (the decoder's configuration).
+ pub config: Vec<u8>,
+ /// Samples the encoder put before the sound's first (hidden by the edit list).
+ pub priming: u32,
+ /// The sound's own samples, after the priming: what plays.
+ pub length: u64,
+ pub units: Vec<Vec<u8>>,
+}
+
+/// Where each track's chunks go in `mdat`, in file order: (audio?, the samples it holds).
+type Layout = Vec<(bool, Range<usize>)>;
+
+impl Mp4 {
+ /// `sps`, `pps`: the stream's parameter sets, without size prefixes.
+ pub fn new(path: &str, width: u32, height: u32, fps: u32, sps: Vec<u8>, pps: Vec<u8>) -> io::Result<Self> {
+ if sps.len() < 4 || pps.is_empty() {
+ return Err(io::Error::new(io::ErrorKind::InvalidData, "H.264 parameter sets are missing"));
+ }
+ let part = format!("{path}.part");
+ let data = BufWriter::with_capacity(1 << 20, File::create(&part)?);
+ Ok(Self { path: path.to_string(), part, data: Some(data), width, height, fps, sps, pps, samples: Vec::new(), bytes: 0 })
+ }
+
+ pub fn sample(&mut self, data: &[u8], pts: i64, dts: i64, key: bool) -> io::Result<()> {
+ if data.is_empty() {
+ return Ok(());
+ }
+ self.data.as_mut().ok_or_else(|| io::Error::other("the video is finished"))?.write_all(data)?;
+ self.bytes += data.len() as u64;
+ self.samples.push(Sample { size: data.len() as u32, pts, dts, key });
+ Ok(())
+ }
+
+ /// Write the file: the video lasts until `end` (in frames — the last hold included), with
+ /// `audio` beside it, if any. A file that could not be written whole is not left behind.
+ pub fn finish(mut self, end: i64, audio: Option<&Audio>) -> io::Result<u64> {
+ if let Some(mut data) = self.data.take() {
+ data.flush()?;
+ }
+ let written = self.write(end, audio);
+ if written.is_err() {
+ let _ = std::fs::remove_file(&self.path);
+ }
+ written
+ }
+
+ /// The chunks: the whole video in one without sound; with sound, a second of each in turn.
+ fn layout(&self, audio: Option<&Audio>) -> Layout {
+ let Some(audio) = audio else {
+ return vec![(false, 0..self.samples.len())];
+ };
+ let first = self.samples.first().map_or(0, |s| s.dts);
+ let (mut chunks, mut v, mut a, mut second) = (Vec::new(), 0, 0, 1i64);
+ while v < self.samples.len() || a < audio.units.len() {
+ let v0 = v;
+ while v < self.samples.len() && self.samples[v].dts - first < second * self.fps as i64 {
+ v += 1;
+ }
+ if v > v0 {
+ chunks.push((false, v0..v));
+ }
+ let a0 = a;
+ while a < audio.units.len() && (a as i64 * 1024) < second * audio.rate as i64 {
+ a += 1;
+ }
+ if a > a0 {
+ chunks.push((true, a0..a));
+ }
+ second += 1;
+ }
+ chunks
+ }
+
+ fn write(&self, end: i64, audio: Option<&Audio>) -> io::Result<u64> {
+ let sound: u64 = audio.map_or(0, |a| a.units.iter().map(|u| u.len() as u64).sum());
+ let body = self.bytes + sound;
+ let wide = body + 16 > u32::MAX as u64;
+ let header = if wide { 16 } else { 8 };
+ let layout = self.layout(audio);
+ let ftyp = ftyp();
+ let probe = self.moov(end, audio, &layout, 0, wide);
+ let start = (ftyp.len() + probe.len() + header) as u64;
+ let moov = self.moov(end, audio, &layout, start, wide);
+ let mut out = BufWriter::with_capacity(1 << 20, File::create(&self.path)?);
+ out.write_all(&ftyp)?;
+ out.write_all(&moov)?;
+ if wide {
+ out.write_all(&1u32.to_be_bytes())?;
+ out.write_all(b"mdat")?;
+ out.write_all(&(body + 16).to_be_bytes())?;
+ } else {
+ out.write_all(&((body + 8) as u32).to_be_bytes())?;
+ out.write_all(b"mdat")?;
+ }
+ let mut video = BufReader::with_capacity(1 << 20, File::open(&self.part)?);
+ for (is_audio, range) in &layout {
+ if *is_audio {
+ for unit in &audio.expect("audio chunks come with audio").units[range.clone()] {
+ out.write_all(unit)?;
+ }
+ } else {
+ let bytes: u64 = self.samples[range.clone()].iter().map(|s| s.size as u64).sum();
+ io::copy(&mut (&mut video).take(bytes), &mut out)?;
+ }
+ }
+ out.flush()?;
+ Ok((ftyp.len() + moov.len() + header) as u64 + body)
+ }
+
+ /// Each track's chunks: (their offsets in the file, the samples each holds).
+ fn chunks(&self, audio: Option<&Audio>, layout: &Layout, offset: u64) -> [(Vec<u64>, Vec<u32>); 2] {
+ let mut tracks: [(Vec<u64>, Vec<u32>); 2] = Default::default();
+ let mut at = offset;
+ for (is_audio, range) in layout {
+ let size: u64 = if *is_audio {
+ audio.map_or(0, |a| a.units[range.clone()].iter().map(|u| u.len() as u64).sum())
+ } else {
+ self.samples[range.clone()].iter().map(|s| s.size as u64).sum()
+ };
+ let track = &mut tracks[*is_audio as usize];
+ track.0.push(at);
+ track.1.push(range.len() as u32);
+ at += size;
+ }
+ tracks
+ }
+
+ /// The header, with the samples starting at byte `offset` of the file.
+ fn moov(&self, end: i64, audio: Option<&Audio>, layout: &Layout, offset: u64, wide: bool) -> Vec<u8> {
+ let first = self.samples.first().map_or(0, |s| s.dts);
+ let decode: Vec<i64> = self.samples.iter().map(|s| s.dts - first).collect();
+ let duration = end.max(1) as u32;
+ let mut deltas: Vec<u32> = decode.windows(2).map(|w| (w[1] - w[0]) as u32).collect();
+ if let Some(&last) = decode.last() {
+ deltas.push((end - last).max(1) as u32);
+ }
+ // composition = decode + offset: the offsets make a sample show at its pts
+ let offsets: Vec<u32> = self.samples.iter().zip(&decode).map(|(s, d)| (s.pts - d).max(0) as u32).collect();
+ let [(video_at, video_counts), (audio_at, audio_counts)] = self.chunks(audio, layout, offset);
+ // the sound's length in the movie's timescale (frames)
+ let heard = audio.map_or(0, |a| (a.length * self.fps as u64).div_ceil(a.rate as u64) as u32);
+ let mut w = Boxes::default();
+ w.boxed(b"moov", |w| {
+ w.full(b"mvhd", 0, 0, |w| {
+ w.u32(0);
+ w.u32(0);
+ w.u32(self.fps);
+ w.u32(duration.max(heard));
+ w.u32(0x0001_0000);
+ w.u16(0x0100);
+ w.zeros(10);
+ w.matrix();
+ w.zeros(24);
+ w.u32(if audio.is_some() { 3 } else { 2 });
+ });
+ w.boxed(b"trak", |w| {
+ w.full(b"tkhd", 0, 3, |w| {
+ w.u32(0);
+ w.u32(0);
+ w.u32(1);
+ w.u32(0);
+ w.u32(duration);
+ w.zeros(8);
+ w.u16(0);
+ w.u16(0);
+ w.u16(0);
+ w.u16(0);
+ w.matrix();
+ w.u32(self.width << 16);
+ w.u32(self.height << 16);
+ });
+ w.boxed(b"mdia", |w| {
+ w.full(b"mdhd", 0, 0, |w| {
+ w.u32(0);
+ w.u32(0);
+ w.u32(self.fps);
+ w.u32(duration);
+ w.u16(0x55c4); // "und"
+ w.u16(0);
+ });
+ w.full(b"hdlr", 0, 0, |w| {
+ w.u32(0);
+ w.bytes(b"vide");
+ w.zeros(12);
+ w.bytes(b"VideoHandler\0");
+ });
+ w.boxed(b"minf", |w| {
+ w.full(b"vmhd", 0, 1, |w| w.zeros(8));
+ w.dinf();
+ w.boxed(b"stbl", |w| {
+ w.full(b"stsd", 0, 0, |w| {
+ w.u32(1);
+ self.avc1(w);
+ });
+ w.full(b"stts", 0, 0, |w| w.runs(&deltas));
+ if offsets.iter().any(|&o| o != 0) {
+ w.full(b"ctts", 0, 0, |w| w.runs(&offsets));
+ }
+ if self.samples.iter().any(|s| !s.key) {
+ let keys: Vec<u32> = (1..).zip(&self.samples).filter(|(_, s)| s.key).map(|(k, _)| k).collect();
+ w.full(b"stss", 0, 0, |w| {
+ w.u32(keys.len() as u32);
+ keys.iter().for_each(|&k| w.u32(k));
+ });
+ }
+ w.stsc(&video_counts);
+ w.full(b"stsz", 0, 0, |w| {
+ w.u32(0);
+ w.u32(self.samples.len() as u32);
+ self.samples.iter().for_each(|s| w.u32(s.size));
+ });
+ w.stco(&video_at, wide);
+ });
+ });
+ });
+ });
+ if let Some(audio) = audio {
+ self.sound(w, audio, heard, &audio_counts, &audio_at, wide);
+ }
+ });
+ w.0
+ }
+
+ /// The sound's track: AAC-LC, its priming cut by an edit list, a unit of pre-roll.
+ fn sound(&self, w: &mut Boxes, audio: &Audio, heard: u32, counts: &[u32], at: &[u64], wide: bool) {
+ let units = audio.units.len() as u32;
+ let bitrate = (audio.units.iter().map(|u| u.len() as u64).sum::<u64>() * 8 * audio.rate as u64 / (units.max(1) as u64 * 1024)) as u32;
+ w.boxed(b"trak", |w| {
+ w.full(b"tkhd", 0, 3, |w| {
+ w.u32(0);
+ w.u32(0);
+ w.u32(2);
+ w.u32(0);
+ w.u32(heard);
+ w.zeros(8);
+ w.u16(0);
+ w.u16(1); // alternate group: sound
+ w.u16(0x0100); // volume
+ w.u16(0);
+ w.matrix();
+ w.u32(0);
+ w.u32(0);
+ });
+ w.boxed(b"edts", |w| {
+ w.full(b"elst", 0, 0, |w| {
+ w.u32(1);
+ w.u32(heard); // in the movie's timescale
+ w.u32(audio.priming); // in the track's: where the sound's first sample is
+ w.u16(1);
+ w.u16(0);
+ });
+ });
+ w.boxed(b"mdia", |w| {
+ w.full(b"mdhd", 0, 0, |w| {
+ w.u32(0);
+ w.u32(0);
+ w.u32(audio.rate);
+ w.u32(units * 1024);
+ w.u16(0x55c4); // "und"
+ w.u16(0);
+ });
+ w.full(b"hdlr", 0, 0, |w| {
+ w.u32(0);
+ w.bytes(b"soun");
+ w.zeros(12);
+ w.bytes(b"SoundHandler\0");
+ });
+ w.boxed(b"minf", |w| {
+ w.full(b"smhd", 0, 0, |w| w.u32(0));
+ w.dinf();
+ w.boxed(b"stbl", |w| {
+ w.full(b"stsd", 0, 0, |w| {
+ w.u32(1);
+ w.boxed(b"mp4a", |w| {
+ w.zeros(6);
+ w.u16(1); // data reference
+ w.zeros(8);
+ w.u16(audio.channels);
+ w.u16(16);
+ w.u32(0);
+ w.u32(audio.rate << 16);
+ w.full(b"esds", 0, 0, |w| {
+ let config = &audio.config;
+ // ES_Descriptor ⊃ DecoderConfigDescriptor ⊃ DecoderSpecificInfo;
+ // SLConfigDescriptor
+ w.u8(0x03);
+ w.u8((3 + 2 + 13 + 2 + config.len() + 3) as u8);
+ w.u16(0);
+ w.u8(0);
+ w.u8(0x04);
+ w.u8((13 + 2 + config.len()) as u8);
+ w.u8(0x40); // MPEG-4 audio
+ w.u8(0x15); // audio stream
+ w.bytes(&[0, 0x18, 0]); // buffer size: 6144 bytes
+ w.u32(bitrate);
+ w.u32(bitrate);
+ w.u8(0x05);
+ w.u8(config.len() as u8);
+ w.bytes(config);
+ w.u8(0x06);
+ w.u8(1);
+ w.u8(0x02);
+ });
+ });
+ });
+ w.full(b"stts", 0, 0, |w| {
+ w.u32(1);
+ w.u32(units);
+ w.u32(1024);
+ });
+ w.stsc(counts);
+ w.full(b"stsz", 0, 0, |w| {
+ w.u32(0);
+ w.u32(units);
+ audio.units.iter().for_each(|u| w.u32(u.len() as u32));
+ });
+ w.stco(at, wide);
+ // a decoder needs the unit before any it starts at
+ w.full(b"sgpd", 1, 0, |w| {
+ w.bytes(b"roll");
+ w.u32(2);
+ w.u32(1);
+ w.u16(0xffff); // -1
+ });
+ w.full(b"sbgp", 0, 0, |w| {
+ w.bytes(b"roll");
+ w.u32(1);
+ w.u32(units);
+ w.u32(1);
+ });
+ });
+ });
+ });
+ });
+ }
+
+ fn avc1(&self, w: &mut Boxes) {
+ w.boxed(b"avc1", |w| {
+ w.zeros(6);
+ w.u16(1); // data reference
+ w.zeros(16);
+ w.u16(self.width as u16);
+ w.u16(self.height as u16);
+ w.u32(0x0048_0000);
+ w.u32(0x0048_0000);
+ w.u32(0);
+ w.u16(1); // frames per sample
+ let name = b"manimgx";
+ w.u8(name.len() as u8);
+ w.bytes(name);
+ w.zeros(31 - name.len());
+ w.u16(0x0018);
+ w.u16(0xffff);
+ w.boxed(b"avcC", |w| {
+ let (sps, pps) = (&self.sps, &self.pps);
+ w.u8(1);
+ w.u8(sps[1]); // profile
+ w.u8(sps[2]); // constraints
+ w.u8(sps[3]); // level
+ w.u8(0xff); // 4-byte NAL sizes
+ w.u8(0xe1); // one SPS
+ w.u16(sps.len() as u16);
+ w.bytes(sps);
+ w.u8(1); // one PPS
+ w.u16(pps.len() as u16);
+ w.bytes(pps);
+ if matches!(sps[1], 100 | 110 | 122 | 144) {
+ w.u8(0xfd); // 4:2:0
+ w.u8(0xf8); // 8-bit luma
+ w.u8(0xf8); // 8-bit chroma
+ w.u8(0);
+ }
+ });
+ });
+ }
+}
+
+impl Drop for Mp4 {
+ /// The samples' file is temporary: finished or not, it goes (closed first, so that it can).
+ fn drop(&mut self) {
+ self.data = None;
+ let _ = std::fs::remove_file(&self.part);
+ }
+}
+
+fn ftyp() -> Vec<u8> {
+ let mut w = Boxes::default();
+ w.boxed(b"ftyp", |w| {
+ w.bytes(b"isom");
+ w.u32(0x200);
+ for brand in [b"isom", b"iso2", b"avc1", b"mp41"] {
+ w.bytes(brand);
+ }
+ });
+ w.0
+}
+
+#[derive(Default)]
+struct Boxes(Vec<u8>);
+
+impl Boxes {
+ fn u8(&mut self, v: u8) {
+ self.0.push(v);
+ }
+ fn u16(&mut self, v: u16) {
+ self.0.extend_from_slice(&v.to_be_bytes());
+ }
+ fn u32(&mut self, v: u32) {
+ self.0.extend_from_slice(&v.to_be_bytes());
+ }
+ fn u64(&mut self, v: u64) {
+ self.0.extend_from_slice(&v.to_be_bytes());
+ }
+ fn bytes(&mut self, b: &[u8]) {
+ self.0.extend_from_slice(b);
+ }
+ fn zeros(&mut self, n: usize) {
+ self.0.resize(self.0.len() + n, 0);
+ }
+ fn matrix(&mut self) {
+ for v in [0x0001_0000, 0, 0, 0, 0x0001_0000, 0, 0, 0, 0x4000_0000u32] {
+ self.u32(v);
+ }
+ }
+ fn dinf(&mut self) {
+ self.boxed(b"dinf", |w| {
+ w.full(b"dref", 0, 0, |w| {
+ w.u32(1);
+ w.full(b"url ", 0, 1, |_| {});
+ })
+ });
+ }
+ /// How many samples a track's chunks hold, run-length: (first chunk, samples each).
+ fn stsc(&mut self, counts: &[u32]) {
+ self.full(b"stsc", 0, 0, |w| {
+ let runs: Vec<(u32, u32)> = (1..).zip(counts).filter(|&(k, &n)| k == 1 || counts[k as usize - 2] != n).map(|(k, &n)| (k, n)).collect();
+ w.u32(runs.len() as u32);
+ for (first, n) in runs {
+ w.u32(first);
+ w.u32(n);
+ w.u32(1);
+ }
+ });
+ }
+ /// Where a track's chunks start in the file.
+ fn stco(&mut self, at: &[u64], wide: bool) {
+ if wide {
+ self.full(b"co64", 0, 0, |w| {
+ w.u32(at.len() as u32);
+ at.iter().for_each(|&o| w.u64(o));
+ });
+ } else {
+ self.full(b"stco", 0, 0, |w| {
+ w.u32(at.len() as u32);
+ at.iter().for_each(|&o| w.u32(o as u32));
+ });
+ }
+ }
+ /// Run-length (count, value) entries.
+ fn runs(&mut self, values: &[u32]) {
+ let mut runs: Vec<(u32, u32)> = Vec::new();
+ for &v in values {
+ match runs.last_mut() {
+ Some((n, last)) if *last == v => *n += 1,
+ _ => runs.push((1, v)),
+ }
+ }
+ self.u32(runs.len() as u32);
+ for (n, v) in runs {
+ self.u32(n);
+ self.u32(v);
+ }
+ }
+ fn boxed(&mut self, kind: &[u8; 4], body: impl FnOnce(&mut Self)) {
+ let at = self.0.len();
+ self.u32(0);
+ self.bytes(kind);
+ body(self);
+ let size = (self.0.len() - at) as u32;
+ self.0[at..at + 4].copy_from_slice(&size.to_be_bytes());
+ }
+ fn full(&mut self, kind: &[u8; 4], version: u8, flags: u32, body: impl FnOnce(&mut Self)) {
+ self.boxed(kind, |w| {
+ w.u32((version as u32) << 24 | flags);
+ body(w);
+ });
+ }
+}