The animation engine for agents: blazingly fast math and 3D videos with Manim's Python API, on a Rust GPU core.
Python
Mux video and audio to MP4
apoogdk990c73de7dcb
diff --git a/rust/engine/src/export.rs b/rust/engine/src/export.rsnew file mode 100644index 00000000..6d6f4ff0--- /dev/null+++ b/rust/engine/src/export.rs@@ -0,0 +1,248 @@+//! A film's video, natively: each frame drawn, converted to NV12 on the GPU where it changed,+//! read back and handed to x264 on a thread of its own (see `encode`), muxed into an MP4.++use std::collections::VecDeque;+use std::sync::Arc;+use std::sync::atomic::{AtomicBool, Ordering};+use std::time::Instant;++use crate::{encode, mp4};+use wgpu::util::DeviceExt;++use crate::render::{COUNT_BYTES, Camera, Frame, Gpu, Player, count};++impl Player {+ /// The macroblocks whose pixels can differ from the previous frame of the same view: the old+ /// and new footprints of every record that changed. Anything else — a new camera, objects+ /// added or removed, what cannot be bounded — is the whole frame.+ fn damage(&self, previous: Option<&Frame>, frame: &Frame, columns: usize, rows: usize) -> Vec<u8> {+ let everything = vec![1u8; columns * rows];+ let Some(previous) = previous else { return everything };+ if previous.view != frame.view || previous.records.len() != frame.records.len() {+ return everything;+ }+ let camera = Camera::new(&frame.view);+ let mut dirty = vec![0u8; columns * rows];+ for (old, new) in previous.records.iter().zip(&frame.records) {+ if bytemuck::bytes_of(old) == bytemuck::bytes_of(new) {+ continue;+ }+ for r in [old, new] {+ let Some([x0, y0, x1, y1]) = self.store.footprint(&camera, r) else { return everything };+ if x1 < 0.0 || y1 < 0.0 || x0 >= camera.size[0] || y0 >= camera.size[1] {+ continue; // off screen+ }+ let (c0, c1) = ((x0.max(0.0) / 16.0) as usize, ((x1 / 16.0) as usize).min(columns - 1));+ let (r0, r1) = ((y0.max(0.0) / 16.0) as usize, ((y1 / 16.0) as usize).min(rows - 1));+ for row in r0..=r1 {+ dirty[row * columns + c0..=row * columns + c1].fill(1);+ }+ }+ }+ dirty+ }++ pub(crate) fn push_frames(&mut self, gpu: &mut Gpu, frames: Vec<Frame>, repeat: u32, key: bool) -> Result<(), String> {+ let mut export = self.export.take().ok_or("no export is open")?;+ let result = self.push_into(gpu, &mut export, frames, repeat, key);+ self.export = Some(export);+ result+ }++ /// Draw a frame into the export and convert only the macroblocks a change can touch — every+ /// one when camera views are drawn too (their pictures change with no record changing).+ fn push_into(&mut self, gpu: &mut Gpu, export: &mut Export, frames: Vec<Frame>, repeat: u32, key: bool) -> Result<(), String> {+ let (columns, rows) = (export.width.div_ceil(16), export.height.div_ceil(16));+ let total = columns * rows;+ if export.pending.len() == RING {+ self.retire(gpu, export)?;+ }+ let slot = export.drawn % RING;+ let frame = frames.last().ok_or("no frame")?;+ let (mut command, grouped, composited) = self.encode(gpu, &frames)?;+ // a frame composited in groups settles the pixels two groups share at the seam between+ // them, which a change anywhere can move+ let whole = frames.len() > 1 || grouped || export.grouped;+ export.grouped = grouped;+ let dirty = if whole { vec![1u8; total] } else { self.damage(export.previous.as_ref(), frame, columns, rows) };+ export.converted += dirty.iter().filter(|&&d| d != 0).count();+ let listed: Vec<u32> = (0..total as u32).filter(|&k| dirty[k as usize] != 0).collect();+ // a frame whose see-through fragments may overflow the lists is kept until its count is+ // read (see `retire`)+ let lists = self.lists.as_ref().filter(|_| composited && !listed.is_empty()).map(|l| l.capacity);+ self.convert(gpu, export, &mut command, &listed, slot, lists.is_some());+ gpu.queue.submit(Some(command.finish()));+ export.map(slot, listed.len());+ gpu.device.poll(wgpu::PollType::Poll).map_err(|e| e.to_string())?;+ let first = export.previous.is_none();+ let kept = lists.map(|capacity| (frames.clone(), capacity));+ export.pending.push_back(Pending { slot, listed, repeat, first, key, kept });+ export.previous = frames.into_iter().last();+ export.drawn += 1;+ Ok(())+ }++ /// Convert the listed macroblocks of the drawn frame into the export's `slot`, followed by+ /// the lists' count when `counted`.+ fn convert(&self, gpu: &Gpu, export: &Export, command: &mut wgpu::CommandEncoder, listed: &[u32], slot: usize, counted: bool) {+ if listed.is_empty() {+ return;+ }+ let columns = export.width.div_ceil(16);+ let list = gpu.device.create_buffer_init(&wgpu::util::BufferInitDescriptor { label: Some("listed"), contents: bytemuck::cast_slice(listed), usage: wgpu::BufferUsages::STORAGE });+ gpu.queue.write_buffer(&export.grid, 0, bytemuck::cast_slice(&[columns as u32, listed.len() as u32, 0, 0]));+ let color = &self.targets.as_ref().expect("targets").frame.color;+ let group = gpu.device.create_bind_group(&wgpu::BindGroupDescriptor {+ label: Some("nv12"),+ layout: &export.convert.get_bind_group_layout(0),+ entries: &[+ wgpu::BindGroupEntry { binding: 0, resource: wgpu::BindingResource::TextureView(color) },+ wgpu::BindGroupEntry { binding: 1, resource: list.as_entire_binding() },+ wgpu::BindGroupEntry { binding: 2, resource: export.packed.as_entire_binding() },+ wgpu::BindGroupEntry { binding: 3, resource: export.grid.as_entire_binding() },+ ],+ });+ {+ let mut pass = command.begin_compute_pass(&Default::default());+ pass.set_pipeline(&export.convert);+ pass.set_bind_group(0, &group, &[]);+ pass.dispatch_workgroups(listed.len() as u32, 1, 1);+ }+ let bytes = listed.len() as u64 * 384;+ command.copy_buffer_to_buffer(&export.packed, 0, &export.ring[slot], 0, Some(bytes));+ if let Some(lists) = self.lists.as_ref().filter(|_| counted) {+ command.copy_buffer_to_buffer(&lists.appended, 0, &export.ring[slot], bytes, Some(COUNT_BYTES));+ }+ }++ /// The oldest frame in flight: its changed macroblocks, read back, to the encoder — drawn+ /// and converted again first if its see-through fragments overflowed the lists.+ pub(crate) fn retire(&mut self, gpu: &mut Gpu, export: &mut Export) -> Result<(), String> {+ let Some(p) = export.pending.pop_front() else { return Ok(()) };+ let mut packets = Vec::new();+ if !p.listed.is_empty() {+ let bytes = p.listed.len() as u64 * 384;+ export.wait(&gpu.device, p.slot)?;+ if let Some((frames, capacity)) = &p.kept {+ let mut capacity = *capacity;+ loop {+ let appended = count(&export.ring[p.slot].slice(bytes..bytes + COUNT_BYTES).get_mapped_range().map_err(|e| e.to_string())?);+ if !self.overflowed(gpu, appended, capacity) {+ break;+ }+ export.ring[p.slot].unmap();+ let (mut command, ..) = self.encode(gpu, frames)?;+ capacity = self.lists.as_ref().map_or(0, |l| l.capacity);+ self.convert(gpu, export, &mut command, &p.listed, p.slot, true);+ gpu.queue.submit(Some(command.finish()));+ export.map(p.slot, p.listed.len());+ export.wait(&gpu.device, p.slot)?;+ }+ }+ packets = export.ring[p.slot].slice(..bytes).get_mapped_range().map_err(|e| e.to_string())?.to_vec();+ export.ring[p.slot].unmap();+ }+ let change = encode::Change { listed: p.listed, packets, first: p.first, repeat: p.repeat, key: p.key };+ export.worker.as_ref().ok_or("the video is finished")?.send(change)+ }+}++/// Frames an export has in flight: drawn and converted, their pixels on the way back (deep+/// enough that the drawing thread rarely waits for the GPU's completion).+const RING: usize = 6;++pub(crate) struct Pending {+ slot: usize,+ listed: Vec<u32>,+ repeat: u32,+ first: bool,+ key: bool,+ kept: Option<(Vec<Frame>, u64)>, // a frame composited through the lists, and their capacity+}++/// A video being written: each frame drawn, converted to NV12 where it changed, read back and+/// handed — the changed macroblocks only — to the encoder's thread, which keeps the picture;+/// `RING` frames in flight: the GPU draws the next frames while x264 encodes this one.+pub(crate) struct Export {+ worker: Option<encode::Worker>,+ convert: wgpu::ComputePipeline, // RGBA → NV12, the listed macroblocks (`nv12.wgsl`)+ packed: wgpu::Buffer,+ ring: Vec<wgpu::Buffer>,+ mapped: Vec<Arc<AtomicBool>>,+ grid: wgpu::Buffer,+ pub(crate) pending: VecDeque<Pending>,+ previous: Option<Frame>,+ grouped: bool, // the previous frame was composited in groups (see `push_into`)+ width: usize,+ height: usize,+ drawn: usize,+ converted: usize,+ start: Instant,+}++impl Export {+ #[allow(clippy::too_many_arguments)]+ pub(crate) fn new(gpu: &Gpu, width: u32, height: u32, fps: u32, preset: &str, crf: f32, options: &[(String, String)], path: &str) -> Result<Self, String> {+ let worker = encode::Worker::new(encode::Encoder::new(width, height, fps, preset, crf, options, path)?, width as usize, height as usize);+ let size = (width.div_ceil(16) * height.div_ceil(16) * 384) as u64;+ let buffer = |label, usage| gpu.device.create_buffer(&wgpu::BufferDescriptor { label: Some(label), size: size + COUNT_BYTES, usage, mapped_at_creation: false });+ let convert = gpu.device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor {+ label: Some("nv12"),+ layout: None, // derived from the shader+ module: &gpu.device.create_shader_module(wgpu::include_wgsl!("nv12.wgsl")),+ entry_point: Some("cs_macroblocks"),+ compilation_options: Default::default(),+ cache: None,+ });+ Ok(Self {+ worker: Some(worker),+ convert,+ packed: buffer("nv12", wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_SRC),+ ring: (0..RING).map(|_| buffer("nv12 readback", wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::COPY_DST)).collect(),+ mapped: (0..RING).map(|_| Default::default()).collect(),+ grid: gpu.device.create_buffer(&wgpu::BufferDescriptor { label: Some("grid"), size: 16, usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, mapped_at_creation: false }),+ pending: VecDeque::with_capacity(RING),+ previous: None,+ grouped: false,+ width: width as usize,+ height: height as usize,+ drawn: 0,+ converted: 0,+ start: Instant::now(),+ })+ }++ /// Read `slot` back once its frame is converted: its `listed` macroblocks, and the lists'+ /// count after them.+ fn map(&self, slot: usize, listed: usize) {+ if listed > 0 {+ let flag = self.mapped[slot].clone();+ self.ring[slot].slice(..listed as u64 * 384 + COUNT_BYTES).map_async(wgpu::MapMode::Read, move |_| flag.store(true, Ordering::Release));+ }+ }++ /// Wait until `slot` is read back.+ fn wait(&self, device: &wgpu::Device, slot: usize) -> Result<(), String> {+ while !self.mapped[slot].load(Ordering::Acquire) {+ device.poll(wgpu::PollType::Wait { submission_index: None, timeout: None }).map_err(|e| e.to_string())?;+ }+ self.mapped[slot].store(false, Ordering::Release);+ Ok(())+ }++ pub(crate) fn finish(mut self, audio: Option<mp4::Audio>) -> Result<(f64, f64, f64, u64), String> {+ let total = self.width.div_ceil(16) * self.height.div_ceil(16);+ let converted = self.converted as f64 / (self.drawn.max(1) * total) as f64;+ let (bytes, x264) = self.worker.take().ok_or("the video is finished")?.join(audio)?;+ Ok((self.start.elapsed().as_secs_f64(), x264, converted, bytes))+ }+}++impl Drop for Export {+ /// An export dropped before it finished (the scene failed) leaves no file.+ fn drop(&mut self) {+ if let Some(worker) = self.worker.take() {+ worker.abort();+ }+ }+}diff --git a/rust/engine/src/mp4.rs b/rust/engine/src/mp4.rsnew file mode 100644index 00000000..71104666--- /dev/null+++ b/rust/engine/src/mp4.rs@@ -0,0 +1,527 @@+//! MP4 (ISO/IEC 14496-12/14/15) holding an H.264 track and, if the film has sound, an AAC-LC track;+//! faststart.+//!+//! A video sample is a picture as x264 writes it (NAL units, each prefixed by its 4-byte size) and a+//! held picture is one sample that lasts longer — times are in frames (timescale = fps). An audio+//! sample is an AAC access unit of 1024 samples; the encoder's priming before the sound's first+//! sample is skipped by an edit list, so the sound starts with the film, to the sample. Video+//! samples stream to a temporary file; `finish` writes the header (`moov`), then the samples,+//! a second of video and a second of sound at a time, so a player can start before the file has+//! arrived.++use std::fs::File;+use std::io::{self, BufReader, BufWriter, Read, Write};+use std::ops::Range;++pub struct Mp4 {+ path: String,+ part: String,+ data: Option<BufWriter<File>>,+ width: u32,+ height: u32,+ fps: u32,+ sps: Vec<u8>,+ pps: Vec<u8>,+ samples: Vec<Sample>,+ bytes: u64,+}++struct Sample {+ size: u32,+ pts: i64,+ dts: i64,+ key: bool,+}++/// A film's sound, encoded: AAC-LC access units of 1024 samples each, in order.+pub struct Audio {+ pub rate: u32,+ pub channels: u16,+ /// The AudioSpecificConfig (the decoder's configuration).+ pub config: Vec<u8>,+ /// Samples the encoder put before the sound's first (hidden by the edit list).+ pub priming: u32,+ /// The sound's own samples, after the priming: what plays.+ pub length: u64,+ pub units: Vec<Vec<u8>>,+}++/// Where each track's chunks go in `mdat`, in file order: (audio?, the samples it holds).+type Layout = Vec<(bool, Range<usize>)>;++impl Mp4 {+ /// `sps`, `pps`: the stream's parameter sets, without size prefixes.+ pub fn new(path: &str, width: u32, height: u32, fps: u32, sps: Vec<u8>, pps: Vec<u8>) -> io::Result<Self> {+ if sps.len() < 4 || pps.is_empty() {+ return Err(io::Error::new(io::ErrorKind::InvalidData, "H.264 parameter sets are missing"));+ }+ let part = format!("{path}.part");+ let data = BufWriter::with_capacity(1 << 20, File::create(&part)?);+ Ok(Self { path: path.to_string(), part, data: Some(data), width, height, fps, sps, pps, samples: Vec::new(), bytes: 0 })+ }++ pub fn sample(&mut self, data: &[u8], pts: i64, dts: i64, key: bool) -> io::Result<()> {+ if data.is_empty() {+ return Ok(());+ }+ self.data.as_mut().ok_or_else(|| io::Error::other("the video is finished"))?.write_all(data)?;+ self.bytes += data.len() as u64;+ self.samples.push(Sample { size: data.len() as u32, pts, dts, key });+ Ok(())+ }++ /// Write the file: the video lasts until `end` (in frames — the last hold included), with+ /// `audio` beside it, if any. A file that could not be written whole is not left behind.+ pub fn finish(mut self, end: i64, audio: Option<&Audio>) -> io::Result<u64> {+ if let Some(mut data) = self.data.take() {+ data.flush()?;+ }+ let written = self.write(end, audio);+ if written.is_err() {+ let _ = std::fs::remove_file(&self.path);+ }+ written+ }++ /// The chunks: the whole video in one without sound; with sound, a second of each in turn.+ fn layout(&self, audio: Option<&Audio>) -> Layout {+ let Some(audio) = audio else {+ return vec![(false, 0..self.samples.len())];+ };+ let first = self.samples.first().map_or(0, |s| s.dts);+ let (mut chunks, mut v, mut a, mut second) = (Vec::new(), 0, 0, 1i64);+ while v < self.samples.len() || a < audio.units.len() {+ let v0 = v;+ while v < self.samples.len() && self.samples[v].dts - first < second * self.fps as i64 {+ v += 1;+ }+ if v > v0 {+ chunks.push((false, v0..v));+ }+ let a0 = a;+ while a < audio.units.len() && (a as i64 * 1024) < second * audio.rate as i64 {+ a += 1;+ }+ if a > a0 {+ chunks.push((true, a0..a));+ }+ second += 1;+ }+ chunks+ }++ fn write(&self, end: i64, audio: Option<&Audio>) -> io::Result<u64> {+ let sound: u64 = audio.map_or(0, |a| a.units.iter().map(|u| u.len() as u64).sum());+ let body = self.bytes + sound;+ let wide = body + 16 > u32::MAX as u64;+ let header = if wide { 16 } else { 8 };+ let layout = self.layout(audio);+ let ftyp = ftyp();+ let probe = self.moov(end, audio, &layout, 0, wide);+ let start = (ftyp.len() + probe.len() + header) as u64;+ let moov = self.moov(end, audio, &layout, start, wide);+ let mut out = BufWriter::with_capacity(1 << 20, File::create(&self.path)?);+ out.write_all(&ftyp)?;+ out.write_all(&moov)?;+ if wide {+ out.write_all(&1u32.to_be_bytes())?;+ out.write_all(b"mdat")?;+ out.write_all(&(body + 16).to_be_bytes())?;+ } else {+ out.write_all(&((body + 8) as u32).to_be_bytes())?;+ out.write_all(b"mdat")?;+ }+ let mut video = BufReader::with_capacity(1 << 20, File::open(&self.part)?);+ for (is_audio, range) in &layout {+ if *is_audio {+ for unit in &audio.expect("audio chunks come with audio").units[range.clone()] {+ out.write_all(unit)?;+ }+ } else {+ let bytes: u64 = self.samples[range.clone()].iter().map(|s| s.size as u64).sum();+ io::copy(&mut (&mut video).take(bytes), &mut out)?;+ }+ }+ out.flush()?;+ Ok((ftyp.len() + moov.len() + header) as u64 + body)+ }++ /// Each track's chunks: (their offsets in the file, the samples each holds).+ fn chunks(&self, audio: Option<&Audio>, layout: &Layout, offset: u64) -> [(Vec<u64>, Vec<u32>); 2] {+ let mut tracks: [(Vec<u64>, Vec<u32>); 2] = Default::default();+ let mut at = offset;+ for (is_audio, range) in layout {+ let size: u64 = if *is_audio {+ audio.map_or(0, |a| a.units[range.clone()].iter().map(|u| u.len() as u64).sum())+ } else {+ self.samples[range.clone()].iter().map(|s| s.size as u64).sum()+ };+ let track = &mut tracks[*is_audio as usize];+ track.0.push(at);+ track.1.push(range.len() as u32);+ at += size;+ }+ tracks+ }++ /// The header, with the samples starting at byte `offset` of the file.+ fn moov(&self, end: i64, audio: Option<&Audio>, layout: &Layout, offset: u64, wide: bool) -> Vec<u8> {+ let first = self.samples.first().map_or(0, |s| s.dts);+ let decode: Vec<i64> = self.samples.iter().map(|s| s.dts - first).collect();+ let duration = end.max(1) as u32;+ let mut deltas: Vec<u32> = decode.windows(2).map(|w| (w[1] - w[0]) as u32).collect();+ if let Some(&last) = decode.last() {+ deltas.push((end - last).max(1) as u32);+ }+ // composition = decode + offset: the offsets make a sample show at its pts+ let offsets: Vec<u32> = self.samples.iter().zip(&decode).map(|(s, d)| (s.pts - d).max(0) as u32).collect();+ let [(video_at, video_counts), (audio_at, audio_counts)] = self.chunks(audio, layout, offset);+ // the sound's length in the movie's timescale (frames)+ let heard = audio.map_or(0, |a| (a.length * self.fps as u64).div_ceil(a.rate as u64) as u32);+ let mut w = Boxes::default();+ w.boxed(b"moov", |w| {+ w.full(b"mvhd", 0, 0, |w| {+ w.u32(0);+ w.u32(0);+ w.u32(self.fps);+ w.u32(duration.max(heard));+ w.u32(0x0001_0000);+ w.u16(0x0100);+ w.zeros(10);+ w.matrix();+ w.zeros(24);+ w.u32(if audio.is_some() { 3 } else { 2 });+ });+ w.boxed(b"trak", |w| {+ w.full(b"tkhd", 0, 3, |w| {+ w.u32(0);+ w.u32(0);+ w.u32(1);+ w.u32(0);+ w.u32(duration);+ w.zeros(8);+ w.u16(0);+ w.u16(0);+ w.u16(0);+ w.u16(0);+ w.matrix();+ w.u32(self.width << 16);+ w.u32(self.height << 16);+ });+ w.boxed(b"mdia", |w| {+ w.full(b"mdhd", 0, 0, |w| {+ w.u32(0);+ w.u32(0);+ w.u32(self.fps);+ w.u32(duration);+ w.u16(0x55c4); // "und"+ w.u16(0);+ });+ w.full(b"hdlr", 0, 0, |w| {+ w.u32(0);+ w.bytes(b"vide");+ w.zeros(12);+ w.bytes(b"VideoHandler\0");+ });+ w.boxed(b"minf", |w| {+ w.full(b"vmhd", 0, 1, |w| w.zeros(8));+ w.dinf();+ w.boxed(b"stbl", |w| {+ w.full(b"stsd", 0, 0, |w| {+ w.u32(1);+ self.avc1(w);+ });+ w.full(b"stts", 0, 0, |w| w.runs(&deltas));+ if offsets.iter().any(|&o| o != 0) {+ w.full(b"ctts", 0, 0, |w| w.runs(&offsets));+ }+ if self.samples.iter().any(|s| !s.key) {+ let keys: Vec<u32> = (1..).zip(&self.samples).filter(|(_, s)| s.key).map(|(k, _)| k).collect();+ w.full(b"stss", 0, 0, |w| {+ w.u32(keys.len() as u32);+ keys.iter().for_each(|&k| w.u32(k));+ });+ }+ w.stsc(&video_counts);+ w.full(b"stsz", 0, 0, |w| {+ w.u32(0);+ w.u32(self.samples.len() as u32);+ self.samples.iter().for_each(|s| w.u32(s.size));+ });+ w.stco(&video_at, wide);+ });+ });+ });+ });+ if let Some(audio) = audio {+ self.sound(w, audio, heard, &audio_counts, &audio_at, wide);+ }+ });+ w.0+ }++ /// The sound's track: AAC-LC, its priming cut by an edit list, a unit of pre-roll.+ fn sound(&self, w: &mut Boxes, audio: &Audio, heard: u32, counts: &[u32], at: &[u64], wide: bool) {+ let units = audio.units.len() as u32;+ let bitrate = (audio.units.iter().map(|u| u.len() as u64).sum::<u64>() * 8 * audio.rate as u64 / (units.max(1) as u64 * 1024)) as u32;+ w.boxed(b"trak", |w| {+ w.full(b"tkhd", 0, 3, |w| {+ w.u32(0);+ w.u32(0);+ w.u32(2);+ w.u32(0);+ w.u32(heard);+ w.zeros(8);+ w.u16(0);+ w.u16(1); // alternate group: sound+ w.u16(0x0100); // volume+ w.u16(0);+ w.matrix();+ w.u32(0);+ w.u32(0);+ });+ w.boxed(b"edts", |w| {+ w.full(b"elst", 0, 0, |w| {+ w.u32(1);+ w.u32(heard); // in the movie's timescale+ w.u32(audio.priming); // in the track's: where the sound's first sample is+ w.u16(1);+ w.u16(0);+ });+ });+ w.boxed(b"mdia", |w| {+ w.full(b"mdhd", 0, 0, |w| {+ w.u32(0);+ w.u32(0);+ w.u32(audio.rate);+ w.u32(units * 1024);+ w.u16(0x55c4); // "und"+ w.u16(0);+ });+ w.full(b"hdlr", 0, 0, |w| {+ w.u32(0);+ w.bytes(b"soun");+ w.zeros(12);+ w.bytes(b"SoundHandler\0");+ });+ w.boxed(b"minf", |w| {+ w.full(b"smhd", 0, 0, |w| w.u32(0));+ w.dinf();+ w.boxed(b"stbl", |w| {+ w.full(b"stsd", 0, 0, |w| {+ w.u32(1);+ w.boxed(b"mp4a", |w| {+ w.zeros(6);+ w.u16(1); // data reference+ w.zeros(8);+ w.u16(audio.channels);+ w.u16(16);+ w.u32(0);+ w.u32(audio.rate << 16);+ w.full(b"esds", 0, 0, |w| {+ let config = &audio.config;+ // ES_Descriptor ⊃ DecoderConfigDescriptor ⊃ DecoderSpecificInfo;+ // SLConfigDescriptor+ w.u8(0x03);+ w.u8((3 + 2 + 13 + 2 + config.len() + 3) as u8);+ w.u16(0);+ w.u8(0);+ w.u8(0x04);+ w.u8((13 + 2 + config.len()) as u8);+ w.u8(0x40); // MPEG-4 audio+ w.u8(0x15); // audio stream+ w.bytes(&[0, 0x18, 0]); // buffer size: 6144 bytes+ w.u32(bitrate);+ w.u32(bitrate);+ w.u8(0x05);+ w.u8(config.len() as u8);+ w.bytes(config);+ w.u8(0x06);+ w.u8(1);+ w.u8(0x02);+ });+ });+ });+ w.full(b"stts", 0, 0, |w| {+ w.u32(1);+ w.u32(units);+ w.u32(1024);+ });+ w.stsc(counts);+ w.full(b"stsz", 0, 0, |w| {+ w.u32(0);+ w.u32(units);+ audio.units.iter().for_each(|u| w.u32(u.len() as u32));+ });+ w.stco(at, wide);+ // a decoder needs the unit before any it starts at+ w.full(b"sgpd", 1, 0, |w| {+ w.bytes(b"roll");+ w.u32(2);+ w.u32(1);+ w.u16(0xffff); // -1+ });+ w.full(b"sbgp", 0, 0, |w| {+ w.bytes(b"roll");+ w.u32(1);+ w.u32(units);+ w.u32(1);+ });+ });+ });+ });+ });+ }++ fn avc1(&self, w: &mut Boxes) {+ w.boxed(b"avc1", |w| {+ w.zeros(6);+ w.u16(1); // data reference+ w.zeros(16);+ w.u16(self.width as u16);+ w.u16(self.height as u16);+ w.u32(0x0048_0000);+ w.u32(0x0048_0000);+ w.u32(0);+ w.u16(1); // frames per sample+ let name = b"manimgx";+ w.u8(name.len() as u8);+ w.bytes(name);+ w.zeros(31 - name.len());+ w.u16(0x0018);+ w.u16(0xffff);+ w.boxed(b"avcC", |w| {+ let (sps, pps) = (&self.sps, &self.pps);+ w.u8(1);+ w.u8(sps[1]); // profile+ w.u8(sps[2]); // constraints+ w.u8(sps[3]); // level+ w.u8(0xff); // 4-byte NAL sizes+ w.u8(0xe1); // one SPS+ w.u16(sps.len() as u16);+ w.bytes(sps);+ w.u8(1); // one PPS+ w.u16(pps.len() as u16);+ w.bytes(pps);+ if matches!(sps[1], 100 | 110 | 122 | 144) {+ w.u8(0xfd); // 4:2:0+ w.u8(0xf8); // 8-bit luma+ w.u8(0xf8); // 8-bit chroma+ w.u8(0);+ }+ });+ });+ }+}++impl Drop for Mp4 {+ /// The samples' file is temporary: finished or not, it goes (closed first, so that it can).+ fn drop(&mut self) {+ self.data = None;+ let _ = std::fs::remove_file(&self.part);+ }+}++fn ftyp() -> Vec<u8> {+ let mut w = Boxes::default();+ w.boxed(b"ftyp", |w| {+ w.bytes(b"isom");+ w.u32(0x200);+ for brand in [b"isom", b"iso2", b"avc1", b"mp41"] {+ w.bytes(brand);+ }+ });+ w.0+}++#[derive(Default)]+struct Boxes(Vec<u8>);++impl Boxes {+ fn u8(&mut self, v: u8) {+ self.0.push(v);+ }+ fn u16(&mut self, v: u16) {+ self.0.extend_from_slice(&v.to_be_bytes());+ }+ fn u32(&mut self, v: u32) {+ self.0.extend_from_slice(&v.to_be_bytes());+ }+ fn u64(&mut self, v: u64) {+ self.0.extend_from_slice(&v.to_be_bytes());+ }+ fn bytes(&mut self, b: &[u8]) {+ self.0.extend_from_slice(b);+ }+ fn zeros(&mut self, n: usize) {+ self.0.resize(self.0.len() + n, 0);+ }+ fn matrix(&mut self) {+ for v in [0x0001_0000, 0, 0, 0, 0x0001_0000, 0, 0, 0, 0x4000_0000u32] {+ self.u32(v);+ }+ }+ fn dinf(&mut self) {+ self.boxed(b"dinf", |w| {+ w.full(b"dref", 0, 0, |w| {+ w.u32(1);+ w.full(b"url ", 0, 1, |_| {});+ })+ });+ }+ /// How many samples a track's chunks hold, run-length: (first chunk, samples each).+ fn stsc(&mut self, counts: &[u32]) {+ self.full(b"stsc", 0, 0, |w| {+ let runs: Vec<(u32, u32)> = (1..).zip(counts).filter(|&(k, &n)| k == 1 || counts[k as usize - 2] != n).map(|(k, &n)| (k, n)).collect();+ w.u32(runs.len() as u32);+ for (first, n) in runs {+ w.u32(first);+ w.u32(n);+ w.u32(1);+ }+ });+ }+ /// Where a track's chunks start in the file.+ fn stco(&mut self, at: &[u64], wide: bool) {+ if wide {+ self.full(b"co64", 0, 0, |w| {+ w.u32(at.len() as u32);+ at.iter().for_each(|&o| w.u64(o));+ });+ } else {+ self.full(b"stco", 0, 0, |w| {+ w.u32(at.len() as u32);+ at.iter().for_each(|&o| w.u32(o as u32));+ });+ }+ }+ /// Run-length (count, value) entries.+ fn runs(&mut self, values: &[u32]) {+ let mut runs: Vec<(u32, u32)> = Vec::new();+ for &v in values {+ match runs.last_mut() {+ Some((n, last)) if *last == v => *n += 1,+ _ => runs.push((1, v)),+ }+ }+ self.u32(runs.len() as u32);+ for (n, v) in runs {+ self.u32(n);+ self.u32(v);+ }+ }+ fn boxed(&mut self, kind: &[u8; 4], body: impl FnOnce(&mut Self)) {+ let at = self.0.len();+ self.u32(0);+ self.bytes(kind);+ body(self);+ let size = (self.0.len() - at) as u32;+ self.0[at..at + 4].copy_from_slice(&size.to_be_bytes());+ }+ fn full(&mut self, kind: &[u8; 4], version: u8, flags: u32, body: impl FnOnce(&mut Self)) {+ self.boxed(kind, |w| {+ w.u32((version as u32) << 24 | flags);+ body(w);+ });+ }+}