The animation engine for agents: blazingly fast math and 3D videos with Manim's Python API, on a Rust GPU core.
Python
Preserve recording timelines and encoded picture durations
Sina Atalayfc31789b5677
diff --git a/rust/engine/src/encode.rs b/rust/engine/src/encode.rsindex b0e86e5b..0bf11a43 100644--- a/rust/engine/src/encode.rs+++ b/rust/engine/src/encode.rs@@ -122,7 +122,7 @@ impl Encoder {let pts = self.pts;self.pts += duration as i64;if let Some(sample) = self.x264.encode(Some(nv12), pts, constant, key)? {- self.mp4.sample(sample.data, sample.pts, sample.dts, sample.key).map_err(|e| e.to_string())?;+ self.mp4.sample(sample.data, sample.pts, sample.key).map_err(|e| e.to_string())?;}Ok(())}@@ -133,7 +133,7 @@ impl Encoder {let Self { mut x264, mut mp4, pts } = self;while x264.delayed() {if let Some(sample) = x264.encode(None, 0, None, false)? {- mp4.sample(sample.data, sample.pts, sample.dts, sample.key).map_err(|e| e.to_string())?;+ mp4.sample(sample.data, sample.pts, sample.key).map_err(|e| e.to_string())?;}}mp4.finish(pts, audio).map_err(|e| e.to_string())diff --git a/rust/engine/src/export.rs b/rust/engine/src/export.rsindex 6d6f4ff0..22eeefdd 100644--- a/rust/engine/src/export.rs+++ b/rust/engine/src/export.rs@@ -42,16 +42,28 @@ impl Player {dirty}- pub(crate) fn push_frames(&mut self, gpu: &mut Gpu, frames: Vec<Frame>, repeat: u32, key: bool) -> Result<(), String> {+ pub(crate) fn push_frames(&mut self, gpu: &mut Gpu, frames: Vec<Frame>, repeat: u32, key: bool, capture: bool) -> Result<(), String> {let mut export = self.export.take().ok_or("no export is open")?;- let result = self.push_into(gpu, &mut export, frames, repeat, key);+ let result = (|| {+ // A pixel consumer waits for this frame. Settle older draws before making it,+ // so their overflow retries cannot replace its captured pixels.+ if capture {+ while !export.pending.is_empty() { self.retire(gpu, &mut export)?; }+ }+ self.push_into(gpu, &mut export, frames, repeat, key, capture)?;+ if capture {+ self.retire(gpu, &mut export)?;+ self.map_pixels(gpu)?;+ }+ Ok(())+ })();self.export = Some(export);result}/// Draw a frame into the export and convert only the macroblocks a change can touch — every/// one when camera views are drawn too (their pictures change with no record changing).- fn push_into(&mut self, gpu: &mut Gpu, export: &mut Export, frames: Vec<Frame>, repeat: u32, key: bool) -> Result<(), String> {+ fn push_into(&mut self, gpu: &mut Gpu, export: &mut Export, frames: Vec<Frame>, repeat: u32, key: bool, capture: bool) -> Result<(), String> {let (columns, rows) = (export.width.div_ceil(16), export.height.div_ceil(16));let total = columns * rows;if export.pending.len() == RING {@@ -71,12 +83,13 @@ impl Player {// read (see `retire`)let lists = self.lists.as_ref().filter(|_| composited && !listed.is_empty()).map(|l| l.capacity);self.convert(gpu, export, &mut command, &listed, slot, lists.is_some());+ if capture { self.copy_pixels(&mut command); }gpu.queue.submit(Some(command.finish()));export.map(slot, listed.len());gpu.device.poll(wgpu::PollType::Poll).map_err(|e| e.to_string())?;let first = export.previous.is_none();let kept = lists.map(|capacity| (frames.clone(), capacity));- export.pending.push_back(Pending { slot, listed, repeat, first, key, kept });+ export.pending.push_back(Pending { slot, listed, repeat, first, key, kept, capture });export.previous = frames.into_iter().last();export.drawn += 1;Ok(())@@ -134,6 +147,7 @@ impl Player {let (mut command, ..) = self.encode(gpu, frames)?;capacity = self.lists.as_ref().map_or(0, |l| l.capacity);self.convert(gpu, export, &mut command, &p.listed, p.slot, true);+ if p.capture { self.copy_pixels(&mut command); }gpu.queue.submit(Some(command.finish()));export.map(p.slot, p.listed.len());export.wait(&gpu.device, p.slot)?;@@ -158,6 +172,7 @@ pub(crate) struct Pending {first: bool,key: bool,kept: Option<(Vec<Frame>, u64)>, // a frame composited through the lists, and their capacity+ capture: bool, // copy its final successful draw for a simultaneous pixel consumer}/// A video being written: each frame drawn, converted to NV12 where it changed, read back anddiff --git a/rust/engine/src/mp4.rs b/rust/engine/src/mp4.rsindex 25c525da..6a7dae4c 100644--- a/rust/engine/src/mp4.rs+++ b/rust/engine/src/mp4.rs@@ -4,8 +4,8 @@//! A video sample is a picture as x264 writes it (NAL units, each prefixed by its 4-byte size) and a//! held picture is one sample that lasts longer — times are in frames (timescale = fps). With//! B-frames (the slower presets), a picture can be decoded before the one it follows: the samples-//! are in decode order, each shown at its own time, and an edit list starts the track at the-//! first picture shown. An audio+//! are in decode order, each retaining its duration; signed composition offsets give its+//! presentation time. An audio//! sample is an AAC access unit of 1024 samples; the encoder's priming before the sound's first//! sample is skipped by an edit list, so the sound starts with the film, to the sample. Video//! samples stream to a temporary file; `finish` writes the header (`moov`), then the samples,@@ -32,7 +32,7 @@ pub struct Mp4 {struct Sample {size: u32,pts: i64,- dts: i64,+ duration: u32,key: bool,}@@ -63,13 +63,13 @@ impl Mp4 {Ok(Self { path: path.to_string(), part, data: Some(data), width, height, fps, sps, pps, samples: Vec::new(), bytes: 0 })}- pub fn sample(&mut self, data: &[u8], pts: i64, dts: i64, key: bool) -> io::Result<()> {+ pub fn sample(&mut self, data: &[u8], pts: i64, key: bool) -> io::Result<()> {if data.is_empty() {return Ok(());}self.data.as_mut().ok_or_else(|| io::Error::other("the video is finished"))?.write_all(data)?;self.bytes += data.len() as u64;- self.samples.push(Sample { size: data.len() as u32, pts, dts, key });+ self.samples.push(Sample { size: data.len() as u32, pts, duration: 0, key });Ok(())}@@ -79,6 +79,16 @@ impl Mp4 {if let Some(mut data) = self.data.take() {data.flush()?;}+ // Recover each picture's authored duration in presentation order, retaining the+ // encoder's sample order. In particular, the final displayed picture owns the hold,+ // even when B-frames follow it in decode order.+ let mut order: Vec<_> = (0..self.samples.len()).collect();+ order.sort_unstable_by_key(|&i| self.samples[i].pts);+ let mut until = end;+ for i in order.into_iter().rev() {+ self.samples[i].duration = (until - self.samples[i].pts) as u32;+ until = self.samples[i].pts;+ }let written = self.write(end, audio);if written.is_err() {let _ = std::fs::remove_file(&self.path);@@ -91,11 +101,17 @@ impl Mp4 {let Some(audio) = audio else {return vec![(false, 0..self.samples.len())];};- let first = self.samples.first().map_or(0, |s| s.dts);+ // A picture must arrive before its own presentation and every later picture in+ // decode order: any of them may need it as a reference. Signed composition offsets+ // make the sample-duration clock unsuitable as a progressive-download deadline.+ let mut deadlines: Vec<_> = self.samples.iter().map(|s| s.pts).collect();+ for i in (1..deadlines.len()).rev() {+ deadlines[i - 1] = deadlines[i - 1].min(deadlines[i]);+ }let (mut chunks, mut v, mut a, mut second) = (Vec::new(), 0, 0, 1i64);while v < self.samples.len() || a < audio.units.len() {let v0 = v;- while v < self.samples.len() && self.samples[v].dts - first < second * self.fps as i64 {+ while v < self.samples.len() && deadlines[v] < second * self.fps as i64 {v += 1;}if v > v0 {@@ -120,9 +136,9 @@ impl Mp4 {let header = if wide { 16 } else { 8 };let layout = self.layout(audio);let ftyp = ftyp();- let probe = self.moov(end, audio, &layout, 0, wide);+ let probe = self.moov(end, audio, &layout, 0, wide)?;let start = (ftyp.len() + probe.len() + header) as u64;- let moov = self.moov(end, audio, &layout, start, wide);+ let moov = self.moov(end, audio, &layout, start, wide)?;let mut out = BufWriter::with_capacity(1 << 20, File::create(&self.path)?);out.write_all(&ftyp)?;out.write_all(&moov)?;@@ -168,21 +184,20 @@ impl Mp4 {}/// The header, with the samples starting at byte `offset` of the file.- fn moov(&self, end: i64, audio: Option<&Audio>, layout: &Layout, offset: u64, wide: bool) -> Vec<u8> {- // The media's clock starts at the first decode. With B-frames, x264 decodes ahead of- // what it shows (its first dts is negative), so a picture shows at its pts minus that- // first dts: an offset of pts − dts (never negative) from its decode, and an edit list- // that starts the track at the first picture shown.- let first = self.samples.first().map_or(0, |s| s.dts);- let start = self.samples.iter().map(|s| s.pts).min().map_or(0, |pts| pts - first);- let decode: Vec<i64> = self.samples.iter().map(|s| s.dts - first).collect();+ fn moov(&self, end: i64, audio: Option<&Audio>, layout: &Layout, offset: u64, wide: bool) -> io::Result<Vec<u8>> {+ // ISO/IEC 14496-12: decoding timestamps order samples; composition timestamps say+ // when they appear. Accumulate their own durations in decode order and preserve every+ // authored presentation time with signed ctts offsets. This also gives the last+ // displayed picture its proper duration, without a decode-preroll edit or extra frames.let duration = end.max(1) as u32;- let media = (end - first).max(1) as u32;- let mut deltas: Vec<u32> = decode.windows(2).map(|w| (w[1] - w[0]) as u32).collect();- if let Some(&last) = decode.last() {- deltas.push((end - first - last).max(1) as u32);- }- let offsets: Vec<u32> = self.samples.iter().map(|s| (s.pts - s.dts).max(0) as u32).collect();+ let deltas: Vec<_> = self.samples.iter().map(|s| s.duration).collect();+ let mut decode = 0;+ let offsets: Vec<_> = self.samples.iter().map(|s| {+ let offset = i32::try_from(s.pts - decode)+ .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "MP4 composition offset exceeds 32-bit range"))?;+ decode += s.duration as i64;+ Ok(offset as u32)+ }).collect::<io::Result<_>>()?;let [(video_at, video_counts), (audio_at, audio_counts)] = self.chunks(audio, layout, offset);// the sound's length in the movie's timescale (frames)let heard = audio.map_or(0, |a| (a.length * self.fps as u64).div_ceil(a.rate as u64) as u32);@@ -216,23 +231,12 @@ impl Mp4 {w.u32(self.width << 16);w.u32(self.height << 16);});- if start > 0 {- w.boxed(b"edts", |w| {- w.full(b"elst", 0, 0, |w| {- w.u32(1);- w.u32(duration); // in the movie's timescale (frames, as the track's)- w.u32(start as u32); // where the first picture shows, in the track's- w.u16(1);- w.u16(0);- });- });- }w.boxed(b"mdia", |w| {w.full(b"mdhd", 0, 0, |w| {w.u32(0);w.u32(0);w.u32(self.fps);- w.u32(media);+ w.u32(duration);w.u16(0x55c4); // "und"w.u16(0);});@@ -252,7 +256,7 @@ impl Mp4 {});w.full(b"stts", 0, 0, |w| w.runs(&deltas));if offsets.iter().any(|&o| o != 0) {- w.full(b"ctts", 0, 0, |w| w.runs(&offsets));+ w.full(b"ctts", 1, 0, |w| w.runs(&offsets));}if self.samples.iter().any(|s| !s.key) {let keys: Vec<u32> = (1..).zip(&self.samples).filter(|(_, s)| s.key).map(|(k, _)| k).collect();@@ -276,7 +280,7 @@ impl Mp4 {self.sound(w, audio, heard, &audio_counts, &audio_at, wide);}});- w.0+ Ok(w.0)}/// The sound's track: AAC-LC, its priming cut by an edit list, a unit of pre-roll.@@ -444,9 +448,9 @@ impl Drop for Mp4 {fn ftyp() -> Vec<u8> {let mut w = Boxes::default();w.boxed(b"ftyp", |w| {- w.bytes(b"isom");+ w.bytes(b"iso4"); // signed composition offsets (ctts version 1)w.u32(0x200);- for brand in [b"isom", b"iso2", b"avc1", b"mp41"] {+ for brand in [b"iso4", b"isom", b"iso2", b"avc1", b"mp41"] {w.bytes(brand);}});@@ -544,3 +548,36 @@ impl Boxes {});}}+++#[cfg(test)]+mod tests {+ use super::*;++ fn movie(pictures: &[(i64, u32)]) -> Mp4 {+ Mp4 {+ path: String::new(), part: String::new(), data: None,+ width: 2, height: 2, fps: 1, sps: vec![0; 4], pps: vec![0], bytes: 0,+ samples: pictures.iter().map(|&(pts, duration)| Sample { size: 0, pts, duration, key: true }).collect(),+ }+ }++ #[test]+ fn progressive_chunks_deliver_references_before_their_earliest_picture() {+ // The P-picture shown at 10s is needed to decode the B-picture shown at 1s.+ // Its 20s hold must not put that B-picture behind 21s of audio.+ let movie = movie(&[(0, 1), (10, 20), (1, 9)]);+ let audio = Audio { rate: 1024, channels: 1, config: vec![], priming: 0, length: 30 * 1024, units: vec![vec![]; 30] };+ let chunks = movie.layout(Some(&audio));+ assert_eq!(&chunks[..4], &[(false, 0..1), (true, 0..1), (false, 1..3), (true, 1..2)]);+ assert!(chunks[4..].iter().all(|(sound, _)| *sound));+ }++ #[test]+ fn composition_offsets_cannot_wrap() {+ let movie = movie(&[(0, 1), (i32::MAX as i64 + 2, 1), (1, i32::MAX as u32 + 1)]);+ let error = movie.moov(i32::MAX as i64 + 3, None, &vec![(false, 0..3)], 0, false).unwrap_err();+ assert_eq!(error.kind(), io::ErrorKind::InvalidInput);+ assert!(error.to_string().contains("composition offset"));+ }+}diff --git a/rust/engine/src/take.rs b/rust/engine/src/take.rsindex 6fa968a9..72fad99b 100644--- a/rust/engine/src/take.rs+++ b/rust/engine/src/take.rs@@ -4,7 +4,8 @@//! reads it back and draws any of its frames on demand (`web`).//!//! A message is its length (u32), its op (u8), then its fields, little-endian; a byte string is-//! its length (u32), then its bytes. A take begins with START and ends with END (whether its+//! its length (u32), then its bytes. A take begins with START: its format version (u32), width+//! and height (u32 each), and frame rate (f64). It ends with END (whether its//! scene failed: a take cut short has none); NOTE carries the director's notes (JSON: the plays,//! the sections, the captions) to whoever shows the take, and SOUND the film's sound, an audio//! file its player plays beside the frames.@@ -18,15 +19,19 @@//! the same record slot drew the frame before: the writer learns what replaces what from the//! frames' records, so a film changing a little from frame to frame is sent as its changes.-#[cfg(any(feature = "python", test))]+#[cfg(any(feature = "python", all(test, not(target_arch = "wasm32"))))]use std::collections::HashMap;-#[cfg(any(feature = "python", test))]+#[cfg(any(feature = "python", all(test, not(target_arch = "wasm32"))))]use std::sync::Arc;-#[cfg(any(feature = "python", test))]+#[cfg(any(feature = "python", all(test, not(target_arch = "wasm32"))))]use crate::pack::{self, Kind};-pub(crate) const START: u8 = 0;+/// The wire contract, including the layouts of views, records, uploads and coded arrays.+/// Bump when a writer's output can no longer be read with the same meaning by an older reader.+pub(crate) const VERSION: u32 = 1;+// Opcode 0 was the unversioned START. A different opcode makes old readers reject this one.+pub(crate) const START: u8 = 14;pub(crate) const PATH: u8 = 1;pub(crate) const POINTS: u8 = 2;pub(crate) const MESH: u8 = 3;@@ -50,7 +55,7 @@ pub(crate) fn slots(records: &[u8]) -> impl Iterator<Item = [u64; KEYS]> + '_ {}/// An upload the writer holds until a frame shows it (so it knows what it replaces).-#[cfg(any(feature = "python", test))]+#[cfg(any(feature = "python", all(test, not(target_arch = "wasm32"))))]struct Upload {op: u8,key: u64,@@ -60,7 +65,7 @@ struct Upload {}/// Writes a take's messages into a buffer, taken out as it fills (`drain`).-#[cfg(any(feature = "python", test))]+#[cfg(any(feature = "python", all(test, not(target_arch = "wasm32"))))]#[derive(Default)]pub(crate) struct Writer {buffer: Vec<u8>,@@ -73,7 +78,7 @@ pub(crate) struct Writer {records: HashMap<u64, u64>, // a view's last records array (0: the frame's; else a camera's)}-#[cfg(any(feature = "python", test))]+#[cfg(any(feature = "python", all(test, not(target_arch = "wasm32"))))]impl Writer {fn begin(&mut self, op: u8) -> usize {let at = self.buffer.len();@@ -99,6 +104,7 @@ impl Writer {pub(crate) fn start(&mut self, width: u32, height: u32, fps: f64) {let at = self.begin(START);+ self.put(&VERSION.to_le_bytes());self.put(&width.to_le_bytes());self.put(&height.to_le_bytes());self.put(&fps.to_le_bytes());@@ -110,10 +116,14 @@ impl Writer {pub(crate) fn points(&mut self, key: u64, vertices: &[u8]) {self.hold(POINTS, key, &[], vec![(Kind::Points, vertices)]);}- #[allow(clippy::too_many_arguments)]- pub(crate) fn mesh(&mut self, key: u64, points: &[u8], uvs: &[u8], normals: &[u8], triangles: &[u8], outline: u32, block: u32) {- let head: Vec<u8> = [outline.to_le_bytes(), block.to_le_bytes()].concat();- self.hold(MESH, key, &head, vec![(Kind::MeshPoints, points), (Kind::MeshUvs, uvs), (Kind::MeshNormals, normals), (Kind::MeshTriangles, triangles)]);+ pub(crate) fn mesh(&mut self, key: u64, mesh: &crate::mesh::Mesh<'_>) {+ let head: Vec<u8> = [mesh.outline.to_le_bytes(), mesh.block.to_le_bytes()].concat();+ self.hold(MESH, key, &head, vec![+ (Kind::MeshPoints, bytemuck::cast_slice(&mesh.points)),+ (Kind::MeshUvs, bytemuck::cast_slice(&mesh.uvs)),+ (Kind::MeshNormals, bytemuck::cast_slice(&mesh.normals)),+ (Kind::MeshTriangles, bytemuck::cast_slice(&mesh.triangles)),+ ]);}pub(crate) fn rows(&mut self, key: u64, rows: &[u8]) {self.hold(ROWS, key, &[], vec![(Kind::Rows, rows)]);@@ -303,7 +313,7 @@ pub(crate) fn messages(stream: &[u8]) -> (Vec<&[u8]>, usize) {(out, at)}-#[cfg(test)]+#[cfg(all(test, not(target_arch = "wasm32")))]mod tests {use super::*;@@ -323,11 +333,15 @@ mod tests {w.start(960, 540, 30.0);let points = |t: f64| -> Vec<u8> { bytemuck::cast_slice(&(0..900).map(|i| ((i as f64) * 0.1 + t).sin()).collect::<Vec<f64>>()).to_vec() };let triangles: Vec<u8> = bytemuck::cast_slice(&(0..300u32).collect::<Vec<u32>>()).to_vec();- w.mesh(7, &points(0.0), &[], &[], &triangles, 0, 0);+ let uvs = [[0.0f64; 2]; 300];+ let normals = [[0.0f64, 0.0, 1.0]; 300];+ let before = points(0.0);+ let after = points(0.01);+ w.mesh(7, &crate::mesh::Mesh::new(&before, bytemuck::cast_slice(&uvs), bytemuck::cast_slice(&normals), &triangles, 0, 0).unwrap());let mut seen_by_camera = record(7);seen_by_camera[56] = 1; // other flags: other recordsw.frame(&[3; 192], &record(7), 1, &[(9, 64, 32, vec![5; 192], seen_by_camera)]);- w.mesh(8, &points(0.01), &[], &[], &triangles, 0, 0); // the next frame's: replaces 7+ w.mesh(8, &crate::mesh::Mesh::new(&after, bytemuck::cast_slice(&uvs), bytemuck::cast_slice(&normals), &triangles, 0, 0).unwrap()); // the next frame's: replaces 7w.frame(&[3; 192], &record(7), 2, &[]);w.frame(&[3; 192], &record(8), 1, &[]);w.sound(&[8; 44]);@@ -370,6 +384,45 @@ mod tests {}assert!(decoded.values().any(|d| d[..] == pack::canonical(Kind::MeshPoints, &points(0.01))[..]));let mut s = Fields(messages[0]);- assert_eq!((s.u8().unwrap(), s.u32().unwrap(), s.u32().unwrap(), s.f64().unwrap()), (START, 960, 540, 30.0));+ assert_eq!((s.u8().unwrap(), s.u32().unwrap(), s.u32().unwrap(), s.u32().unwrap(), s.f64().unwrap()), (START, VERSION, 960, 540, 30.0));+ }++ #[test]+ fn a_recorded_mesh_keeps_normals_derived_before_positions_are_rounded() {+ let points = [[-0.6f64, -0.6, 1e8], [0.6, -0.6, 1e8 + 1.0], [0.0, 0.6, 1e8]];+ let uvs = [[0.25f64, 0.5], [0.5, 0.75], [0.75, 1.0]];+ let mesh = crate::mesh::Mesh::new(bytemuck::cast_slice(&points), bytemuck::cast_slice(&uvs), &[], bytemuck::cast_slice(&[0u32, 1, 2]), 0, 0).unwrap();+ // The GPU positions have lost their small z difference. Deriving normals from those+ // positions would turn this tilted face into a flat one and change its lighting.+ assert!(mesh.points.iter().all(|v| v[2] as f32 == 1e8));+ assert_eq!(mesh.normals[0], [-1.2, 0.6, 1.44]);+ let mut writer = Writer::default();+ writer.start(320, 240, 10.0);+ writer.mesh(1, &mesh);+ writer.frame(&[0; 192], &record(1), 1, &[]);+ writer.close(false);+ let stream = writer.drain();+ let mut arrays = HashMap::new();+ for message in messages(&stream).0 {+ let mut fields = Fields(message);+ if fields.u8().unwrap() != ARRAY { continue; }+ let (_key, base, kind, mode, data) = (fields.u64().unwrap(), fields.u64().unwrap(), fields.u8().unwrap(), fields.u8().unwrap(), fields.string().unwrap());+ assert_eq!(base, 0);+ arrays.insert(kind, pack::decode(Kind::from_u8(kind).unwrap(), mode, data, None, None).unwrap());+ }+ let wide = |kind: Kind| {+ let f: Vec<f32> = bytemuck::pod_collect_to_vec(&arrays[&(kind as u8)]);+ bytemuck::cast_slice(&f.into_iter().map(f64::from).collect::<Vec<_>>()).to_vec()+ };+ let (points, uvs, normals) = (wide(Kind::MeshPoints), wide(Kind::MeshUvs), wide(Kind::MeshNormals));+ let replay = crate::mesh::Mesh::new(&points, &uvs, &normals, &arrays[&(Kind::MeshTriangles as u8)], 0, 0).unwrap();+ for (before, after) in mesh.points.iter().zip(&*replay.points) {+ assert_eq!(before.map(|v| v as f32), after.map(|v| v as f32));+ }+ for (before, after) in mesh.normals.iter().zip(&*replay.normals) {+ assert_eq!(before.map(|v| v as f32), after.map(|v| v as f32));+ }+ assert_eq!(replay.uvs, mesh.uvs);+ assert_eq!(replay.triangles, mesh.triangles);}}diff --git a/src/manimgx/rendering/film.py b/src/manimgx/rendering/film.pyindex c957e56e..8ed56c6c 100644--- a/src/manimgx/rendering/film.py+++ b/src/manimgx/rendering/film.py@@ -15,6 +15,7 @@ from typing import TYPE_CHECKING, Literal, get_argsfrom warnings import deprecatedfrom manimgx import _engine+from manimgx.animation import clockfrom manimgx.config import configfrom manimgx.rendering import feedfrom manimgx.rendering.feed import CameraView, Feeder@@ -260,7 +261,7 @@ class Film:raise ValueError(f"a video needs an even width and height, not {width}x{height}")- self.fps = Fraction(fps).limit_denominator(1000)+ self.fps = clock.rational(fps)"""How many frames a second the film has: the configuration's when it began."""# the GPU draws the film; or its take is recorded, to be drawn by manimgx's player: by# choice, or where the engine has no GPU (in Pyodide)@@ -412,6 +413,8 @@ class Film:view, records, cameras = self.feeder.frame(camera, mobjects)if self._player is None:raise RuntimeError("a film recorded as a take is drawn by manimgx's player")+ if self._player.pressured():+ self.feeder.sweep([records, *[c[4] for c in cameras]], pressed=True)return self._player.render(view, records, cameras)@deprecated("manimgx's machinery: the scene calls it", category=None)@@ -479,17 +482,38 @@ class Film:self._flush()if (player := self._player) is None:return- if self.video is not None:- player.push(view, records, repeat, cameras, self._pending_key)- if self.frames is not None:- self.frames(- Frame(- self._first,- repeat,- lambda: player.render(view, records, cameras),- self._pending_key,- )- )+ if (self.video is not None or self.frames is not None) and player.pressured():+ self.feeder.sweep([records, *[c[4] for c in cameras]], pressed=True)+ key = self._pending_key+ if self.frames is None:+ if self.video is not None:+ player.push(view, records, repeat, cameras, key)+ return+ sent = self.video is None++ def push() -> None:+ nonlocal sent+ if not sent:+ player.push(view, records, repeat, cameras, key)+ sent = True++ def draw() -> bytes:+ nonlocal sent+ if sent:+ return player.render(view, records, cameras)+ # A pixel consumer and the encoder share one completed GPU draw. Defer export+ # until the callback asks: metadata-only callbacks keep the asynchronous path.+ pixels = player.push(view, records, repeat, cameras, key, capture=True)+ sent = True+ assert pixels is not None+ return pixels++ try:+ self.frames(Frame(self._first, repeat, draw, key))+ except Cut:+ push() # the frame that cuts a film still belongs to its video+ raise+ push()@deprecated("manimgx's machinery: the scene calls it", category=None)def close(self) -> None:@@ -500,19 +524,17 @@ class Film:any other error abandons the video."""try:- self._send()- except Cut:- pass+ with contextlib.suppress(Cut):+ self._send()+ with contextlib.suppress(Cut): # its take cut it: it hears no more+ self._end()except BaseException:self.abort()raise- with contextlib.suppress(Cut): # its take cut it as it ended: it hears no more- self._end()def _end(self) -> None:"""Write the closed film's video, if any, and tell its take how it ends."""if self.video is not None and self._player is not None:- self.video = Nonesound = self.soundtrack()if sound is None:self.export = Export(*self._player.end_export())@@ -525,6 +547,7 @@ class Film:sound.tobytes(), channels, RATE, BITRATE * channels))+ self.video = Noneif self._recorder is not None:# what the player plays and shows beside the frames; then the take's endif (sound := self.soundtrack()) is not None:diff --git a/tests/integration/test_film.py b/tests/integration/test_film.pyindex dccd294a..a2b74030 100644--- a/tests/integration/test_film.py+++ b/tests/integration/test_film.py@@ -47,6 +47,28 @@ def test_an_error_at_the_last_frame_abandons_the_video(tmp_path: Path) -> None:assert list(tmp_path.iterdir()) == [] # not the video, nor a part of it+def test_a_soundtrack_error_closes_the_native_export(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch+) -> None:+ def failed_soundtrack(self: m.Film) -> None:+ raise ValueError("the soundtrack failed")++ monkeypatch.setattr(m.Film, "soundtrack", failed_soundtrack)+ scene = Held()+ try:+ with pytest.raises(ValueError, match="the soundtrack failed"):+ scene.render(tmp_path / "failed.mp4")+ player = scene.film._player+ assert player is not None+ # Retain the scene and player: cleanup must not depend on garbage collection.+ player.begin_export(str(tmp_path / "next.mp4"))+ player.abort_export()+ assert list(tmp_path.iterdir()) == []+ finally:+ if (player := scene.film._player) is not None:+ player.abort_export()++def test_a_video_needs_even_sides(tmp_path: Path) -> None:config.pixel_width = 321 # H.264's 4:2:0 halves both sideswith pytest.raises(ValueError, match="an even width and height, not 321x180"):diff --git a/tests/integration/test_replay.py b/tests/integration/test_replay.pynew file mode 100644index 00000000..2056904f--- /dev/null+++ b/tests/integration/test_replay.py@@ -0,0 +1,180 @@+"""Recorded frames use the same decoder and renderer as native playback, in any order."""++import numpy as np+import pytest++import manimgx as m+from manimgx._engine import TAKE_VERSION, Player, Recorder, Replay, adapter_info+from manimgx.rendering.feed import Feeder+++def test_recorded_geometry_and_images_replay_exactly_in_any_order() -> None:+ adapter = adapter_info()+ assert set(adapter) == {+ "name",+ "vendor",+ "device",+ "device_type",+ "driver",+ "driver_info",+ "backend",+ }+ assert adapter["name"]+ assert adapter["backend"]+ assert int(adapter["vendor"]) >= 0+ assert int(adapter["device"]) >= 0+ width, height, fps = 130, 74, 29.97 # readback rows require padding+ player, recorder = Player(width, height), Recorder(width, height, fps)+ direct, recorded = Feeder(width, height, player), Feeder(width, height, recorder)+ camera = m.Camera(three_d=True, phi=0.4, theta=-1.2)+ path = m.VMobject(color=m.BLUE, stroke_width=8).set_points_as_corners(+ [(-3, -1, 0), (-2, 1, 0)]+ )+ mesh = m.MeshMobject(+ np.array([[-0.8, -1, 0.1], [0.9, -0.8, -0.2], [0.6, 1, 0.6], [-0.7, 0.9, 0]]),+ np.array([[0, 1, 2], [0, 2, 3]]),+ color=m.RED,+ shade_in_3d=True,+ )+ image = (+ m.ImageMobject(+ np.array(+ [+ [[255, 0, 0, 255], [0, 255, 0, 255]],+ [[0, 0, 255, 255], [255, 255, 0, 255]],+ ],+ dtype=np.uint8,+ )+ )+ .scale_to_fit_height(1.6)+ .shift(2 * m.RIGHT)+ )+ objects: list[m.Mobject] = [path, mesh, image]+ expected: list[bytes] = []+ for step, repeat in enumerate([3, 1, 2, 1]):+ path.add_points_as_corners([(-1.8 + step * 0.2, 0.5 - step * 0.3, 0)])+ mesh.rotate(0.1, axis=m.UP)+ expected.extend([player.render(*direct.frame(camera, objects))] * repeat)+ view, records, cameras = recorded.frame(camera, objects)+ recorder.frame(view, records, repeat, cameras)+ recorder.end()+ replay = Replay(recorder.drain())+ assert replay.size == (width, height)+ assert replay.frames == len(expected)+ assert replay.fps == fps+ assert replay.timeline == [(0, 3), (3, 1), (4, 2), (6, 1)]+ for frame in [6, 0, 4, 1, 3, 2, 5, 0, 6]:+ assert replay.render(frame) == expected[frame]+ for frame in [-1, replay.frames]:+ with pytest.raises(IndexError, match="outside"):+ replay.render(frame)+ assert adapter_info() == adapter+++def test_independent_takes_own_their_resources_when_keys_overlap() -> None:+ image = m.ImageMobject(np.full((2, 2, 4), 255, np.uint8)).scale_to_fit_height(4)+ camera = m.Camera()++ def take(color: bytes) -> tuple[Replay, bytes, int]:+ player, recorder = Player(64, 36), Recorder(64, 36, 30)+ direct = Feeder(64, 36, player)+ recorded = Feeder(64, 36, recorder)+ direct_frame = direct.frame(camera, [image])+ view, records, cameras = recorded.frame(camera, [image])+ key = recorded.texture(image.pixel_array)+ assert direct.texture(image.pixel_array) == key+ # Identical keys intentionally name different pixels in independent takes. Both+ # receive ordinary native uploads; the test neither reads nor rewrites the format.+ player.add_texture(key, 2, 2, color * 4)+ recorder.add_texture(key, 2, 2, color * 4)+ recorder.frame(view, records, 1, cameras)+ recorder.end()+ return Replay(recorder.drain()), player.render(*direct_frame), key++ first, red, first_key = take(bytes([255, 0, 0, 255]))+ second, green, second_key = take(bytes([0, 255, 0, 255]))+ assert first_key == second_key+ assert red != green+ for _ in range(3):+ assert second.render(0) == green+ assert first.render(0) == red++[email protected](pixel_width=960, pixel_height=540, frame_rate=10)[email protected]("closes", [False, True], ids=["curved", "closed-then-open"])+def test_growing_paths_keep_their_historical_picture_when_seeking(closes: bool) -> None:+ class Growing(m.Scene):+ def construct(self) -> None:+ starts = [m.LEFT + m.UP, m.LEFT + m.DOWN, 2 * m.LEFT]+ ends = [m.RIGHT, m.RIGHT + m.UP, m.RIGHT + m.DOWN]+ colors = [m.RED, m.GREEN, m.BLUE]+ dots = m.VGroup(+ *[m.Dot(point, color=color) for point, color in zip(starts, colors)]+ )+ targets = m.VGroup(+ *[m.Dot(point, color=color) for point, color in zip(ends, colors)]+ )+ self.add(dots, targets)+ for dot in dots:+ self.add(m.TracedPath(dot.get_center, stroke_color=dot.get_color()))+ self.wait()+ if closes:+ self.play(m.Rotate(dots, m.TAU), run_time=2)+ self.play(dots.animate.shift(m.RIGHT), run_time=1)+ else:+ self.play(+ m.Transform(dots, targets, path_func=m.counterclockwise_path()),+ run_time=2,+ )+ self.wait()++ expected: list[bytes] = []+ Growing().render(+ frames=lambda frame: expected.extend([frame.pixels()] * frame.repeat)+ )+ chunks: list[bytes] = []+ Growing().render(take=chunks.append)+ replay = Replay(b"".join(chunks))+ assert replay.frames == len(expected)+ forward = list(range(replay.frames))+ for index in [+ *reversed(forward),+ *forward,+ *np.random.default_rng(7).permutation(forward),+ ]:+ assert replay.render(int(index)) == expected[index]+++def test_a_replay_rejects_incomplete_failed_and_multiple_takes() -> None:+ recorder = Recorder(32, 24, 30)+ view, records, cameras = Feeder(32, 24, recorder).frame(m.Camera(), [])+ recorder.frame(view, records, 2, cameras)+ unfinished = recorder.drain()+ recorder.end()+ complete = unfinished + recorder.drain()+ assert Replay(complete).frames == 2+ for data, message in [+ (b"", "exactly one"),+ (unfinished, "successfully"),+ (complete[:-1], "ends early"),+ (complete + complete, "exactly one"),+ ]:+ with pytest.raises(ValueError, match=message):+ Replay(data)+ failed = Recorder(32, 24, 30)+ failed.frame(view, records, 1, cameras)+ failed.end(failed=True)+ with pytest.raises(ValueError, match="successfully"):+ Replay(failed.drain())+++def test_take_versions_are_explicit_and_rejected_before_playback() -> None:+ recorder = Recorder(32, 24, 30)+ start = bytearray(recorder.drain())+ assert int.from_bytes(start[5:9], "little") == TAKE_VERSION+ start[5:9] = (TAKE_VERSION + 1).to_bytes(4, "little")+ with pytest.raises(ValueError, match=f"unsupported take format {TAKE_VERSION + 1}"):+ Replay(bytes(start))+ # The old unversioned START: its length, opcode0, width, height and fps.+ with pytest.raises(ValueError, match="unversioned takes are unsupported"):+ Replay(bytes.fromhex("11000000 00 20000000 18000000 0000000000003e40"))diff --git a/tests/integration/test_take.py b/tests/integration/test_take.pyindex 22ac93a9..30ae687f 100644--- a/tests/integration/test_take.py+++ b/tests/integration/test_take.py@@ -20,7 +20,7 @@ from manimgx import _enginefrom manimgx.rendering import feedfrom manimgx.rendering.film import Cut, Frame-START, FRAME, NOTE, SOUND, END = 0, 8, 9, 10, 12+START, FRAME, NOTE, SOUND, END = 14, 8, 9, 10, 12class Held(m.Scene):@@ -61,7 +61,7 @@ def test_a_take_holds_the_frames_a_video_would() -> None:take, count = record(Held())(op, start), *rest = messages(take)assert op == START- assert struct.unpack("<IId", start) == (320, 180, 30.0)+ assert struct.unpack("<IIId", start) == (_engine.TAKE_VERSION, 320, 180, 30.0)repeats = [struct.unpack_from("<I", fields)[0] for op, fields in rest if op == FRAME]diff --git a/tests/integration/test_video.py b/tests/integration/test_video.pyindex 7cb0713b..934b6269 100644--- a/tests/integration/test_video.py+++ b/tests/integration/test_video.py@@ -3,14 +3,18 @@ conversion to NV12, x264 (built into the engine), and what the MP4 says of its c"""from pathlib import Path+from unittest.mock import Mock, patchimport avimport numpy as npimport pytestfrom tests.integration.corpus.frames import decode+from tests.integration.corpus.run_manimgx import mp4_framesimport manimgx as m-from manimgx.rendering.film import Frame, X264Preset+from manimgx._engine import Player+from manimgx.rendering.feed import Feeder+from manimgx.rendering.film import Cut, Frame, X264PresetSIZE = (320, 180)pytestmark = pytest.mark.config(pixel_width=SIZE[0], pixel_height=SIZE[1])@@ -57,3 +61,214 @@ def test_each_picture_shows_at_its_frame(tmp_path: Path, preset: X264Preset) ->if frame.pts is not None]assert shown == starts++[email protected]("pixels", [False, True])[email protected]("cut", [False, True])+def test_video_and_pixel_sinks_share_one_draw(+ tmp_path: Path, pixels: bool, cut: bool+) -> None:+ player = Mock(wraps=Player(*SIZE))+ sent: list[Frame] = []++ def sink(frame: Frame) -> None:+ sent.append(frame)+ if pixels:+ assert len(frame.pixels()) == SIZE[0] * SIZE[1] * 4+ if cut and len(sent) == 3:+ raise Cut++ video = tmp_path / "shared.mp4"+ with patch("manimgx.rendering.feed.Player", return_value=player):+ Filled().render(video, frames=sink)+ assert player.push.call_count == len(sent)+ assert all(+ call.kwargs.get("capture", False) == pixels+ for call in player.push.call_args_list+ )+ player.render.assert_not_called()+ assert len(list(decode(video, SIZE))) == len(sent)+ # A frame retained by a metadata consumer must no longer try to export it when read.+ assert len(sent[-1].pixels()) == SIZE[0] * SIZE[1] * 4+ assert player.push.call_count == len(sent)++[email protected]("queued_layers", [False, True])[email protected]("samples", [1, 4])+def test_captured_exports_preserve_pixels_and_video_across_pending_draws(+ tmp_path: Path, queued_layers: bool, samples: int+) -> None:+ # RGBA rows require padding; NV12 edges are partial macroblocks.+ width, height = 130, 74+ capture, plain, direct = (Player(width, height, samples) for _ in range(3))+ feeders = [Feeder(width, height, player) for player in (capture, plain, direct)]+ camera = m.Camera(three_d=True)+ # Cover the picture in see-through mesh layers: the first draw must grow its fragment+ # lists. Capturing must return the successful retry, including after older queued draws.+ layers: list[m.Mobject] = [+ m.MeshMobject(+ np.array([[-8, -5, z], [8, -5, z], [8, 5, z], [-8, 5, z]]),+ np.array([[0, 1, 2], [0, 2, 3]]),+ color=m.BLUE if i % 2 else m.RED,+ fill_opacity=0.2,+ )+ for i, z in enumerate(np.linspace(-0.2, 0.2, 12))+ ]+ captured_video, plain_video = tmp_path / "captured.mp4", tmp_path / "plain.mp4"+ capture.begin_export(str(captured_video), 30)+ plain.begin_export(str(plain_video), 30)+ for index, wanted in enumerate([False, False, False, True, True, False, True]):+ objects = (+ layers+ if (index >= 3) != queued_layers+ else [m.Square(fill_opacity=1).shift((index % 3) * m.RIGHT)]+ )+ prepared = [feeder.frame(camera, objects) for feeder in feeders]+ expected = direct.render(*prepared[2])+ assert any(expected[0::4]) # geometry is visible, including the mesh layers+ view, records, cameras = prepared[0]+ actual = capture.push(view, records, cameras=cameras, capture=wanted)+ assert actual == (expected if wanted else None)+ view, records, cameras = prepared[1]+ assert plain.push(view, records, cameras=cameras) is None+ capture.end_export()+ plain.end_export()+ assert captured_video.read_bytes() == plain_video.read_bytes()++[email protected](frame_rate=10)+ "story", ["empty", "static", "hold", "off_grid", "sections", "cut"]+)[email protected]("preset", ["ultrafast", "slow"])+def test_video_duration_covers_the_complete_film(+ tmp_path: Path, story: str, preset: X264Preset+) -> None:+ """Every callback hold, including the closing frame, survives MP4 sample timing."""++ class Story(m.Scene):+ def construct(self) -> None:+ if story == "empty":+ return+ dot = m.Dot()+ self.add(dot)+ if story == "static":+ return+ if story == "hold":+ self.wait(1)+ else:+ self.play(dot.animate.shift(m.RIGHT), run_time=0.137)+ if story == "sections":+ self.next_section("held")+ self.wait(0.53)++ sent: list[tuple[int, int]] = []++ def observe(frame: Frame) -> None:+ sent.append((frame.index, frame.repeat))+ if story == "cut" and len(sent) == 2:+ raise Cut++ path = tmp_path / "timing.mp4"+ film = Story().render(path, frames=observe, preset=preset)+ covered = 0+ for index, repeat in sent:+ assert index == covered+ assert repeat > 0+ covered += repeat+ assert mp4_frames(path) == film.frame_count == covered+ with av.open(str(path)) as container:+ stream = container.streams.video[0]+ assert stream.time_base is not None+ shown = [+ (frame.pts * stream.time_base * 10, frame.duration * stream.time_base * 10)+ for frame in container.decode(stream)+ if frame.pts is not None+ ]+ assert shown == sent+ if story in {"empty", "static"}:+ assert covered == 1+ if story in {"hold", "sections"}:+ assert any(repeat > 1 for _, repeat in sent)++[email protected]("sound", [False, True])+ ("bframes", "pyramid"), [(0, "none"), (3, "none"), (3, "normal")]+)+ "durations",+ [+ [1] * 9 + [60],+ [40] + [1] * 9,+ [1, 8, 1, 1, 32, 1, 1, 2, 1, 16],+ ],+ ids=["last-hold", "first-hold", "mixed-holds"],+)+def test_reordered_video_preserves_each_pictures_duration(+ tmp_path: Path, bframes: int, pyramid: str, durations: list[int], sound: bool+) -> None:+ player = Player(*SIZE)+ feeder, camera = Feeder(*SIZE, player), m.Camera()+ path = tmp_path / "reordered.mp4"+ player.begin_export(+ str(path),+ 10,+ "slow",+ options=[+ ("bframes", str(bframes)),+ ("b-adapt", "0"),+ ("b-pyramid", pyramid),+ ("scenecut", "0"),+ ],+ )+ expected, at = [], 0+ for index, duration in enumerate(durations):+ view, records, cameras = feeder.frame(+ camera, [m.Dot().shift((index / 10) * m.RIGHT)]+ )+ player.push(view, records, repeat=duration, cameras=cameras, key=index == 5)+ expected.append((at, duration))+ at += duration+ player.end_export(np.zeros(at * 4800, np.float32).tobytes() if sound else None)+ with av.open(str(path)) as container:+ stream = container.streams.video[0]+ assert stream.time_base is not None+ assert stream.duration is not None+ assert stream.duration * stream.time_base * 10 == at+ actual = [+ (frame.pts * stream.time_base * 10, frame.duration * stream.time_base * 10)+ for frame in container.decode(stream)+ if frame.pts is not None+ ]+ assert actual == expected+ # These controls really exercise reordering, including B-pyramid reference pictures.+ with av.open(str(path)) as container:+ order = [+ packet.pts+ for packet in container.demux(video=0)+ if packet.size and packet.pts is not None+ ]+ assert len(order) == len(durations)+ assert (order != sorted(order)) == bool(bframes)+ if sound:+ with av.open(str(path)) as container:+ audio = container.streams.audio[0]+ assert audio.duration == at * 4800+ audio_packets = [packet for packet in container.demux(audio) if packet.size]+ with av.open(str(path)) as container:+ video_packets = [+ packet for packet in container.demux(video=0) if packet.size+ ]+ # Every decoding predecessor arrives before the audio chunk containing the+ # earliest picture still needing it, even if it owns a long display hold.+ for i, packet in enumerate(video_packets):+ deadline = min(order[i:]) // 10+ audible = next(+ p+ for p in audio_packets+ if p.pts is not None and p.pts >= deadline * 48000+ )+ assert packet.pos is not None+ assert audible.pos is not None+ assert packet.pos < audible.pos