Skip to content

The animation engine for agents: blazingly fast math and 3D videos with Manim's Python API, on a Rust GPU core.

Python

Preserve recording timelines and encoded picture durations

Sina Atalayfc31789b5677
diff --git a/rust/engine/src/encode.rs b/rust/engine/src/encode.rs
index b0e86e5b..0bf11a43 100644
--- a/rust/engine/src/encode.rs
+++ b/rust/engine/src/encode.rs
@@ -122,7 +122,7 @@ impl Encoder {
let pts = self.pts;
self.pts += duration as i64;
if let Some(sample) = self.x264.encode(Some(nv12), pts, constant, key)? {
- self.mp4.sample(sample.data, sample.pts, sample.dts, sample.key).map_err(|e| e.to_string())?;
+ self.mp4.sample(sample.data, sample.pts, sample.key).map_err(|e| e.to_string())?;
}
Ok(())
}
@@ -133,7 +133,7 @@ impl Encoder {
let Self { mut x264, mut mp4, pts } = self;
while x264.delayed() {
if let Some(sample) = x264.encode(None, 0, None, false)? {
- mp4.sample(sample.data, sample.pts, sample.dts, sample.key).map_err(|e| e.to_string())?;
+ mp4.sample(sample.data, sample.pts, sample.key).map_err(|e| e.to_string())?;
}
}
mp4.finish(pts, audio).map_err(|e| e.to_string())
diff --git a/rust/engine/src/export.rs b/rust/engine/src/export.rs
index 6d6f4ff0..22eeefdd 100644
--- a/rust/engine/src/export.rs
+++ b/rust/engine/src/export.rs
@@ -42,16 +42,28 @@ impl Player {
dirty
}
- pub(crate) fn push_frames(&mut self, gpu: &mut Gpu, frames: Vec<Frame>, repeat: u32, key: bool) -> Result<(), String> {
+ pub(crate) fn push_frames(&mut self, gpu: &mut Gpu, frames: Vec<Frame>, repeat: u32, key: bool, capture: bool) -> Result<(), String> {
let mut export = self.export.take().ok_or("no export is open")?;
- let result = self.push_into(gpu, &mut export, frames, repeat, key);
+ let result = (|| {
+ // A pixel consumer waits for this frame. Settle older draws before making it,
+ // so their overflow retries cannot replace its captured pixels.
+ if capture {
+ while !export.pending.is_empty() { self.retire(gpu, &mut export)?; }
+ }
+ self.push_into(gpu, &mut export, frames, repeat, key, capture)?;
+ if capture {
+ self.retire(gpu, &mut export)?;
+ self.map_pixels(gpu)?;
+ }
+ Ok(())
+ })();
self.export = Some(export);
result
}
/// Draw a frame into the export and convert only the macroblocks a change can touch — every
/// one when camera views are drawn too (their pictures change with no record changing).
- fn push_into(&mut self, gpu: &mut Gpu, export: &mut Export, frames: Vec<Frame>, repeat: u32, key: bool) -> Result<(), String> {
+ fn push_into(&mut self, gpu: &mut Gpu, export: &mut Export, frames: Vec<Frame>, repeat: u32, key: bool, capture: bool) -> Result<(), String> {
let (columns, rows) = (export.width.div_ceil(16), export.height.div_ceil(16));
let total = columns * rows;
if export.pending.len() == RING {
@@ -71,12 +83,13 @@ impl Player {
// read (see `retire`)
let lists = self.lists.as_ref().filter(|_| composited && !listed.is_empty()).map(|l| l.capacity);
self.convert(gpu, export, &mut command, &listed, slot, lists.is_some());
+ if capture { self.copy_pixels(&mut command); }
gpu.queue.submit(Some(command.finish()));
export.map(slot, listed.len());
gpu.device.poll(wgpu::PollType::Poll).map_err(|e| e.to_string())?;
let first = export.previous.is_none();
let kept = lists.map(|capacity| (frames.clone(), capacity));
- export.pending.push_back(Pending { slot, listed, repeat, first, key, kept });
+ export.pending.push_back(Pending { slot, listed, repeat, first, key, kept, capture });
export.previous = frames.into_iter().last();
export.drawn += 1;
Ok(())
@@ -134,6 +147,7 @@ impl Player {
let (mut command, ..) = self.encode(gpu, frames)?;
capacity = self.lists.as_ref().map_or(0, |l| l.capacity);
self.convert(gpu, export, &mut command, &p.listed, p.slot, true);
+ if p.capture { self.copy_pixels(&mut command); }
gpu.queue.submit(Some(command.finish()));
export.map(p.slot, p.listed.len());
export.wait(&gpu.device, p.slot)?;
@@ -158,6 +172,7 @@ pub(crate) struct Pending {
first: bool,
key: bool,
kept: Option<(Vec<Frame>, u64)>, // a frame composited through the lists, and their capacity
+ capture: bool, // copy its final successful draw for a simultaneous pixel consumer
}
/// A video being written: each frame drawn, converted to NV12 where it changed, read back and
diff --git a/rust/engine/src/mp4.rs b/rust/engine/src/mp4.rs
index 25c525da..6a7dae4c 100644
--- a/rust/engine/src/mp4.rs
+++ b/rust/engine/src/mp4.rs
@@ -4,8 +4,8 @@
//! A video sample is a picture as x264 writes it (NAL units, each prefixed by its 4-byte size) and a
//! held picture is one sample that lasts longer — times are in frames (timescale = fps). With
//! B-frames (the slower presets), a picture can be decoded before the one it follows: the samples
-//! are in decode order, each shown at its own time, and an edit list starts the track at the
-//! first picture shown. An audio
+//! are in decode order, each retaining its duration; signed composition offsets give its
+//! presentation time. An audio
//! sample is an AAC access unit of 1024 samples; the encoder's priming before the sound's first
//! sample is skipped by an edit list, so the sound starts with the film, to the sample. Video
//! samples stream to a temporary file; `finish` writes the header (`moov`), then the samples,
@@ -32,7 +32,7 @@ pub struct Mp4 {
struct Sample {
size: u32,
pts: i64,
- dts: i64,
+ duration: u32,
key: bool,
}
@@ -63,13 +63,13 @@ impl Mp4 {
Ok(Self { path: path.to_string(), part, data: Some(data), width, height, fps, sps, pps, samples: Vec::new(), bytes: 0 })
}
- pub fn sample(&mut self, data: &[u8], pts: i64, dts: i64, key: bool) -> io::Result<()> {
+ pub fn sample(&mut self, data: &[u8], pts: i64, key: bool) -> io::Result<()> {
if data.is_empty() {
return Ok(());
}
self.data.as_mut().ok_or_else(|| io::Error::other("the video is finished"))?.write_all(data)?;
self.bytes += data.len() as u64;
- self.samples.push(Sample { size: data.len() as u32, pts, dts, key });
+ self.samples.push(Sample { size: data.len() as u32, pts, duration: 0, key });
Ok(())
}
@@ -79,6 +79,16 @@ impl Mp4 {
if let Some(mut data) = self.data.take() {
data.flush()?;
}
+ // Recover each picture's authored duration in presentation order, retaining the
+ // encoder's sample order. In particular, the final displayed picture owns the hold,
+ // even when B-frames follow it in decode order.
+ let mut order: Vec<_> = (0..self.samples.len()).collect();
+ order.sort_unstable_by_key(|&i| self.samples[i].pts);
+ let mut until = end;
+ for i in order.into_iter().rev() {
+ self.samples[i].duration = (until - self.samples[i].pts) as u32;
+ until = self.samples[i].pts;
+ }
let written = self.write(end, audio);
if written.is_err() {
let _ = std::fs::remove_file(&self.path);
@@ -91,11 +101,17 @@ impl Mp4 {
let Some(audio) = audio else {
return vec![(false, 0..self.samples.len())];
};
- let first = self.samples.first().map_or(0, |s| s.dts);
+ // A picture must arrive before its own presentation and every later picture in
+ // decode order: any of them may need it as a reference. Signed composition offsets
+ // make the sample-duration clock unsuitable as a progressive-download deadline.
+ let mut deadlines: Vec<_> = self.samples.iter().map(|s| s.pts).collect();
+ for i in (1..deadlines.len()).rev() {
+ deadlines[i - 1] = deadlines[i - 1].min(deadlines[i]);
+ }
let (mut chunks, mut v, mut a, mut second) = (Vec::new(), 0, 0, 1i64);
while v < self.samples.len() || a < audio.units.len() {
let v0 = v;
- while v < self.samples.len() && self.samples[v].dts - first < second * self.fps as i64 {
+ while v < self.samples.len() && deadlines[v] < second * self.fps as i64 {
v += 1;
}
if v > v0 {
@@ -120,9 +136,9 @@ impl Mp4 {
let header = if wide { 16 } else { 8 };
let layout = self.layout(audio);
let ftyp = ftyp();
- let probe = self.moov(end, audio, &layout, 0, wide);
+ let probe = self.moov(end, audio, &layout, 0, wide)?;
let start = (ftyp.len() + probe.len() + header) as u64;
- let moov = self.moov(end, audio, &layout, start, wide);
+ let moov = self.moov(end, audio, &layout, start, wide)?;
let mut out = BufWriter::with_capacity(1 << 20, File::create(&self.path)?);
out.write_all(&ftyp)?;
out.write_all(&moov)?;
@@ -168,21 +184,20 @@ impl Mp4 {
}
/// The header, with the samples starting at byte `offset` of the file.
- fn moov(&self, end: i64, audio: Option<&Audio>, layout: &Layout, offset: u64, wide: bool) -> Vec<u8> {
- // The media's clock starts at the first decode. With B-frames, x264 decodes ahead of
- // what it shows (its first dts is negative), so a picture shows at its pts minus that
- // first dts: an offset of pts − dts (never negative) from its decode, and an edit list
- // that starts the track at the first picture shown.
- let first = self.samples.first().map_or(0, |s| s.dts);
- let start = self.samples.iter().map(|s| s.pts).min().map_or(0, |pts| pts - first);
- let decode: Vec<i64> = self.samples.iter().map(|s| s.dts - first).collect();
+ fn moov(&self, end: i64, audio: Option<&Audio>, layout: &Layout, offset: u64, wide: bool) -> io::Result<Vec<u8>> {
+ // ISO/IEC 14496-12: decoding timestamps order samples; composition timestamps say
+ // when they appear. Accumulate their own durations in decode order and preserve every
+ // authored presentation time with signed ctts offsets. This also gives the last
+ // displayed picture its proper duration, without a decode-preroll edit or extra frames.
let duration = end.max(1) as u32;
- let media = (end - first).max(1) as u32;
- let mut deltas: Vec<u32> = decode.windows(2).map(|w| (w[1] - w[0]) as u32).collect();
- if let Some(&last) = decode.last() {
- deltas.push((end - first - last).max(1) as u32);
- }
- let offsets: Vec<u32> = self.samples.iter().map(|s| (s.pts - s.dts).max(0) as u32).collect();
+ let deltas: Vec<_> = self.samples.iter().map(|s| s.duration).collect();
+ let mut decode = 0;
+ let offsets: Vec<_> = self.samples.iter().map(|s| {
+ let offset = i32::try_from(s.pts - decode)
+ .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "MP4 composition offset exceeds 32-bit range"))?;
+ decode += s.duration as i64;
+ Ok(offset as u32)
+ }).collect::<io::Result<_>>()?;
let [(video_at, video_counts), (audio_at, audio_counts)] = self.chunks(audio, layout, offset);
// the sound's length in the movie's timescale (frames)
let heard = audio.map_or(0, |a| (a.length * self.fps as u64).div_ceil(a.rate as u64) as u32);
@@ -216,23 +231,12 @@ impl Mp4 {
w.u32(self.width << 16);
w.u32(self.height << 16);
});
- if start > 0 {
- w.boxed(b"edts", |w| {
- w.full(b"elst", 0, 0, |w| {
- w.u32(1);
- w.u32(duration); // in the movie's timescale (frames, as the track's)
- w.u32(start as u32); // where the first picture shows, in the track's
- w.u16(1);
- w.u16(0);
- });
- });
- }
w.boxed(b"mdia", |w| {
w.full(b"mdhd", 0, 0, |w| {
w.u32(0);
w.u32(0);
w.u32(self.fps);
- w.u32(media);
+ w.u32(duration);
w.u16(0x55c4); // "und"
w.u16(0);
});
@@ -252,7 +256,7 @@ impl Mp4 {
});
w.full(b"stts", 0, 0, |w| w.runs(&deltas));
if offsets.iter().any(|&o| o != 0) {
- w.full(b"ctts", 0, 0, |w| w.runs(&offsets));
+ w.full(b"ctts", 1, 0, |w| w.runs(&offsets));
}
if self.samples.iter().any(|s| !s.key) {
let keys: Vec<u32> = (1..).zip(&self.samples).filter(|(_, s)| s.key).map(|(k, _)| k).collect();
@@ -276,7 +280,7 @@ impl Mp4 {
self.sound(w, audio, heard, &audio_counts, &audio_at, wide);
}
});
- w.0
+ Ok(w.0)
}
/// The sound's track: AAC-LC, its priming cut by an edit list, a unit of pre-roll.
@@ -444,9 +448,9 @@ impl Drop for Mp4 {
fn ftyp() -> Vec<u8> {
let mut w = Boxes::default();
w.boxed(b"ftyp", |w| {
- w.bytes(b"isom");
+ w.bytes(b"iso4"); // signed composition offsets (ctts version 1)
w.u32(0x200);
- for brand in [b"isom", b"iso2", b"avc1", b"mp41"] {
+ for brand in [b"iso4", b"isom", b"iso2", b"avc1", b"mp41"] {
w.bytes(brand);
}
});
@@ -544,3 +548,36 @@ impl Boxes {
});
}
}
+
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn movie(pictures: &[(i64, u32)]) -> Mp4 {
+ Mp4 {
+ path: String::new(), part: String::new(), data: None,
+ width: 2, height: 2, fps: 1, sps: vec![0; 4], pps: vec![0], bytes: 0,
+ samples: pictures.iter().map(|&(pts, duration)| Sample { size: 0, pts, duration, key: true }).collect(),
+ }
+ }
+
+ #[test]
+ fn progressive_chunks_deliver_references_before_their_earliest_picture() {
+ // The P-picture shown at 10s is needed to decode the B-picture shown at 1s.
+ // Its 20s hold must not put that B-picture behind 21s of audio.
+ let movie = movie(&[(0, 1), (10, 20), (1, 9)]);
+ let audio = Audio { rate: 1024, channels: 1, config: vec![], priming: 0, length: 30 * 1024, units: vec![vec![]; 30] };
+ let chunks = movie.layout(Some(&audio));
+ assert_eq!(&chunks[..4], &[(false, 0..1), (true, 0..1), (false, 1..3), (true, 1..2)]);
+ assert!(chunks[4..].iter().all(|(sound, _)| *sound));
+ }
+
+ #[test]
+ fn composition_offsets_cannot_wrap() {
+ let movie = movie(&[(0, 1), (i32::MAX as i64 + 2, 1), (1, i32::MAX as u32 + 1)]);
+ let error = movie.moov(i32::MAX as i64 + 3, None, &vec![(false, 0..3)], 0, false).unwrap_err();
+ assert_eq!(error.kind(), io::ErrorKind::InvalidInput);
+ assert!(error.to_string().contains("composition offset"));
+ }
+}
diff --git a/rust/engine/src/take.rs b/rust/engine/src/take.rs
index 6fa968a9..72fad99b 100644
--- a/rust/engine/src/take.rs
+++ b/rust/engine/src/take.rs
@@ -4,7 +4,8 @@
//! reads it back and draws any of its frames on demand (`web`).
//!
//! A message is its length (u32), its op (u8), then its fields, little-endian; a byte string is
-//! its length (u32), then its bytes. A take begins with START and ends with END (whether its
+//! its length (u32), then its bytes. A take begins with START: its format version (u32), width
+//! and height (u32 each), and frame rate (f64). It ends with END (whether its
//! scene failed: a take cut short has none); NOTE carries the director's notes (JSON: the plays,
//! the sections, the captions) to whoever shows the take, and SOUND the film's sound, an audio
//! file its player plays beside the frames.
@@ -18,15 +19,19 @@
//! the same record slot drew the frame before: the writer learns what replaces what from the
//! frames' records, so a film changing a little from frame to frame is sent as its changes.
-#[cfg(any(feature = "python", test))]
+#[cfg(any(feature = "python", all(test, not(target_arch = "wasm32"))))]
use std::collections::HashMap;
-#[cfg(any(feature = "python", test))]
+#[cfg(any(feature = "python", all(test, not(target_arch = "wasm32"))))]
use std::sync::Arc;
-#[cfg(any(feature = "python", test))]
+#[cfg(any(feature = "python", all(test, not(target_arch = "wasm32"))))]
use crate::pack::{self, Kind};
-pub(crate) const START: u8 = 0;
+/// The wire contract, including the layouts of views, records, uploads and coded arrays.
+/// Bump when a writer's output can no longer be read with the same meaning by an older reader.
+pub(crate) const VERSION: u32 = 1;
+// Opcode 0 was the unversioned START. A different opcode makes old readers reject this one.
+pub(crate) const START: u8 = 14;
pub(crate) const PATH: u8 = 1;
pub(crate) const POINTS: u8 = 2;
pub(crate) const MESH: u8 = 3;
@@ -50,7 +55,7 @@ pub(crate) fn slots(records: &[u8]) -> impl Iterator<Item = [u64; KEYS]> + '_ {
}
/// An upload the writer holds until a frame shows it (so it knows what it replaces).
-#[cfg(any(feature = "python", test))]
+#[cfg(any(feature = "python", all(test, not(target_arch = "wasm32"))))]
struct Upload {
op: u8,
key: u64,
@@ -60,7 +65,7 @@ struct Upload {
}
/// Writes a take's messages into a buffer, taken out as it fills (`drain`).
-#[cfg(any(feature = "python", test))]
+#[cfg(any(feature = "python", all(test, not(target_arch = "wasm32"))))]
#[derive(Default)]
pub(crate) struct Writer {
buffer: Vec<u8>,
@@ -73,7 +78,7 @@ pub(crate) struct Writer {
records: HashMap<u64, u64>, // a view's last records array (0: the frame's; else a camera's)
}
-#[cfg(any(feature = "python", test))]
+#[cfg(any(feature = "python", all(test, not(target_arch = "wasm32"))))]
impl Writer {
fn begin(&mut self, op: u8) -> usize {
let at = self.buffer.len();
@@ -99,6 +104,7 @@ impl Writer {
pub(crate) fn start(&mut self, width: u32, height: u32, fps: f64) {
let at = self.begin(START);
+ self.put(&VERSION.to_le_bytes());
self.put(&width.to_le_bytes());
self.put(&height.to_le_bytes());
self.put(&fps.to_le_bytes());
@@ -110,10 +116,14 @@ impl Writer {
pub(crate) fn points(&mut self, key: u64, vertices: &[u8]) {
self.hold(POINTS, key, &[], vec![(Kind::Points, vertices)]);
}
- #[allow(clippy::too_many_arguments)]
- pub(crate) fn mesh(&mut self, key: u64, points: &[u8], uvs: &[u8], normals: &[u8], triangles: &[u8], outline: u32, block: u32) {
- let head: Vec<u8> = [outline.to_le_bytes(), block.to_le_bytes()].concat();
- self.hold(MESH, key, &head, vec![(Kind::MeshPoints, points), (Kind::MeshUvs, uvs), (Kind::MeshNormals, normals), (Kind::MeshTriangles, triangles)]);
+ pub(crate) fn mesh(&mut self, key: u64, mesh: &crate::mesh::Mesh<'_>) {
+ let head: Vec<u8> = [mesh.outline.to_le_bytes(), mesh.block.to_le_bytes()].concat();
+ self.hold(MESH, key, &head, vec![
+ (Kind::MeshPoints, bytemuck::cast_slice(&mesh.points)),
+ (Kind::MeshUvs, bytemuck::cast_slice(&mesh.uvs)),
+ (Kind::MeshNormals, bytemuck::cast_slice(&mesh.normals)),
+ (Kind::MeshTriangles, bytemuck::cast_slice(&mesh.triangles)),
+ ]);
}
pub(crate) fn rows(&mut self, key: u64, rows: &[u8]) {
self.hold(ROWS, key, &[], vec![(Kind::Rows, rows)]);
@@ -303,7 +313,7 @@ pub(crate) fn messages(stream: &[u8]) -> (Vec<&[u8]>, usize) {
(out, at)
}
-#[cfg(test)]
+#[cfg(all(test, not(target_arch = "wasm32")))]
mod tests {
use super::*;
@@ -323,11 +333,15 @@ mod tests {
w.start(960, 540, 30.0);
let points = |t: f64| -> Vec<u8> { bytemuck::cast_slice(&(0..900).map(|i| ((i as f64) * 0.1 + t).sin()).collect::<Vec<f64>>()).to_vec() };
let triangles: Vec<u8> = bytemuck::cast_slice(&(0..300u32).collect::<Vec<u32>>()).to_vec();
- w.mesh(7, &points(0.0), &[], &[], &triangles, 0, 0);
+ let uvs = [[0.0f64; 2]; 300];
+ let normals = [[0.0f64, 0.0, 1.0]; 300];
+ let before = points(0.0);
+ let after = points(0.01);
+ w.mesh(7, &crate::mesh::Mesh::new(&before, bytemuck::cast_slice(&uvs), bytemuck::cast_slice(&normals), &triangles, 0, 0).unwrap());
let mut seen_by_camera = record(7);
seen_by_camera[56] = 1; // other flags: other records
w.frame(&[3; 192], &record(7), 1, &[(9, 64, 32, vec![5; 192], seen_by_camera)]);
- w.mesh(8, &points(0.01), &[], &[], &triangles, 0, 0); // the next frame's: replaces 7
+ w.mesh(8, &crate::mesh::Mesh::new(&after, bytemuck::cast_slice(&uvs), bytemuck::cast_slice(&normals), &triangles, 0, 0).unwrap()); // the next frame's: replaces 7
w.frame(&[3; 192], &record(7), 2, &[]);
w.frame(&[3; 192], &record(8), 1, &[]);
w.sound(&[8; 44]);
@@ -370,6 +384,45 @@ mod tests {
}
assert!(decoded.values().any(|d| d[..] == pack::canonical(Kind::MeshPoints, &points(0.01))[..]));
let mut s = Fields(messages[0]);
- assert_eq!((s.u8().unwrap(), s.u32().unwrap(), s.u32().unwrap(), s.f64().unwrap()), (START, 960, 540, 30.0));
+ assert_eq!((s.u8().unwrap(), s.u32().unwrap(), s.u32().unwrap(), s.u32().unwrap(), s.f64().unwrap()), (START, VERSION, 960, 540, 30.0));
+ }
+
+ #[test]
+ fn a_recorded_mesh_keeps_normals_derived_before_positions_are_rounded() {
+ let points = [[-0.6f64, -0.6, 1e8], [0.6, -0.6, 1e8 + 1.0], [0.0, 0.6, 1e8]];
+ let uvs = [[0.25f64, 0.5], [0.5, 0.75], [0.75, 1.0]];
+ let mesh = crate::mesh::Mesh::new(bytemuck::cast_slice(&points), bytemuck::cast_slice(&uvs), &[], bytemuck::cast_slice(&[0u32, 1, 2]), 0, 0).unwrap();
+ // The GPU positions have lost their small z difference. Deriving normals from those
+ // positions would turn this tilted face into a flat one and change its lighting.
+ assert!(mesh.points.iter().all(|v| v[2] as f32 == 1e8));
+ assert_eq!(mesh.normals[0], [-1.2, 0.6, 1.44]);
+ let mut writer = Writer::default();
+ writer.start(320, 240, 10.0);
+ writer.mesh(1, &mesh);
+ writer.frame(&[0; 192], &record(1), 1, &[]);
+ writer.close(false);
+ let stream = writer.drain();
+ let mut arrays = HashMap::new();
+ for message in messages(&stream).0 {
+ let mut fields = Fields(message);
+ if fields.u8().unwrap() != ARRAY { continue; }
+ let (_key, base, kind, mode, data) = (fields.u64().unwrap(), fields.u64().unwrap(), fields.u8().unwrap(), fields.u8().unwrap(), fields.string().unwrap());
+ assert_eq!(base, 0);
+ arrays.insert(kind, pack::decode(Kind::from_u8(kind).unwrap(), mode, data, None, None).unwrap());
+ }
+ let wide = |kind: Kind| {
+ let f: Vec<f32> = bytemuck::pod_collect_to_vec(&arrays[&(kind as u8)]);
+ bytemuck::cast_slice(&f.into_iter().map(f64::from).collect::<Vec<_>>()).to_vec()
+ };
+ let (points, uvs, normals) = (wide(Kind::MeshPoints), wide(Kind::MeshUvs), wide(Kind::MeshNormals));
+ let replay = crate::mesh::Mesh::new(&points, &uvs, &normals, &arrays[&(Kind::MeshTriangles as u8)], 0, 0).unwrap();
+ for (before, after) in mesh.points.iter().zip(&*replay.points) {
+ assert_eq!(before.map(|v| v as f32), after.map(|v| v as f32));
+ }
+ for (before, after) in mesh.normals.iter().zip(&*replay.normals) {
+ assert_eq!(before.map(|v| v as f32), after.map(|v| v as f32));
+ }
+ assert_eq!(replay.uvs, mesh.uvs);
+ assert_eq!(replay.triangles, mesh.triangles);
}
}
diff --git a/src/manimgx/rendering/film.py b/src/manimgx/rendering/film.py
index c957e56e..8ed56c6c 100644
--- a/src/manimgx/rendering/film.py
+++ b/src/manimgx/rendering/film.py
@@ -15,6 +15,7 @@ from typing import TYPE_CHECKING, Literal, get_args
from warnings import deprecated
from manimgx import _engine
+from manimgx.animation import clock
from manimgx.config import config
from manimgx.rendering import feed
from manimgx.rendering.feed import CameraView, Feeder
@@ -260,7 +261,7 @@ class Film:
raise ValueError(
f"a video needs an even width and height, not {width}x{height}"
)
- self.fps = Fraction(fps).limit_denominator(1000)
+ self.fps = clock.rational(fps)
"""How many frames a second the film has: the configuration's when it began."""
# the GPU draws the film; or its take is recorded, to be drawn by manimgx's player: by
# choice, or where the engine has no GPU (in Pyodide)
@@ -412,6 +413,8 @@ class Film:
view, records, cameras = self.feeder.frame(camera, mobjects)
if self._player is None:
raise RuntimeError("a film recorded as a take is drawn by manimgx's player")
+ if self._player.pressured():
+ self.feeder.sweep([records, *[c[4] for c in cameras]], pressed=True)
return self._player.render(view, records, cameras)
@deprecated("manimgx's machinery: the scene calls it", category=None)
@@ -479,17 +482,38 @@ class Film:
self._flush()
if (player := self._player) is None:
return
- if self.video is not None:
- player.push(view, records, repeat, cameras, self._pending_key)
- if self.frames is not None:
- self.frames(
- Frame(
- self._first,
- repeat,
- lambda: player.render(view, records, cameras),
- self._pending_key,
- )
- )
+ if (self.video is not None or self.frames is not None) and player.pressured():
+ self.feeder.sweep([records, *[c[4] for c in cameras]], pressed=True)
+ key = self._pending_key
+ if self.frames is None:
+ if self.video is not None:
+ player.push(view, records, repeat, cameras, key)
+ return
+ sent = self.video is None
+
+ def push() -> None:
+ nonlocal sent
+ if not sent:
+ player.push(view, records, repeat, cameras, key)
+ sent = True
+
+ def draw() -> bytes:
+ nonlocal sent
+ if sent:
+ return player.render(view, records, cameras)
+ # A pixel consumer and the encoder share one completed GPU draw. Defer export
+ # until the callback asks: metadata-only callbacks keep the asynchronous path.
+ pixels = player.push(view, records, repeat, cameras, key, capture=True)
+ sent = True
+ assert pixels is not None
+ return pixels
+
+ try:
+ self.frames(Frame(self._first, repeat, draw, key))
+ except Cut:
+ push() # the frame that cuts a film still belongs to its video
+ raise
+ push()
@deprecated("manimgx's machinery: the scene calls it", category=None)
def close(self) -> None:
@@ -500,19 +524,17 @@ class Film:
any other error abandons the video.
"""
try:
- self._send()
- except Cut:
- pass
+ with contextlib.suppress(Cut):
+ self._send()
+ with contextlib.suppress(Cut): # its take cut it: it hears no more
+ self._end()
except BaseException:
self.abort()
raise
- with contextlib.suppress(Cut): # its take cut it as it ended: it hears no more
- self._end()
def _end(self) -> None:
"""Write the closed film's video, if any, and tell its take how it ends."""
if self.video is not None and self._player is not None:
- self.video = None
sound = self.soundtrack()
if sound is None:
self.export = Export(*self._player.end_export())
@@ -525,6 +547,7 @@ class Film:
sound.tobytes(), channels, RATE, BITRATE * channels
)
)
+ self.video = None
if self._recorder is not None:
# what the player plays and shows beside the frames; then the take's end
if (sound := self.soundtrack()) is not None:
diff --git a/tests/integration/test_film.py b/tests/integration/test_film.py
index dccd294a..a2b74030 100644
--- a/tests/integration/test_film.py
+++ b/tests/integration/test_film.py
@@ -47,6 +47,28 @@ def test_an_error_at_the_last_frame_abandons_the_video(tmp_path: Path) -> None:
assert list(tmp_path.iterdir()) == [] # not the video, nor a part of it
+def test_a_soundtrack_error_closes_the_native_export(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ def failed_soundtrack(self: m.Film) -> None:
+ raise ValueError("the soundtrack failed")
+
+ monkeypatch.setattr(m.Film, "soundtrack", failed_soundtrack)
+ scene = Held()
+ try:
+ with pytest.raises(ValueError, match="the soundtrack failed"):
+ scene.render(tmp_path / "failed.mp4")
+ player = scene.film._player
+ assert player is not None
+ # Retain the scene and player: cleanup must not depend on garbage collection.
+ player.begin_export(str(tmp_path / "next.mp4"))
+ player.abort_export()
+ assert list(tmp_path.iterdir()) == []
+ finally:
+ if (player := scene.film._player) is not None:
+ player.abort_export()
+
+
def test_a_video_needs_even_sides(tmp_path: Path) -> None:
config.pixel_width = 321 # H.264's 4:2:0 halves both sides
with pytest.raises(ValueError, match="an even width and height, not 321x180"):
diff --git a/tests/integration/test_replay.py b/tests/integration/test_replay.py
new file mode 100644
index 00000000..2056904f
--- /dev/null
+++ b/tests/integration/test_replay.py
@@ -0,0 +1,180 @@
+"""Recorded frames use the same decoder and renderer as native playback, in any order."""
+
+import numpy as np
+import pytest
+
+import manimgx as m
+from manimgx._engine import TAKE_VERSION, Player, Recorder, Replay, adapter_info
+from manimgx.rendering.feed import Feeder
+
+
+def test_recorded_geometry_and_images_replay_exactly_in_any_order() -> None:
+ adapter = adapter_info()
+ assert set(adapter) == {
+ "name",
+ "vendor",
+ "device",
+ "device_type",
+ "driver",
+ "driver_info",
+ "backend",
+ }
+ assert adapter["name"]
+ assert adapter["backend"]
+ assert int(adapter["vendor"]) >= 0
+ assert int(adapter["device"]) >= 0
+ width, height, fps = 130, 74, 29.97 # readback rows require padding
+ player, recorder = Player(width, height), Recorder(width, height, fps)
+ direct, recorded = Feeder(width, height, player), Feeder(width, height, recorder)
+ camera = m.Camera(three_d=True, phi=0.4, theta=-1.2)
+ path = m.VMobject(color=m.BLUE, stroke_width=8).set_points_as_corners(
+ [(-3, -1, 0), (-2, 1, 0)]
+ )
+ mesh = m.MeshMobject(
+ np.array([[-0.8, -1, 0.1], [0.9, -0.8, -0.2], [0.6, 1, 0.6], [-0.7, 0.9, 0]]),
+ np.array([[0, 1, 2], [0, 2, 3]]),
+ color=m.RED,
+ shade_in_3d=True,
+ )
+ image = (
+ m.ImageMobject(
+ np.array(
+ [
+ [[255, 0, 0, 255], [0, 255, 0, 255]],
+ [[0, 0, 255, 255], [255, 255, 0, 255]],
+ ],
+ dtype=np.uint8,
+ )
+ )
+ .scale_to_fit_height(1.6)
+ .shift(2 * m.RIGHT)
+ )
+ objects: list[m.Mobject] = [path, mesh, image]
+ expected: list[bytes] = []
+ for step, repeat in enumerate([3, 1, 2, 1]):
+ path.add_points_as_corners([(-1.8 + step * 0.2, 0.5 - step * 0.3, 0)])
+ mesh.rotate(0.1, axis=m.UP)
+ expected.extend([player.render(*direct.frame(camera, objects))] * repeat)
+ view, records, cameras = recorded.frame(camera, objects)
+ recorder.frame(view, records, repeat, cameras)
+ recorder.end()
+ replay = Replay(recorder.drain())
+ assert replay.size == (width, height)
+ assert replay.frames == len(expected)
+ assert replay.fps == fps
+ assert replay.timeline == [(0, 3), (3, 1), (4, 2), (6, 1)]
+ for frame in [6, 0, 4, 1, 3, 2, 5, 0, 6]:
+ assert replay.render(frame) == expected[frame]
+ for frame in [-1, replay.frames]:
+ with pytest.raises(IndexError, match="outside"):
+ replay.render(frame)
+ assert adapter_info() == adapter
+
+
+def test_independent_takes_own_their_resources_when_keys_overlap() -> None:
+ image = m.ImageMobject(np.full((2, 2, 4), 255, np.uint8)).scale_to_fit_height(4)
+ camera = m.Camera()
+
+ def take(color: bytes) -> tuple[Replay, bytes, int]:
+ player, recorder = Player(64, 36), Recorder(64, 36, 30)
+ direct = Feeder(64, 36, player)
+ recorded = Feeder(64, 36, recorder)
+ direct_frame = direct.frame(camera, [image])
+ view, records, cameras = recorded.frame(camera, [image])
+ key = recorded.texture(image.pixel_array)
+ assert direct.texture(image.pixel_array) == key
+ # Identical keys intentionally name different pixels in independent takes. Both
+ # receive ordinary native uploads; the test neither reads nor rewrites the format.
+ player.add_texture(key, 2, 2, color * 4)
+ recorder.add_texture(key, 2, 2, color * 4)
+ recorder.frame(view, records, 1, cameras)
+ recorder.end()
+ return Replay(recorder.drain()), player.render(*direct_frame), key
+
+ first, red, first_key = take(bytes([255, 0, 0, 255]))
+ second, green, second_key = take(bytes([0, 255, 0, 255]))
+ assert first_key == second_key
+ assert red != green
+ for _ in range(3):
+ assert second.render(0) == green
+ assert first.render(0) == red
+
+
[email protected](pixel_width=960, pixel_height=540, frame_rate=10)
[email protected]("closes", [False, True], ids=["curved", "closed-then-open"])
+def test_growing_paths_keep_their_historical_picture_when_seeking(closes: bool) -> None:
+ class Growing(m.Scene):
+ def construct(self) -> None:
+ starts = [m.LEFT + m.UP, m.LEFT + m.DOWN, 2 * m.LEFT]
+ ends = [m.RIGHT, m.RIGHT + m.UP, m.RIGHT + m.DOWN]
+ colors = [m.RED, m.GREEN, m.BLUE]
+ dots = m.VGroup(
+ *[m.Dot(point, color=color) for point, color in zip(starts, colors)]
+ )
+ targets = m.VGroup(
+ *[m.Dot(point, color=color) for point, color in zip(ends, colors)]
+ )
+ self.add(dots, targets)
+ for dot in dots:
+ self.add(m.TracedPath(dot.get_center, stroke_color=dot.get_color()))
+ self.wait()
+ if closes:
+ self.play(m.Rotate(dots, m.TAU), run_time=2)
+ self.play(dots.animate.shift(m.RIGHT), run_time=1)
+ else:
+ self.play(
+ m.Transform(dots, targets, path_func=m.counterclockwise_path()),
+ run_time=2,
+ )
+ self.wait()
+
+ expected: list[bytes] = []
+ Growing().render(
+ frames=lambda frame: expected.extend([frame.pixels()] * frame.repeat)
+ )
+ chunks: list[bytes] = []
+ Growing().render(take=chunks.append)
+ replay = Replay(b"".join(chunks))
+ assert replay.frames == len(expected)
+ forward = list(range(replay.frames))
+ for index in [
+ *reversed(forward),
+ *forward,
+ *np.random.default_rng(7).permutation(forward),
+ ]:
+ assert replay.render(int(index)) == expected[index]
+
+
+def test_a_replay_rejects_incomplete_failed_and_multiple_takes() -> None:
+ recorder = Recorder(32, 24, 30)
+ view, records, cameras = Feeder(32, 24, recorder).frame(m.Camera(), [])
+ recorder.frame(view, records, 2, cameras)
+ unfinished = recorder.drain()
+ recorder.end()
+ complete = unfinished + recorder.drain()
+ assert Replay(complete).frames == 2
+ for data, message in [
+ (b"", "exactly one"),
+ (unfinished, "successfully"),
+ (complete[:-1], "ends early"),
+ (complete + complete, "exactly one"),
+ ]:
+ with pytest.raises(ValueError, match=message):
+ Replay(data)
+ failed = Recorder(32, 24, 30)
+ failed.frame(view, records, 1, cameras)
+ failed.end(failed=True)
+ with pytest.raises(ValueError, match="successfully"):
+ Replay(failed.drain())
+
+
+def test_take_versions_are_explicit_and_rejected_before_playback() -> None:
+ recorder = Recorder(32, 24, 30)
+ start = bytearray(recorder.drain())
+ assert int.from_bytes(start[5:9], "little") == TAKE_VERSION
+ start[5:9] = (TAKE_VERSION + 1).to_bytes(4, "little")
+ with pytest.raises(ValueError, match=f"unsupported take format {TAKE_VERSION + 1}"):
+ Replay(bytes(start))
+ # The old unversioned START: its length, opcode0, width, height and fps.
+ with pytest.raises(ValueError, match="unversioned takes are unsupported"):
+ Replay(bytes.fromhex("11000000 00 20000000 18000000 0000000000003e40"))
diff --git a/tests/integration/test_take.py b/tests/integration/test_take.py
index 22ac93a9..30ae687f 100644
--- a/tests/integration/test_take.py
+++ b/tests/integration/test_take.py
@@ -20,7 +20,7 @@ from manimgx import _engine
from manimgx.rendering import feed
from manimgx.rendering.film import Cut, Frame
-START, FRAME, NOTE, SOUND, END = 0, 8, 9, 10, 12
+START, FRAME, NOTE, SOUND, END = 14, 8, 9, 10, 12
class Held(m.Scene):
@@ -61,7 +61,7 @@ def test_a_take_holds_the_frames_a_video_would() -> None:
take, count = record(Held())
(op, start), *rest = messages(take)
assert op == START
- assert struct.unpack("<IId", start) == (320, 180, 30.0)
+ assert struct.unpack("<IIId", start) == (_engine.TAKE_VERSION, 320, 180, 30.0)
repeats = [
struct.unpack_from("<I", fields)[0] for op, fields in rest if op == FRAME
]
diff --git a/tests/integration/test_video.py b/tests/integration/test_video.py
index 7cb0713b..934b6269 100644
--- a/tests/integration/test_video.py
+++ b/tests/integration/test_video.py
@@ -3,14 +3,18 @@ conversion to NV12, x264 (built into the engine), and what the MP4 says of its c
"""
from pathlib import Path
+from unittest.mock import Mock, patch
import av
import numpy as np
import pytest
from tests.integration.corpus.frames import decode
+from tests.integration.corpus.run_manimgx import mp4_frames
import manimgx as m
-from manimgx.rendering.film import Frame, X264Preset
+from manimgx._engine import Player
+from manimgx.rendering.feed import Feeder
+from manimgx.rendering.film import Cut, Frame, X264Preset
SIZE = (320, 180)
pytestmark = pytest.mark.config(pixel_width=SIZE[0], pixel_height=SIZE[1])
@@ -57,3 +61,214 @@ def test_each_picture_shows_at_its_frame(tmp_path: Path, preset: X264Preset) ->
if frame.pts is not None
]
assert shown == starts
+
+
[email protected]("pixels", [False, True])
[email protected]("cut", [False, True])
+def test_video_and_pixel_sinks_share_one_draw(
+ tmp_path: Path, pixels: bool, cut: bool
+) -> None:
+ player = Mock(wraps=Player(*SIZE))
+ sent: list[Frame] = []
+
+ def sink(frame: Frame) -> None:
+ sent.append(frame)
+ if pixels:
+ assert len(frame.pixels()) == SIZE[0] * SIZE[1] * 4
+ if cut and len(sent) == 3:
+ raise Cut
+
+ video = tmp_path / "shared.mp4"
+ with patch("manimgx.rendering.feed.Player", return_value=player):
+ Filled().render(video, frames=sink)
+ assert player.push.call_count == len(sent)
+ assert all(
+ call.kwargs.get("capture", False) == pixels
+ for call in player.push.call_args_list
+ )
+ player.render.assert_not_called()
+ assert len(list(decode(video, SIZE))) == len(sent)
+ # A frame retained by a metadata consumer must no longer try to export it when read.
+ assert len(sent[-1].pixels()) == SIZE[0] * SIZE[1] * 4
+ assert player.push.call_count == len(sent)
+
+
[email protected]("queued_layers", [False, True])
[email protected]("samples", [1, 4])
+def test_captured_exports_preserve_pixels_and_video_across_pending_draws(
+ tmp_path: Path, queued_layers: bool, samples: int
+) -> None:
+ # RGBA rows require padding; NV12 edges are partial macroblocks.
+ width, height = 130, 74
+ capture, plain, direct = (Player(width, height, samples) for _ in range(3))
+ feeders = [Feeder(width, height, player) for player in (capture, plain, direct)]
+ camera = m.Camera(three_d=True)
+ # Cover the picture in see-through mesh layers: the first draw must grow its fragment
+ # lists. Capturing must return the successful retry, including after older queued draws.
+ layers: list[m.Mobject] = [
+ m.MeshMobject(
+ np.array([[-8, -5, z], [8, -5, z], [8, 5, z], [-8, 5, z]]),
+ np.array([[0, 1, 2], [0, 2, 3]]),
+ color=m.BLUE if i % 2 else m.RED,
+ fill_opacity=0.2,
+ )
+ for i, z in enumerate(np.linspace(-0.2, 0.2, 12))
+ ]
+ captured_video, plain_video = tmp_path / "captured.mp4", tmp_path / "plain.mp4"
+ capture.begin_export(str(captured_video), 30)
+ plain.begin_export(str(plain_video), 30)
+ for index, wanted in enumerate([False, False, False, True, True, False, True]):
+ objects = (
+ layers
+ if (index >= 3) != queued_layers
+ else [m.Square(fill_opacity=1).shift((index % 3) * m.RIGHT)]
+ )
+ prepared = [feeder.frame(camera, objects) for feeder in feeders]
+ expected = direct.render(*prepared[2])
+ assert any(expected[0::4]) # geometry is visible, including the mesh layers
+ view, records, cameras = prepared[0]
+ actual = capture.push(view, records, cameras=cameras, capture=wanted)
+ assert actual == (expected if wanted else None)
+ view, records, cameras = prepared[1]
+ assert plain.push(view, records, cameras=cameras) is None
+ capture.end_export()
+ plain.end_export()
+ assert captured_video.read_bytes() == plain_video.read_bytes()
+
+
[email protected](frame_rate=10)
+ "story", ["empty", "static", "hold", "off_grid", "sections", "cut"]
+)
[email protected]("preset", ["ultrafast", "slow"])
+def test_video_duration_covers_the_complete_film(
+ tmp_path: Path, story: str, preset: X264Preset
+) -> None:
+ """Every callback hold, including the closing frame, survives MP4 sample timing."""
+
+ class Story(m.Scene):
+ def construct(self) -> None:
+ if story == "empty":
+ return
+ dot = m.Dot()
+ self.add(dot)
+ if story == "static":
+ return
+ if story == "hold":
+ self.wait(1)
+ else:
+ self.play(dot.animate.shift(m.RIGHT), run_time=0.137)
+ if story == "sections":
+ self.next_section("held")
+ self.wait(0.53)
+
+ sent: list[tuple[int, int]] = []
+
+ def observe(frame: Frame) -> None:
+ sent.append((frame.index, frame.repeat))
+ if story == "cut" and len(sent) == 2:
+ raise Cut
+
+ path = tmp_path / "timing.mp4"
+ film = Story().render(path, frames=observe, preset=preset)
+ covered = 0
+ for index, repeat in sent:
+ assert index == covered
+ assert repeat > 0
+ covered += repeat
+ assert mp4_frames(path) == film.frame_count == covered
+ with av.open(str(path)) as container:
+ stream = container.streams.video[0]
+ assert stream.time_base is not None
+ shown = [
+ (frame.pts * stream.time_base * 10, frame.duration * stream.time_base * 10)
+ for frame in container.decode(stream)
+ if frame.pts is not None
+ ]
+ assert shown == sent
+ if story in {"empty", "static"}:
+ assert covered == 1
+ if story in {"hold", "sections"}:
+ assert any(repeat > 1 for _, repeat in sent)
+
+
[email protected]("sound", [False, True])
+ ("bframes", "pyramid"), [(0, "none"), (3, "none"), (3, "normal")]
+)
+ "durations",
+ [
+ [1] * 9 + [60],
+ [40] + [1] * 9,
+ [1, 8, 1, 1, 32, 1, 1, 2, 1, 16],
+ ],
+ ids=["last-hold", "first-hold", "mixed-holds"],
+)
+def test_reordered_video_preserves_each_pictures_duration(
+ tmp_path: Path, bframes: int, pyramid: str, durations: list[int], sound: bool
+) -> None:
+ player = Player(*SIZE)
+ feeder, camera = Feeder(*SIZE, player), m.Camera()
+ path = tmp_path / "reordered.mp4"
+ player.begin_export(
+ str(path),
+ 10,
+ "slow",
+ options=[
+ ("bframes", str(bframes)),
+ ("b-adapt", "0"),
+ ("b-pyramid", pyramid),
+ ("scenecut", "0"),
+ ],
+ )
+ expected, at = [], 0
+ for index, duration in enumerate(durations):
+ view, records, cameras = feeder.frame(
+ camera, [m.Dot().shift((index / 10) * m.RIGHT)]
+ )
+ player.push(view, records, repeat=duration, cameras=cameras, key=index == 5)
+ expected.append((at, duration))
+ at += duration
+ player.end_export(np.zeros(at * 4800, np.float32).tobytes() if sound else None)
+ with av.open(str(path)) as container:
+ stream = container.streams.video[0]
+ assert stream.time_base is not None
+ assert stream.duration is not None
+ assert stream.duration * stream.time_base * 10 == at
+ actual = [
+ (frame.pts * stream.time_base * 10, frame.duration * stream.time_base * 10)
+ for frame in container.decode(stream)
+ if frame.pts is not None
+ ]
+ assert actual == expected
+ # These controls really exercise reordering, including B-pyramid reference pictures.
+ with av.open(str(path)) as container:
+ order = [
+ packet.pts
+ for packet in container.demux(video=0)
+ if packet.size and packet.pts is not None
+ ]
+ assert len(order) == len(durations)
+ assert (order != sorted(order)) == bool(bframes)
+ if sound:
+ with av.open(str(path)) as container:
+ audio = container.streams.audio[0]
+ assert audio.duration == at * 4800
+ audio_packets = [packet for packet in container.demux(audio) if packet.size]
+ with av.open(str(path)) as container:
+ video_packets = [
+ packet for packet in container.demux(video=0) if packet.size
+ ]
+ # Every decoding predecessor arrives before the audio chunk containing the
+ # earliest picture still needing it, even if it owns a long display hold.
+ for i, packet in enumerate(video_packets):
+ deadline = min(order[i:]) // 10
+ audible = next(
+ p
+ for p in audio_packets
+ if p.pts is not None and p.pts >= deadline * 48000
+ )
+ assert packet.pos is not None
+ assert audible.pos is not None
+ assert packet.pos < audible.pos