The animation engine for agents: blazingly fast math and 3D videos with Manim's Python API, on a Rust GPU core.
Python
Add film take capture and replay
apoogdk6bd080f77d62
diff --git a/rust/engine/src/take.rs b/rust/engine/src/take.rsnew file mode 100644index 00000000..6fa968a9--- /dev/null+++ b/rust/engine/src/take.rs@@ -0,0 +1,375 @@+//! A take: a scene's run as the engine's calls — shapes, brushes and images uploaded under their+//! keys, then frames (a view and its records, shown for some frames) — written as one stream of+//! messages. Python records it wherever it runs (natively or in Pyodide: `Recorder`); a projector+//! reads it back and draws any of its frames on demand (`web`).+//!+//! A message is its length (u32), its op (u8), then its fields, little-endian; a byte string is+//! its length (u32), then its bytes. A take begins with START and ends with END (whether its+//! scene failed: a take cut short has none); NOTE carries the director's notes (JSON: the plays,+//! the sections, the captions) to whoever shows the take, and SOUND the film's sound, an audio+//! file its player plays beside the frames.+//!+//! ENVIRONMENT carries an environment's picture (the light around a 3D view: `environment`), sent+//! when it is added, before any frame that shows it.+//!+//! What an upload holds goes as ARRAY messages (see `pack`): each array under its own key, sent+//! once, coded against the array it replaces. The upload itself names its arrays by key, and a+//! FRAME names its records array. An array's base is the array of the same role in the upload+//! the same record slot drew the frame before: the writer learns what replaces what from the+//! frames' records, so a film changing a little from frame to frame is sent as its changes.++#[cfg(any(feature = "python", test))]+use std::collections::HashMap;+#[cfg(any(feature = "python", test))]+use std::sync::Arc;++#[cfg(any(feature = "python", test))]+use crate::pack::{self, Kind};++pub(crate) const START: u8 = 0;+pub(crate) const PATH: u8 = 1;+pub(crate) const POINTS: u8 = 2;+pub(crate) const MESH: u8 = 3;+pub(crate) const ROWS: u8 = 4;+pub(crate) const TEXTURE: u8 = 5;+pub(crate) const GROW: u8 = 6;+pub(crate) const FRAME: u8 = 8;+pub(crate) const NOTE: u8 = 9;+pub(crate) const SOUND: u8 = 10;+pub(crate) const ARRAY: u8 = 11;+pub(crate) const END: u8 = 12;+pub(crate) const ENVIRONMENT: u8 = 13;++/// The keys a record draws: shapes, brushes (fill and stroke, of both paints), texture.+pub(crate) const KEYS: usize = 7;+pub(crate) const RECORD: usize = 336; // (`render::Record`'s size: checked where it is defined)++/// The keys of each record in `records`.+pub(crate) fn slots(records: &[u8]) -> impl Iterator<Item = [u64; KEYS]> + '_ {+ records.chunks_exact(RECORD).map(|r| std::array::from_fn(|k| u64::from_le_bytes(r[8 * k..8 * k + 8].try_into().unwrap())))+}++/// An upload the writer holds until a frame shows it (so it knows what it replaces).+#[cfg(any(feature = "python", test))]+struct Upload {+ op: u8,+ key: u64,+ head: Vec<u8>, // its fields other than its arrays+ arrays: Vec<(Kind, Vec<u8>)>,+ waited: bool, // a frame has passed without showing it+}++/// Writes a take's messages into a buffer, taken out as it fills (`drain`).+#[cfg(any(feature = "python", test))]+#[derive(Default)]+pub(crate) struct Writer {+ buffer: Vec<u8>,+ sent: HashMap<u64, u8>, // every array sent: its depth in its chain+ arrays: HashMap<u64, Arc<[u8]>>, // arrays that may be bases: those the last frames drew+ parents: HashMap<u64, u64>, // their bases+ uploads: HashMap<u64, (u8, Vec<u64>)>, // an upload's op and arrays (uploads the last frames drew)+ pending: Vec<Upload>,+ slots: Vec<[u64; KEYS]>, // the last frame's records' keys (the frame's, then its cameras')+ records: HashMap<u64, u64>, // a view's last records array (0: the frame's; else a camera's)+}++#[cfg(any(feature = "python", test))]+impl Writer {+ fn begin(&mut self, op: u8) -> usize {+ let at = self.buffer.len();+ self.buffer.extend_from_slice(&[0; 4]);+ self.buffer.push(op);+ at+ }+ fn end(&mut self, at: usize) {+ let n = (self.buffer.len() - at - 4) as u32;+ self.buffer[at..at + 4].copy_from_slice(&n.to_le_bytes());+ }+ fn put(&mut self, bytes: &[u8]) {+ self.buffer.extend_from_slice(bytes);+ }+ fn string(&mut self, bytes: &[u8]) {+ self.put(&(bytes.len() as u32).to_le_bytes());+ self.put(bytes);+ }+ fn hold(&mut self, op: u8, key: u64, head: &[u8], arrays: Vec<(Kind, &[u8])>) {+ let arrays = arrays.into_iter().map(|(kind, data)| (kind, pack::canonical(kind, data))).collect();+ self.pending.push(Upload { op, key, head: head.to_vec(), arrays, waited: false });+ }++ pub(crate) fn start(&mut self, width: u32, height: u32, fps: f64) {+ let at = self.begin(START);+ self.put(&width.to_le_bytes());+ self.put(&height.to_le_bytes());+ self.put(&fps.to_le_bytes());+ self.end(at);+ }+ pub(crate) fn path(&mut self, key: u64, points: &[u8], subpaths: &[u8], centroid_area: [f32; 6]) {+ self.hold(PATH, key, bytemuck::cast_slice(¢roid_area), vec![(Kind::PathPoints, points), (Kind::PathSubpaths, subpaths)]);+ }+ pub(crate) fn points(&mut self, key: u64, vertices: &[u8]) {+ self.hold(POINTS, key, &[], vec![(Kind::Points, vertices)]);+ }+ #[allow(clippy::too_many_arguments)]+ pub(crate) fn mesh(&mut self, key: u64, points: &[u8], uvs: &[u8], normals: &[u8], triangles: &[u8], outline: u32, block: u32) {+ let head: Vec<u8> = [outline.to_le_bytes(), block.to_le_bytes()].concat();+ self.hold(MESH, key, &head, vec![(Kind::MeshPoints, points), (Kind::MeshUvs, uvs), (Kind::MeshNormals, normals), (Kind::MeshTriangles, triangles)]);+ }+ pub(crate) fn rows(&mut self, key: u64, rows: &[u8]) {+ self.hold(ROWS, key, &[], vec![(Kind::Rows, rows)]);+ }+ pub(crate) fn texture(&mut self, key: u64, width: u32, height: u32, rgba: &[u8]) {+ let head: Vec<u8> = [width.to_le_bytes(), height.to_le_bytes()].concat();+ self.hold(TEXTURE, key, &head, vec![(Kind::Texture, rgba)]);+ }+ /// An environment, written at once (frames show it by its id, in their views: none waits to show it).+ pub(crate) fn environment(&mut self, id: u32, width: u32, height: u32, rgbe: &[u8]) {+ let key = self.array(Kind::Texture, rgbe.to_vec(), None);+ let at = self.begin(ENVIRONMENT);+ self.put(&id.to_le_bytes());+ self.put(&width.to_le_bytes());+ self.put(&height.to_le_bytes());+ self.put(&key.to_le_bytes());+ self.end(at);+ }+ pub(crate) fn grow(&mut self, key: u64, points: &[u8], closed: bool) {+ self.hold(GROW, key, &[closed as u8], vec![(Kind::PathPoints, points)]);+ }++ /// Send an array (or, sent before, nothing): its key. Coded against `base`, if that is at+ /// hand and its chain is short enough.+ fn array(&mut self, kind: Kind, data: Vec<u8>, base: Option<u64>) -> u64 {+ let key = pack::key(kind, &data);+ if self.sent.contains_key(&key) {+ self.arrays.entry(key).or_insert_with(|| data.into());+ return key;+ }+ let base = base.filter(|b| self.sent.get(b).is_some_and(|&d| d < pack::DEPTH) && self.arrays.contains_key(b));+ let older = base.and_then(|b| self.parents.get(&b)).and_then(|o| self.arrays.get(o)).cloned();+ let (mode, blob) = pack::encode(kind, &data, base.map(|b| &self.arrays[&b][..]), older.as_deref());+ let base = base.filter(|_| mode != pack::ALONE);+ let at = self.begin(ARRAY);+ self.put(&key.to_le_bytes());+ self.put(&base.unwrap_or(0).to_le_bytes());+ self.put(&[kind as u8, mode]);+ self.string(&blob);+ self.end(at);+ self.sent.insert(key, base.map_or(0, |b| self.sent[&b] + 1));+ if let Some(b) = base {+ self.parents.insert(key, b);+ }+ self.arrays.insert(key, data.into());+ key+ }++ fn upload(&mut self, u: Upload, replaces: Option<u64>) {+ // an upload of another kind has other arrays: no bases+ let bases = replaces.and_then(|r| self.uploads.get(&r)).filter(|(op, _)| *op == u.op).map(|(_, keys)| keys.clone()).unwrap_or_default();+ let keys: Vec<u64> = u.arrays.into_iter().enumerate().map(|(i, (kind, data))| self.array(kind, data, bases.get(i).copied())).collect();+ let at = self.begin(u.op);+ self.put(&u.key.to_le_bytes());+ self.put(&u.head);+ for k in &keys {+ self.put(&k.to_le_bytes());+ }+ self.end(at);+ if u.op != GROW {+ self.uploads.insert(u.key, (u.op, keys));+ }+ }++ pub(crate) fn frame(&mut self, view: &[u8], records: &[u8], repeat: u32, cameras: &[crate::CameraView]) {+ // the views' records, and the keys each draws+ let views: Vec<(u64, &[u8])> = std::iter::once((0, records)).chain(cameras.iter().map(|c| (c.0, &c.4[..]))).collect();+ let slots: Vec<[u64; KEYS]> = views.iter().flat_map(|(_, r)| slots(r)).collect();+ // the uploads it shows first: each replaces what its slot drew the frame before+ let mut replaces: HashMap<u64, Option<u64>> = HashMap::new();+ for (i, keys) in slots.iter().enumerate() {+ for (role, &key) in keys.iter().enumerate() {+ if key != 0 && !replaces.contains_key(&key) && self.pending.iter().any(|u| u.key == key) {+ replaces.insert(key, self.slots.get(i).map(|was| was[role]).filter(|&was| was != 0 && was != key));+ }+ }+ }+ // in the order they came (a path before its growth); one no frame shows goes after a frame+ for mut u in std::mem::take(&mut self.pending) {+ match replaces.get(&u.key) {+ Some(&was) => self.upload(u, was),+ None if u.waited => self.upload(u, None),+ None => {+ u.waited = true;+ self.pending.push(u);+ }+ }+ }+ let mut last = HashMap::new();+ let record_keys: Vec<u64> = views+ .iter()+ .map(|&(view, records)| {+ let key = self.array(Kind::Records, records.to_vec(), self.records.get(&view).copied());+ last.insert(view, key);+ key+ })+ .collect();+ let at = self.begin(FRAME);+ self.put(&repeat.to_le_bytes());+ self.string(view);+ self.put(&record_keys[0].to_le_bytes());+ self.put(&(cameras.len() as u32).to_le_bytes());+ for ((key, width, height, v, _), records) in cameras.iter().zip(&record_keys[1..]) {+ self.put(&key.to_le_bytes());+ self.put(&width.to_le_bytes());+ self.put(&height.to_le_bytes());+ self.string(v);+ self.put(&records.to_le_bytes());+ }+ self.end(at);+ // keep what this frame drew (the bases of the next), and the bases of those+ self.uploads.retain(|k, _| slots.iter().any(|s| s.contains(k)));+ let mut keep: std::collections::HashSet<u64> = self.uploads.values().flat_map(|(_, keys)| keys).copied().chain(last.values().copied()).collect();+ keep.extend(keep.iter().filter_map(|k| self.parents.get(k)).copied().collect::<Vec<_>>());+ self.arrays.retain(|k, _| keep.contains(k));+ self.parents.retain(|k, _| keep.contains(k));+ (self.slots, self.records) = (slots, last);+ }+ pub(crate) fn note(&mut self, json: &str) {+ let at = self.begin(NOTE);+ self.string(json.as_bytes());+ self.end(at);+ }+ pub(crate) fn sound(&mut self, file: &[u8]) {+ let at = self.begin(SOUND);+ self.string(file);+ self.end(at);+ }+ /// The take's end: its film ended, or its scene `failed`.+ pub(crate) fn close(&mut self, failed: bool) {+ let at = self.begin(END);+ self.put(&[failed as u8]);+ self.end(at);+ }++ /// The messages written since the last drain.+ pub(crate) fn drain(&mut self) -> Vec<u8> {+ std::mem::take(&mut self.buffer)+ }+}++/// Reads a message's fields in order.+#[cfg(any(feature = "player", test))]+pub(crate) struct Fields<'a>(pub(crate) &'a [u8]);++#[cfg(any(feature = "player", test))]+impl<'a> Fields<'a> {+ pub(crate) fn take(&mut self, n: usize) -> Result<&'a [u8], String> {+ if self.0.len() < n {+ return Err("a message of the take ends early".into());+ }+ let (head, rest) = self.0.split_at(n);+ self.0 = rest;+ Ok(head)+ }+ pub(crate) fn u8(&mut self) -> Result<u8, String> {+ Ok(self.take(1)?[0])+ }+ pub(crate) fn u32(&mut self) -> Result<u32, String> {+ Ok(u32::from_le_bytes(self.take(4)?.try_into().unwrap()))+ }+ pub(crate) fn u64(&mut self) -> Result<u64, String> {+ Ok(u64::from_le_bytes(self.take(8)?.try_into().unwrap()))+ }+ pub(crate) fn f64(&mut self) -> Result<f64, String> {+ Ok(f64::from_le_bytes(self.take(8)?.try_into().unwrap()))+ }+ pub(crate) fn f32s<const N: usize>(&mut self) -> Result<[f32; N], String> {+ Ok(bytemuck::pod_read_unaligned(self.take(4 * N)?))+ }+ pub(crate) fn string(&mut self) -> Result<&'a [u8], String> {+ let n = self.u32()? as usize;+ self.take(n)+ }+}++/// The whole messages at the start of `stream`, and how many bytes they take (the rest is a+/// message still arriving).+#[cfg(any(feature = "player", test))]+pub(crate) fn messages(stream: &[u8]) -> (Vec<&[u8]>, usize) {+ let (mut out, mut at) = (Vec::new(), 0);+ while let Some(n) = stream.get(at..at + 4).map(|b| u32::from_le_bytes(b.try_into().unwrap()) as usize) {+ let Some(message) = stream.get(at + 4..at + 4 + n) else { break };+ out.push(message);+ at += 4 + n;+ }+ (out, at)+}++#[cfg(test)]+mod tests {+ use super::*;++ fn record(shape: u64) -> Vec<u8> {+ let mut r = vec![0u8; RECORD];+ r[..8].copy_from_slice(&shape.to_le_bytes());+ r+ }++ /// What a writer writes, a reader reads back: every message whole, in order, however the+ /// stream is cut into chunks; an upload waits for the frame that shows it, and replaces what+ /// its slot drew before: its arrays are coded against those, decode to them exactly, and an+ /// array sent before is not sent again.+ #[test]+ fn round_trip() {+ let mut w = Writer::default();+ w.start(960, 540, 30.0);+ let points = |t: f64| -> Vec<u8> { bytemuck::cast_slice(&(0..900).map(|i| ((i as f64) * 0.1 + t).sin()).collect::<Vec<f64>>()).to_vec() };+ let triangles: Vec<u8> = bytemuck::cast_slice(&(0..300u32).collect::<Vec<u32>>()).to_vec();+ w.mesh(7, &points(0.0), &[], &[], &triangles, 0, 0);+ let mut seen_by_camera = record(7);+ seen_by_camera[56] = 1; // other flags: other records+ w.frame(&[3; 192], &record(7), 1, &[(9, 64, 32, vec![5; 192], seen_by_camera)]);+ w.mesh(8, &points(0.01), &[], &[], &triangles, 0, 0); // the next frame's: replaces 7+ w.frame(&[3; 192], &record(7), 2, &[]);+ w.frame(&[3; 192], &record(8), 1, &[]);+ w.sound(&[8; 44]);+ w.note("{\"captions\": []}");+ w.close(false);+ let stream = w.drain();+ for cut in [1, 5, 100, stream.len()] {+ let (mut seen, mut pending) = (Vec::new(), Vec::new());+ for chunk in stream.chunks(cut) {+ pending.extend_from_slice(chunk);+ let (messages, used) = messages(&pending);+ seen.extend(messages.iter().map(|m| m[0]));+ pending.drain(..used);+ }+ let (a, m, f) = (ARRAY, MESH, FRAME);+ // mesh 7: its 4 arrays; the frame's records and its camera's; frame; frame 2 (its+ // records as before); mesh 8 (its points only, then 3 known arrays), records, frame 3+ assert_eq!(seen, [START, a, a, a, a, m, a, a, f, f, a, m, a, f, SOUND, NOTE, END]);+ assert!(pending.is_empty());+ }+ let (messages, _) = messages(&stream);+ // decode every array, as a projector would (each in order, its base before it)+ let mut decoded: HashMap<u64, Vec<u8>> = HashMap::new();+ let mut bases: HashMap<u64, u64> = HashMap::new();+ for m in &messages {+ let mut f = Fields(m);+ if f.u8().unwrap() != ARRAY {+ continue;+ }+ let (key, base, kind, mode, blob) = (f.u64().unwrap(), f.u64().unwrap(), f.u8().unwrap(), f.u8().unwrap(), f.string().unwrap());+ let kind = Kind::from_u8(kind).unwrap();+ let older = bases.get(&base).and_then(|o| decoded.get(o)).map(|v| &v[..]);+ let data = pack::decode(kind, mode, blob, decoded.get(&base).map(|v| &v[..]), older).unwrap();+ assert_eq!(pack::key(kind, &data), key);+ if base != 0 {+ assert_eq!(mode, pack::FIRST, "mesh 8's points are coded against mesh 7's");+ bases.insert(key, base);+ }+ decoded.insert(key, data);+ }+ assert!(decoded.values().any(|d| d[..] == pack::canonical(Kind::MeshPoints, &points(0.01))[..]));+ let mut s = Fields(messages[0]);+ assert_eq!((s.u8().unwrap(), s.u32().unwrap(), s.u32().unwrap(), s.f64().unwrap()), (START, 960, 540, 30.0));+ }+}