Skip to content

The animation engine for agents: blazingly fast math and 3D videos with Manim's Python API, on a Rust GPU core.

Python

Add native preview window

apoogdkcbfe82a672e9
diff --git a/rust/bridge/crates/manimgx-metal/Cargo.toml b/rust/bridge/crates/manimgx-metal/Cargo.toml
new file mode 100644
index 00000000..1607f777
--- /dev/null
+++ b/rust/bridge/crates/manimgx-metal/Cargo.toml
@@ -0,0 +1,21 @@
+[package]
+name = "manimgx-metal"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+log = { workspace = true }
+
+[target.'cfg(target_os = "macos")'.dependencies]
+manimgx-core = { path = "../manimgx-core" }
+manimgx-encode = { path = "../manimgx-encode" }
+metal = "0.31"
+objc = "0.2"
+bytemuck = { workspace = true }
+glam = { workspace = true }
+
+[features]
+time_profile = []
+
+[lints.rust]
+unexpected_cfgs = { level = "warn", check-cfg = ['cfg(feature, values("time_profile", "cargo-clippy"))'] }
diff --git a/rust/bridge/crates/manimgx-metal/src/backend.rs b/rust/bridge/crates/manimgx-metal/src/backend.rs
new file mode 100644
index 00000000..068d6399
--- /dev/null
+++ b/rust/bridge/crates/manimgx-metal/src/backend.rs
@@ -0,0 +1,1008 @@
+use std::collections::HashMap;
+use std::ffi::c_void;
+use std::sync::Arc;
+
+use manimgx_core::color::srgb_to_linear;
+use manimgx_core::scene::{ObjectId, RenderHint, Scene, TextureData};
+use manimgx_core::uniform::CameraUniform;
+use manimgx_core::vertex::Vertex;
+use metal::foreign_types::{ForeignType, ForeignTypeRef};
+use metal::*;
+use objc::runtime::Object;
+#[allow(unused_imports)]
+use objc::{sel, sel_impl};
+
+// ── Cached GPU resources ──
+
+struct CachedMetalBuffers {
+ vertex_buffer: Buffer,
+ index_buffer: Buffer,
+ index_count: u32,
+}
+
+struct CachedMetalTexture {
+ texture: Texture,
+}
+
+// ── Per-frame slot for N-buffering ──
+//
+// Two backing modes:
+// Buffer: MTLBuffer (shared) backs output_texture. After GPU completes,
+// buffer.contents() is a direct pointer to BGRA pixels → pipe to ffmpeg.
+// IOSurface: IOSurface (shared) backs output_texture. Same memory is wrapped
+// by a CVPixelBuffer → fed directly to VideoToolbox. Zero readback.
+
+enum SlotBacking {
+ /// CPU-readable MTLBuffer backing store for ffmpeg encoding path.
+ Buffer(Buffer),
+ /// IOSurface + CVPixelBuffer for VideoToolbox encoding path.
+ IOSurface {
+ surface: crate::metal_encoder::IOSurfaceRef,
+ cv_pixel_buffer: crate::metal_encoder::CVPixelBufferRef,
+ },
+}
+
+impl Drop for SlotBacking {
+ fn drop(&mut self) {
+ if let SlotBacking::IOSurface { surface, cv_pixel_buffer } = self {
+ unsafe {
+ crate::metal_encoder::CVPixelBufferRelease(*cv_pixel_buffer);
+ crate::metal_encoder::CFRelease(*surface as crate::metal_encoder::CFTypeRef);
+ }
+ }
+ // Buffer variant: metal crate's Drop handles it
+ }
+}
+
+struct FrameSlot {
+ backing: SlotBacking,
+ output_texture: Texture, // buffer-backed or IOSurface-backed texture (resolve/render target)
+ msaa_texture: Option<Texture>, // private mode, sample_count>1
+ depth_texture: Texture, // private mode
+ camera_buffer: Buffer, // shared mode, per-slot
+ object_buffer: Buffer, // shared mode, per-slot
+ pending_cmd_buffer: Option<*mut c_void>, // retained raw MTLCommandBuffer pointer
+ /// Signaled by encoding thread after it finishes reading pixels from
+ /// pixel_buffer.contents(). Main thread waits on this before reusing the slot.
+ pub encoding_done: Arc<EncodingSignal>,
+}
+
+unsafe impl Send for FrameSlot {}
+
+/// Condition-variable-based signal for encoding thread → main thread synchronization.
+/// Replaces AtomicBool spin loop to avoid burning a CPU core.
+pub struct EncodingSignal {
+ done: std::sync::Mutex<bool>,
+ cvar: std::sync::Condvar,
+}
+
+impl EncodingSignal {
+ pub fn new(initial: bool) -> Self {
+ Self {
+ done: std::sync::Mutex::new(initial),
+ cvar: std::sync::Condvar::new(),
+ }
+ }
+
+ /// Mark as done and wake the waiting thread.
+ pub fn signal_done(&self) {
+ let mut done = self.done.lock().unwrap();
+ *done = true;
+ self.cvar.notify_one();
+ }
+
+ /// Wait until done, then return.
+ pub fn wait_done(&self) {
+ let mut done = self.done.lock().unwrap();
+ while !*done {
+ done = self.cvar.wait(done).unwrap();
+ }
+ }
+
+ /// Reset to not-done (called before handing slot to encoding thread).
+ pub fn reset(&self) {
+ let mut done = self.done.lock().unwrap();
+ *done = false;
+ }
+}
+
+
+// ── Shader compilation ──
+
+/// Replace `#include "types.h"` with actual types content for runtime MSL compilation.
+fn prepare_shader(types_src: &str, shader_src: &str) -> String {
+ shader_src.replace("#include \"types.h\"", types_src)
+}
+
+// ── MetalBackend ──
+
+pub struct MetalBackend {
+ device: Device,
+ queue: CommandQueue,
+
+ default_pipeline: RenderPipelineState,
+ flat_no_cull_pipeline: RenderPipelineState,
+ textured_pipeline: RenderPipelineState,
+
+ depth_stencil_state: DepthStencilState,
+ sampler_state: SamplerState,
+
+ frame_slots: Vec<FrameSlot>,
+
+ buffer_cache: HashMap<ObjectId, CachedMetalBuffers>,
+ texture_cache: HashMap<ObjectId, CachedMetalTexture>,
+
+ width: u32,
+ height: u32,
+ pixel_size: usize,
+
+ // NV12 compute conversion (optional, only when pix_fmt="nv12")
+ nv12_pipeline: Option<ComputePipelineState>,
+ nv12_buffers: Vec<Buffer>, // one per frame slot, W*H*3/2 bytes
+ nv12_dims_buffer: Option<Buffer>, // constant [u32; 2] = [width, height]
+ nv12_size: usize,
+}
+
+impl MetalBackend {
+ /// Query the default Metal device for supported MSAA sample counts.
+ /// Returns an error string if `sample_count` is not supported.
+ pub fn validate_sample_count(sample_count: u32) -> Result<(), String> {
+ let device = Device::system_default().ok_or("No Metal device found")?;
+ if !device.supports_texture_sample_count(sample_count as u64) {
+ let supported: Vec<u32> = [1, 2, 4, 8, 16]
+ .iter()
+ .copied()
+ .filter(|&c| device.supports_texture_sample_count(c as u64))
+ .collect();
+ return Err(format!(
+ "Metal device '{}' does not support msaa={}. Supported values: {:?}.",
+ device.name(), sample_count, supported
+ ));
+ }
+ Ok(())
+ }
+
+ pub fn new(width: u32, height: u32, sample_count: u32, n_buffers: u32, pix_fmt: &str) -> Self {
+ let device = Device::system_default().expect("No Metal device found");
+ let queue = device.new_command_queue();
+
+ let use_nv12 = pix_fmt == "nv12";
+ log::info!("Metal native backend: {} ({}x{}, MSAA={}, pix_fmt={})", device.name(), width, height, sample_count, pix_fmt);
+
+ let pixel_format = MTLPixelFormat::BGRA8Unorm_sRGB;
+ let bytes_per_row = (width * 4) as u64;
+ let pixel_size = (width * height * 4) as usize;
+
+ // Compile MSL shaders
+ let types_src = include_str!("../../../shaders/metal/types.h");
+ let opts = CompileOptions::new();
+
+ let default_lib = device
+ .new_library_with_source(&prepare_shader(types_src, include_str!("../../../shaders/metal/default.metal")), &opts)
+ .unwrap_or_else(|e| panic!("default.metal compilation failed:\n{}", e));
+ let flat_color_lib = device
+ .new_library_with_source(&prepare_shader(types_src, include_str!("../../../shaders/metal/flat_color.metal")), &opts)
+ .unwrap_or_else(|e| panic!("flat_color.metal compilation failed:\n{}", e));
+ let text_lib = device
+ .new_library_with_source(&prepare_shader(types_src, include_str!("../../../shaders/metal/text.metal")), &opts)
+ .unwrap_or_else(|e| panic!("text.metal compilation failed:\n{}", e));
+
+ // Vertex descriptor (shared across all pipelines)
+ let vertex_desc = Self::create_vertex_descriptor();
+
+ // Create 5 pipelines
+ let default_pipeline = Self::create_pipeline(
+ &device, &default_lib, vertex_desc, pixel_format, sample_count,
+ );
+ let flat_no_cull_pipeline = Self::create_pipeline(
+ &device, &flat_color_lib, vertex_desc, pixel_format, sample_count,
+ );
+ let textured_pipeline = Self::create_pipeline(
+ &device, &text_lib, vertex_desc, pixel_format, sample_count,
+ );
+
+ // Depth stencil state
+ let depth_desc = DepthStencilDescriptor::new();
+ depth_desc.set_depth_compare_function(MTLCompareFunction::Less);
+ depth_desc.set_depth_write_enabled(true);
+ let depth_stencil_state = device.new_depth_stencil_state(&depth_desc);
+
+ // Sampler for text textures
+ let sampler_desc = SamplerDescriptor::new();
+ sampler_desc.set_address_mode_s(MTLSamplerAddressMode::ClampToEdge);
+ sampler_desc.set_address_mode_t(MTLSamplerAddressMode::ClampToEdge);
+ sampler_desc.set_mag_filter(MTLSamplerMinMagFilter::Linear);
+ sampler_desc.set_min_filter(MTLSamplerMinMagFilter::Linear);
+ let sampler_state = device.new_sampler(&sampler_desc);
+
+ // NV12 compute pipeline (optional)
+ let nv12_size = (width * height * 3 / 2) as usize;
+ let (nv12_pipeline, nv12_dims_buffer) = if use_nv12 {
+ assert!(width.is_multiple_of(2) && height.is_multiple_of(2), "NV12 requires even width and height");
+ let nv12_src = include_str!("../../../shaders/metal/bgra_to_nv12.metal");
+ let nv12_lib = device
+ .new_library_with_source(nv12_src, &opts)
+ .unwrap_or_else(|e| panic!("bgra_to_nv12.metal compilation failed:\n{}", e));
+ let nv12_fn = nv12_lib.get_function("bgra_to_nv12", None)
+ .expect("bgra_to_nv12 function not found");
+ let pipeline = device.new_compute_pipeline_state_with_function(&nv12_fn)
+ .unwrap_or_else(|e| panic!("Failed to create NV12 compute pipeline: {}", e));
+
+ let dims: [u32; 2] = [width, height];
+ let dims_buf = device.new_buffer_with_data(
+ dims.as_ptr() as *const c_void,
+ std::mem::size_of::<[u32; 2]>() as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+
+ (Some(pipeline), Some(dims_buf))
+ } else {
+ (None, None)
+ };
+
+ // Create N frame slots with buffer-backed textures
+ let n = n_buffers.max(1) as usize;
+ let max_objects: usize = 512;
+ let uniform_buf_size = (max_objects * 256) as u64;
+
+ let mut nv12_buffers = Vec::new();
+
+ let frame_slots: Vec<FrameSlot> = (0..n)
+ .map(|_| {
+ // Zero-copy pixel readback: create MTLBuffer, then create MTLTexture
+ // backed by that buffer. GPU writes into the texture → pixels land in
+ // the buffer's memory. After completion, buffer.contents() is a direct
+ // pointer to the pixel data.
+ let pixel_buffer = device.new_buffer(
+ pixel_size as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+
+ let tex_desc = TextureDescriptor::new();
+ tex_desc.set_pixel_format(pixel_format);
+ tex_desc.set_width(width as u64);
+ tex_desc.set_height(height as u64);
+ tex_desc.set_texture_type(MTLTextureType::D2);
+ tex_desc.set_storage_mode(MTLStorageMode::Shared);
+ tex_desc.set_usage(MTLTextureUsage::RenderTarget);
+
+ let output_texture = pixel_buffer.new_texture_with_descriptor(
+ &tex_desc,
+ 0, // offset
+ bytes_per_row, // bytesPerRow
+ );
+
+ // NV12 output buffer per slot
+ if use_nv12 {
+ nv12_buffers.push(device.new_buffer(
+ nv12_size as u64,
+ MTLResourceOptions::StorageModeShared,
+ ));
+ }
+
+ let msaa_texture = if sample_count > 1 {
+ Some(Self::create_msaa_texture(&device, width, height, pixel_format, sample_count))
+ } else {
+ None
+ };
+
+ let depth_texture = Self::create_depth_texture(&device, width, height, sample_count);
+
+ let camera_buffer = device.new_buffer(
+ std::mem::size_of::<CameraUniform>() as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+
+ let object_buffer = device.new_buffer(
+ uniform_buf_size,
+ MTLResourceOptions::StorageModeShared,
+ );
+
+ FrameSlot {
+ backing: SlotBacking::Buffer(pixel_buffer),
+ output_texture,
+ msaa_texture,
+ depth_texture,
+ camera_buffer,
+ object_buffer,
+ pending_cmd_buffer: None,
+ encoding_done: Arc::new(EncodingSignal::new(true)), // initially "done" (no work pending)
+ }
+ })
+ .collect();
+
+ Self {
+ device,
+ queue,
+ default_pipeline,
+ flat_no_cull_pipeline,
+ textured_pipeline,
+ depth_stencil_state,
+ sampler_state,
+ frame_slots,
+ buffer_cache: HashMap::new(),
+ texture_cache: HashMap::new(),
+ width,
+ height,
+ pixel_size,
+ nv12_pipeline,
+ nv12_buffers,
+ nv12_dims_buffer,
+ nv12_size,
+ }
+ }
+
+ pub fn pixel_size(&self) -> usize {
+ self.pixel_size
+ }
+
+ /// Create a MetalBackend with IOSurface-backed frame slots for VideoToolbox encoding.
+ /// Each slot gets its own IOSurface + CVPixelBuffer. No pixel readback needed.
+ pub fn new_metal_encoder(width: u32, height: u32, sample_count: u32, n_buffers: u32) -> Self {
+ let device = Device::system_default().expect("No Metal device found");
+ let queue = device.new_command_queue();
+
+ log::info!("Metal native backend (VT encoder): {} ({}x{}, MSAA={})", device.name(), width, height, sample_count);
+
+ let pixel_format = MTLPixelFormat::BGRA8Unorm_sRGB;
+
+ // Compile MSL shaders
+ let types_src = include_str!("../../../shaders/metal/types.h");
+ let opts = CompileOptions::new();
+
+ let default_lib = device
+ .new_library_with_source(&prepare_shader(types_src, include_str!("../../../shaders/metal/default.metal")), &opts)
+ .unwrap_or_else(|e| panic!("default.metal compilation failed:\n{}", e));
+ let flat_color_lib = device
+ .new_library_with_source(&prepare_shader(types_src, include_str!("../../../shaders/metal/flat_color.metal")), &opts)
+ .unwrap_or_else(|e| panic!("flat_color.metal compilation failed:\n{}", e));
+ let text_lib = device
+ .new_library_with_source(&prepare_shader(types_src, include_str!("../../../shaders/metal/text.metal")), &opts)
+ .unwrap_or_else(|e| panic!("text.metal compilation failed:\n{}", e));
+
+ let vertex_desc = Self::create_vertex_descriptor();
+
+ let default_pipeline = Self::create_pipeline(&device, &default_lib, vertex_desc, pixel_format, sample_count);
+ let flat_no_cull_pipeline = Self::create_pipeline(&device, &flat_color_lib, vertex_desc, pixel_format, sample_count);
+ let textured_pipeline = Self::create_pipeline(&device, &text_lib, vertex_desc, pixel_format, sample_count);
+
+ let depth_desc = DepthStencilDescriptor::new();
+ depth_desc.set_depth_compare_function(MTLCompareFunction::Less);
+ depth_desc.set_depth_write_enabled(true);
+ let depth_stencil_state = device.new_depth_stencil_state(&depth_desc);
+
+ let sampler_desc = SamplerDescriptor::new();
+ sampler_desc.set_address_mode_s(MTLSamplerAddressMode::ClampToEdge);
+ sampler_desc.set_address_mode_t(MTLSamplerAddressMode::ClampToEdge);
+ sampler_desc.set_mag_filter(MTLSamplerMinMagFilter::Linear);
+ sampler_desc.set_min_filter(MTLSamplerMinMagFilter::Linear);
+ let sampler_state = device.new_sampler(&sampler_desc);
+
+ let n = n_buffers.max(1) as usize;
+ let max_objects: usize = 512;
+ let uniform_buf_size = (max_objects * 256) as u64;
+
+ let frame_slots: Vec<FrameSlot> = (0..n)
+ .map(|_| {
+ // Create IOSurface → IOSurface-backed MTLTexture → CVPixelBuffer
+ let surface = crate::metal_encoder::create_iosurface(width, height);
+ let cv_pixel_buffer = crate::metal_encoder::create_cv_pixel_buffer(surface, width, height);
+
+ // Create MTLTextureDescriptor for IOSurface-backed texture
+ let tex_desc = TextureDescriptor::new();
+ tex_desc.set_pixel_format(pixel_format);
+ tex_desc.set_width(width as u64);
+ tex_desc.set_height(height as u64);
+ tex_desc.set_texture_type(MTLTextureType::D2);
+ tex_desc.set_storage_mode(MTLStorageMode::Shared);
+ tex_desc.set_usage(MTLTextureUsage::RenderTarget);
+
+ // Create Metal texture backed by IOSurface via objc runtime
+ let output_texture = unsafe {
+ let raw_texture: *mut Object = objc::msg_send![
+ device.as_ptr() as *mut Object,
+ newTextureWithDescriptor: tex_desc.as_ptr() as *mut Object
+ iosurface: surface as *mut Object
+ plane: 0usize
+ ];
+ assert!(!raw_texture.is_null(), "Failed to create IOSurface-backed Metal texture");
+ Texture::from_ptr(raw_texture as *mut MTLTexture)
+ };
+
+ let msaa_texture = if sample_count > 1 {
+ Some(Self::create_msaa_texture(&device, width, height, pixel_format, sample_count))
+ } else {
+ None
+ };
+
+ let depth_texture = Self::create_depth_texture(&device, width, height, sample_count);
+
+ let camera_buffer = device.new_buffer(
+ std::mem::size_of::<CameraUniform>() as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+
+ let object_buffer = device.new_buffer(
+ uniform_buf_size,
+ MTLResourceOptions::StorageModeShared,
+ );
+
+ FrameSlot {
+ backing: SlotBacking::IOSurface { surface, cv_pixel_buffer },
+ output_texture,
+ msaa_texture,
+ depth_texture,
+ camera_buffer,
+ object_buffer,
+ pending_cmd_buffer: None,
+ encoding_done: Arc::new(EncodingSignal::new(true)),
+ }
+ })
+ .collect();
+
+ Self {
+ device,
+ queue,
+ default_pipeline,
+ flat_no_cull_pipeline,
+ textured_pipeline,
+ depth_stencil_state,
+ sampler_state,
+ frame_slots,
+ buffer_cache: HashMap::new(),
+ texture_cache: HashMap::new(),
+ width,
+ height,
+ pixel_size: 0, // not used for IOSurface path
+ nv12_pipeline: None,
+ nv12_buffers: Vec::new(),
+ nv12_dims_buffer: None,
+ nv12_size: 0,
+ }
+ }
+
+ // ── Texture creation helpers ──
+
+ fn create_msaa_texture(device: &Device, width: u32, height: u32, format: MTLPixelFormat, sample_count: u32) -> Texture {
+ let desc = TextureDescriptor::new();
+ desc.set_pixel_format(format);
+ desc.set_width(width as u64);
+ desc.set_height(height as u64);
+ desc.set_sample_count(sample_count as u64);
+ desc.set_texture_type(MTLTextureType::D2Multisample);
+ desc.set_storage_mode(MTLStorageMode::Private);
+ desc.set_usage(MTLTextureUsage::RenderTarget);
+ device.new_texture(&desc)
+ }
+
+ fn create_depth_texture(device: &Device, width: u32, height: u32, sample_count: u32) -> Texture {
+ let desc = TextureDescriptor::new();
+ desc.set_pixel_format(MTLPixelFormat::Depth32Float);
+ desc.set_width(width as u64);
+ desc.set_height(height as u64);
+ desc.set_storage_mode(MTLStorageMode::Private);
+ desc.set_usage(MTLTextureUsage::RenderTarget);
+ if sample_count > 1 {
+ desc.set_sample_count(sample_count as u64);
+ desc.set_texture_type(MTLTextureType::D2Multisample);
+ } else {
+ desc.set_texture_type(MTLTextureType::D2);
+ }
+ device.new_texture(&desc)
+ }
+
+ // ── Pipeline creation ──
+
+ fn create_vertex_descriptor() -> &'static VertexDescriptorRef {
+ let desc = VertexDescriptor::new();
+
+ // attribute(0): position — float3 at offset 0
+ let attr0 = desc.attributes().object_at(0).unwrap();
+ attr0.set_format(MTLVertexFormat::Float3);
+ attr0.set_offset(0);
+ attr0.set_buffer_index(0);
+
+ // attribute(1): normal — float3 at offset 12
+ let attr1 = desc.attributes().object_at(1).unwrap();
+ attr1.set_format(MTLVertexFormat::Float3);
+ attr1.set_offset(12);
+ attr1.set_buffer_index(0);
+
+ // attribute(2): color — float4 at offset 24
+ let attr2 = desc.attributes().object_at(2).unwrap();
+ attr2.set_format(MTLVertexFormat::Float4);
+ attr2.set_offset(24);
+ attr2.set_buffer_index(0);
+
+ // layout(0): stride=40, per-vertex
+ let layout0 = desc.layouts().object_at(0).unwrap();
+ layout0.set_stride(40);
+ layout0.set_step_function(MTLVertexStepFunction::PerVertex);
+
+ desc
+ }
+
+ fn create_pipeline(
+ device: &Device,
+ library: &Library,
+ vertex_desc: &VertexDescriptorRef,
+ color_format: MTLPixelFormat,
+ sample_count: u32,
+ ) -> RenderPipelineState {
+ let vertex_fn = library.get_function("vs_main", None)
+ .expect("vs_main function not found in MSL library");
+ let fragment_fn = library.get_function("fs_main", None)
+ .expect("fs_main function not found in MSL library");
+
+ let pipe_desc = RenderPipelineDescriptor::new();
+ pipe_desc.set_vertex_function(Some(&vertex_fn));
+ pipe_desc.set_fragment_function(Some(&fragment_fn));
+ pipe_desc.set_vertex_descriptor(Some(vertex_desc));
+ pipe_desc.set_sample_count(sample_count as u64);
+ pipe_desc.set_depth_attachment_pixel_format(MTLPixelFormat::Depth32Float);
+
+ // Color attachment with alpha blending (matches wgpu's ALPHA_BLENDING)
+ let color_att = pipe_desc.color_attachments().object_at(0).unwrap();
+ color_att.set_pixel_format(color_format);
+ color_att.set_blending_enabled(true);
+ color_att.set_source_rgb_blend_factor(MTLBlendFactor::SourceAlpha);
+ color_att.set_destination_rgb_blend_factor(MTLBlendFactor::OneMinusSourceAlpha);
+ color_att.set_source_alpha_blend_factor(MTLBlendFactor::SourceAlpha);
+ color_att.set_destination_alpha_blend_factor(MTLBlendFactor::OneMinusSourceAlpha);
+ color_att.set_rgb_blend_operation(MTLBlendOperation::Add);
+ color_att.set_alpha_blend_operation(MTLBlendOperation::Add);
+
+ device.new_render_pipeline_state(&pipe_desc)
+ .unwrap_or_else(|e| panic!("Failed to create Metal render pipeline: {}", e))
+ }
+
+ // ── Buffer caching ──
+
+ fn get_or_cache_buffers(&mut self, obj_id: ObjectId, vertices: &[Vertex], indices: &[u32]) -> &CachedMetalBuffers {
+ self.buffer_cache.entry(obj_id).or_insert_with(|| {
+ let vertex_data = bytemuck::cast_slice::<Vertex, u8>(vertices);
+ let index_data = bytemuck::cast_slice::<u32, u8>(indices);
+
+ let vertex_buffer = self.device.new_buffer_with_data(
+ vertex_data.as_ptr() as *const c_void,
+ vertex_data.len() as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+ let index_buffer = self.device.new_buffer_with_data(
+ index_data.as_ptr() as *const c_void,
+ index_data.len() as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+
+ CachedMetalBuffers {
+ vertex_buffer,
+ index_buffer,
+ index_count: indices.len() as u32,
+ }
+ })
+ }
+
+ fn get_or_cache_texture(&mut self, obj_id: ObjectId, texture_data: &TextureData) -> &CachedMetalTexture {
+ self.texture_cache.entry(obj_id).or_insert_with(|| {
+ let desc = TextureDescriptor::new();
+ desc.set_pixel_format(MTLPixelFormat::RGBA8Unorm_sRGB);
+ desc.set_width(texture_data.width as u64);
+ desc.set_height(texture_data.height as u64);
+ desc.set_storage_mode(MTLStorageMode::Shared);
+ desc.set_usage(MTLTextureUsage::ShaderRead);
+ desc.set_texture_type(MTLTextureType::D2);
+
+ let texture = self.device.new_texture(&desc);
+
+ let region = MTLRegion::new_2d(0, 0, texture_data.width as u64, texture_data.height as u64);
+ texture.replace_region(
+ region,
+ 0, // mipmap level
+ texture_data.rgba.as_ptr() as *const c_void,
+ (4 * texture_data.width) as u64, // bytes per row
+ );
+
+ CachedMetalTexture { texture }
+ })
+ }
+
+ // ── Render a single frame into a slot ──
+
+ /// Build clear color from scene background (sRGB → linear for sRGB framebuffer).
+ pub fn clear_color(scene: &Scene) -> MTLClearColor {
+ MTLClearColor {
+ red: srgb_to_linear(scene.background[0]) as f64,
+ green: srgb_to_linear(scene.background[1]) as f64,
+ blue: srgb_to_linear(scene.background[2]) as f64,
+ alpha: scene.background[3] as f64,
+ }
+ }
+
+ /// Get the encoding_done signal for a slot (used by render_loop for synchronization).
+ pub fn slot_encoding_done(&self, slot_idx: usize) -> &Arc<EncodingSignal> {
+ &self.frame_slots[slot_idx].encoding_done
+ }
+
+ /// Wait until the encoding thread has finished reading from this slot's pixel buffer.
+ /// Must be called before submitting new GPU work to this slot.
+ pub fn wait_encoding_done(&self, slot_idx: usize) {
+ self.frame_slots[slot_idx].encoding_done.wait_done();
+ }
+
+ /// Encode and submit a render pass for one frame. Does NOT wait for completion.
+ /// NV12 compute writes to self.nv12_buffers[slot_idx] (per-slot buffer).
+ pub fn submit_frame(
+ &mut self,
+ scene: &Scene,
+ cam_bytes: &[u8],
+ uniform_staging: &[u8],
+ total_uniform_bytes: usize,
+ draw_cmds: &[(usize, usize)],
+ clear_color: MTLClearColor,
+ slot_idx: usize,
+ ) {
+ self.submit_frame_impl(scene, cam_bytes, uniform_staging, total_uniform_bytes, draw_cmds, clear_color, slot_idx, None);
+ }
+
+ /// Like submit_frame, but NV12 compute writes to an external pool buffer instead.
+ pub fn submit_frame_nv12(
+ &mut self,
+ scene: &Scene,
+ cam_bytes: &[u8],
+ uniform_staging: &[u8],
+ total_uniform_bytes: usize,
+ draw_cmds: &[(usize, usize)],
+ clear_color: MTLClearColor,
+ slot_idx: usize,
+ nv12_target: &Buffer,
+ ) {
+ self.submit_frame_impl(scene, cam_bytes, uniform_staging, total_uniform_bytes, draw_cmds, clear_color, slot_idx, Some(nv12_target));
+ }
+
+ fn submit_frame_impl(
+ &mut self,
+ scene: &Scene,
+ cam_bytes: &[u8],
+ uniform_staging: &[u8],
+ total_uniform_bytes: usize,
+ draw_cmds: &[(usize, usize)], // (obj_idx, uniform_slot)
+ clear_color: MTLClearColor,
+ slot_idx: usize,
+ nv12_override: Option<&Buffer>,
+ ) {
+ let slot = &self.frame_slots[slot_idx];
+
+ // Write camera uniform via direct memcpy
+ unsafe {
+ let dst = slot.camera_buffer.contents() as *mut u8;
+ std::ptr::copy_nonoverlapping(cam_bytes.as_ptr(), dst, cam_bytes.len());
+ }
+
+ // Write object uniforms
+ if total_uniform_bytes > 0 {
+ unsafe {
+ let dst = slot.object_buffer.contents() as *mut u8;
+ std::ptr::copy_nonoverlapping(
+ uniform_staging.as_ptr(),
+ dst,
+ total_uniform_bytes,
+ );
+ }
+ }
+
+ // Pre-cache textures (must happen before encoding)
+ for &(obj_idx, _) in draw_cmds {
+ let obj = &scene.objects[obj_idx];
+ if let RenderHint::Textured = &obj.render_hint {
+ if let Some(ref tex_data) = obj.texture_data {
+ self.get_or_cache_texture(obj.id, tex_data);
+ }
+ }
+ }
+
+ // Pre-cache geometry
+ for &(obj_idx, _) in draw_cmds {
+ let obj = &scene.objects[obj_idx];
+ self.get_or_cache_buffers(obj.id, &obj.vertices, &obj.indices);
+ }
+
+ let cmd_buf = self.queue.new_command_buffer();
+
+ // Configure render pass descriptor
+ let pass_desc = RenderPassDescriptor::new();
+
+ // Color attachment
+ let color_att = pass_desc.color_attachments().object_at(0).unwrap();
+ if let Some(ref msaa_tex) = self.frame_slots[slot_idx].msaa_texture {
+ color_att.set_texture(Some(msaa_tex));
+ color_att.set_resolve_texture(Some(&self.frame_slots[slot_idx].output_texture));
+ color_att.set_store_action(MTLStoreAction::MultisampleResolve);
+ } else {
+ color_att.set_texture(Some(&self.frame_slots[slot_idx].output_texture));
+ color_att.set_store_action(MTLStoreAction::Store);
+ }
+ color_att.set_load_action(MTLLoadAction::Clear);
+ color_att.set_clear_color(clear_color);
+
+ // Depth attachment
+ let depth_att = pass_desc.depth_attachment().unwrap();
+ depth_att.set_texture(Some(&self.frame_slots[slot_idx].depth_texture));
+ depth_att.set_load_action(MTLLoadAction::Clear);
+ depth_att.set_clear_depth(1.0);
+ depth_att.set_store_action(MTLStoreAction::DontCare);
+
+ let encoder = cmd_buf.new_render_command_encoder(pass_desc);
+
+ // Set viewport
+ encoder.set_viewport(MTLViewport {
+ originX: 0.0,
+ originY: 0.0,
+ width: self.width as f64,
+ height: self.height as f64,
+ znear: 0.0,
+ zfar: 1.0,
+ });
+
+ // Set depth stencil state
+ encoder.set_depth_stencil_state(&self.depth_stencil_state);
+
+ // Set front face winding (CCW, matching wgpu)
+ encoder.set_front_facing_winding(MTLWinding::CounterClockwise);
+
+ // Draw each object
+ let slot = &self.frame_slots[slot_idx];
+ let uniform_alignment: usize = 256;
+
+ for &(obj_idx, uniform_slot) in draw_cmds {
+ let obj = &scene.objects[obj_idx];
+ let is_textured = matches!(&obj.render_hint, RenderHint::Textured);
+
+ // Select pipeline and cull mode based on render hint
+ let (pipeline, cull_mode) = match &obj.render_hint {
+ RenderHint::Default => (&self.default_pipeline, MTLCullMode::None),
+ RenderHint::FlatUnlit => (&self.flat_no_cull_pipeline, MTLCullMode::None),
+ RenderHint::Textured => (&self.textured_pipeline, MTLCullMode::None),
+ };
+
+ encoder.set_render_pipeline_state(pipeline);
+ encoder.set_cull_mode(cull_mode);
+
+ let cached = self.buffer_cache.get(&obj.id).unwrap();
+ let obj_offset = (uniform_slot * uniform_alignment) as u64;
+
+ // Buffer index 0: vertex data
+ encoder.set_vertex_buffer(0, Some(&cached.vertex_buffer), 0);
+ // Buffer index 1: camera uniform (vertex shader only)
+ encoder.set_vertex_buffer(1, Some(&slot.camera_buffer), 0);
+ // Buffer index 2: object uniform (vertex AND fragment)
+ encoder.set_vertex_buffer(2, Some(&slot.object_buffer), obj_offset);
+ encoder.set_fragment_buffer(2, Some(&slot.object_buffer), obj_offset);
+
+ // Texture binding for text objects
+ if is_textured {
+ if let Some(tex_cached) = self.texture_cache.get(&obj.id) {
+ encoder.set_fragment_texture(0, Some(&tex_cached.texture));
+ encoder.set_fragment_sampler_state(0, Some(&self.sampler_state));
+ }
+ }
+
+ // Draw indexed primitives
+ encoder.draw_indexed_primitives(
+ MTLPrimitiveType::Triangle,
+ cached.index_count as u64,
+ MTLIndexType::UInt32,
+ &cached.index_buffer,
+ 0,
+ );
+ }
+
+ encoder.end_encoding();
+
+ // NV12 compute pass: convert BGRA → NV12 on the same command buffer
+ if let Some(ref nv12_pipeline) = self.nv12_pipeline {
+ let pixel_buf = match &self.frame_slots[slot_idx].backing {
+ SlotBacking::Buffer(buf) => buf,
+ _ => panic!("NV12 compute requires Buffer-backed slot"),
+ };
+ let nv12_buf = nv12_override.unwrap_or(&self.nv12_buffers[slot_idx]);
+ let dims_buf = self.nv12_dims_buffer.as_ref().unwrap();
+
+ let compute_enc = cmd_buf.new_compute_command_encoder();
+ compute_enc.set_compute_pipeline_state(nv12_pipeline);
+ compute_enc.set_buffer(0, Some(pixel_buf), 0);
+ compute_enc.set_buffer(1, Some(nv12_buf), 0);
+ compute_enc.set_buffer(2, Some(dims_buf), 0);
+
+ // Each thread processes a 4×2 pixel block, threadgroup is 8×8 threads (32×16 pixels)
+ let threadgroup_size = MTLSize::new(8, 8, 1);
+ let grid_w = (self.width as u64).div_ceil(4); // ceil(width / 4)
+ let grid_h = (self.height as u64).div_ceil(2); // ceil(height / 2)
+ compute_enc.dispatch_thread_groups(
+ MTLSize::new(
+ grid_w.div_ceil(8), // ceil(grid_w / threadgroup_w)
+ grid_h.div_ceil(8), // ceil(grid_h / threadgroup_h)
+ 1,
+ ),
+ threadgroup_size,
+ );
+ compute_enc.end_encoding();
+ }
+
+ cmd_buf.commit();
+
+ // Retain the command buffer so we can wait on it later
+ let raw_ptr = cmd_buf.as_ptr() as *mut c_void;
+ unsafe {
+ let _: *mut Object = objc::msg_send![raw_ptr as *mut Object, retain];
+ }
+ self.frame_slots[slot_idx].pending_cmd_buffer = Some(raw_ptr);
+ }
+
+ /// Wait for a slot's command buffer to complete, then return a direct pointer
+ /// to the pixel data in the buffer-backed texture's MTLBuffer.
+ ///
+ /// ZERO COPY: The returned pointer points directly into the MTLBuffer's shared
+ /// memory. The GPU wrote pixels there during the render pass resolve. No getBytes,
+ /// no staging buffer, no memcpy. The pointer is valid until the next GPU submission
+ /// to this slot.
+ ///
+ /// The caller MUST call `mark_encoding_start` before passing this pointer to
+ /// the encoding thread, and the encoding thread MUST set `encoding_done` when
+ /// finished reading. The main thread will spin on `wait_encoding_done` before
+ /// reusing this slot.
+ pub fn drain_slot(&mut self, slot_idx: usize) -> *const u8 {
+ let slot = &mut self.frame_slots[slot_idx];
+
+ if let Some(raw_ptr) = slot.pending_cmd_buffer.take() {
+ unsafe {
+ // Wait for GPU completion
+ let _: () = objc::msg_send![raw_ptr as *mut Object, waitUntilCompleted];
+ // Release the retained command buffer
+ let _: () = objc::msg_send![raw_ptr as *mut Object, release];
+ }
+ }
+
+ // Return direct pointer to pixel data in the buffer-backed texture's backing store
+ match &slot.backing {
+ SlotBacking::Buffer(buf) => buf.contents() as *const u8,
+ SlotBacking::IOSurface { .. } => panic!("drain_slot called on IOSurface-backed slot; use drain_slot_metal"),
+ }
+ }
+
+ /// Wait for GPU completion on an IOSurface-backed slot. No pixel readback —
+ /// the IOSurface already has the data for VideoToolbox to read.
+ pub fn drain_slot_metal(&mut self, slot_idx: usize) {
+ let slot = &mut self.frame_slots[slot_idx];
+
+ if let Some(raw_ptr) = slot.pending_cmd_buffer.take() {
+ unsafe {
+ let _: () = objc::msg_send![raw_ptr as *mut Object, waitUntilCompleted];
+ let _: () = objc::msg_send![raw_ptr as *mut Object, release];
+ }
+ }
+ }
+
+ /// Wait for GPU completion on an IOSurface-backed slot and return GPU timing.
+ /// Used by the profiling render loop.
+ pub fn drain_slot_metal_timed(&mut self, slot_idx: usize) -> f64 {
+ let slot = &mut self.frame_slots[slot_idx];
+ let mut gpu_duration = 0.0;
+
+ if let Some(raw_ptr) = slot.pending_cmd_buffer.take() {
+ unsafe {
+ let _: () = objc::msg_send![raw_ptr as *mut Object, waitUntilCompleted];
+
+ let gpu_start: f64 = objc::msg_send![raw_ptr as *mut Object, GPUStartTime];
+ let gpu_end: f64 = objc::msg_send![raw_ptr as *mut Object, GPUEndTime];
+ if gpu_end > gpu_start {
+ gpu_duration = gpu_end - gpu_start;
+ }
+
+ let _: () = objc::msg_send![raw_ptr as *mut Object, release];
+ }
+ }
+
+ gpu_duration
+ }
+
+ /// Get the CVPixelBuffer for a slot (IOSurface-backed slots only).
+ pub fn slot_cv_pixel_buffer(&self, slot_idx: usize) -> crate::metal_encoder::CVPixelBufferRef {
+ match &self.frame_slots[slot_idx].backing {
+ SlotBacking::IOSurface { cv_pixel_buffer, .. } => *cv_pixel_buffer,
+ SlotBacking::Buffer(_) => panic!("slot_cv_pixel_buffer called on Buffer-backed slot"),
+ }
+ }
+
+ /// Mark that the encoding thread is about to start reading from this slot's pixel buffer.
+ pub fn mark_encoding_start(&self, slot_idx: usize) {
+ self.frame_slots[slot_idx].encoding_done.reset();
+ }
+
+ /// Wait for a slot's command buffer to complete, then return a direct pointer
+ /// to the pixel data AND the GPU execution time in seconds.
+ ///
+ /// Same as `drain_slot` but reads `MTLCommandBuffer.GPUStartTime/GPUEndTime`
+ /// before releasing the command buffer. Used by the profiling render loop.
+ pub fn drain_slot_timed(&mut self, slot_idx: usize) -> (*const u8, f64) {
+ let slot = &mut self.frame_slots[slot_idx];
+ let mut gpu_duration = 0.0;
+
+ if let Some(raw_ptr) = slot.pending_cmd_buffer.take() {
+ unsafe {
+ let _: () = objc::msg_send![raw_ptr as *mut Object, waitUntilCompleted];
+
+ // Read GPU timing from completed command buffer
+ let gpu_start: f64 = objc::msg_send![raw_ptr as *mut Object, GPUStartTime];
+ let gpu_end: f64 = objc::msg_send![raw_ptr as *mut Object, GPUEndTime];
+ if gpu_end > gpu_start {
+ gpu_duration = gpu_end - gpu_start;
+ }
+
+ let _: () = objc::msg_send![raw_ptr as *mut Object, release];
+ }
+ }
+
+ match &slot.backing {
+ SlotBacking::Buffer(buf) => (buf.contents() as *const u8, gpu_duration),
+ SlotBacking::IOSurface { .. } => panic!("drain_slot_timed called on IOSurface-backed slot"),
+ }
+ }
+
+ /// Wait for GPU completion, then return pointer to the NV12 buffer for this slot.
+ pub fn drain_slot_nv12(&mut self, slot_idx: usize) -> *const u8 {
+ let slot = &mut self.frame_slots[slot_idx];
+
+ if let Some(raw_ptr) = slot.pending_cmd_buffer.take() {
+ unsafe {
+ let _: () = objc::msg_send![raw_ptr as *mut Object, waitUntilCompleted];
+ let _: () = objc::msg_send![raw_ptr as *mut Object, release];
+ }
+ }
+
+ self.nv12_buffers[slot_idx].contents() as *const u8
+ }
+
+ /// Wait for GPU completion, return NV12 buffer pointer + GPU timing.
+ pub fn drain_slot_nv12_timed(&mut self, slot_idx: usize) -> (*const u8, f64) {
+ let slot = &mut self.frame_slots[slot_idx];
+ let mut gpu_duration = 0.0;
+
+ if let Some(raw_ptr) = slot.pending_cmd_buffer.take() {
+ unsafe {
+ let _: () = objc::msg_send![raw_ptr as *mut Object, waitUntilCompleted];
+
+ let gpu_start: f64 = objc::msg_send![raw_ptr as *mut Object, GPUStartTime];
+ let gpu_end: f64 = objc::msg_send![raw_ptr as *mut Object, GPUEndTime];
+ if gpu_end > gpu_start {
+ gpu_duration = gpu_end - gpu_start;
+ }
+
+ let _: () = objc::msg_send![raw_ptr as *mut Object, release];
+ }
+ }
+
+ (self.nv12_buffers[slot_idx].contents() as *const u8, gpu_duration)
+ }
+
+ /// NV12 frame size in bytes (W*H*3/2). Zero if NV12 not enabled.
+ pub fn nv12_size(&self) -> usize {
+ self.nv12_size
+ }
+
+ /// Whether NV12 compute conversion is enabled.
+ pub fn has_nv12(&self) -> bool {
+ self.nv12_pipeline.is_some()
+ }
+
+ /// Create standalone NV12 MTLBuffers for the elastic pool (not tied to frame slots).
+ pub fn create_nv12_buffers(&self, count: usize) -> Vec<Buffer> {
+ (0..count).map(|_| {
+ self.device.new_buffer(self.nv12_size as u64, MTLResourceOptions::StorageModeShared)
+ }).collect()
+ }
+
+ /// Number of frame slots.
+ pub fn n_slots(&self) -> usize {
+ self.frame_slots.len()
+ }
+}
diff --git a/rust/bridge/crates/manimgx-metal/src/lib.rs b/rust/bridge/crates/manimgx-metal/src/lib.rs
new file mode 100644
index 00000000..07c7c8d6
--- /dev/null
+++ b/rust/bridge/crates/manimgx-metal/src/lib.rs
@@ -0,0 +1,19 @@
+#![cfg(target_os = "macos")]
+
+mod backend;
+pub(crate) mod metal_encoder;
+
+#[cfg(not(feature = "time_profile"))]
+mod render_loop_x264;
+#[cfg(feature = "time_profile")]
+#[path = "render_loop_x264_profile.rs"]
+mod render_loop_x264;
+#[cfg(not(feature = "time_profile"))]
+mod render_loop_metal;
+#[cfg(feature = "time_profile")]
+#[path = "render_loop_metal_profile.rs"]
+mod render_loop_metal;
+
+pub use backend::MetalBackend;
+pub use render_loop_metal::render_video_metal;
+pub use render_loop_x264::render_video_x264;
diff --git a/rust/bridge/crates/manimgx-metal/src/render_loop_metal.rs b/rust/bridge/crates/manimgx-metal/src/render_loop_metal.rs
new file mode 100644
index 00000000..88edfe3b
--- /dev/null
+++ b/rust/bridge/crates/manimgx-metal/src/render_loop_metal.rs
@@ -0,0 +1,217 @@
+use manimgx_core::scene::Scene;
+use manimgx_core::timeline_eval::TimelineEval;
+
+use manimgx_core::uniform::{CameraUniform, ObjectUniform};
+use crate::backend::MetalBackend;
+use crate::metal_encoder::MetalEncoder;
+
+/// Render a video using the native Metal backend with VideoToolbox encoding.
+///
+/// Zero-copy path: GPU renders into IOSurface-backed textures, VideoToolbox
+/// reads the same IOSurface memory for hardware H.264 encoding. No pixel
+/// readback, no ffmpeg, no CPU copies.
+pub fn render_video_metal(
+ scene: &mut Scene,
+ timeline: &dyn TimelineEval,
+ output: &str,
+ total_frames: u32,
+ fps: u32,
+ sample_count: u32,
+ n_buffers: u32,
+ check_cancelled: &dyn Fn() -> bool,
+) {
+ if total_frames == 0 {
+ return;
+ }
+
+ let width = scene.width;
+ let height = scene.height;
+ let n = n_buffers.max(1) as usize;
+
+ // Create Metal backend with IOSurface-backed frame slots
+ let mut backend = MetalBackend::new_metal_encoder(width, height, sample_count, n_buffers);
+
+ // Create VideoToolbox encoder
+ let mut encoder = MetalEncoder::new(output, width, height, fps);
+
+ // Uniform staging
+ let uniform_alignment: usize = 256;
+ let max_objects = scene.objects.len().max(1);
+ let uniform_size = std::mem::size_of::<ObjectUniform>();
+ let cam_size = std::mem::size_of::<CameraUniform>();
+ let mut uniform_staging = vec![0u8; max_objects * uniform_alignment];
+
+ // Frame dedup state
+ let mut prev_cam_bytes = vec![0u8; cam_size];
+ let mut prev_uniform_bytes = vec![0u8; uniform_staging.len()];
+ let mut prev_draw_count: usize = 0;
+ let mut first_frame = true;
+ let mut dup_count = 0u32;
+ let mut render_count = 0usize;
+
+ // Track last rendered slot for dup frame re-encoding
+ let mut last_rendered_slot: usize = 0;
+
+ // N-buffering: pending slot queue
+ let mut pending: std::collections::VecDeque<usize> = std::collections::VecDeque::new();
+
+ let clear_color = MetalBackend::clear_color(scene);
+ let render_start = std::time::Instant::now();
+ let mut last_progress = std::time::Instant::now();
+
+ let mut draw_cmds: Vec<(usize, usize)> = Vec::with_capacity(scene.objects.len());
+
+ for frame_idx in 0..total_frames {
+ // Progress bar
+ print_progress(frame_idx, total_frames, &render_start, &mut last_progress);
+ if check_cancelled() { eprintln!("\n Render interrupted at frame {}/{}", frame_idx, total_frames); break; }
+
+ let time = frame_idx as f32 / fps as f32;
+ timeline.evaluate(scene, time);
+
+ // Build camera uniform
+ let view_proj = scene.camera.view_projection_matrix(scene.aspect_ratio());
+ let cam_uniform = CameraUniform {
+ view_proj: view_proj.to_cols_array_2d(),
+ };
+ let cam_bytes = bytemuck::bytes_of(&cam_uniform);
+
+ // Build draw commands + object uniforms
+ draw_cmds.clear();
+ for (obj_idx, obj) in scene.objects.iter().enumerate() {
+ if obj.material.opacity <= 0.0 {
+ continue;
+ }
+ let slot = draw_cmds.len();
+ let byte_offset = slot * uniform_alignment;
+
+ if byte_offset + uniform_size > uniform_staging.len() {
+ let new_len = (byte_offset + uniform_size).next_power_of_two();
+ uniform_staging.resize(new_len, 0);
+ prev_uniform_bytes.resize(new_len, 0);
+ }
+
+ let mut material_color = obj.material.color;
+ material_color[3] = obj.material.color[3] * obj.material.opacity;
+
+ let uniform = ObjectUniform {
+ model: obj.transform.to_matrix().to_cols_array_2d(),
+ material_color,
+ use_vertex_colors: if obj.material.use_vertex_colors { 1 } else { 0 },
+ _pad: [0; 3],
+ };
+ uniform_staging[byte_offset..byte_offset + uniform_size]
+ .copy_from_slice(bytemuck::bytes_of(&uniform));
+ draw_cmds.push((obj_idx, slot));
+ }
+
+ let total_uniform_bytes = draw_cmds.len() * uniform_alignment;
+
+ // Frame deduplication
+ let is_dup = !first_frame
+ && draw_cmds.len() == prev_draw_count
+ && cam_bytes == &prev_cam_bytes[..]
+ && (total_uniform_bytes == 0
+ || uniform_staging[..total_uniform_bytes]
+ == prev_uniform_bytes[..total_uniform_bytes]);
+
+ if is_dup {
+ // Drain any pending GPU slots before encoding dup
+ while let Some(drain_idx) = pending.pop_front() {
+ backend.drain_slot_metal(drain_idx);
+ encoder.encode_frame(backend.slot_cv_pixel_buffer(drain_idx));
+ last_rendered_slot = drain_idx;
+ }
+ // Re-encode from last rendered slot's IOSurface (still has the data)
+ encoder.encode_frame(backend.slot_cv_pixel_buffer(last_rendered_slot));
+ dup_count += 1;
+ continue;
+ }
+
+ let slot_idx = render_count % n;
+
+ // Drain oldest pending slot if all slots are in use
+ if pending.len() >= n {
+ let drain_idx = pending.pop_front().unwrap();
+ backend.drain_slot_metal(drain_idx);
+ encoder.encode_frame(backend.slot_cv_pixel_buffer(drain_idx));
+ last_rendered_slot = drain_idx;
+ }
+
+ // Submit this frame's render work
+ backend.submit_frame(
+ scene,
+ cam_bytes,
+ &uniform_staging,
+ total_uniform_bytes,
+ &draw_cmds,
+ clear_color,
+ slot_idx,
+ );
+ pending.push_back(slot_idx);
+
+ // Update dedup state
+ prev_cam_bytes.copy_from_slice(cam_bytes);
+ if total_uniform_bytes > 0 {
+ prev_uniform_bytes[..total_uniform_bytes]
+ .copy_from_slice(&uniform_staging[..total_uniform_bytes]);
+ }
+ prev_draw_count = draw_cmds.len();
+ first_frame = false;
+ render_count += 1;
+ }
+
+ // Drain remaining pending slots
+ while let Some(drain_idx) = pending.pop_front() {
+ backend.drain_slot_metal(drain_idx);
+ encoder.encode_frame(backend.slot_cv_pixel_buffer(drain_idx));
+ }
+
+ encoder.finish();
+
+ eprintln!();
+ let rendered = total_frames - dup_count;
+ eprintln!(
+ " frames: {} rendered, {} duplicates skipped ({:.0}%)",
+ rendered,
+ dup_count,
+ dup_count as f64 / total_frames as f64 * 100.0,
+ );
+}
+
+fn print_progress(
+ frame_idx: u32,
+ total_frames: u32,
+ render_start: &std::time::Instant,
+ last_progress: &mut std::time::Instant,
+) {
+ if last_progress.elapsed().as_millis() >= 250 || frame_idx == total_frames - 1 {
+ let done = frame_idx + 1;
+ let elapsed = render_start.elapsed().as_secs_f64();
+ let pct = done as f64 / total_frames as f64;
+ let fps_now = if elapsed > 0.0 {
+ done as f64 / elapsed
+ } else {
+ 0.0
+ };
+ let eta = if pct > 0.0 {
+ elapsed / pct - elapsed
+ } else {
+ 0.0
+ };
+ let bar_width = 30;
+ let filled = (pct * bar_width as f64) as usize;
+ let bar: String = "█".repeat(filled) + &"░".repeat(bar_width - filled);
+ eprint!(
+ "\r [{}] {:>3.0}% | {}/{} | {:.1}s | ~{:.1}s left | {:.0} fps\x1b[K",
+ bar,
+ pct * 100.0,
+ done,
+ total_frames,
+ elapsed,
+ eta,
+ fps_now,
+ );
+ *last_progress = std::time::Instant::now();
+ }
+}
diff --git a/rust/bridge/shaders/default.wgsl b/rust/bridge/shaders/default.wgsl
new file mode 100644
index 00000000..310e11c4
--- /dev/null
+++ b/rust/bridge/shaders/default.wgsl
@@ -0,0 +1,56 @@
+struct CameraUniform {
+ view_proj: mat4x4<f32>,
+};
+
+struct ObjectUniform {
+ model: mat4x4<f32>,
+ material_color: vec4<f32>,
+ use_vertex_colors: u32,
+};
+
+@group(0) @binding(0) var<uniform> camera: CameraUniform;
+@group(1) @binding(0) var<uniform> object: ObjectUniform;
+
+struct VertexInput {
+ @location(0) position: vec3<f32>,
+ @location(1) normal: vec3<f32>,
+ @location(2) color: vec4<f32>,
+};
+
+struct VertexOutput {
+ @builtin(position) clip_position: vec4<f32>,
+ @location(0) world_normal: vec3<f32>,
+ @location(1) color: vec4<f32>,
+};
+
+@vertex
+fn vs_main(in: VertexInput) -> VertexOutput {
+ var out: VertexOutput;
+ let world_pos = object.model * vec4<f32>(in.position, 1.0);
+ out.clip_position = camera.view_proj * world_pos;
+ // Transform normal (ignoring non-uniform scale for simplicity)
+ out.world_normal = (object.model * vec4<f32>(in.normal, 0.0)).xyz;
+ out.color = in.color;
+ return out;
+}
+
+@fragment
+fn fs_main(@builtin(front_facing) front_facing: bool, in: VertexOutput) -> @location(0) vec4<f32> {
+ let light_dir = normalize(vec3<f32>(0.5, 1.0, 0.3));
+ var normal = normalize(in.world_normal);
+ if (!front_facing) {
+ normal = -normal;
+ }
+ let ndotl = max(dot(normal, light_dir), 0.0);
+ let ambient = 0.3;
+ let diffuse = ndotl * 0.7;
+ let lighting = ambient + diffuse;
+
+ var base_color: vec4<f32>;
+ if (object.use_vertex_colors > 0u) {
+ base_color = in.color * object.material_color;
+ } else {
+ base_color = object.material_color;
+ }
+ return vec4<f32>(base_color.rgb * lighting, base_color.a);
+}
diff --git a/rust/bridge/shaders/metal/default.metal b/rust/bridge/shaders/metal/default.metal
new file mode 100644
index 00000000..056bfe9c
--- /dev/null
+++ b/rust/bridge/shaders/metal/default.metal
@@ -0,0 +1,47 @@
+// 3D lit shading — port of default.wgsl.
+// Diffuse + ambient lighting, double-sided via [[front_facing]].
+
+#include "types.h"
+
+struct VertexOut {
+ float4 position [[position]];
+ float3 world_normal;
+ float4 color;
+};
+
+vertex VertexOut vs_main(
+ VertexIn in [[stage_in]],
+ constant CameraUniform& camera [[buffer(1)]],
+ constant ObjectUniform& object [[buffer(2)]]
+) {
+ VertexOut out;
+ float4 world_pos = object.model * float4(in.position, 1.0);
+ out.position = camera.view_proj * world_pos;
+ out.world_normal = (object.model * float4(in.normal, 0.0)).xyz;
+ out.color = in.color;
+ return out;
+}
+
+fragment float4 fs_main(
+ VertexOut in [[stage_in]],
+ bool front_facing [[front_facing]],
+ constant ObjectUniform& object [[buffer(2)]]
+) {
+ float3 light_dir = normalize(float3(0.5, 1.0, 0.3));
+ float3 normal = normalize(in.world_normal);
+ if (!front_facing) {
+ normal = -normal;
+ }
+ float ndotl = max(dot(normal, light_dir), 0.0);
+ float ambient = 0.3;
+ float diffuse = ndotl * 0.7;
+ float lighting = ambient + diffuse;
+
+ float4 base_color;
+ if (object.use_vertex_colors > 0u) {
+ base_color = in.color * object.material_color;
+ } else {
+ base_color = object.material_color;
+ }
+ return float4(base_color.rgb * lighting, base_color.a);
+}
diff --git a/src/manimgx/rendering/film.py b/src/manimgx/rendering/film.py
index 7b113b99..c957e56e 100644
--- a/src/manimgx/rendering/film.py
+++ b/src/manimgx/rendering/film.py
@@ -11,7 +11,8 @@ import os
from collections.abc import Callable
from dataclasses import asdict, dataclass
from fractions import Fraction
-from typing import TYPE_CHECKING, Literal
+from typing import TYPE_CHECKING, Literal, get_args
+from warnings import deprecated
from manimgx import _engine
from manimgx.config import config
@@ -28,6 +29,21 @@ if TYPE_CHECKING:
from manimgx.scene import Camera
+X264Preset = Literal[
+ "ultrafast",
+ "superfast",
+ "veryfast",
+ "faster",
+ "fast",
+ "medium",
+ "slow",
+ "slower",
+ "veryslow",
+ "placebo",
+]
+"""The x264 presets, from fastest to slowest."""
+
+
class Frame:
"""A frame of a film, as the film sends it: shown as frames `index` to
`index + repeat - 1` of the video.
@@ -214,8 +230,8 @@ class Film:
if the film is recorded as a take (see [`Take`][manimgx.rendering.film.Take]).
preset: The x264 preset the video is encoded with: faster presets make larger
files.
- crf: The video's quality, as x264's constant rate factor: lower is better, and
- 18 looks lossless.
+ crf: The video's quality, as x264's constant rate factor, from 0 to 51:
+ lower is better, the default value is 18.
"""
def __init__(
@@ -225,9 +241,15 @@ class Film:
frames: FrameSink | None = None,
plays: PlayHook | None = None,
take: Take | None = None,
- preset: str = "ultrafast",
+ preset: X264Preset = "ultrafast",
crf: float = 18.0,
) -> None:
+ if preset not in get_args(X264Preset):
+ raise ValueError(f"unknown x264 preset {preset!r}")
+ if not 0 <= crf <= 51:
+ raise ValueError(
+ f"x264 CRF must be a finite number from 0 to 51, not {crf}"
+ )
fps = config.frame_rate
if video is not None and fps != int(fps):
raise ValueError(
@@ -304,6 +326,7 @@ class Film:
return mix(self.clips, self.frame_count / self.fps)
+ @deprecated("manimgx's machinery: the scene calls it", category=None)
def section(
self, name: str, start: Fraction, type: SectionType, notes: str
) -> None:
@@ -340,9 +363,7 @@ class Film:
def stamp(t: float) -> str:
ms = round(t * 1000)
- return (
- f"{ms // 3_600_000:02d}:{ms // 60_000 % 60:02d}:{ms // 1000 % 60:02d},{ms % 1000:03d}"
- )
+ return f"{ms // 3_600_000:02d}:{ms // 60_000 % 60:02d}:{ms // 1000 % 60:02d},{ms % 1000:03d}"
return "".join(
f"{i}\n{stamp(c.start)} --> {stamp(c.end)}\n{c.text}\n\n"
@@ -362,6 +383,7 @@ class Film:
]
return sorted(self.subcaptions + spoken, key=lambda c: c.start)
+ @deprecated("manimgx's machinery: the scene calls it", category=None)
def played(self, play: Play, animations: "tuple[Animation, ...]") -> None:
"""Keep a play that has ended, and hand it to the play hook with the animations
it played: the scene calls it as each play ends, the world as the play left it.
@@ -376,6 +398,7 @@ class Film:
if self._hook is not None:
self._hook(play, animations)
+ @deprecated("manimgx's machinery: the scene calls it", category=None)
def picture(self, camera: "Camera", mobjects: "list[Mobject]") -> bytes:
"""Draw the world as `camera` sees it now: a picture, not a frame of the film.
@@ -391,6 +414,7 @@ class Film:
raise RuntimeError("a film recorded as a take is drawn by manimgx's player")
return self._player.render(view, records, cameras)
+ @deprecated("manimgx's machinery: the scene calls it", category=None)
def record(
self, camera: "Camera", mobjects: "list[Mobject]", repeat: int = 1
) -> None:
@@ -408,6 +432,7 @@ class Film:
view, records, cameras = self.feeder.frame(camera, mobjects)
self._add(view, records, cameras, repeat)
+ @deprecated("manimgx's machinery: the scene calls it", category=None)
def tween(
self,
camera: "Camera",
@@ -466,6 +491,7 @@ class Film:
)
)
+ @deprecated("manimgx's machinery: the scene calls it", category=None)
def close(self) -> None:
"""Close the film: send its last frame, and write its video, if any.
@@ -508,6 +534,7 @@ class Film:
self._recorder.end()
self._flush()
+ @deprecated("manimgx's machinery: the scene calls it", category=None)
def abort(self) -> None:
"""Stop the film without writing its video; its take, if it is recorded as one, ends
saying that its scene failed.
diff --git a/src/manimgx/rendering/window.py b/src/manimgx/rendering/window.py
new file mode 100644
index 00000000..914f6d08
--- /dev/null
+++ b/src/manimgx/rendering/window.py
@@ -0,0 +1,178 @@
+"""Window: manimgx's player in a window of its own, on this machine's screen, drawn by its GPU.
+
+A window plays the takes it is sent (see [`Film`][manimgx.Film]): each frame as soon as it is
+recorded, any frame again at once. A newer take — the scene run again — replaces the one shown
+at the same moment, once it gets there: an edit never loses the place.
+
+ with m.Window() as window:
+ MyScene().render(take=window)
+ window.wait() # until it is closed
+
+The window is a process of its own, which the take reaches through a pipe: it plays on while
+the scene runs, and a scene's process stays free for the scene. It closes when it is closed, or
+when the process that opened it closes it, or ends.
+"""
+
+import contextlib
+import json
+import queue
+import subprocess
+import sys
+import threading
+from collections.abc import Sequence
+from types import TracebackType
+from typing import Self
+
+from manimgx import _engine
+from manimgx.rendering.film import Cut
+
+
+class Window:
+ """manimgx's player in a window on this machine's screen, which plays the takes it is sent.
+
+ A window is a [`Take`][manimgx.rendering.film.Take]: hand it to
+ [`Scene.render`][manimgx.Scene.render] as `take`, and it plays the film as it is recorded.
+ Once it is closed, a film sent to it is cut (see [`Cut`][manimgx.rendering.film.Cut]).
+
+ Its viewer can ask for another of the file's scenes (N and P): see
+ [`scenes`][manimgx.Window.scenes] and [`asked`][manimgx.Window.asked].
+
+ Args:
+ title: The window's title.
+ time: Where its playhead starts, in seconds.
+ """
+
+ def __init__(self, title: str = "manimgx", *, time: float = 0.0) -> None:
+ self._process = subprocess.Popen(
+ [
+ sys.executable,
+ "-c",
+ "from manimgx.rendering.window import _main; _main()",
+ ]
+ + [title, repr(float(time))],
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ )
+ self._asked: queue.Queue[str] = queue.Queue()
+ self.failure: str | None = None
+ """Why the window failed, if it did (no screen to open on, no GPU): it is closed."""
+ self._listening = threading.Thread(target=self._listen, daemon=True)
+ self._listening.start()
+
+ def __call__(self, data: bytes) -> None:
+ """Send the window the take's next bytes.
+
+ Raises:
+ Cut: The window is closed: the film is cut there.
+ """
+ pipe = self._process.stdin
+ if pipe is None or pipe.closed:
+ raise Cut
+ try:
+ pipe.write(data) # whole: a buffered pipe writes it all
+ pipe.flush()
+ except (BrokenPipeError, ValueError):
+ with contextlib.suppress(OSError):
+ pipe.close()
+ raise Cut from None
+
+ def scenes(self, names: Sequence[str], playing: str) -> None:
+ """Tell the window the file's scenes, and the one it plays: its viewer can ask for
+ the others (N, P; see [`asked`][manimgx.Window.asked]).
+
+ Args:
+ names: The file's scenes, in order.
+ playing: The one its takes are of now.
+ """
+ self._say({"scenes": list(names), "scene": playing})
+
+ def failed(self, error: BaseException, trace: str, line: int | None = None) -> None:
+ """Show the scene's error over the last picture it made, until the next take.
+
+ Args:
+ trace: What to show: the lines of the scene's code it went through, and the error.
+ line: The line of the scene's file it failed at, if any.
+ """
+ fault = {"message": str(error), "type": type(error).__name__}
+ self._say({"error": {**fault, "trace": trace, "line": line}})
+
+ def _say(self, note: dict[str, object]) -> None:
+ """A note between takes (the director's JSON, see `Film`): nothing, to a closed
+ window."""
+ with contextlib.suppress(Cut):
+ self(_engine.note(json.dumps(note)))
+
+ @property
+ def open(self) -> bool:
+ """Whether the window is open: until its viewer closes it, or it is closed."""
+ return self._process.poll() is None
+
+ def asked(self) -> str | None:
+ """The scene the viewer asked for last, if they asked since (N, P: the next or the
+ previous of the file's scenes)."""
+ name = None
+ while not self._asked.empty():
+ name = self._asked.get_nowait()
+ return name
+
+ def wait(self, timeout: float | None = None) -> bool:
+ """Wait until the window is closed, at most `timeout` seconds; whether it was."""
+ try:
+ self._process.wait(timeout)
+ except subprocess.TimeoutExpired:
+ return False
+ self._listening.join() # what it said before it closed, `failure` among it
+ return True
+
+ def close(self) -> None:
+ """Close the window."""
+ pipe = self._process.stdin
+ if pipe is not None and not pipe.closed:
+ with contextlib.suppress(OSError):
+ pipe.close() # its end of the take: it closes
+ if not self.wait(10):
+ self._process.kill()
+ self.wait()
+
+ def __enter__(self) -> Self:
+ return self
+
+ def __exit__(
+ self,
+ kind: type[BaseException] | None,
+ error: BaseException | None,
+ trace: TracebackType | None,
+ ) -> None:
+ self.close()
+
+ def _listen(self) -> None:
+ """What the window says, a line of JSON each, as it comes: a scene its viewer asks
+ for, or why it failed."""
+ assert self._process.stdout is not None
+ for line in self._process.stdout:
+ try:
+ said = json.loads(line)
+ except ValueError:
+ continue
+ if not isinstance(said, dict):
+ continue
+ if isinstance(scene := said.get("scene"), str):
+ self._asked.put(scene)
+ if isinstance(failure := said.get("failure"), str):
+ self.failure = failure
+
+
+def _main() -> None:
+ """The window's own process (`Window` starts it): the window, until it is closed."""
+ import signal
+
+ from manimgx.drawing.typesetting import FONTS
+
+ # Ctrl+C reaches every process of the terminal: the window's director closes it
+ signal.signal(signal.SIGINT, signal.SIG_IGN)
+ title, time = sys.argv[1], float(sys.argv[2])
+ try:
+ _engine.window(title, time, list(FONTS))
+ except BaseException as failed: # a panic of the engine's too
+ print(json.dumps({"failure": str(failed) or type(failed).__name__}), flush=True)
+ sys.exit(1)