A high-performance inference engine for AI models
Sparse buffers (#379)
Aleksandr Golokoze32d68a07d1b
diff --git a/Cargo.lock b/Cargo.lockindex cb1fe3632..9ab1ad99b 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -508,9 +508,9 @@ dependencies = [[[package]]name = "avif-serialize"-version = "0.8.8"+version = "0.8.9"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "375082f007bd67184fb9c0374614b29f9aaa604ec301635f72338bb65386a53d"+checksum = "e7178fe5f7d460b13895ebb9dcb28a3a6216d2df2574a0806cb51b555d297f38"dependencies = ["arrayvec",]@@ -813,6 +813,15 @@ version = "0.2.4"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c"+[[package]]+name = "bs58"+version = "0.5.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4"+dependencies = [+ "tinyvec",+]+[[package]]name = "bstr"version = "1.12.1"@@ -954,9 +963,9 @@ dependencies = [[[package]]name = "cc"-version = "1.2.61"+version = "1.2.62"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d16d90359e986641506914ba71350897565610e87ce0ad9e6f28569db3dd5c6d"+checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98"dependencies = ["find-msvc-tools","jobserver",@@ -2597,9 +2606,9 @@ dependencies = [[[package]]name = "hashbrown"-version = "0.17.0"+version = "0.17.1"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51"+checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"[[package]]name = "heck"@@ -3011,7 +3020,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9"dependencies = ["equivalent",- "hashbrown 0.17.0",+ "hashbrown 0.17.1","serde","serde_core",]@@ -3283,9 +3292,9 @@ dependencies = [[[package]]name = "js-sys"-version = "0.3.97"+version = "0.3.98"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a1840c94c045fbcf8ba2812c95db44499f7c64910a912551aaaa541decebcacf"+checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08"dependencies = ["cfg-if","futures-util",@@ -6089,11 +6098,12 @@ dependencies = [[[package]]name = "serde_with"-version = "3.19.0"+version = "3.20.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "f05839ce67618e14a09b286535c0d9c94e85ef25469b0e13cb4f844e5593eb19"+checksum = "e72c1c2cb7b223fafb600a619537a871c2818583d619401b785e7c0b746ccde2"dependencies = ["base64 0.22.1",+ "bs58","chrono","hex","indexmap 1.9.3",@@ -6108,9 +6118,9 @@ dependencies = [[[package]]name = "serde_with_macros"-version = "3.19.0"+version = "3.20.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "cf2ebbe86054f9b45bc3881e865683ccfaccce97b9b4cb53f3039d67f355a334"+checksum = "b90c488738ecb4fb0262f41f43bc40efc5868d9fb744319ddf5f5317f417bfac"dependencies = ["darling 0.23.0","proc-macro2",@@ -6851,9 +6861,9 @@ dependencies = [[[package]]name = "tokio"-version = "1.52.2"+version = "1.52.3"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "110a78583f19d5cdb2c5ccf321d1290344e71313c6c37d43520d386027d18386"+checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe"dependencies = ["bytes","libc",@@ -7704,9 +7714,9 @@ dependencies = [[[package]]name = "wasm-bindgen"-version = "0.2.120"+version = "0.2.121"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "df52b6d9b87e0c74c9edfa1eb2d9bf85e5d63515474513aa50fa181b3c4f5db1"+checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790"dependencies = ["cfg-if","once_cell",@@ -7719,9 +7729,9 @@ dependencies = [[[package]]name = "wasm-bindgen-futures"-version = "0.4.70"+version = "0.4.71"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "af934872acec734c2d80e6617bbb5ff4f12b052dd8e6332b0817bce889516084"+checksum = "96492d0d3ffba25305a7dc88720d250b1401d7edca02cc3bcd50633b424673b8"dependencies = ["js-sys","wasm-bindgen",@@ -7729,9 +7739,9 @@ dependencies = [[[package]]name = "wasm-bindgen-macro"-version = "0.2.120"+version = "0.2.121"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "78b1041f495fb322e64aca85f5756b2172e35cd459376e67f2a6c9dffcedb103"+checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578"dependencies = ["quote","wasm-bindgen-macro-support",@@ -7739,9 +7749,9 @@ dependencies = [[[package]]name = "wasm-bindgen-macro-support"-version = "0.2.120"+version = "0.2.121"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "9dcd0ff20416988a18ac686d4d4d0f6aae9ebf08a389ff5d29012b05af2a1b41"+checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2"dependencies = ["bumpalo","proc-macro2",@@ -7752,9 +7762,9 @@ dependencies = [[[package]]name = "wasm-bindgen-shared"-version = "0.2.120"+version = "0.2.121"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "49757b3c82ebf16c57d69365a142940b384176c24df52a087fb748e2085359ea"+checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441"dependencies = ["unicode-ident",]@@ -7821,9 +7831,9 @@ dependencies = [[[package]]name = "web-sys"-version = "0.3.97"+version = "0.3.98"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "2eadbac71025cd7b0834f20d1fe8472e8495821b4e9801eb0a60bd1f19827602"+checksum = "4b572dff8bcf38bad0fa19729c89bb5748b2b9b1d8be70cf90df697e3a8f32aa"dependencies = ["js-sys","wasm-bindgen",diff --git a/crates/backend-uzu/src/backends/common/backend.rs b/crates/backend-uzu/src/backends/common/backend.rsindex ebea0122a..3bb337516 100644--- a/crates/backend-uzu/src/backends/common/backend.rs+++ b/crates/backend-uzu/src/backends/common/backend.rs@@ -1,11 +1,12 @@use std::{error::Error, fmt::Debug};-use super::{CommandBuffer, Context, DenseBuffer, Event, Kernels, kernel::ManualKernels};+use super::{CommandBuffer, Context, DenseBuffer, Event, Kernels, SparseBuffer, kernel::ManualKernels};pub trait Backend: Debug + Clone + 'static {type Context: Context<Backend = Self>;type CommandBuffer: CommandBuffer<Backend = Self>;type DenseBuffer: DenseBuffer<Backend = Self>;+ type SparseBuffer: SparseBuffer<Backend = Self>;type Event: Event<Backend = Self>;type Kernels: Kernels<Backend = Self> + ManualKernels;type Error: Error + Debug;diff --git a/crates/backend-uzu/src/backends/common/context.rs b/crates/backend-uzu/src/backends/common/context.rsindex 60d6ea19e..0e2593ad0 100644--- a/crates/backend-uzu/src/backends/common/context.rs+++ b/crates/backend-uzu/src/backends/common/context.rs@@ -25,6 +25,11 @@ pub trait Context: Sized {size: usize,) -> Result<<Self::Backend as Backend>::DenseBuffer, <Self::Backend as Backend>::Error>;+ fn create_buffer_with_data(+ &self,+ data: &[u8],+ ) -> Result<<Self::Backend as Backend>::DenseBuffer, <Self::Backend as Backend>::Error>;+fn create_allocation(&self,size: usize,@@ -38,6 +43,11 @@ pub trait Context: Sized {fn create_event(&self) -> Result<<Self::Backend as Backend>::Event, <Self::Backend as Backend>::Error>;+ fn create_sparse_buffer(+ &self,+ capacity: usize,+ ) -> Result<<Self::Backend as Backend>::SparseBuffer, <Self::Backend as Backend>::Error>;+fn peak_memory_usage(&self) -> Option<usize>;fn enable_capture();diff --git a/crates/backend-uzu/src/backends/common/mod.rs b/crates/backend-uzu/src/backends/common/mod.rsindex 6e7533d97..e0a2de665 100644--- a/crates/backend-uzu/src/backends/common/mod.rs+++ b/crates/backend-uzu/src/backends/common/mod.rs@@ -12,6 +12,7 @@ mod event;pub mod gpu_types;mod hazard_tracker;pub mod kernel;+mod sparse_buffer;pub use activation_config::ActivationConfig;pub use allocation_access::{@@ -31,3 +32,4 @@ pub use encoder::{Completed, Encoder, Executable, Pending};pub use event::Event;pub use hazard_tracker::Access;pub use kernel::Kernels;+pub use sparse_buffer::SparseBuffer;diff --git a/crates/backend-uzu/src/backends/common/sparse_buffer.rs b/crates/backend-uzu/src/backends/common/sparse_buffer.rsnew file mode 100644index 000000000..ec2ae87f2--- /dev/null+++ b/crates/backend-uzu/src/backends/common/sparse_buffer.rs@@ -0,0 +1,17 @@+use std::ops::Range;++use crate::backends::common::{Backend, Buffer};++pub trait SparseBuffer: Buffer<Backend: Backend<SparseBuffer = Self>> {+ fn map(+ &mut self,+ context: &<Self::Backend as Backend>::Context,+ pages: &Range<usize>,+ ) -> Result<(), <Self::Backend as Backend>::Error>;++ fn unmap(+ &mut self,+ context: &<Self::Backend as Backend>::Context,+ pages: &Range<usize>,+ ) -> Result<(), <Self::Backend as Backend>::Error>;+}diff --git a/crates/backend-uzu/src/backends/cpu/backend.rs b/crates/backend-uzu/src/backends/cpu/backend.rsindex 557e3ea5c..aeb1caa63 100644--- a/crates/backend-uzu/src/backends/cpu/backend.rs+++ b/crates/backend-uzu/src/backends/cpu/backend.rs@@ -1,6 +1,8 @@use std::{cell::UnsafeCell, pin::Pin, sync::atomic::AtomicU64};-use super::{command_buffer::CpuCommandBuffer, context::CpuContext, error::CpuError, kernel::CpuKernels};+use super::{+ command_buffer::CpuCommandBuffer, context::CpuContext, error::CpuError, kernel::CpuKernels, sparse::CpuSparseBuffer,+};use crate::backends::common::Backend;#[derive(Debug, Clone)]@@ -10,6 +12,7 @@ impl Backend for Cpu {type Context = CpuContext;type CommandBuffer = CpuCommandBuffer;type DenseBuffer = UnsafeCell<Pin<Box<[u8]>>>;+ type SparseBuffer = CpuSparseBuffer;type Event = Pin<Box<AtomicU64>>;type Kernels = CpuKernels;type Error = CpuError;diff --git a/crates/backend-uzu/src/backends/cpu/context.rs b/crates/backend-uzu/src/backends/cpu/context.rsindex 2e2f96c18..5e1975a8a 100644--- a/crates/backend-uzu/src/backends/cpu/context.rs+++ b/crates/backend-uzu/src/backends/cpu/context.rs@@ -8,7 +8,7 @@ use std::{};use crate::backends::{- common::{Allocation, AllocationPool, AllocationType, Allocator, Context},+ common::{Allocation, AllocationPool, AllocationType, Allocator, Backend, Context},cpu::{Cpu, command_buffer::CpuCommandBufferInitial, error::CpuError},};@@ -58,6 +58,13 @@ impl Context for CpuContext {Ok(UnsafeCell::new(Pin::new(vec![0; size].into_boxed_slice())))}+ fn create_buffer_with_data(+ &self,+ data: &[u8],+ ) -> Result<UnsafeCell<Pin<Box<[u8]>>>, CpuError> {+ Ok(UnsafeCell::new(Pin::new(data.to_vec().into_boxed_slice())))+ }+fn create_allocation(&self,size: usize,@@ -81,6 +88,13 @@ impl Context for CpuContext {Ok(Box::pin(AtomicU64::new(0)))}+ fn create_sparse_buffer(+ &self,+ _capacity: usize,+ ) -> Result<<Self::Backend as Backend>::SparseBuffer, <Self::Backend as Backend>::Error> {+ Err(CpuError::NotSupported)+ }+fn peak_memory_usage(&self) -> Option<usize> {None}@@ -89,9 +103,8 @@ impl Context for CpuContext {fn start_capture(&self,- trace_path: &std::path::Path,+ _trace_path: &Path,) -> Result<(), CpuError> {- let _ = trace_path;Err(CpuError::NotSupported)}diff --git a/crates/backend-uzu/src/backends/cpu/mod.rs b/crates/backend-uzu/src/backends/cpu/mod.rsindex 93121f103..c8f54618b 100644--- a/crates/backend-uzu/src/backends/cpu/mod.rs+++ b/crates/backend-uzu/src/backends/cpu/mod.rs@@ -5,7 +5,8 @@ mod dense_buffer;mod error;mod event;mod kernel;--pub use backend::Cpu;+mod sparse;pub mod argmax;++pub use backend::Cpu;diff --git a/crates/backend-uzu/src/backends/cpu/sparse/mod.rs b/crates/backend-uzu/src/backends/cpu/sparse/mod.rsnew file mode 100644index 000000000..e27fbbe61--- /dev/null+++ b/crates/backend-uzu/src/backends/cpu/sparse/mod.rs@@ -0,0 +1,3 @@+mod sparse_buffer;++pub use sparse_buffer::CpuSparseBuffer;diff --git a/crates/backend-uzu/src/backends/cpu/sparse/sparse_buffer.rs b/crates/backend-uzu/src/backends/cpu/sparse/sparse_buffer.rsnew file mode 100644index 000000000..c1a166c74--- /dev/null+++ b/crates/backend-uzu/src/backends/cpu/sparse/sparse_buffer.rs@@ -0,0 +1,46 @@+use std::ops::Range;++use crate::backends::{+ common::{Backend, Buffer, SparseBuffer},+ cpu::Cpu,+};++#[derive(Debug)]+pub struct CpuSparseBuffer {}++impl Buffer for CpuSparseBuffer {+ type Backend = Cpu;++ fn gpu_ptr(&self) -> usize {+ todo!()+ }++ fn size(&self) -> usize {+ todo!()+ }++ fn set_label(+ &mut self,+ _label: Option<&str>,+ ) {+ todo!()+ }+}++impl SparseBuffer for CpuSparseBuffer {+ fn map(+ &mut self,+ _context: &<Self::Backend as Backend>::Context,+ _pages: &Range<usize>,+ ) -> Result<(), <Self::Backend as Backend>::Error> {+ todo!()+ }++ fn unmap(+ &mut self,+ _context: &<Self::Backend as Backend>::Context,+ _pages: &Range<usize>,+ ) -> Result<(), <Self::Backend as Backend>::Error> {+ todo!()+ }+}diff --git a/crates/backend-uzu/src/backends/metal/backend.rs b/crates/backend-uzu/src/backends/metal/backend.rsindex a5e6891e9..b7cb72365 100644--- a/crates/backend-uzu/src/backends/metal/backend.rs+++ b/crates/backend-uzu/src/backends/metal/backend.rs@@ -2,7 +2,7 @@ use metal::{MTLBuffer, MTLEvent};use objc2::{rc::Retained, runtime::ProtocolObject};use super::{command_buffer::MetalCommandBuffer, context::MetalContext, error::MetalError, kernel::MetalKernels};-use crate::backends::common::Backend;+use crate::backends::{common::Backend, metal::sparse::MetalSparseBuffer};#[derive(Debug, Clone)]pub struct Metal;@@ -11,6 +11,7 @@ impl Backend for Metal {type Context = MetalContext;type CommandBuffer = MetalCommandBuffer;type DenseBuffer = Retained<ProtocolObject<dyn MTLBuffer>>;+ type SparseBuffer = MetalSparseBuffer;type Event = Retained<ProtocolObject<dyn MTLEvent>>;type Kernels = MetalKernels;type Error = MetalError;diff --git a/crates/backend-uzu/src/backends/metal/context.rs b/crates/backend-uzu/src/backends/metal/context.rsindex d8fc64358..af770818d 100644--- a/crates/backend-uzu/src/backends/metal/context.rs+++ b/crates/backend-uzu/src/backends/metal/context.rs@@ -1,9 +1,16 @@-use std::{cell::RefCell, collections::HashMap, path::Path, rc::Rc};+#[cfg(test)]+use std::cell::Ref;+use std::{+ cell::{RefCell, RefMut},+ collections::HashMap,+ path::Path,+ rc::{Rc, Weak},+};use metal::{- MTLBuffer, MTLCaptureDescriptor, MTLCaptureDestination, MTLCaptureManager, MTLCommandQueue, MTLCommandQueueExt,- MTLComputePipelineState, MTLDevice, MTLDeviceExt, MTLEvent, MTLFunctionConstantValues, MTLLibrary,- MTLResourceOptions,+ MTL4CommandQueue, MTLBuffer, MTLCaptureDescriptor, MTLCaptureDestination, MTLCaptureManager, MTLCommandQueue,+ MTLCommandQueueExt, MTLComputePipelineState, MTLDevice, MTLDeviceExt, MTLEvent, MTLFunctionConstantValues,+ MTLLibrary, MTLResourceOptions, MTLSparsePageSize,};use objc2::{rc::Retained, runtime::ProtocolObject};@@ -16,8 +23,12 @@ use super::{};use crate::{backends::{- common::{Allocation, AllocationPool, AllocationType, Allocator, Context},- metal::command_buffer::MetalCommandBufferInitial,+ common::{Allocation, AllocationPool, AllocationType, Allocator, Backend, Context},+ metal::{+ command_buffer::MetalCommandBufferInitial,+ metal_extensions::SparsePageSizeExt,+ sparse::{MetalSparseBuffer, MetalSparseHeapPool},+ },},utils::model_size::ModelSize,};@@ -25,11 +36,14 @@ use crate::{pub struct MetalContext {pub device: Retained<ProtocolObject<dyn MTLDevice>>,pub command_queue: Retained<ProtocolObject<dyn MTLCommandQueue>>,+ pub command_queue4: Retained<ProtocolObject<dyn MTL4CommandQueue>>,allocator: Rc<Allocator<Metal>>,peak_memory_usage: RefCell<usize>,device_capabilities: MetalDeviceCapabilities,library: Retained<ProtocolObject<dyn MTLLibrary>>,pipeline_cache: RefCell<HashMap<String, Retained<ProtocolObject<dyn MTLComputePipelineState>>>>,+ sparse_heap_pool: RefCell<MetalSparseHeapPool>,+ weak_self: Weak<MetalContext>,}impl MetalContext {@@ -53,6 +67,15 @@ impl MetalContext {Ok(pipeline)}++ #[cfg(test)]+ pub(super) fn sparse_heap_pool(&self) -> Ref<'_, MetalSparseHeapPool> {+ self.sparse_heap_pool.borrow()+ }++ pub(super) fn sparse_heap_pool_mut(&self) -> RefMut<'_, MetalSparseHeapPool> {+ self.sparse_heap_pool.borrow_mut()+ }}impl Context for MetalContext {@@ -60,25 +83,34 @@ impl Context for MetalContext {fn new() -> Result<Rc<Self>, MetalError> {let device: Retained<ProtocolObject<dyn MTLDevice>> =- <dyn metal::MTLDevice>::system_default().ok_or(MetalError::CannotOpenDevice)?;+ <dyn MTLDevice>::system_default().ok_or(MetalError::CannotOpenDevice)?;let command_queue =device.new_command_queue_with_max_command_buffer_count(1024).ok_or(MetalError::CannotCreateCommandQueue)?;+ let command_queue4 = device.new_mtl4_command_queue().ok_or(MetalError::CannotCreateCommandQueueMtl4)?;+let library = device.new_library_with_data(kernel::MTLB).map_err(|nserror| MetalError::CannotCreateLibrary(nserror.to_string()))?;let device_capabilities = MetalDeviceCapabilities::from_device(&device);+ let page_size = MTLSparsePageSize::KB256;+ let heap_capacity = 64 * 4 * page_size.in_bytes();+ let sparse_pool = MetalSparseHeapPool::new(page_size, heap_capacity);+Ok(Rc::new_cyclic(|weak_self| Self {device,command_queue,+ command_queue4,allocator: Allocator::new(weak_self.clone()),peak_memory_usage: RefCell::new(0),device_capabilities,library,pipeline_cache: RefCell::new(HashMap::new()),+ sparse_heap_pool: RefCell::new(sparse_pool),+ weak_self: weak_self.clone(),}))}@@ -111,11 +143,29 @@ impl Context for MetalContext {&self,size: usize,) -> Result<Retained<ProtocolObject<dyn MTLBuffer>>, MetalError> {- let buffer =- self.device.new_buffer(size, MTLResourceOptions::STORAGE_MODE_SHARED).ok_or(MetalError::CannotCreateBuffer);+ let buffer = self+ .device+ .new_buffer(size, MTLResourceOptions::STORAGE_MODE_SHARED)+ .ok_or(MetalError::CannotCreateBuffer)?;+let mut peak_memory_usage_borrow = self.peak_memory_usage.borrow_mut();*peak_memory_usage_borrow = peak_memory_usage_borrow.max(self.device.current_allocated_size());- buffer++ Ok(buffer)+ }++ fn create_buffer_with_data(+ &self,+ data: &[u8],+ ) -> Result<Retained<ProtocolObject<dyn MTLBuffer>>, MetalError> {+ let buffer = self+ .device+ .new_buffer_with_data(data, MTLResourceOptions::STORAGE_MODE_SHARED)+ .ok_or(MetalError::CannotCreateBuffer)?;++ let mut peak_memory_usage_borrow = self.peak_memory_usage.borrow_mut();+ *peak_memory_usage_borrow = peak_memory_usage_borrow.max(self.device.current_allocated_size());+ Ok(buffer)}fn create_allocation(@@ -143,6 +193,15 @@ impl Context for MetalContext {self.device.new_event().ok_or(MetalError::CannotCreateEvent)}+ fn create_sparse_buffer(+ &self,+ capacity: usize,+ ) -> Result<<Self::Backend as Backend>::SparseBuffer, <Self::Backend as Backend>::Error> {+ let sparse_page_size = self.sparse_heap_pool.borrow().page_size();+ let context = self.weak_self.upgrade().ok_or(MetalError::CannotCreateBuffer)?;+ Ok(MetalSparseBuffer::new(context, capacity, sparse_page_size)?)+ }+fn peak_memory_usage(&self) -> Option<usize> {Some(*self.peak_memory_usage.borrow())}@@ -155,8 +214,8 @@ impl Context for MetalContext {fn start_capture(&self,- trace_path: &std::path::Path,- ) -> Result<(), <Self::Backend as crate::backends::common::Backend>::Error> {+ trace_path: &Path,+ ) -> Result<(), <Self::Backend as Backend>::Error> {let capture_manager = MTLCaptureManager::shared_capture_manager();let capture_descriptor = MTLCaptureDescriptor::new();capture_descriptor.set_destination(MTLCaptureDestination::GPUTraceDocument);@@ -172,7 +231,7 @@ impl Context for MetalContext {Ok(())}- fn stop_capture(&self) -> Result<(), <Self::Backend as crate::backends::common::Backend>::Error> {+ fn stop_capture(&self) -> Result<(), <Self::Backend as Backend>::Error> {MTLCaptureManager::shared_capture_manager().stop_capture();Ok(())diff --git a/crates/backend-uzu/src/backends/metal/device_capabilities.rs b/crates/backend-uzu/src/backends/metal/device_capabilities.rsindex 6908d8b91..5a74d6144 100644--- a/crates/backend-uzu/src/backends/metal/device_capabilities.rs+++ b/crates/backend-uzu/src/backends/metal/device_capabilities.rs@@ -1,5 +1,5 @@use bytesize::ByteSize;-use metal::{MTLDevice, MTLDeviceExt as _};+use metal::{MTLDevice, MTLDeviceExt};use objc2::runtime::ProtocolObject;use super::metal_extensions::{DeviceExt, DeviceGeneration};diff --git a/crates/backend-uzu/src/backends/metal/error.rs b/crates/backend-uzu/src/backends/metal/error.rsindex 208238b05..a14e3459a 100644--- a/crates/backend-uzu/src/backends/metal/error.rs+++ b/crates/backend-uzu/src/backends/metal/error.rs@@ -10,6 +10,8 @@ pub enum MetalError {CannotCreateLibrary(String),#[error("Cannot create command queue")]CannotCreateCommandQueue,+ #[error("Cannot create command Metal 4 queue")]+ CannotCreateCommandQueueMtl4,#[error("Cannot create buffer")]CannotCreateBuffer,#[error("Cannot create command buffer")]@@ -22,4 +24,8 @@ pub enum MetalError {CannotCreateFunction,#[error("Cannot create pipeline state: {0}")]CannotCreatePipelineState(String),+ #[error("Can not allocate buffer with size={0}")]+ SparseBufferAlloc(usize),+ #[error("Can not allocate heap with size={0} and page size={1}")]+ SparseHeapAlloc(usize, usize),}diff --git a/crates/backend-uzu/src/backends/metal/metal_extensions/mod.rs b/crates/backend-uzu/src/backends/metal/metal_extensions/mod.rsindex 3bcccae95..6ddbead8f 100644--- a/crates/backend-uzu/src/backends/metal/metal_extensions/mod.rs+++ b/crates/backend-uzu/src/backends/metal/metal_extensions/mod.rs@@ -3,9 +3,11 @@ mod data_type;mod device_extensions;mod function_constant_values_extensions_set_value;mod library_extensions_pipeline;+mod sparse_page_size_extensions;pub use compute_command_encoder_extensions_set_value::ComputeEncoderSetValue;pub use data_type::MetalDataTypeExt;pub use device_extensions::{DeviceExt, DeviceGeneration};pub use function_constant_values_extensions_set_value::FunctionConstantValuesSetValue;pub use library_extensions_pipeline::LibraryPipelineExtensions;+pub use sparse_page_size_extensions::SparsePageSizeExt;diff --git a/crates/backend-uzu/src/backends/metal/metal_extensions/sparse_page_size_extensions.rs b/crates/backend-uzu/src/backends/metal/metal_extensions/sparse_page_size_extensions.rsnew file mode 100644index 000000000..2dc91318a--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/metal_extensions/sparse_page_size_extensions.rs@@ -0,0 +1,15 @@+use metal::MTLSparsePageSize;++pub trait SparsePageSizeExt {+ fn in_bytes(&self) -> usize;+}++impl SparsePageSizeExt for MTLSparsePageSize {+ fn in_bytes(&self) -> usize {+ match self {+ MTLSparsePageSize::KB16 => 16 * 1024,+ MTLSparsePageSize::KB64 => 64 * 1024,+ MTLSparsePageSize::KB256 => 256 * 1024,+ }+ }+}diff --git a/crates/backend-uzu/src/backends/metal/mod.rs b/crates/backend-uzu/src/backends/metal/mod.rsindex df926e84b..43a52655b 100644--- a/crates/backend-uzu/src/backends/metal/mod.rs+++ b/crates/backend-uzu/src/backends/metal/mod.rs@@ -7,6 +7,7 @@ mod error;mod event;mod kernel;mod metal_extensions;+mod sparse;pub use backend::Metal;pub use context::MetalContext;diff --git a/crates/backend-uzu/src/backends/metal/sparse/mod.rs b/crates/backend-uzu/src/backends/metal/sparse/mod.rsnew file mode 100644index 000000000..dd4321d32--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/sparse/mod.rs@@ -0,0 +1,7 @@+mod sparse_buffer;+mod sparse_heap;+mod sparse_heap_pool;+mod sparse_utils;++pub(crate) use sparse_buffer::MetalSparseBuffer;+pub(crate) use sparse_heap_pool::MetalSparseHeapPool;diff --git a/crates/backend-uzu/src/backends/metal/sparse/sparse_buffer.rs b/crates/backend-uzu/src/backends/metal/sparse/sparse_buffer.rsnew file mode 100644index 000000000..cc2fedd94--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/sparse/sparse_buffer.rs@@ -0,0 +1,123 @@+use std::{+ cmp::{max, min},+ fmt::Debug,+ ops::Range,+ rc::Rc,+};++use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions, MTLSparsePageSize};+use objc2::{rc::Retained, runtime::ProtocolObject};+use rangemap::RangeSet;++use crate::{+ backends::{+ common::{Backend, Buffer, SparseBuffer},+ metal::{Metal, error::MetalError, metal_extensions::SparsePageSizeExt},+ },+ prelude::MetalContext,+};++pub struct MetalSparseBuffer {+ buffer: Retained<ProtocolObject<dyn MTLBuffer>>,+ mapped_pages: RangeSet<usize>,+ context: Rc<MetalContext>,+}++impl MetalSparseBuffer {+ pub(crate) fn new(+ context: Rc<MetalContext>,+ capacity: usize,+ page_size: MTLSparsePageSize,+ ) -> Result<Self, MetalError> {+ let aligned_capacity = capacity.next_multiple_of(page_size.in_bytes());+ let buffer = context+ .device+ .new_buffer_with_length_options_placement_sparse_page_size(+ aligned_capacity,+ MTLResourceOptions::STORAGE_MODE_PRIVATE,+ page_size,+ )+ .ok_or(MetalError::SparseBufferAlloc(aligned_capacity))?;++ Ok(Self {+ buffer,+ mapped_pages: RangeSet::new(),+ context,+ })+ }+}++impl Debug for MetalSparseBuffer {+ fn fmt(+ &self,+ f: &mut std::fmt::Formatter<'_>,+ ) -> std::fmt::Result {+ f.debug_struct("MetalSparseBuffer")+ .field("mapped_pages", &self.mapped_pages)+ .field("buffer", &self.buffer)+ .finish_non_exhaustive()+ }+}++impl Drop for MetalSparseBuffer {+ fn drop(&mut self) {+ let context = self.context.clone();+ for range in self.mapped_pages.iter() {+ context.sparse_heap_pool_mut().unmap(&context, &self.buffer, &range).expect("Failed to unmap");+ }+ }+}++impl Buffer for MetalSparseBuffer {+ type Backend = Metal;++ fn gpu_ptr(&self) -> usize {+ self.buffer.gpu_ptr()+ }++ fn size(&self) -> usize {+ self.buffer.size()+ }++ fn set_label(+ &mut self,+ label: Option<&str>,+ ) {+ self.buffer.set_label(label)+ }+}++impl SparseBuffer for MetalSparseBuffer {+ fn map(+ &mut self,+ context: &<Self::Backend as Backend>::Context,+ pages: &Range<usize>,+ ) -> Result<(), <Self::Backend as Backend>::Error> {+ self.mapped_pages.gaps(pages).collect::<Vec<_>>().into_iter().try_for_each(|gap| {+ context.sparse_heap_pool_mut().map(context, &self.buffer, &gap)?;+ self.mapped_pages.insert(gap);+ Ok(())+ })+ }++ fn unmap(+ &mut self,+ context: &<Self::Backend as Backend>::Context,+ pages: &Range<usize>,+ ) -> Result<(), <Self::Backend as Backend>::Error> {+ self.mapped_pages+ .overlapping(pages)+ .map(|range| max(range.start, pages.start)..min(range.end, pages.end))+ .collect::<Vec<_>>()+ .into_iter()+ .try_for_each(|range| {+ context.sparse_heap_pool_mut().unmap(context, &self.buffer, &range)?;+ self.mapped_pages.remove(range);+ Ok(())+ })+ }+}++#[cfg(test)]+#[path = "../../../../tests/unit/backends/metal/sparse/sparse_buffer_test.rs"]+mod tests;diff --git a/crates/backend-uzu/src/backends/metal/sparse/sparse_heap.rs b/crates/backend-uzu/src/backends/metal/sparse/sparse_heap.rsnew file mode 100644index 000000000..241ca0c32--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/sparse/sparse_heap.rs@@ -0,0 +1,123 @@+use std::{collections::HashMap, ops::Range};++use metal::{+ MTL4CommandQueue, MTL4CommandQueueExt, MTL4UpdateSparseBufferMappingOperation, MTLBuffer, MTLDeviceExt, MTLHeap,+ MTLHeapDescriptor, MTLHeapType, MTLSparsePageSize, MTLSparseTextureMappingMode, MTLStorageMode,+};+use objc2::{rc::Retained, runtime::ProtocolObject};+use rangemap::{RangeMap, RangeSet};++use crate::{+ backends::metal::{+ error::MetalError,+ metal_extensions::SparsePageSizeExt,+ sparse::sparse_utils::{MetalSparseHeapBufferMapping, MetalSparseHeapMappingParameters},+ },+ prelude::MetalContext,+};++pub(super) struct MetalSparseHeap {+ heap: Retained<ProtocolObject<dyn MTLHeap>>,+ // Mappings are tracked per buffer (keyed by gpu_address) so multiple+ // buffers can alias overlapping heap page ranges without losing track of+ // either side.+ buffer_mappings: HashMap<u64, RangeMap<usize, MetalSparseHeapBufferMapping>>,+ free_pages: RangeSet<usize>,+}++impl MetalSparseHeap {+ pub fn new(+ context: &MetalContext,+ capacity_bytes: usize,+ page_size: MTLSparsePageSize,+ ) -> Result<Self, MetalError> {+ let aligned_capacity = capacity_bytes.next_multiple_of(page_size.in_bytes());++ let heap_desc = MTLHeapDescriptor::new();+ // Sparse buffers must be backed by a Placement heap with a sparse page size set;+ // `MTLHeapType::Sparse` is for sparse textures and trips a runtime assertion when+ // passed to updateBufferMappings.+ heap_desc.set_type(MTLHeapType::Placement);+ heap_desc.set_storage_mode(MTLStorageMode::Private);+ heap_desc.set_size(aligned_capacity);+ heap_desc.set_max_compatible_placement_sparse_page_size(page_size);+ let heap = context+ .device+ .new_heap_with_descriptor(&heap_desc)+ .ok_or(MetalError::SparseHeapAlloc(aligned_capacity, page_size.in_bytes()))?;++ let mut free_pages = RangeSet::new();+ free_pages.insert(0..(aligned_capacity / page_size.in_bytes()));+ Ok(Self {+ heap,+ buffer_mappings: HashMap::new(),+ free_pages,+ })+ }++ pub fn free_pages(&self) -> &RangeSet<usize> {+ &self.free_pages+ }++ /// Iterates `(heap_range, mapping)` pairs that belong to a single buffer.+ pub fn mappings_for(+ &self,+ buffer_address: u64,+ ) -> impl Iterator<Item = (Range<usize>, &MetalSparseHeapBufferMapping)> + '_ {+ self.buffer_mappings+ .get(&buffer_address)+ .into_iter()+ .flat_map(|m| m.iter().map(|(r, mapping)| (r.clone(), mapping)))+ }++ pub fn is_empty(&self) -> bool {+ self.buffer_mappings.is_empty()+ }++ /// Command queue calls doesn't have any synchronization.+ /// It is the responsibility of the caller.+ pub fn execute(+ &mut self,+ buffer: &ProtocolObject<dyn MTLBuffer>,+ cmd_queue: &ProtocolObject<dyn MTL4CommandQueue>,+ operations: &[MetalSparseHeapMappingParameters],+ map: bool,+ ) {+ if operations.is_empty() {+ return;+ }++ let mtl_operations: Vec<MTL4UpdateSparseBufferMappingOperation> = operations+ .iter()+ .map(|op| {+ let mode = if map {+ MTLSparseTextureMappingMode::Map+ } else {+ MTLSparseTextureMappingMode::Unmap+ };+ MTL4UpdateSparseBufferMappingOperation::new(mode, op.buffer_pages.clone(), op.heap_page_offset)+ })+ .collect();++ cmd_queue.update_buffer_mappings(buffer, Some(&self.heap), &mtl_operations);++ let buffer_address = buffer.gpu_address();+ let entry = self.buffer_mappings.entry(buffer_address).or_default();+ mtl_operations.iter().for_each(|mtl_op| {+ let heap_range = mtl_op.heap_offset..(mtl_op.heap_offset + mtl_op.buffer_range().len());+ if map {+ let buffer_range = mtl_op.buffer_range();+ let buffer_mapping = MetalSparseHeapBufferMapping::new(mtl_op.heap_offset, buffer_range.start);+ entry.insert(heap_range.clone(), buffer_mapping);+ self.free_pages.remove(heap_range);+ } else {+ entry.remove(heap_range.clone());+ self.free_pages.insert(heap_range);+ }+ });++ if entry.is_empty() {+ self.buffer_mappings.remove(&buffer_address);+ }+ }+}diff --git a/crates/backend-uzu/src/backends/metal/sparse/sparse_heap_pool.rs b/crates/backend-uzu/src/backends/metal/sparse/sparse_heap_pool.rsnew file mode 100644index 000000000..fe63a06fd--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/sparse/sparse_heap_pool.rs@@ -0,0 +1,165 @@+use std::{+ cmp::{max, min},+ ops::Range,+};++use metal::{MTLBuffer, MTLSparsePageSize};+use objc2::runtime::ProtocolObject;++use crate::{+ backends::metal::{+ error::MetalError,+ metal_extensions::SparsePageSizeExt,+ sparse::{sparse_heap::MetalSparseHeap, sparse_utils::MetalSparseHeapMappingParameters},+ },+ prelude::MetalContext,+};++pub struct MetalSparseHeapPool {+ heaps: Vec<MetalSparseHeap>,+ heap_capacity: usize,+ page_size: MTLSparsePageSize,+}++impl MetalSparseHeapPool {+ pub fn new(+ page_size: MTLSparsePageSize,+ heap_capacity: usize,+ ) -> Self {+ Self {+ heaps: Vec::new(),+ page_size,+ heap_capacity,+ }+ }++ pub fn map(+ &mut self,+ context: &MetalContext,+ buffer: &ProtocolObject<dyn MTLBuffer>,+ buffer_pages: &Range<usize>,+ ) -> Result<(), MetalError> {+ if buffer_pages.len() == 0 {+ return Ok(());+ }++ let mut pages_to_map = buffer_pages.clone();+ let heap_capacity_pages = self.heap_capacity_pages();++ // Try to find pages in existing heaps.+ // While `pages_to_map` is not empty in each heap find unmapped pages, collect them into `mappings` and map+ let mut existing_heaps_mappings: Vec<(usize, Box<[MetalSparseHeapMappingParameters]>)> = Vec::new();+ for (i, heap) in self.heaps.iter_mut().enumerate() {+ let mut mappings: Vec<MetalSparseHeapMappingParameters> = Vec::new();++ for free_pages in heap.free_pages().iter() {+ let map_pages_count = min(free_pages.len(), pages_to_map.len());+ let mapping = MetalSparseHeapMappingParameters {+ buffer_pages: pages_to_map.start..(pages_to_map.start + map_pages_count),+ heap_page_offset: free_pages.start,+ };+ mappings.push(mapping);++ pages_to_map.start += map_pages_count;+ if pages_to_map.len() == 0 {+ break;+ }+ }++ heap.execute(buffer, &context.command_queue4, &mappings, true);+ existing_heaps_mappings.push((i, mappings.into_boxed_slice()));++ if pages_to_map.len() == 0 {+ break;+ }+ }++ // If `pages_to_map` still not empty, it's necessary to allocate new heaps+ let new_heaps_required = pages_to_map.len().div_ceil(heap_capacity_pages);+ for _ in 0..new_heaps_required {+ let map_pages_count = min(heap_capacity_pages, pages_to_map.len());+ let buffer_pages = pages_to_map.start..(pages_to_map.start + map_pages_count);+ let op_params = MetalSparseHeapMappingParameters {+ buffer_pages,+ heap_page_offset: 0,+ };++ match MetalSparseHeap::new(context, self.heap_capacity, self.page_size) {+ Ok(mut heap) => {+ let operations = [op_params];+ heap.execute(buffer, &context.command_queue4, &operations, true);+ self.heaps.push(heap);+ pages_to_map.start += map_pages_count;+ existing_heaps_mappings.push((self.heaps.len() - 1, Box::new(operations)));+ },+ Err(err) => {+ // it's necessary to unmap previously mapped pages in existing heaps+ for (heap_pos, mappings) in existing_heaps_mappings {+ self.heaps[heap_pos].execute(buffer, &context.command_queue4, &mappings, false);+ }+ return Err(err);+ },+ };+ }++ Ok(())+ }++ pub fn unmap(+ &mut self,+ context: &MetalContext,+ buffer: &ProtocolObject<dyn MTLBuffer>,+ buffer_pages: &Range<usize>,+ ) -> Result<(), MetalError> {+ let buffer_address = buffer.gpu_address();++ // Iterate over heaps, find mappings for `buffer_address`, unmap them+ self.heaps.iter_mut().for_each(|heap| {+ let unmappings: Vec<MetalSparseHeapMappingParameters> = heap+ .mappings_for(buffer_address)+ .filter_map(|(heap_range, mapping)| {+ let mapped_buffer_pages = mapping.buffer_pages_for(&heap_range);+ let unmap_start = max(buffer_pages.start, mapped_buffer_pages.start);+ let unmap_end = min(buffer_pages.end, mapped_buffer_pages.end);+ if unmap_start >= unmap_end {+ return None;+ };++ let offset_within_range = unmap_start - mapped_buffer_pages.start;+ Some(MetalSparseHeapMappingParameters {+ buffer_pages: unmap_start..unmap_end,+ heap_page_offset: heap_range.start + offset_within_range,+ })+ })+ .collect();+ heap.execute(buffer, &context.command_queue4, &unmappings, false);+ });++ // Remove empty heaps+ self.heaps.retain(|heap| !heap.is_empty());++ Ok(())+ }++ #[cfg(test)]+ pub fn heap_capacity_bytes(&self) -> usize {+ self.heap_capacity+ }++ pub fn heap_capacity_pages(&self) -> usize {+ self.heap_capacity / self.page_size.in_bytes()+ }++ #[cfg(test)]+ pub fn heaps_count(&self) -> usize {+ self.heaps.len()+ }++ pub fn page_size(&self) -> MTLSparsePageSize {+ self.page_size+ }+}++#[cfg(test)]+#[path = "../../../../tests/unit/backends/metal/sparse/sparse_heap_pool_test.rs"]+mod tests;diff --git a/crates/backend-uzu/src/backends/metal/sparse/sparse_utils.rs b/crates/backend-uzu/src/backends/metal/sparse/sparse_utils.rsnew file mode 100644index 000000000..103de7d1f--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/sparse/sparse_utils.rs@@ -0,0 +1,36 @@+use std::ops::Range;++#[derive(Clone, PartialEq)]+pub(super) struct MetalSparseHeapBufferMapping {+ // Anchors for an originally inserted contiguous mapping. They survive+ // rangemap splits unchanged, so the heap↔buffer correspondence can always+ // be recovered from any surviving heap subrange via `buffer_pages_for`.+ heap_page_anchor: usize,+ buffer_page_anchor: usize,+}++impl MetalSparseHeapBufferMapping {+ pub fn new(+ heap_page_anchor: usize,+ buffer_page_anchor: usize,+ ) -> Self {+ Self {+ heap_page_anchor,+ buffer_page_anchor,+ }+ }++ pub fn buffer_pages_for(+ &self,+ heap_range: &Range<usize>,+ ) -> Range<usize> {+ let start = self.buffer_page_anchor + (heap_range.start - self.heap_page_anchor);+ let end = self.buffer_page_anchor + (heap_range.end - self.heap_page_anchor);+ start..end+ }+}++pub(super) struct MetalSparseHeapMappingParameters {+ pub(super) buffer_pages: Range<usize>,+ pub(super) heap_page_offset: usize,+}diff --git a/crates/backend-uzu/tests/unit/backends/metal/sparse/sparse_buffer_test.rs b/crates/backend-uzu/tests/unit/backends/metal/sparse/sparse_buffer_test.rsnew file mode 100644index 000000000..22f5315bf--- /dev/null+++ b/crates/backend-uzu/tests/unit/backends/metal/sparse/sparse_buffer_test.rs@@ -0,0 +1,203 @@+use std::{ops::Range, rc::Rc};++use crate::backends::{+ common::{Backend, Context, SparseBuffer},+ metal::Metal,+};++fn create_context() -> Rc<<Metal as Backend>::Context> {+ <Metal as Backend>::Context::new().expect("Failed to create Metal context")+}++/// Capacity sized to fit `heap_count` worth of heaps.+fn buffer_capacity(+ ctx: &<Metal as Backend>::Context,+ heap_count: usize,+) -> usize {+ heap_count * ctx.sparse_heap_pool().heap_capacity_bytes()+}++/// Buffer-page range covering `heap_count` worth of heaps starting at `heap_offset`.+fn pages_for_heaps(+ ctx: &<Metal as Backend>::Context,+ heap_offset: usize,+ heap_count: usize,+) -> Range<usize> {+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();+ (heap_offset * pages_per_heap)..((heap_offset + heap_count) * pages_per_heap)+}++#[test]+fn test_mapping_succeeds() {+ let ctx = create_context();+ let capacity = buffer_capacity(ctx.as_ref(), 4);+ let pages = pages_for_heaps(ctx.as_ref(), 0, 4);++ let mut sparse_buffer = ctx.create_sparse_buffer(capacity).expect("Failed to create sparse buffer");+ sparse_buffer.map(ctx.as_ref(), &pages).expect("Failed to map sparse buffer");+}++#[test]+fn test_unmapping_succeeds() {+ let ctx = create_context();+ let capacity = buffer_capacity(ctx.as_ref(), 4);+ let pages = pages_for_heaps(ctx.as_ref(), 0, 4);++ let mut sparse_buffer = ctx.create_sparse_buffer(capacity).expect("Failed to create sparse buffer");+ sparse_buffer.map(ctx.as_ref(), &pages).expect("Failed to map sparse buffer");+ sparse_buffer.unmap(ctx.as_ref(), &pages).expect("Failed to unmap sparse buffer");+}++#[test]+fn test_partial_unmapping_succeeds() {+ let ctx = create_context();+ let capacity = buffer_capacity(ctx.as_ref(), 4);+ let mapped = pages_for_heaps(ctx.as_ref(), 0, 4);+ let unmapped = pages_for_heaps(ctx.as_ref(), 1, 2);++ let mut sparse_buffer = ctx.create_sparse_buffer(capacity).expect("Failed to create sparse buffer");+ sparse_buffer.map(ctx.as_ref(), &mapped).expect("Failed to map sparse buffer");+ sparse_buffer.unmap(ctx.as_ref(), &unmapped).expect("Failed to unmap sparse buffer");+}++#[test]+fn test_mapping_uses_minimum_heaps() {+ let ctx = create_context();+ let heap_count = 4;+ let capacity = buffer_capacity(ctx.as_ref(), heap_count);+ let pages = pages_for_heaps(ctx.as_ref(), 0, heap_count);++ let mut sparse_buffer = ctx.create_sparse_buffer(capacity).expect("Failed to create sparse buffer");+ sparse_buffer.map(ctx.as_ref(), &pages).expect("Failed to map sparse buffer");++ let heaps = ctx.sparse_heap_pool();+ let expected = pages.len().div_ceil(heaps.heap_capacity_pages());+ assert_eq!(heaps.heaps_count(), expected, "mapping should allocate the minimum number of heaps");+}++#[test]+fn test_remapping_reuses_freed_pages() {+ let ctx = create_context();+ let capacity = buffer_capacity(ctx.as_ref(), 6);+ let initial_pages = pages_for_heaps(ctx.as_ref(), 0, 4);+ let unmapped_pages = pages_for_heaps(ctx.as_ref(), 1, 1);+ let extra_pages = pages_for_heaps(ctx.as_ref(), 4, 1);++ let mut sparse_buffer = ctx.create_sparse_buffer(capacity).expect("Failed to create sparse buffer");+ sparse_buffer.map(ctx.as_ref(), &initial_pages).expect("Failed to map sparse buffer");+ let initial_heaps = ctx.sparse_heap_pool().heaps_count();++ sparse_buffer.unmap(ctx.as_ref(), &unmapped_pages).expect("Failed to unmap sparse buffer");+ sparse_buffer.map(ctx.as_ref(), &extra_pages).expect("Failed to remap sparse buffer");++ assert_eq!(+ ctx.sparse_heap_pool().heaps_count(),+ initial_heaps,+ "remapping after unmap should reuse freed heap pages instead of allocating new heaps",+ );+}++#[test]+fn test_full_unmap_then_remap_reuses_heaps() {+ let ctx = create_context();+ let capacity = buffer_capacity(ctx.as_ref(), 4);+ let pages = pages_for_heaps(ctx.as_ref(), 0, 4);++ let mut sparse_buffer = ctx.create_sparse_buffer(capacity).expect("Failed to create sparse buffer");+ sparse_buffer.map(ctx.as_ref(), &pages).expect("Failed to map sparse buffer");+ let initial_heaps = ctx.sparse_heap_pool().heaps_count();++ sparse_buffer.unmap(ctx.as_ref(), &pages).expect("Failed to unmap sparse buffer");+ sparse_buffer.map(ctx.as_ref(), &pages).expect("Failed to remap sparse buffer");++ assert_eq!(+ ctx.sparse_heap_pool().heaps_count(),+ initial_heaps,+ "remapping the same range after a full unmap should not grow the heap pool",+ );+}++#[test]+fn test_remapping_same_pages_is_noop() {+ let ctx = create_context();+ let capacity = buffer_capacity(ctx.as_ref(), 4);+ let pages = pages_for_heaps(ctx.as_ref(), 0, 4);++ let mut sparse_buffer = ctx.create_sparse_buffer(capacity).expect("Failed to create sparse buffer");+ sparse_buffer.map(ctx.as_ref(), &pages).expect("Failed to map sparse buffer");+ let initial_heaps = ctx.sparse_heap_pool().heaps_count();++ sparse_buffer.map(ctx.as_ref(), &pages).expect("Failed to remap sparse buffer");++ assert_eq!(+ ctx.sparse_heap_pool().heaps_count(),+ initial_heaps,+ "remapping the same pages should not allocate new heaps",+ );+}++#[test]+fn test_drop_releases_pool_heaps() {+ // Regression: a mapped buffer dropped without an explicit unmap must+ // still release its heap pages back to the shared pool.+ let ctx = create_context();+ let capacity = buffer_capacity(ctx.as_ref(), 2);+ let pages = pages_for_heaps(ctx.as_ref(), 0, 2);++ {+ let mut sparse_buffer = ctx.create_sparse_buffer(capacity).expect("Failed to create sparse buffer");+ sparse_buffer.map(ctx.as_ref(), &pages).expect("Failed to map sparse buffer");+ assert!(ctx.sparse_heap_pool().heaps_count() > 0);+ }++ assert_eq!(+ ctx.sparse_heap_pool().heaps_count(),+ 0,+ "dropping a mapped sparse buffer should release its heap pages",+ );+}++#[test]+fn test_drop_does_not_disturb_other_buffer_mappings() {+ // Dropping one buffer must not unmap heap pages held by another buffer+ // sharing the same pool.+ let ctx = create_context();+ let capacity = buffer_capacity(ctx.as_ref(), 1);+ let pages = pages_for_heaps(ctx.as_ref(), 0, 1);++ let mut keeper = ctx.create_sparse_buffer(capacity).expect("Failed to create keeper buffer");+ keeper.map(ctx.as_ref(), &pages).expect("Failed to map keeper buffer");+ let baseline = ctx.sparse_heap_pool().heaps_count();++ {+ let mut transient = ctx.create_sparse_buffer(capacity).expect("Failed to create transient buffer");+ transient.map(ctx.as_ref(), &pages).expect("Failed to map transient buffer");+ }++ assert_eq!(+ ctx.sparse_heap_pool().heaps_count(),+ baseline,+ "dropping a transient buffer must leave keeper buffer's mappings intact",+ );+}++#[test]+fn test_sequential_mappings_are_compact() {+ let ctx = create_context();+ let capacity = buffer_capacity(ctx.as_ref(), 4);+ let first = pages_for_heaps(ctx.as_ref(), 0, 2);+ let second = pages_for_heaps(ctx.as_ref(), 2, 2);++ let mut sparse_buffer = ctx.create_sparse_buffer(capacity).expect("Failed to create sparse buffer");+ sparse_buffer.map(ctx.as_ref(), &first).expect("Failed to map first range");+ sparse_buffer.map(ctx.as_ref(), &second).expect("Failed to map second range");++ let heaps = ctx.sparse_heap_pool();+ let total_pages = first.len() + second.len();+ let expected = total_pages.div_ceil(heaps.heap_capacity_pages());+ assert_eq!(+ heaps.heaps_count(),+ expected,+ "two adjacent mappings should pack into the same minimum number of heaps",+ );+}diff --git a/crates/backend-uzu/tests/unit/backends/metal/sparse/sparse_heap_pool_test.rs b/crates/backend-uzu/tests/unit/backends/metal/sparse/sparse_heap_pool_test.rsnew file mode 100644index 000000000..0615e5a21--- /dev/null+++ b/crates/backend-uzu/tests/unit/backends/metal/sparse/sparse_heap_pool_test.rs@@ -0,0 +1,399 @@+use std::rc::Rc;++use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};+use objc2::{rc::Retained, runtime::ProtocolObject};++use crate::{+ backends::{+ common::{Backend, Context},+ metal::{Metal, metal_extensions::SparsePageSizeExt},+ },+ prelude::MetalContext,+};++fn create_context() -> Rc<MetalContext> {+ <Metal as Backend>::Context::new().expect("Failed to create Metal context")+}++fn create_sparse_buffer(+ ctx: &MetalContext,+ capacity_bytes: usize,+) -> Retained<ProtocolObject<dyn MTLBuffer>> {+ let page_size = ctx.sparse_heap_pool().page_size();+ let aligned = capacity_bytes.next_multiple_of(page_size.in_bytes());+ ctx.device+ .new_buffer_with_length_options_placement_sparse_page_size(+ aligned,+ MTLResourceOptions::STORAGE_MODE_PRIVATE,+ page_size,+ )+ .expect("Failed to create sparse buffer")+}++#[test]+fn test_new_pool_is_empty() {+ let ctx = create_context();+ assert_eq!(ctx.sparse_heap_pool().heaps_count(), 0);+}++#[test]+fn test_map_empty_range_allocates_no_heaps() {+ let ctx = create_context();+ let cap = ctx.sparse_heap_pool().heap_capacity_bytes();+ let buffer = create_sparse_buffer(ctx.as_ref(), cap);++ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(0..0)).expect("map empty range");++ assert_eq!(ctx.sparse_heap_pool().heaps_count(), 0);+}++#[test]+fn test_map_single_heap_allocates_one_heap() {+ let ctx = create_context();+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();+ let cap = ctx.sparse_heap_pool().heap_capacity_bytes();+ let buffer = create_sparse_buffer(ctx.as_ref(), cap);++ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(0..pages_per_heap)).expect("map");++ assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);+}++#[test]+fn test_map_multi_heap_allocates_minimum_heaps() {+ let ctx = create_context();+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();+ let cap = 4 * ctx.sparse_heap_pool().heap_capacity_bytes();+ let buffer = create_sparse_buffer(ctx.as_ref(), cap);++ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(0..4 * pages_per_heap)).expect("map");++ assert_eq!(ctx.sparse_heap_pool().heaps_count(), 4);+}++#[test]+fn test_map_partial_heap_rounds_up_heap_count() {+ let ctx = create_context();+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();+ let cap = 2 * ctx.sparse_heap_pool().heap_capacity_bytes();+ let buffer = create_sparse_buffer(ctx.as_ref(), cap);++ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(0..pages_per_heap + 1)).expect("map");++ assert_eq!(ctx.sparse_heap_pool().heaps_count(), 2);+}++#[test]+fn test_sequential_mappings_pack_into_one_heap() {+ let ctx = create_context();+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();+ let half = pages_per_heap / 2;+ let cap = ctx.sparse_heap_pool().heap_capacity_bytes();+ let buffer = create_sparse_buffer(ctx.as_ref(), cap);++ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(0..half)).expect("map first");+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(half..pages_per_heap)).expect("map second");++ assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);+}++#[test]+fn test_map_overflows_existing_heap_into_new_heap() {+ let ctx = create_context();+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();+ let half = pages_per_heap / 2;+ let cap = 2 * ctx.sparse_heap_pool().heap_capacity_bytes();+ let buffer = create_sparse_buffer(ctx.as_ref(), cap);++ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(0..half)).expect("map first");+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(half..half + pages_per_heap)).expect("map overflow");++ assert_eq!(ctx.sparse_heap_pool().heaps_count(), 2);+}++#[test]+fn test_two_buffers_share_one_heap() {+ let ctx = create_context();+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();+ let half = pages_per_heap / 2;+ let cap = ctx.sparse_heap_pool().heap_capacity_bytes();+ let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);+ let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);++ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..half)).expect("map a");+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..half)).expect("map b");++ assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);+}++#[test]+fn test_full_unmap_removes_heaps() {+ let ctx = create_context();+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();+ let cap = 2 * ctx.sparse_heap_pool().heap_capacity_bytes();+ let buffer = create_sparse_buffer(ctx.as_ref(), cap);+ let pages = 0..2 * pages_per_heap;++ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &pages).expect("map");+ ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer, &pages).expect("unmap");++ assert_eq!(ctx.sparse_heap_pool().heaps_count(), 0);+}++#[test]+fn test_partial_unmap_keeps_heap() {+ let ctx = create_context();+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();+ let cap = ctx.sparse_heap_pool().heap_capacity_bytes();+ let buffer = create_sparse_buffer(ctx.as_ref(), cap);++ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(0..pages_per_heap)).expect("map");+ ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer, &(0..pages_per_heap / 2)).expect("partial unmap");++ assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);+}++#[test]+fn test_unmap_with_other_buffer_keeps_mappings() {+ let ctx = create_context();+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();+ let cap = ctx.sparse_heap_pool().heap_capacity_bytes();+ let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);+ let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);+ let pages = 0..pages_per_heap;++ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &pages).expect("map a");+ ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer_b, &pages).expect("unmap b");++ assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);+}++#[test]+fn test_unmap_one_buffer_keeps_heap_for_other() {+ let ctx = create_context();+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();+ let half = pages_per_heap / 2;+ let cap = ctx.sparse_heap_pool().heap_capacity_bytes();+ let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);+ let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);++ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..half)).expect("map a");+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..half)).expect("map b");+ ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer_a, &(0..half)).expect("unmap a");++ assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);+}++#[test]+fn test_remap_after_unmap_does_not_grow_pool() {+ let ctx = create_context();+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();+ let cap = 2 * ctx.sparse_heap_pool().heap_capacity_bytes();+ let buffer = create_sparse_buffer(ctx.as_ref(), cap);+ let pages = 0..2 * pages_per_heap;++ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &pages).expect("map");+ let initial = ctx.sparse_heap_pool().heaps_count();+ ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer, &pages).expect("unmap");+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &pages).expect("remap");++ assert_eq!(ctx.sparse_heap_pool().heaps_count(), initial);+}++#[test]+fn test_remap_into_freed_gap_reuses_heap() {+ let ctx = create_context();+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();+ let half = pages_per_heap / 2;+ let cap = ctx.sparse_heap_pool().heap_capacity_bytes();+ let buffer = create_sparse_buffer(ctx.as_ref(), cap);++ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(0..pages_per_heap)).expect("map full");+ ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer, &(0..half)).expect("unmap half");+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(0..half)).expect("remap into gap");++ assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);+}++#[test]+fn test_sequential_partial_unmaps_release_full_heap() {+ // Regression: after a partial unmap splits a rangemap entry, the surviving+ // entry must still report the correct buffer↔heap correspondence so that a+ // subsequent unmap of the remaining suffix targets the right heap pages.+ let ctx = create_context();+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();+ let half = pages_per_heap / 2;+ let cap = ctx.sparse_heap_pool().heap_capacity_bytes();+ let buffer = create_sparse_buffer(ctx.as_ref(), cap);++ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(0..pages_per_heap)).expect("map");+ ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer, &(0..half)).expect("unmap prefix");+ ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer, &(half..pages_per_heap)).expect("unmap suffix");++ assert_eq!(ctx.sparse_heap_pool().heaps_count(), 0);+}++#[test]+fn test_three_buffers_share_one_heap() {+ let ctx = create_context();+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();+ let third = pages_per_heap / 3;+ let cap = ctx.sparse_heap_pool().heap_capacity_bytes();+ let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);+ let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);+ let buffer_c = create_sparse_buffer(ctx.as_ref(), cap);++ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..third)).expect("map a");+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..third)).expect("map b");+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_c, &(0..third)).expect("map c");++ assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);+}++#[test]+fn test_two_buffers_exactly_fill_one_heap() {+ let ctx = create_context();+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();+ let half = pages_per_heap / 2;+ let cap = ctx.sparse_heap_pool().heap_capacity_bytes();+ let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);+ let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);++ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..half)).expect("map a");+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..pages_per_heap - half)).expect("map b fills heap");++ assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);+}++#[test]+fn test_third_buffer_spills_when_heap_filled_by_others() {+ let ctx = create_context();+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();+ let half = pages_per_heap / 2;+ let cap = ctx.sparse_heap_pool().heap_capacity_bytes();+ let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);+ let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);+ let buffer_c = create_sparse_buffer(ctx.as_ref(), cap);++ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..half)).expect("map a");+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..pages_per_heap - half)).expect("map b fills heap");+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_c, &(0..1)).expect("map c spills");++ assert_eq!(ctx.sparse_heap_pool().heaps_count(), 2);+}++#[test]+fn test_unmap_buffer_frees_pages_for_another_in_same_heap() {+ let ctx = create_context();+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();+ let half = pages_per_heap / 2;+ let cap = ctx.sparse_heap_pool().heap_capacity_bytes();+ let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);+ let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);+ let buffer_c = create_sparse_buffer(ctx.as_ref(), cap);++ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..half)).expect("map a");+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..pages_per_heap - half)).expect("map b fills heap");+ ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer_a, &(0..half)).expect("unmap a");+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_c, &(0..half)).expect("map c into gap");++ assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);+}++#[test]+fn test_unmap_middle_of_three_buffers_keeps_heap() {+ let ctx = create_context();+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();+ let third = pages_per_heap / 3;+ let cap = ctx.sparse_heap_pool().heap_capacity_bytes();+ let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);+ let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);+ let buffer_c = create_sparse_buffer(ctx.as_ref(), cap);++ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..third)).expect("map a");+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..third)).expect("map b");+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_c, &(0..third)).expect("map c");+ ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer_b, &(0..third)).expect("unmap b");++ assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);+}++#[test]+fn test_unmap_all_shared_buffers_releases_heap() {+ let ctx = create_context();+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();+ let third = pages_per_heap / 3;+ let cap = ctx.sparse_heap_pool().heap_capacity_bytes();+ let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);+ let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);+ let buffer_c = create_sparse_buffer(ctx.as_ref(), cap);++ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..third)).expect("map a");+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..third)).expect("map b");+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_c, &(0..third)).expect("map c");+ ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer_a, &(0..third)).expect("unmap a");+ ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer_b, &(0..third)).expect("unmap b");+ ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer_c, &(0..third)).expect("unmap c");++ assert_eq!(ctx.sparse_heap_pool().heaps_count(), 0);+}++#[test]+fn test_second_buffer_spans_existing_and_new_heap() {+ let ctx = create_context();+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();+ let half = pages_per_heap / 2;+ let cap = 2 * ctx.sparse_heap_pool().heap_capacity_bytes();+ let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);+ let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);++ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..half)).expect("map a");+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..pages_per_heap)).expect("map b spans heaps");++ assert_eq!(ctx.sparse_heap_pool().heaps_count(), 2);+}++#[test]+fn test_unmap_spanning_buffer_keeps_heap_with_other_buffer() {+ let ctx = create_context();+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();+ let half = pages_per_heap / 2;+ let cap = 2 * ctx.sparse_heap_pool().heap_capacity_bytes();+ let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);+ let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);++ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..half)).expect("map a");+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..pages_per_heap)).expect("map b spans heaps");+ ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer_b, &(0..pages_per_heap)).expect("unmap b");++ assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);+}++#[test]+fn test_remap_two_buffers_after_unmap_does_not_grow_pool() {+ let ctx = create_context();+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();+ let half = pages_per_heap / 2;+ let cap = ctx.sparse_heap_pool().heap_capacity_bytes();+ let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);+ let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);++ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..half)).expect("map a");+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..half)).expect("map b");+ let initial = ctx.sparse_heap_pool().heaps_count();+ ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer_a, &(0..half)).expect("unmap a");+ ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer_b, &(0..half)).expect("unmap b");+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..half)).expect("remap a");+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..half)).expect("remap b");++ assert_eq!(ctx.sparse_heap_pool().heaps_count(), initial);+}++#[test]+fn test_heap_capacity_pages_matches_byte_capacity() {+ let ctx = create_context();+ let pool = ctx.sparse_heap_pool();+ let page_size_bytes = pool.page_size().in_bytes();++ assert_eq!(pool.heap_capacity_pages(), pool.heap_capacity_bytes() / page_size_bytes);+}