Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Sparse buffers (#379)

Aleksandr Golokoze32d68a07d1b
diff --git a/Cargo.lock b/Cargo.lock
index cb1fe3632..9ab1ad99b 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -508,9 +508,9 @@ dependencies = [
[[package]]
name = "avif-serialize"
-version = "0.8.8"
+version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "375082f007bd67184fb9c0374614b29f9aaa604ec301635f72338bb65386a53d"
+checksum = "e7178fe5f7d460b13895ebb9dcb28a3a6216d2df2574a0806cb51b555d297f38"
dependencies = [
"arrayvec",
]
@@ -813,6 +813,15 @@ version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c"
+[[package]]
+name = "bs58"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4"
+dependencies = [
+ "tinyvec",
+]
+
[[package]]
name = "bstr"
version = "1.12.1"
@@ -954,9 +963,9 @@ dependencies = [
[[package]]
name = "cc"
-version = "1.2.61"
+version = "1.2.62"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d16d90359e986641506914ba71350897565610e87ce0ad9e6f28569db3dd5c6d"
+checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98"
dependencies = [
"find-msvc-tools",
"jobserver",
@@ -2597,9 +2606,9 @@ dependencies = [
[[package]]
name = "hashbrown"
-version = "0.17.0"
+version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51"
+checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
[[package]]
name = "heck"
@@ -3011,7 +3020,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9"
dependencies = [
"equivalent",
- "hashbrown 0.17.0",
+ "hashbrown 0.17.1",
"serde",
"serde_core",
]
@@ -3283,9 +3292,9 @@ dependencies = [
[[package]]
name = "js-sys"
-version = "0.3.97"
+version = "0.3.98"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1840c94c045fbcf8ba2812c95db44499f7c64910a912551aaaa541decebcacf"
+checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08"
dependencies = [
"cfg-if",
"futures-util",
@@ -6089,11 +6098,12 @@ dependencies = [
[[package]]
name = "serde_with"
-version = "3.19.0"
+version = "3.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f05839ce67618e14a09b286535c0d9c94e85ef25469b0e13cb4f844e5593eb19"
+checksum = "e72c1c2cb7b223fafb600a619537a871c2818583d619401b785e7c0b746ccde2"
dependencies = [
"base64 0.22.1",
+ "bs58",
"chrono",
"hex",
"indexmap 1.9.3",
@@ -6108,9 +6118,9 @@ dependencies = [
[[package]]
name = "serde_with_macros"
-version = "3.19.0"
+version = "3.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf2ebbe86054f9b45bc3881e865683ccfaccce97b9b4cb53f3039d67f355a334"
+checksum = "b90c488738ecb4fb0262f41f43bc40efc5868d9fb744319ddf5f5317f417bfac"
dependencies = [
"darling 0.23.0",
"proc-macro2",
@@ -6851,9 +6861,9 @@ dependencies = [
[[package]]
name = "tokio"
-version = "1.52.2"
+version = "1.52.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "110a78583f19d5cdb2c5ccf321d1290344e71313c6c37d43520d386027d18386"
+checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe"
dependencies = [
"bytes",
"libc",
@@ -7704,9 +7714,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen"
-version = "0.2.120"
+version = "0.2.121"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "df52b6d9b87e0c74c9edfa1eb2d9bf85e5d63515474513aa50fa181b3c4f5db1"
+checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790"
dependencies = [
"cfg-if",
"once_cell",
@@ -7719,9 +7729,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-futures"
-version = "0.4.70"
+version = "0.4.71"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af934872acec734c2d80e6617bbb5ff4f12b052dd8e6332b0817bce889516084"
+checksum = "96492d0d3ffba25305a7dc88720d250b1401d7edca02cc3bcd50633b424673b8"
dependencies = [
"js-sys",
"wasm-bindgen",
@@ -7729,9 +7739,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro"
-version = "0.2.120"
+version = "0.2.121"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "78b1041f495fb322e64aca85f5756b2172e35cd459376e67f2a6c9dffcedb103"
+checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
@@ -7739,9 +7749,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro-support"
-version = "0.2.120"
+version = "0.2.121"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9dcd0ff20416988a18ac686d4d4d0f6aae9ebf08a389ff5d29012b05af2a1b41"
+checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2"
dependencies = [
"bumpalo",
"proc-macro2",
@@ -7752,9 +7762,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-shared"
-version = "0.2.120"
+version = "0.2.121"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49757b3c82ebf16c57d69365a142940b384176c24df52a087fb748e2085359ea"
+checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441"
dependencies = [
"unicode-ident",
]
@@ -7821,9 +7831,9 @@ dependencies = [
[[package]]
name = "web-sys"
-version = "0.3.97"
+version = "0.3.98"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2eadbac71025cd7b0834f20d1fe8472e8495821b4e9801eb0a60bd1f19827602"
+checksum = "4b572dff8bcf38bad0fa19729c89bb5748b2b9b1d8be70cf90df697e3a8f32aa"
dependencies = [
"js-sys",
"wasm-bindgen",
diff --git a/crates/backend-uzu/src/backends/common/backend.rs b/crates/backend-uzu/src/backends/common/backend.rs
index ebea0122a..3bb337516 100644
--- a/crates/backend-uzu/src/backends/common/backend.rs
+++ b/crates/backend-uzu/src/backends/common/backend.rs
@@ -1,11 +1,12 @@
use std::{error::Error, fmt::Debug};
-use super::{CommandBuffer, Context, DenseBuffer, Event, Kernels, kernel::ManualKernels};
+use super::{CommandBuffer, Context, DenseBuffer, Event, Kernels, SparseBuffer, kernel::ManualKernels};
pub trait Backend: Debug + Clone + 'static {
type Context: Context<Backend = Self>;
type CommandBuffer: CommandBuffer<Backend = Self>;
type DenseBuffer: DenseBuffer<Backend = Self>;
+ type SparseBuffer: SparseBuffer<Backend = Self>;
type Event: Event<Backend = Self>;
type Kernels: Kernels<Backend = Self> + ManualKernels;
type Error: Error + Debug;
diff --git a/crates/backend-uzu/src/backends/common/context.rs b/crates/backend-uzu/src/backends/common/context.rs
index 60d6ea19e..0e2593ad0 100644
--- a/crates/backend-uzu/src/backends/common/context.rs
+++ b/crates/backend-uzu/src/backends/common/context.rs
@@ -25,6 +25,11 @@ pub trait Context: Sized {
size: usize,
) -> Result<<Self::Backend as Backend>::DenseBuffer, <Self::Backend as Backend>::Error>;
+ fn create_buffer_with_data(
+ &self,
+ data: &[u8],
+ ) -> Result<<Self::Backend as Backend>::DenseBuffer, <Self::Backend as Backend>::Error>;
+
fn create_allocation(
&self,
size: usize,
@@ -38,6 +43,11 @@ pub trait Context: Sized {
fn create_event(&self) -> Result<<Self::Backend as Backend>::Event, <Self::Backend as Backend>::Error>;
+ fn create_sparse_buffer(
+ &self,
+ capacity: usize,
+ ) -> Result<<Self::Backend as Backend>::SparseBuffer, <Self::Backend as Backend>::Error>;
+
fn peak_memory_usage(&self) -> Option<usize>;
fn enable_capture();
diff --git a/crates/backend-uzu/src/backends/common/mod.rs b/crates/backend-uzu/src/backends/common/mod.rs
index 6e7533d97..e0a2de665 100644
--- a/crates/backend-uzu/src/backends/common/mod.rs
+++ b/crates/backend-uzu/src/backends/common/mod.rs
@@ -12,6 +12,7 @@ mod event;
pub mod gpu_types;
mod hazard_tracker;
pub mod kernel;
+mod sparse_buffer;
pub use activation_config::ActivationConfig;
pub use allocation_access::{
@@ -31,3 +32,4 @@ pub use encoder::{Completed, Encoder, Executable, Pending};
pub use event::Event;
pub use hazard_tracker::Access;
pub use kernel::Kernels;
+pub use sparse_buffer::SparseBuffer;
diff --git a/crates/backend-uzu/src/backends/common/sparse_buffer.rs b/crates/backend-uzu/src/backends/common/sparse_buffer.rs
new file mode 100644
index 000000000..ec2ae87f2
--- /dev/null
+++ b/crates/backend-uzu/src/backends/common/sparse_buffer.rs
@@ -0,0 +1,17 @@
+use std::ops::Range;
+
+use crate::backends::common::{Backend, Buffer};
+
+pub trait SparseBuffer: Buffer<Backend: Backend<SparseBuffer = Self>> {
+ fn map(
+ &mut self,
+ context: &<Self::Backend as Backend>::Context,
+ pages: &Range<usize>,
+ ) -> Result<(), <Self::Backend as Backend>::Error>;
+
+ fn unmap(
+ &mut self,
+ context: &<Self::Backend as Backend>::Context,
+ pages: &Range<usize>,
+ ) -> Result<(), <Self::Backend as Backend>::Error>;
+}
diff --git a/crates/backend-uzu/src/backends/cpu/backend.rs b/crates/backend-uzu/src/backends/cpu/backend.rs
index 557e3ea5c..aeb1caa63 100644
--- a/crates/backend-uzu/src/backends/cpu/backend.rs
+++ b/crates/backend-uzu/src/backends/cpu/backend.rs
@@ -1,6 +1,8 @@
use std::{cell::UnsafeCell, pin::Pin, sync::atomic::AtomicU64};
-use super::{command_buffer::CpuCommandBuffer, context::CpuContext, error::CpuError, kernel::CpuKernels};
+use super::{
+ command_buffer::CpuCommandBuffer, context::CpuContext, error::CpuError, kernel::CpuKernels, sparse::CpuSparseBuffer,
+};
use crate::backends::common::Backend;
#[derive(Debug, Clone)]
@@ -10,6 +12,7 @@ impl Backend for Cpu {
type Context = CpuContext;
type CommandBuffer = CpuCommandBuffer;
type DenseBuffer = UnsafeCell<Pin<Box<[u8]>>>;
+ type SparseBuffer = CpuSparseBuffer;
type Event = Pin<Box<AtomicU64>>;
type Kernels = CpuKernels;
type Error = CpuError;
diff --git a/crates/backend-uzu/src/backends/cpu/context.rs b/crates/backend-uzu/src/backends/cpu/context.rs
index 2e2f96c18..5e1975a8a 100644
--- a/crates/backend-uzu/src/backends/cpu/context.rs
+++ b/crates/backend-uzu/src/backends/cpu/context.rs
@@ -8,7 +8,7 @@ use std::{
};
use crate::backends::{
- common::{Allocation, AllocationPool, AllocationType, Allocator, Context},
+ common::{Allocation, AllocationPool, AllocationType, Allocator, Backend, Context},
cpu::{Cpu, command_buffer::CpuCommandBufferInitial, error::CpuError},
};
@@ -58,6 +58,13 @@ impl Context for CpuContext {
Ok(UnsafeCell::new(Pin::new(vec![0; size].into_boxed_slice())))
}
+ fn create_buffer_with_data(
+ &self,
+ data: &[u8],
+ ) -> Result<UnsafeCell<Pin<Box<[u8]>>>, CpuError> {
+ Ok(UnsafeCell::new(Pin::new(data.to_vec().into_boxed_slice())))
+ }
+
fn create_allocation(
&self,
size: usize,
@@ -81,6 +88,13 @@ impl Context for CpuContext {
Ok(Box::pin(AtomicU64::new(0)))
}
+ fn create_sparse_buffer(
+ &self,
+ _capacity: usize,
+ ) -> Result<<Self::Backend as Backend>::SparseBuffer, <Self::Backend as Backend>::Error> {
+ Err(CpuError::NotSupported)
+ }
+
fn peak_memory_usage(&self) -> Option<usize> {
None
}
@@ -89,9 +103,8 @@ impl Context for CpuContext {
fn start_capture(
&self,
- trace_path: &std::path::Path,
+ _trace_path: &Path,
) -> Result<(), CpuError> {
- let _ = trace_path;
Err(CpuError::NotSupported)
}
diff --git a/crates/backend-uzu/src/backends/cpu/mod.rs b/crates/backend-uzu/src/backends/cpu/mod.rs
index 93121f103..c8f54618b 100644
--- a/crates/backend-uzu/src/backends/cpu/mod.rs
+++ b/crates/backend-uzu/src/backends/cpu/mod.rs
@@ -5,7 +5,8 @@ mod dense_buffer;
mod error;
mod event;
mod kernel;
-
-pub use backend::Cpu;
+mod sparse;
pub mod argmax;
+
+pub use backend::Cpu;
diff --git a/crates/backend-uzu/src/backends/cpu/sparse/mod.rs b/crates/backend-uzu/src/backends/cpu/sparse/mod.rs
new file mode 100644
index 000000000..e27fbbe61
--- /dev/null
+++ b/crates/backend-uzu/src/backends/cpu/sparse/mod.rs
@@ -0,0 +1,3 @@
+mod sparse_buffer;
+
+pub use sparse_buffer::CpuSparseBuffer;
diff --git a/crates/backend-uzu/src/backends/cpu/sparse/sparse_buffer.rs b/crates/backend-uzu/src/backends/cpu/sparse/sparse_buffer.rs
new file mode 100644
index 000000000..c1a166c74
--- /dev/null
+++ b/crates/backend-uzu/src/backends/cpu/sparse/sparse_buffer.rs
@@ -0,0 +1,46 @@
+use std::ops::Range;
+
+use crate::backends::{
+ common::{Backend, Buffer, SparseBuffer},
+ cpu::Cpu,
+};
+
+#[derive(Debug)]
+pub struct CpuSparseBuffer {}
+
+impl Buffer for CpuSparseBuffer {
+ type Backend = Cpu;
+
+ fn gpu_ptr(&self) -> usize {
+ todo!()
+ }
+
+ fn size(&self) -> usize {
+ todo!()
+ }
+
+ fn set_label(
+ &mut self,
+ _label: Option<&str>,
+ ) {
+ todo!()
+ }
+}
+
+impl SparseBuffer for CpuSparseBuffer {
+ fn map(
+ &mut self,
+ _context: &<Self::Backend as Backend>::Context,
+ _pages: &Range<usize>,
+ ) -> Result<(), <Self::Backend as Backend>::Error> {
+ todo!()
+ }
+
+ fn unmap(
+ &mut self,
+ _context: &<Self::Backend as Backend>::Context,
+ _pages: &Range<usize>,
+ ) -> Result<(), <Self::Backend as Backend>::Error> {
+ todo!()
+ }
+}
diff --git a/crates/backend-uzu/src/backends/metal/backend.rs b/crates/backend-uzu/src/backends/metal/backend.rs
index a5e6891e9..b7cb72365 100644
--- a/crates/backend-uzu/src/backends/metal/backend.rs
+++ b/crates/backend-uzu/src/backends/metal/backend.rs
@@ -2,7 +2,7 @@ use metal::{MTLBuffer, MTLEvent};
use objc2::{rc::Retained, runtime::ProtocolObject};
use super::{command_buffer::MetalCommandBuffer, context::MetalContext, error::MetalError, kernel::MetalKernels};
-use crate::backends::common::Backend;
+use crate::backends::{common::Backend, metal::sparse::MetalSparseBuffer};
#[derive(Debug, Clone)]
pub struct Metal;
@@ -11,6 +11,7 @@ impl Backend for Metal {
type Context = MetalContext;
type CommandBuffer = MetalCommandBuffer;
type DenseBuffer = Retained<ProtocolObject<dyn MTLBuffer>>;
+ type SparseBuffer = MetalSparseBuffer;
type Event = Retained<ProtocolObject<dyn MTLEvent>>;
type Kernels = MetalKernels;
type Error = MetalError;
diff --git a/crates/backend-uzu/src/backends/metal/context.rs b/crates/backend-uzu/src/backends/metal/context.rs
index d8fc64358..af770818d 100644
--- a/crates/backend-uzu/src/backends/metal/context.rs
+++ b/crates/backend-uzu/src/backends/metal/context.rs
@@ -1,9 +1,16 @@
-use std::{cell::RefCell, collections::HashMap, path::Path, rc::Rc};
+#[cfg(test)]
+use std::cell::Ref;
+use std::{
+ cell::{RefCell, RefMut},
+ collections::HashMap,
+ path::Path,
+ rc::{Rc, Weak},
+};
use metal::{
- MTLBuffer, MTLCaptureDescriptor, MTLCaptureDestination, MTLCaptureManager, MTLCommandQueue, MTLCommandQueueExt,
- MTLComputePipelineState, MTLDevice, MTLDeviceExt, MTLEvent, MTLFunctionConstantValues, MTLLibrary,
- MTLResourceOptions,
+ MTL4CommandQueue, MTLBuffer, MTLCaptureDescriptor, MTLCaptureDestination, MTLCaptureManager, MTLCommandQueue,
+ MTLCommandQueueExt, MTLComputePipelineState, MTLDevice, MTLDeviceExt, MTLEvent, MTLFunctionConstantValues,
+ MTLLibrary, MTLResourceOptions, MTLSparsePageSize,
};
use objc2::{rc::Retained, runtime::ProtocolObject};
@@ -16,8 +23,12 @@ use super::{
};
use crate::{
backends::{
- common::{Allocation, AllocationPool, AllocationType, Allocator, Context},
- metal::command_buffer::MetalCommandBufferInitial,
+ common::{Allocation, AllocationPool, AllocationType, Allocator, Backend, Context},
+ metal::{
+ command_buffer::MetalCommandBufferInitial,
+ metal_extensions::SparsePageSizeExt,
+ sparse::{MetalSparseBuffer, MetalSparseHeapPool},
+ },
},
utils::model_size::ModelSize,
};
@@ -25,11 +36,14 @@ use crate::{
pub struct MetalContext {
pub device: Retained<ProtocolObject<dyn MTLDevice>>,
pub command_queue: Retained<ProtocolObject<dyn MTLCommandQueue>>,
+ pub command_queue4: Retained<ProtocolObject<dyn MTL4CommandQueue>>,
allocator: Rc<Allocator<Metal>>,
peak_memory_usage: RefCell<usize>,
device_capabilities: MetalDeviceCapabilities,
library: Retained<ProtocolObject<dyn MTLLibrary>>,
pipeline_cache: RefCell<HashMap<String, Retained<ProtocolObject<dyn MTLComputePipelineState>>>>,
+ sparse_heap_pool: RefCell<MetalSparseHeapPool>,
+ weak_self: Weak<MetalContext>,
}
impl MetalContext {
@@ -53,6 +67,15 @@ impl MetalContext {
Ok(pipeline)
}
+
+ #[cfg(test)]
+ pub(super) fn sparse_heap_pool(&self) -> Ref<'_, MetalSparseHeapPool> {
+ self.sparse_heap_pool.borrow()
+ }
+
+ pub(super) fn sparse_heap_pool_mut(&self) -> RefMut<'_, MetalSparseHeapPool> {
+ self.sparse_heap_pool.borrow_mut()
+ }
}
impl Context for MetalContext {
@@ -60,25 +83,34 @@ impl Context for MetalContext {
fn new() -> Result<Rc<Self>, MetalError> {
let device: Retained<ProtocolObject<dyn MTLDevice>> =
- <dyn metal::MTLDevice>::system_default().ok_or(MetalError::CannotOpenDevice)?;
+ <dyn MTLDevice>::system_default().ok_or(MetalError::CannotOpenDevice)?;
let command_queue =
device.new_command_queue_with_max_command_buffer_count(1024).ok_or(MetalError::CannotCreateCommandQueue)?;
+ let command_queue4 = device.new_mtl4_command_queue().ok_or(MetalError::CannotCreateCommandQueueMtl4)?;
+
let library = device
.new_library_with_data(kernel::MTLB)
.map_err(|nserror| MetalError::CannotCreateLibrary(nserror.to_string()))?;
let device_capabilities = MetalDeviceCapabilities::from_device(&device);
+ let page_size = MTLSparsePageSize::KB256;
+ let heap_capacity = 64 * 4 * page_size.in_bytes();
+ let sparse_pool = MetalSparseHeapPool::new(page_size, heap_capacity);
+
Ok(Rc::new_cyclic(|weak_self| Self {
device,
command_queue,
+ command_queue4,
allocator: Allocator::new(weak_self.clone()),
peak_memory_usage: RefCell::new(0),
device_capabilities,
library,
pipeline_cache: RefCell::new(HashMap::new()),
+ sparse_heap_pool: RefCell::new(sparse_pool),
+ weak_self: weak_self.clone(),
}))
}
@@ -111,11 +143,29 @@ impl Context for MetalContext {
&self,
size: usize,
) -> Result<Retained<ProtocolObject<dyn MTLBuffer>>, MetalError> {
- let buffer =
- self.device.new_buffer(size, MTLResourceOptions::STORAGE_MODE_SHARED).ok_or(MetalError::CannotCreateBuffer);
+ let buffer = self
+ .device
+ .new_buffer(size, MTLResourceOptions::STORAGE_MODE_SHARED)
+ .ok_or(MetalError::CannotCreateBuffer)?;
+
let mut peak_memory_usage_borrow = self.peak_memory_usage.borrow_mut();
*peak_memory_usage_borrow = peak_memory_usage_borrow.max(self.device.current_allocated_size());
- buffer
+
+ Ok(buffer)
+ }
+
+ fn create_buffer_with_data(
+ &self,
+ data: &[u8],
+ ) -> Result<Retained<ProtocolObject<dyn MTLBuffer>>, MetalError> {
+ let buffer = self
+ .device
+ .new_buffer_with_data(data, MTLResourceOptions::STORAGE_MODE_SHARED)
+ .ok_or(MetalError::CannotCreateBuffer)?;
+
+ let mut peak_memory_usage_borrow = self.peak_memory_usage.borrow_mut();
+ *peak_memory_usage_borrow = peak_memory_usage_borrow.max(self.device.current_allocated_size());
+ Ok(buffer)
}
fn create_allocation(
@@ -143,6 +193,15 @@ impl Context for MetalContext {
self.device.new_event().ok_or(MetalError::CannotCreateEvent)
}
+ fn create_sparse_buffer(
+ &self,
+ capacity: usize,
+ ) -> Result<<Self::Backend as Backend>::SparseBuffer, <Self::Backend as Backend>::Error> {
+ let sparse_page_size = self.sparse_heap_pool.borrow().page_size();
+ let context = self.weak_self.upgrade().ok_or(MetalError::CannotCreateBuffer)?;
+ Ok(MetalSparseBuffer::new(context, capacity, sparse_page_size)?)
+ }
+
fn peak_memory_usage(&self) -> Option<usize> {
Some(*self.peak_memory_usage.borrow())
}
@@ -155,8 +214,8 @@ impl Context for MetalContext {
fn start_capture(
&self,
- trace_path: &std::path::Path,
- ) -> Result<(), <Self::Backend as crate::backends::common::Backend>::Error> {
+ trace_path: &Path,
+ ) -> Result<(), <Self::Backend as Backend>::Error> {
let capture_manager = MTLCaptureManager::shared_capture_manager();
let capture_descriptor = MTLCaptureDescriptor::new();
capture_descriptor.set_destination(MTLCaptureDestination::GPUTraceDocument);
@@ -172,7 +231,7 @@ impl Context for MetalContext {
Ok(())
}
- fn stop_capture(&self) -> Result<(), <Self::Backend as crate::backends::common::Backend>::Error> {
+ fn stop_capture(&self) -> Result<(), <Self::Backend as Backend>::Error> {
MTLCaptureManager::shared_capture_manager().stop_capture();
Ok(())
diff --git a/crates/backend-uzu/src/backends/metal/device_capabilities.rs b/crates/backend-uzu/src/backends/metal/device_capabilities.rs
index 6908d8b91..5a74d6144 100644
--- a/crates/backend-uzu/src/backends/metal/device_capabilities.rs
+++ b/crates/backend-uzu/src/backends/metal/device_capabilities.rs
@@ -1,5 +1,5 @@
use bytesize::ByteSize;
-use metal::{MTLDevice, MTLDeviceExt as _};
+use metal::{MTLDevice, MTLDeviceExt};
use objc2::runtime::ProtocolObject;
use super::metal_extensions::{DeviceExt, DeviceGeneration};
diff --git a/crates/backend-uzu/src/backends/metal/error.rs b/crates/backend-uzu/src/backends/metal/error.rs
index 208238b05..a14e3459a 100644
--- a/crates/backend-uzu/src/backends/metal/error.rs
+++ b/crates/backend-uzu/src/backends/metal/error.rs
@@ -10,6 +10,8 @@ pub enum MetalError {
CannotCreateLibrary(String),
#[error("Cannot create command queue")]
CannotCreateCommandQueue,
+ #[error("Cannot create command Metal 4 queue")]
+ CannotCreateCommandQueueMtl4,
#[error("Cannot create buffer")]
CannotCreateBuffer,
#[error("Cannot create command buffer")]
@@ -22,4 +24,8 @@ pub enum MetalError {
CannotCreateFunction,
#[error("Cannot create pipeline state: {0}")]
CannotCreatePipelineState(String),
+ #[error("Can not allocate buffer with size={0}")]
+ SparseBufferAlloc(usize),
+ #[error("Can not allocate heap with size={0} and page size={1}")]
+ SparseHeapAlloc(usize, usize),
}
diff --git a/crates/backend-uzu/src/backends/metal/metal_extensions/mod.rs b/crates/backend-uzu/src/backends/metal/metal_extensions/mod.rs
index 3bcccae95..6ddbead8f 100644
--- a/crates/backend-uzu/src/backends/metal/metal_extensions/mod.rs
+++ b/crates/backend-uzu/src/backends/metal/metal_extensions/mod.rs
@@ -3,9 +3,11 @@ mod data_type;
mod device_extensions;
mod function_constant_values_extensions_set_value;
mod library_extensions_pipeline;
+mod sparse_page_size_extensions;
pub use compute_command_encoder_extensions_set_value::ComputeEncoderSetValue;
pub use data_type::MetalDataTypeExt;
pub use device_extensions::{DeviceExt, DeviceGeneration};
pub use function_constant_values_extensions_set_value::FunctionConstantValuesSetValue;
pub use library_extensions_pipeline::LibraryPipelineExtensions;
+pub use sparse_page_size_extensions::SparsePageSizeExt;
diff --git a/crates/backend-uzu/src/backends/metal/metal_extensions/sparse_page_size_extensions.rs b/crates/backend-uzu/src/backends/metal/metal_extensions/sparse_page_size_extensions.rs
new file mode 100644
index 000000000..2dc91318a
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/metal_extensions/sparse_page_size_extensions.rs
@@ -0,0 +1,15 @@
+use metal::MTLSparsePageSize;
+
+pub trait SparsePageSizeExt {
+ fn in_bytes(&self) -> usize;
+}
+
+impl SparsePageSizeExt for MTLSparsePageSize {
+ fn in_bytes(&self) -> usize {
+ match self {
+ MTLSparsePageSize::KB16 => 16 * 1024,
+ MTLSparsePageSize::KB64 => 64 * 1024,
+ MTLSparsePageSize::KB256 => 256 * 1024,
+ }
+ }
+}
diff --git a/crates/backend-uzu/src/backends/metal/mod.rs b/crates/backend-uzu/src/backends/metal/mod.rs
index df926e84b..43a52655b 100644
--- a/crates/backend-uzu/src/backends/metal/mod.rs
+++ b/crates/backend-uzu/src/backends/metal/mod.rs
@@ -7,6 +7,7 @@ mod error;
mod event;
mod kernel;
mod metal_extensions;
+mod sparse;
pub use backend::Metal;
pub use context::MetalContext;
diff --git a/crates/backend-uzu/src/backends/metal/sparse/mod.rs b/crates/backend-uzu/src/backends/metal/sparse/mod.rs
new file mode 100644
index 000000000..dd4321d32
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/sparse/mod.rs
@@ -0,0 +1,7 @@
+mod sparse_buffer;
+mod sparse_heap;
+mod sparse_heap_pool;
+mod sparse_utils;
+
+pub(crate) use sparse_buffer::MetalSparseBuffer;
+pub(crate) use sparse_heap_pool::MetalSparseHeapPool;
diff --git a/crates/backend-uzu/src/backends/metal/sparse/sparse_buffer.rs b/crates/backend-uzu/src/backends/metal/sparse/sparse_buffer.rs
new file mode 100644
index 000000000..cc2fedd94
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/sparse/sparse_buffer.rs
@@ -0,0 +1,123 @@
+use std::{
+ cmp::{max, min},
+ fmt::Debug,
+ ops::Range,
+ rc::Rc,
+};
+
+use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions, MTLSparsePageSize};
+use objc2::{rc::Retained, runtime::ProtocolObject};
+use rangemap::RangeSet;
+
+use crate::{
+ backends::{
+ common::{Backend, Buffer, SparseBuffer},
+ metal::{Metal, error::MetalError, metal_extensions::SparsePageSizeExt},
+ },
+ prelude::MetalContext,
+};
+
+pub struct MetalSparseBuffer {
+ buffer: Retained<ProtocolObject<dyn MTLBuffer>>,
+ mapped_pages: RangeSet<usize>,
+ context: Rc<MetalContext>,
+}
+
+impl MetalSparseBuffer {
+ pub(crate) fn new(
+ context: Rc<MetalContext>,
+ capacity: usize,
+ page_size: MTLSparsePageSize,
+ ) -> Result<Self, MetalError> {
+ let aligned_capacity = capacity.next_multiple_of(page_size.in_bytes());
+ let buffer = context
+ .device
+ .new_buffer_with_length_options_placement_sparse_page_size(
+ aligned_capacity,
+ MTLResourceOptions::STORAGE_MODE_PRIVATE,
+ page_size,
+ )
+ .ok_or(MetalError::SparseBufferAlloc(aligned_capacity))?;
+
+ Ok(Self {
+ buffer,
+ mapped_pages: RangeSet::new(),
+ context,
+ })
+ }
+}
+
+impl Debug for MetalSparseBuffer {
+ fn fmt(
+ &self,
+ f: &mut std::fmt::Formatter<'_>,
+ ) -> std::fmt::Result {
+ f.debug_struct("MetalSparseBuffer")
+ .field("mapped_pages", &self.mapped_pages)
+ .field("buffer", &self.buffer)
+ .finish_non_exhaustive()
+ }
+}
+
+impl Drop for MetalSparseBuffer {
+ fn drop(&mut self) {
+ let context = self.context.clone();
+ for range in self.mapped_pages.iter() {
+ context.sparse_heap_pool_mut().unmap(&context, &self.buffer, &range).expect("Failed to unmap");
+ }
+ }
+}
+
+impl Buffer for MetalSparseBuffer {
+ type Backend = Metal;
+
+ fn gpu_ptr(&self) -> usize {
+ self.buffer.gpu_ptr()
+ }
+
+ fn size(&self) -> usize {
+ self.buffer.size()
+ }
+
+ fn set_label(
+ &mut self,
+ label: Option<&str>,
+ ) {
+ self.buffer.set_label(label)
+ }
+}
+
+impl SparseBuffer for MetalSparseBuffer {
+ fn map(
+ &mut self,
+ context: &<Self::Backend as Backend>::Context,
+ pages: &Range<usize>,
+ ) -> Result<(), <Self::Backend as Backend>::Error> {
+ self.mapped_pages.gaps(pages).collect::<Vec<_>>().into_iter().try_for_each(|gap| {
+ context.sparse_heap_pool_mut().map(context, &self.buffer, &gap)?;
+ self.mapped_pages.insert(gap);
+ Ok(())
+ })
+ }
+
+ fn unmap(
+ &mut self,
+ context: &<Self::Backend as Backend>::Context,
+ pages: &Range<usize>,
+ ) -> Result<(), <Self::Backend as Backend>::Error> {
+ self.mapped_pages
+ .overlapping(pages)
+ .map(|range| max(range.start, pages.start)..min(range.end, pages.end))
+ .collect::<Vec<_>>()
+ .into_iter()
+ .try_for_each(|range| {
+ context.sparse_heap_pool_mut().unmap(context, &self.buffer, &range)?;
+ self.mapped_pages.remove(range);
+ Ok(())
+ })
+ }
+}
+
+#[cfg(test)]
+#[path = "../../../../tests/unit/backends/metal/sparse/sparse_buffer_test.rs"]
+mod tests;
diff --git a/crates/backend-uzu/src/backends/metal/sparse/sparse_heap.rs b/crates/backend-uzu/src/backends/metal/sparse/sparse_heap.rs
new file mode 100644
index 000000000..241ca0c32
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/sparse/sparse_heap.rs
@@ -0,0 +1,123 @@
+use std::{collections::HashMap, ops::Range};
+
+use metal::{
+ MTL4CommandQueue, MTL4CommandQueueExt, MTL4UpdateSparseBufferMappingOperation, MTLBuffer, MTLDeviceExt, MTLHeap,
+ MTLHeapDescriptor, MTLHeapType, MTLSparsePageSize, MTLSparseTextureMappingMode, MTLStorageMode,
+};
+use objc2::{rc::Retained, runtime::ProtocolObject};
+use rangemap::{RangeMap, RangeSet};
+
+use crate::{
+ backends::metal::{
+ error::MetalError,
+ metal_extensions::SparsePageSizeExt,
+ sparse::sparse_utils::{MetalSparseHeapBufferMapping, MetalSparseHeapMappingParameters},
+ },
+ prelude::MetalContext,
+};
+
+pub(super) struct MetalSparseHeap {
+ heap: Retained<ProtocolObject<dyn MTLHeap>>,
+ // Mappings are tracked per buffer (keyed by gpu_address) so multiple
+ // buffers can alias overlapping heap page ranges without losing track of
+ // either side.
+ buffer_mappings: HashMap<u64, RangeMap<usize, MetalSparseHeapBufferMapping>>,
+ free_pages: RangeSet<usize>,
+}
+
+impl MetalSparseHeap {
+ pub fn new(
+ context: &MetalContext,
+ capacity_bytes: usize,
+ page_size: MTLSparsePageSize,
+ ) -> Result<Self, MetalError> {
+ let aligned_capacity = capacity_bytes.next_multiple_of(page_size.in_bytes());
+
+ let heap_desc = MTLHeapDescriptor::new();
+ // Sparse buffers must be backed by a Placement heap with a sparse page size set;
+ // `MTLHeapType::Sparse` is for sparse textures and trips a runtime assertion when
+ // passed to updateBufferMappings.
+ heap_desc.set_type(MTLHeapType::Placement);
+ heap_desc.set_storage_mode(MTLStorageMode::Private);
+ heap_desc.set_size(aligned_capacity);
+ heap_desc.set_max_compatible_placement_sparse_page_size(page_size);
+ let heap = context
+ .device
+ .new_heap_with_descriptor(&heap_desc)
+ .ok_or(MetalError::SparseHeapAlloc(aligned_capacity, page_size.in_bytes()))?;
+
+ let mut free_pages = RangeSet::new();
+ free_pages.insert(0..(aligned_capacity / page_size.in_bytes()));
+ Ok(Self {
+ heap,
+ buffer_mappings: HashMap::new(),
+ free_pages,
+ })
+ }
+
+ pub fn free_pages(&self) -> &RangeSet<usize> {
+ &self.free_pages
+ }
+
+ /// Iterates `(heap_range, mapping)` pairs that belong to a single buffer.
+ pub fn mappings_for(
+ &self,
+ buffer_address: u64,
+ ) -> impl Iterator<Item = (Range<usize>, &MetalSparseHeapBufferMapping)> + '_ {
+ self.buffer_mappings
+ .get(&buffer_address)
+ .into_iter()
+ .flat_map(|m| m.iter().map(|(r, mapping)| (r.clone(), mapping)))
+ }
+
+ pub fn is_empty(&self) -> bool {
+ self.buffer_mappings.is_empty()
+ }
+
+ /// Command queue calls doesn't have any synchronization.
+ /// It is the responsibility of the caller.
+ pub fn execute(
+ &mut self,
+ buffer: &ProtocolObject<dyn MTLBuffer>,
+ cmd_queue: &ProtocolObject<dyn MTL4CommandQueue>,
+ operations: &[MetalSparseHeapMappingParameters],
+ map: bool,
+ ) {
+ if operations.is_empty() {
+ return;
+ }
+
+ let mtl_operations: Vec<MTL4UpdateSparseBufferMappingOperation> = operations
+ .iter()
+ .map(|op| {
+ let mode = if map {
+ MTLSparseTextureMappingMode::Map
+ } else {
+ MTLSparseTextureMappingMode::Unmap
+ };
+ MTL4UpdateSparseBufferMappingOperation::new(mode, op.buffer_pages.clone(), op.heap_page_offset)
+ })
+ .collect();
+
+ cmd_queue.update_buffer_mappings(buffer, Some(&self.heap), &mtl_operations);
+
+ let buffer_address = buffer.gpu_address();
+ let entry = self.buffer_mappings.entry(buffer_address).or_default();
+ mtl_operations.iter().for_each(|mtl_op| {
+ let heap_range = mtl_op.heap_offset..(mtl_op.heap_offset + mtl_op.buffer_range().len());
+ if map {
+ let buffer_range = mtl_op.buffer_range();
+ let buffer_mapping = MetalSparseHeapBufferMapping::new(mtl_op.heap_offset, buffer_range.start);
+ entry.insert(heap_range.clone(), buffer_mapping);
+ self.free_pages.remove(heap_range);
+ } else {
+ entry.remove(heap_range.clone());
+ self.free_pages.insert(heap_range);
+ }
+ });
+
+ if entry.is_empty() {
+ self.buffer_mappings.remove(&buffer_address);
+ }
+ }
+}
diff --git a/crates/backend-uzu/src/backends/metal/sparse/sparse_heap_pool.rs b/crates/backend-uzu/src/backends/metal/sparse/sparse_heap_pool.rs
new file mode 100644
index 000000000..fe63a06fd
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/sparse/sparse_heap_pool.rs
@@ -0,0 +1,165 @@
+use std::{
+ cmp::{max, min},
+ ops::Range,
+};
+
+use metal::{MTLBuffer, MTLSparsePageSize};
+use objc2::runtime::ProtocolObject;
+
+use crate::{
+ backends::metal::{
+ error::MetalError,
+ metal_extensions::SparsePageSizeExt,
+ sparse::{sparse_heap::MetalSparseHeap, sparse_utils::MetalSparseHeapMappingParameters},
+ },
+ prelude::MetalContext,
+};
+
+pub struct MetalSparseHeapPool {
+ heaps: Vec<MetalSparseHeap>,
+ heap_capacity: usize,
+ page_size: MTLSparsePageSize,
+}
+
+impl MetalSparseHeapPool {
+ pub fn new(
+ page_size: MTLSparsePageSize,
+ heap_capacity: usize,
+ ) -> Self {
+ Self {
+ heaps: Vec::new(),
+ page_size,
+ heap_capacity,
+ }
+ }
+
+ pub fn map(
+ &mut self,
+ context: &MetalContext,
+ buffer: &ProtocolObject<dyn MTLBuffer>,
+ buffer_pages: &Range<usize>,
+ ) -> Result<(), MetalError> {
+ if buffer_pages.len() == 0 {
+ return Ok(());
+ }
+
+ let mut pages_to_map = buffer_pages.clone();
+ let heap_capacity_pages = self.heap_capacity_pages();
+
+ // Try to find pages in existing heaps.
+ // While `pages_to_map` is not empty in each heap find unmapped pages, collect them into `mappings` and map
+ let mut existing_heaps_mappings: Vec<(usize, Box<[MetalSparseHeapMappingParameters]>)> = Vec::new();
+ for (i, heap) in self.heaps.iter_mut().enumerate() {
+ let mut mappings: Vec<MetalSparseHeapMappingParameters> = Vec::new();
+
+ for free_pages in heap.free_pages().iter() {
+ let map_pages_count = min(free_pages.len(), pages_to_map.len());
+ let mapping = MetalSparseHeapMappingParameters {
+ buffer_pages: pages_to_map.start..(pages_to_map.start + map_pages_count),
+ heap_page_offset: free_pages.start,
+ };
+ mappings.push(mapping);
+
+ pages_to_map.start += map_pages_count;
+ if pages_to_map.len() == 0 {
+ break;
+ }
+ }
+
+ heap.execute(buffer, &context.command_queue4, &mappings, true);
+ existing_heaps_mappings.push((i, mappings.into_boxed_slice()));
+
+ if pages_to_map.len() == 0 {
+ break;
+ }
+ }
+
+ // If `pages_to_map` still not empty, it's necessary to allocate new heaps
+ let new_heaps_required = pages_to_map.len().div_ceil(heap_capacity_pages);
+ for _ in 0..new_heaps_required {
+ let map_pages_count = min(heap_capacity_pages, pages_to_map.len());
+ let buffer_pages = pages_to_map.start..(pages_to_map.start + map_pages_count);
+ let op_params = MetalSparseHeapMappingParameters {
+ buffer_pages,
+ heap_page_offset: 0,
+ };
+
+ match MetalSparseHeap::new(context, self.heap_capacity, self.page_size) {
+ Ok(mut heap) => {
+ let operations = [op_params];
+ heap.execute(buffer, &context.command_queue4, &operations, true);
+ self.heaps.push(heap);
+ pages_to_map.start += map_pages_count;
+ existing_heaps_mappings.push((self.heaps.len() - 1, Box::new(operations)));
+ },
+ Err(err) => {
+ // it's necessary to unmap previously mapped pages in existing heaps
+ for (heap_pos, mappings) in existing_heaps_mappings {
+ self.heaps[heap_pos].execute(buffer, &context.command_queue4, &mappings, false);
+ }
+ return Err(err);
+ },
+ };
+ }
+
+ Ok(())
+ }
+
+ pub fn unmap(
+ &mut self,
+ context: &MetalContext,
+ buffer: &ProtocolObject<dyn MTLBuffer>,
+ buffer_pages: &Range<usize>,
+ ) -> Result<(), MetalError> {
+ let buffer_address = buffer.gpu_address();
+
+ // Iterate over heaps, find mappings for `buffer_address`, unmap them
+ self.heaps.iter_mut().for_each(|heap| {
+ let unmappings: Vec<MetalSparseHeapMappingParameters> = heap
+ .mappings_for(buffer_address)
+ .filter_map(|(heap_range, mapping)| {
+ let mapped_buffer_pages = mapping.buffer_pages_for(&heap_range);
+ let unmap_start = max(buffer_pages.start, mapped_buffer_pages.start);
+ let unmap_end = min(buffer_pages.end, mapped_buffer_pages.end);
+ if unmap_start >= unmap_end {
+ return None;
+ };
+
+ let offset_within_range = unmap_start - mapped_buffer_pages.start;
+ Some(MetalSparseHeapMappingParameters {
+ buffer_pages: unmap_start..unmap_end,
+ heap_page_offset: heap_range.start + offset_within_range,
+ })
+ })
+ .collect();
+ heap.execute(buffer, &context.command_queue4, &unmappings, false);
+ });
+
+ // Remove empty heaps
+ self.heaps.retain(|heap| !heap.is_empty());
+
+ Ok(())
+ }
+
+ #[cfg(test)]
+ pub fn heap_capacity_bytes(&self) -> usize {
+ self.heap_capacity
+ }
+
+ pub fn heap_capacity_pages(&self) -> usize {
+ self.heap_capacity / self.page_size.in_bytes()
+ }
+
+ #[cfg(test)]
+ pub fn heaps_count(&self) -> usize {
+ self.heaps.len()
+ }
+
+ pub fn page_size(&self) -> MTLSparsePageSize {
+ self.page_size
+ }
+}
+
+#[cfg(test)]
+#[path = "../../../../tests/unit/backends/metal/sparse/sparse_heap_pool_test.rs"]
+mod tests;
diff --git a/crates/backend-uzu/src/backends/metal/sparse/sparse_utils.rs b/crates/backend-uzu/src/backends/metal/sparse/sparse_utils.rs
new file mode 100644
index 000000000..103de7d1f
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/sparse/sparse_utils.rs
@@ -0,0 +1,36 @@
+use std::ops::Range;
+
+#[derive(Clone, PartialEq)]
+pub(super) struct MetalSparseHeapBufferMapping {
+ // Anchors for an originally inserted contiguous mapping. They survive
+ // rangemap splits unchanged, so the heap↔buffer correspondence can always
+ // be recovered from any surviving heap subrange via `buffer_pages_for`.
+ heap_page_anchor: usize,
+ buffer_page_anchor: usize,
+}
+
+impl MetalSparseHeapBufferMapping {
+ pub fn new(
+ heap_page_anchor: usize,
+ buffer_page_anchor: usize,
+ ) -> Self {
+ Self {
+ heap_page_anchor,
+ buffer_page_anchor,
+ }
+ }
+
+ pub fn buffer_pages_for(
+ &self,
+ heap_range: &Range<usize>,
+ ) -> Range<usize> {
+ let start = self.buffer_page_anchor + (heap_range.start - self.heap_page_anchor);
+ let end = self.buffer_page_anchor + (heap_range.end - self.heap_page_anchor);
+ start..end
+ }
+}
+
+pub(super) struct MetalSparseHeapMappingParameters {
+ pub(super) buffer_pages: Range<usize>,
+ pub(super) heap_page_offset: usize,
+}
diff --git a/crates/backend-uzu/tests/unit/backends/metal/sparse/sparse_buffer_test.rs b/crates/backend-uzu/tests/unit/backends/metal/sparse/sparse_buffer_test.rs
new file mode 100644
index 000000000..22f5315bf
--- /dev/null
+++ b/crates/backend-uzu/tests/unit/backends/metal/sparse/sparse_buffer_test.rs
@@ -0,0 +1,203 @@
+use std::{ops::Range, rc::Rc};
+
+use crate::backends::{
+ common::{Backend, Context, SparseBuffer},
+ metal::Metal,
+};
+
+fn create_context() -> Rc<<Metal as Backend>::Context> {
+ <Metal as Backend>::Context::new().expect("Failed to create Metal context")
+}
+
+/// Capacity sized to fit `heap_count` worth of heaps.
+fn buffer_capacity(
+ ctx: &<Metal as Backend>::Context,
+ heap_count: usize,
+) -> usize {
+ heap_count * ctx.sparse_heap_pool().heap_capacity_bytes()
+}
+
+/// Buffer-page range covering `heap_count` worth of heaps starting at `heap_offset`.
+fn pages_for_heaps(
+ ctx: &<Metal as Backend>::Context,
+ heap_offset: usize,
+ heap_count: usize,
+) -> Range<usize> {
+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();
+ (heap_offset * pages_per_heap)..((heap_offset + heap_count) * pages_per_heap)
+}
+
+#[test]
+fn test_mapping_succeeds() {
+ let ctx = create_context();
+ let capacity = buffer_capacity(ctx.as_ref(), 4);
+ let pages = pages_for_heaps(ctx.as_ref(), 0, 4);
+
+ let mut sparse_buffer = ctx.create_sparse_buffer(capacity).expect("Failed to create sparse buffer");
+ sparse_buffer.map(ctx.as_ref(), &pages).expect("Failed to map sparse buffer");
+}
+
+#[test]
+fn test_unmapping_succeeds() {
+ let ctx = create_context();
+ let capacity = buffer_capacity(ctx.as_ref(), 4);
+ let pages = pages_for_heaps(ctx.as_ref(), 0, 4);
+
+ let mut sparse_buffer = ctx.create_sparse_buffer(capacity).expect("Failed to create sparse buffer");
+ sparse_buffer.map(ctx.as_ref(), &pages).expect("Failed to map sparse buffer");
+ sparse_buffer.unmap(ctx.as_ref(), &pages).expect("Failed to unmap sparse buffer");
+}
+
+#[test]
+fn test_partial_unmapping_succeeds() {
+ let ctx = create_context();
+ let capacity = buffer_capacity(ctx.as_ref(), 4);
+ let mapped = pages_for_heaps(ctx.as_ref(), 0, 4);
+ let unmapped = pages_for_heaps(ctx.as_ref(), 1, 2);
+
+ let mut sparse_buffer = ctx.create_sparse_buffer(capacity).expect("Failed to create sparse buffer");
+ sparse_buffer.map(ctx.as_ref(), &mapped).expect("Failed to map sparse buffer");
+ sparse_buffer.unmap(ctx.as_ref(), &unmapped).expect("Failed to unmap sparse buffer");
+}
+
+#[test]
+fn test_mapping_uses_minimum_heaps() {
+ let ctx = create_context();
+ let heap_count = 4;
+ let capacity = buffer_capacity(ctx.as_ref(), heap_count);
+ let pages = pages_for_heaps(ctx.as_ref(), 0, heap_count);
+
+ let mut sparse_buffer = ctx.create_sparse_buffer(capacity).expect("Failed to create sparse buffer");
+ sparse_buffer.map(ctx.as_ref(), &pages).expect("Failed to map sparse buffer");
+
+ let heaps = ctx.sparse_heap_pool();
+ let expected = pages.len().div_ceil(heaps.heap_capacity_pages());
+ assert_eq!(heaps.heaps_count(), expected, "mapping should allocate the minimum number of heaps");
+}
+
+#[test]
+fn test_remapping_reuses_freed_pages() {
+ let ctx = create_context();
+ let capacity = buffer_capacity(ctx.as_ref(), 6);
+ let initial_pages = pages_for_heaps(ctx.as_ref(), 0, 4);
+ let unmapped_pages = pages_for_heaps(ctx.as_ref(), 1, 1);
+ let extra_pages = pages_for_heaps(ctx.as_ref(), 4, 1);
+
+ let mut sparse_buffer = ctx.create_sparse_buffer(capacity).expect("Failed to create sparse buffer");
+ sparse_buffer.map(ctx.as_ref(), &initial_pages).expect("Failed to map sparse buffer");
+ let initial_heaps = ctx.sparse_heap_pool().heaps_count();
+
+ sparse_buffer.unmap(ctx.as_ref(), &unmapped_pages).expect("Failed to unmap sparse buffer");
+ sparse_buffer.map(ctx.as_ref(), &extra_pages).expect("Failed to remap sparse buffer");
+
+ assert_eq!(
+ ctx.sparse_heap_pool().heaps_count(),
+ initial_heaps,
+ "remapping after unmap should reuse freed heap pages instead of allocating new heaps",
+ );
+}
+
+#[test]
+fn test_full_unmap_then_remap_reuses_heaps() {
+ let ctx = create_context();
+ let capacity = buffer_capacity(ctx.as_ref(), 4);
+ let pages = pages_for_heaps(ctx.as_ref(), 0, 4);
+
+ let mut sparse_buffer = ctx.create_sparse_buffer(capacity).expect("Failed to create sparse buffer");
+ sparse_buffer.map(ctx.as_ref(), &pages).expect("Failed to map sparse buffer");
+ let initial_heaps = ctx.sparse_heap_pool().heaps_count();
+
+ sparse_buffer.unmap(ctx.as_ref(), &pages).expect("Failed to unmap sparse buffer");
+ sparse_buffer.map(ctx.as_ref(), &pages).expect("Failed to remap sparse buffer");
+
+ assert_eq!(
+ ctx.sparse_heap_pool().heaps_count(),
+ initial_heaps,
+ "remapping the same range after a full unmap should not grow the heap pool",
+ );
+}
+
+#[test]
+fn test_remapping_same_pages_is_noop() {
+ let ctx = create_context();
+ let capacity = buffer_capacity(ctx.as_ref(), 4);
+ let pages = pages_for_heaps(ctx.as_ref(), 0, 4);
+
+ let mut sparse_buffer = ctx.create_sparse_buffer(capacity).expect("Failed to create sparse buffer");
+ sparse_buffer.map(ctx.as_ref(), &pages).expect("Failed to map sparse buffer");
+ let initial_heaps = ctx.sparse_heap_pool().heaps_count();
+
+ sparse_buffer.map(ctx.as_ref(), &pages).expect("Failed to remap sparse buffer");
+
+ assert_eq!(
+ ctx.sparse_heap_pool().heaps_count(),
+ initial_heaps,
+ "remapping the same pages should not allocate new heaps",
+ );
+}
+
+#[test]
+fn test_drop_releases_pool_heaps() {
+ // Regression: a mapped buffer dropped without an explicit unmap must
+ // still release its heap pages back to the shared pool.
+ let ctx = create_context();
+ let capacity = buffer_capacity(ctx.as_ref(), 2);
+ let pages = pages_for_heaps(ctx.as_ref(), 0, 2);
+
+ {
+ let mut sparse_buffer = ctx.create_sparse_buffer(capacity).expect("Failed to create sparse buffer");
+ sparse_buffer.map(ctx.as_ref(), &pages).expect("Failed to map sparse buffer");
+ assert!(ctx.sparse_heap_pool().heaps_count() > 0);
+ }
+
+ assert_eq!(
+ ctx.sparse_heap_pool().heaps_count(),
+ 0,
+ "dropping a mapped sparse buffer should release its heap pages",
+ );
+}
+
+#[test]
+fn test_drop_does_not_disturb_other_buffer_mappings() {
+ // Dropping one buffer must not unmap heap pages held by another buffer
+ // sharing the same pool.
+ let ctx = create_context();
+ let capacity = buffer_capacity(ctx.as_ref(), 1);
+ let pages = pages_for_heaps(ctx.as_ref(), 0, 1);
+
+ let mut keeper = ctx.create_sparse_buffer(capacity).expect("Failed to create keeper buffer");
+ keeper.map(ctx.as_ref(), &pages).expect("Failed to map keeper buffer");
+ let baseline = ctx.sparse_heap_pool().heaps_count();
+
+ {
+ let mut transient = ctx.create_sparse_buffer(capacity).expect("Failed to create transient buffer");
+ transient.map(ctx.as_ref(), &pages).expect("Failed to map transient buffer");
+ }
+
+ assert_eq!(
+ ctx.sparse_heap_pool().heaps_count(),
+ baseline,
+ "dropping a transient buffer must leave keeper buffer's mappings intact",
+ );
+}
+
+#[test]
+fn test_sequential_mappings_are_compact() {
+ let ctx = create_context();
+ let capacity = buffer_capacity(ctx.as_ref(), 4);
+ let first = pages_for_heaps(ctx.as_ref(), 0, 2);
+ let second = pages_for_heaps(ctx.as_ref(), 2, 2);
+
+ let mut sparse_buffer = ctx.create_sparse_buffer(capacity).expect("Failed to create sparse buffer");
+ sparse_buffer.map(ctx.as_ref(), &first).expect("Failed to map first range");
+ sparse_buffer.map(ctx.as_ref(), &second).expect("Failed to map second range");
+
+ let heaps = ctx.sparse_heap_pool();
+ let total_pages = first.len() + second.len();
+ let expected = total_pages.div_ceil(heaps.heap_capacity_pages());
+ assert_eq!(
+ heaps.heaps_count(),
+ expected,
+ "two adjacent mappings should pack into the same minimum number of heaps",
+ );
+}
diff --git a/crates/backend-uzu/tests/unit/backends/metal/sparse/sparse_heap_pool_test.rs b/crates/backend-uzu/tests/unit/backends/metal/sparse/sparse_heap_pool_test.rs
new file mode 100644
index 000000000..0615e5a21
--- /dev/null
+++ b/crates/backend-uzu/tests/unit/backends/metal/sparse/sparse_heap_pool_test.rs
@@ -0,0 +1,399 @@
+use std::rc::Rc;
+
+use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};
+use objc2::{rc::Retained, runtime::ProtocolObject};
+
+use crate::{
+ backends::{
+ common::{Backend, Context},
+ metal::{Metal, metal_extensions::SparsePageSizeExt},
+ },
+ prelude::MetalContext,
+};
+
+fn create_context() -> Rc<MetalContext> {
+ <Metal as Backend>::Context::new().expect("Failed to create Metal context")
+}
+
+fn create_sparse_buffer(
+ ctx: &MetalContext,
+ capacity_bytes: usize,
+) -> Retained<ProtocolObject<dyn MTLBuffer>> {
+ let page_size = ctx.sparse_heap_pool().page_size();
+ let aligned = capacity_bytes.next_multiple_of(page_size.in_bytes());
+ ctx.device
+ .new_buffer_with_length_options_placement_sparse_page_size(
+ aligned,
+ MTLResourceOptions::STORAGE_MODE_PRIVATE,
+ page_size,
+ )
+ .expect("Failed to create sparse buffer")
+}
+
+#[test]
+fn test_new_pool_is_empty() {
+ let ctx = create_context();
+ assert_eq!(ctx.sparse_heap_pool().heaps_count(), 0);
+}
+
+#[test]
+fn test_map_empty_range_allocates_no_heaps() {
+ let ctx = create_context();
+ let cap = ctx.sparse_heap_pool().heap_capacity_bytes();
+ let buffer = create_sparse_buffer(ctx.as_ref(), cap);
+
+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(0..0)).expect("map empty range");
+
+ assert_eq!(ctx.sparse_heap_pool().heaps_count(), 0);
+}
+
+#[test]
+fn test_map_single_heap_allocates_one_heap() {
+ let ctx = create_context();
+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();
+ let cap = ctx.sparse_heap_pool().heap_capacity_bytes();
+ let buffer = create_sparse_buffer(ctx.as_ref(), cap);
+
+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(0..pages_per_heap)).expect("map");
+
+ assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);
+}
+
+#[test]
+fn test_map_multi_heap_allocates_minimum_heaps() {
+ let ctx = create_context();
+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();
+ let cap = 4 * ctx.sparse_heap_pool().heap_capacity_bytes();
+ let buffer = create_sparse_buffer(ctx.as_ref(), cap);
+
+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(0..4 * pages_per_heap)).expect("map");
+
+ assert_eq!(ctx.sparse_heap_pool().heaps_count(), 4);
+}
+
+#[test]
+fn test_map_partial_heap_rounds_up_heap_count() {
+ let ctx = create_context();
+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();
+ let cap = 2 * ctx.sparse_heap_pool().heap_capacity_bytes();
+ let buffer = create_sparse_buffer(ctx.as_ref(), cap);
+
+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(0..pages_per_heap + 1)).expect("map");
+
+ assert_eq!(ctx.sparse_heap_pool().heaps_count(), 2);
+}
+
+#[test]
+fn test_sequential_mappings_pack_into_one_heap() {
+ let ctx = create_context();
+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();
+ let half = pages_per_heap / 2;
+ let cap = ctx.sparse_heap_pool().heap_capacity_bytes();
+ let buffer = create_sparse_buffer(ctx.as_ref(), cap);
+
+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(0..half)).expect("map first");
+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(half..pages_per_heap)).expect("map second");
+
+ assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);
+}
+
+#[test]
+fn test_map_overflows_existing_heap_into_new_heap() {
+ let ctx = create_context();
+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();
+ let half = pages_per_heap / 2;
+ let cap = 2 * ctx.sparse_heap_pool().heap_capacity_bytes();
+ let buffer = create_sparse_buffer(ctx.as_ref(), cap);
+
+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(0..half)).expect("map first");
+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(half..half + pages_per_heap)).expect("map overflow");
+
+ assert_eq!(ctx.sparse_heap_pool().heaps_count(), 2);
+}
+
+#[test]
+fn test_two_buffers_share_one_heap() {
+ let ctx = create_context();
+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();
+ let half = pages_per_heap / 2;
+ let cap = ctx.sparse_heap_pool().heap_capacity_bytes();
+ let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);
+ let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);
+
+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..half)).expect("map a");
+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..half)).expect("map b");
+
+ assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);
+}
+
+#[test]
+fn test_full_unmap_removes_heaps() {
+ let ctx = create_context();
+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();
+ let cap = 2 * ctx.sparse_heap_pool().heap_capacity_bytes();
+ let buffer = create_sparse_buffer(ctx.as_ref(), cap);
+ let pages = 0..2 * pages_per_heap;
+
+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &pages).expect("map");
+ ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer, &pages).expect("unmap");
+
+ assert_eq!(ctx.sparse_heap_pool().heaps_count(), 0);
+}
+
+#[test]
+fn test_partial_unmap_keeps_heap() {
+ let ctx = create_context();
+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();
+ let cap = ctx.sparse_heap_pool().heap_capacity_bytes();
+ let buffer = create_sparse_buffer(ctx.as_ref(), cap);
+
+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(0..pages_per_heap)).expect("map");
+ ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer, &(0..pages_per_heap / 2)).expect("partial unmap");
+
+ assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);
+}
+
+#[test]
+fn test_unmap_with_other_buffer_keeps_mappings() {
+ let ctx = create_context();
+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();
+ let cap = ctx.sparse_heap_pool().heap_capacity_bytes();
+ let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);
+ let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);
+ let pages = 0..pages_per_heap;
+
+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &pages).expect("map a");
+ ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer_b, &pages).expect("unmap b");
+
+ assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);
+}
+
+#[test]
+fn test_unmap_one_buffer_keeps_heap_for_other() {
+ let ctx = create_context();
+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();
+ let half = pages_per_heap / 2;
+ let cap = ctx.sparse_heap_pool().heap_capacity_bytes();
+ let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);
+ let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);
+
+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..half)).expect("map a");
+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..half)).expect("map b");
+ ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer_a, &(0..half)).expect("unmap a");
+
+ assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);
+}
+
+#[test]
+fn test_remap_after_unmap_does_not_grow_pool() {
+ let ctx = create_context();
+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();
+ let cap = 2 * ctx.sparse_heap_pool().heap_capacity_bytes();
+ let buffer = create_sparse_buffer(ctx.as_ref(), cap);
+ let pages = 0..2 * pages_per_heap;
+
+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &pages).expect("map");
+ let initial = ctx.sparse_heap_pool().heaps_count();
+ ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer, &pages).expect("unmap");
+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &pages).expect("remap");
+
+ assert_eq!(ctx.sparse_heap_pool().heaps_count(), initial);
+}
+
+#[test]
+fn test_remap_into_freed_gap_reuses_heap() {
+ let ctx = create_context();
+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();
+ let half = pages_per_heap / 2;
+ let cap = ctx.sparse_heap_pool().heap_capacity_bytes();
+ let buffer = create_sparse_buffer(ctx.as_ref(), cap);
+
+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(0..pages_per_heap)).expect("map full");
+ ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer, &(0..half)).expect("unmap half");
+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(0..half)).expect("remap into gap");
+
+ assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);
+}
+
+#[test]
+fn test_sequential_partial_unmaps_release_full_heap() {
+ // Regression: after a partial unmap splits a rangemap entry, the surviving
+ // entry must still report the correct buffer↔heap correspondence so that a
+ // subsequent unmap of the remaining suffix targets the right heap pages.
+ let ctx = create_context();
+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();
+ let half = pages_per_heap / 2;
+ let cap = ctx.sparse_heap_pool().heap_capacity_bytes();
+ let buffer = create_sparse_buffer(ctx.as_ref(), cap);
+
+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(0..pages_per_heap)).expect("map");
+ ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer, &(0..half)).expect("unmap prefix");
+ ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer, &(half..pages_per_heap)).expect("unmap suffix");
+
+ assert_eq!(ctx.sparse_heap_pool().heaps_count(), 0);
+}
+
+#[test]
+fn test_three_buffers_share_one_heap() {
+ let ctx = create_context();
+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();
+ let third = pages_per_heap / 3;
+ let cap = ctx.sparse_heap_pool().heap_capacity_bytes();
+ let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);
+ let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);
+ let buffer_c = create_sparse_buffer(ctx.as_ref(), cap);
+
+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..third)).expect("map a");
+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..third)).expect("map b");
+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_c, &(0..third)).expect("map c");
+
+ assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);
+}
+
+#[test]
+fn test_two_buffers_exactly_fill_one_heap() {
+ let ctx = create_context();
+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();
+ let half = pages_per_heap / 2;
+ let cap = ctx.sparse_heap_pool().heap_capacity_bytes();
+ let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);
+ let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);
+
+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..half)).expect("map a");
+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..pages_per_heap - half)).expect("map b fills heap");
+
+ assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);
+}
+
+#[test]
+fn test_third_buffer_spills_when_heap_filled_by_others() {
+ let ctx = create_context();
+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();
+ let half = pages_per_heap / 2;
+ let cap = ctx.sparse_heap_pool().heap_capacity_bytes();
+ let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);
+ let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);
+ let buffer_c = create_sparse_buffer(ctx.as_ref(), cap);
+
+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..half)).expect("map a");
+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..pages_per_heap - half)).expect("map b fills heap");
+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_c, &(0..1)).expect("map c spills");
+
+ assert_eq!(ctx.sparse_heap_pool().heaps_count(), 2);
+}
+
+#[test]
+fn test_unmap_buffer_frees_pages_for_another_in_same_heap() {
+ let ctx = create_context();
+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();
+ let half = pages_per_heap / 2;
+ let cap = ctx.sparse_heap_pool().heap_capacity_bytes();
+ let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);
+ let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);
+ let buffer_c = create_sparse_buffer(ctx.as_ref(), cap);
+
+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..half)).expect("map a");
+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..pages_per_heap - half)).expect("map b fills heap");
+ ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer_a, &(0..half)).expect("unmap a");
+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_c, &(0..half)).expect("map c into gap");
+
+ assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);
+}
+
+#[test]
+fn test_unmap_middle_of_three_buffers_keeps_heap() {
+ let ctx = create_context();
+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();
+ let third = pages_per_heap / 3;
+ let cap = ctx.sparse_heap_pool().heap_capacity_bytes();
+ let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);
+ let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);
+ let buffer_c = create_sparse_buffer(ctx.as_ref(), cap);
+
+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..third)).expect("map a");
+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..third)).expect("map b");
+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_c, &(0..third)).expect("map c");
+ ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer_b, &(0..third)).expect("unmap b");
+
+ assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);
+}
+
+#[test]
+fn test_unmap_all_shared_buffers_releases_heap() {
+ let ctx = create_context();
+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();
+ let third = pages_per_heap / 3;
+ let cap = ctx.sparse_heap_pool().heap_capacity_bytes();
+ let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);
+ let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);
+ let buffer_c = create_sparse_buffer(ctx.as_ref(), cap);
+
+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..third)).expect("map a");
+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..third)).expect("map b");
+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_c, &(0..third)).expect("map c");
+ ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer_a, &(0..third)).expect("unmap a");
+ ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer_b, &(0..third)).expect("unmap b");
+ ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer_c, &(0..third)).expect("unmap c");
+
+ assert_eq!(ctx.sparse_heap_pool().heaps_count(), 0);
+}
+
+#[test]
+fn test_second_buffer_spans_existing_and_new_heap() {
+ let ctx = create_context();
+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();
+ let half = pages_per_heap / 2;
+ let cap = 2 * ctx.sparse_heap_pool().heap_capacity_bytes();
+ let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);
+ let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);
+
+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..half)).expect("map a");
+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..pages_per_heap)).expect("map b spans heaps");
+
+ assert_eq!(ctx.sparse_heap_pool().heaps_count(), 2);
+}
+
+#[test]
+fn test_unmap_spanning_buffer_keeps_heap_with_other_buffer() {
+ let ctx = create_context();
+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();
+ let half = pages_per_heap / 2;
+ let cap = 2 * ctx.sparse_heap_pool().heap_capacity_bytes();
+ let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);
+ let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);
+
+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..half)).expect("map a");
+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..pages_per_heap)).expect("map b spans heaps");
+ ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer_b, &(0..pages_per_heap)).expect("unmap b");
+
+ assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);
+}
+
+#[test]
+fn test_remap_two_buffers_after_unmap_does_not_grow_pool() {
+ let ctx = create_context();
+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();
+ let half = pages_per_heap / 2;
+ let cap = ctx.sparse_heap_pool().heap_capacity_bytes();
+ let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);
+ let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);
+
+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..half)).expect("map a");
+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..half)).expect("map b");
+ let initial = ctx.sparse_heap_pool().heaps_count();
+ ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer_a, &(0..half)).expect("unmap a");
+ ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer_b, &(0..half)).expect("unmap b");
+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..half)).expect("remap a");
+ ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..half)).expect("remap b");
+
+ assert_eq!(ctx.sparse_heap_pool().heaps_count(), initial);
+}
+
+#[test]
+fn test_heap_capacity_pages_matches_byte_capacity() {
+ let ctx = create_context();
+ let pool = ctx.sparse_heap_pool();
+ let page_size_bytes = pool.page_size().in_bytes();
+
+ assert_eq!(pool.heap_capacity_pages(), pool.heap_capacity_bytes() / page_size_bytes);
+}