A high-performance inference engine for AI models
KV cache sparse buffers (#434)
Aleksandr Golokoz28fed454ee07
diff --git a/Cargo.lock b/Cargo.lockindex 3fd5ea8d9..91095f9e7 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -3831,9 +3831,9 @@ dependencies = [[[package]]name = "mtl-rs"-version = "0.1.11"+version = "0.1.12"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "88e5c70772dfcf1f3f89d873fbd7e039d9d291b192b84df2b5f462e3d3597fba"+checksum = "b5da07ba3cea3adad7b479c3d1b097875e8c56e9711830b72c8c2b5f5cf0ce8f"dependencies = ["bitflags 2.11.1","block2",diff --git a/Cargo.toml b/Cargo.tomlindex 60c112317..63e88a91e 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -69,7 +69,7 @@ tokenizers = { version = "0.22", features = [objc2 = "0.6.4"objc2-foundation = "0.3.2"objc2-security = "0.3.2"-metal = { package = "mtl-rs", version = "0.1" }+metal = { package = "mtl-rs", version = "0.1.12" }block2 = "0.6"bytemuck = "1.23"bytesize = "2"diff --git a/crates/backend-uzu/src/backends/common/command_buffer.rs b/crates/backend-uzu/src/backends/common/command_buffer.rsindex 2702dbdb7..a4190b047 100644--- a/crates/backend-uzu/src/backends/common/command_buffer.rs+++ b/crates/backend-uzu/src/backends/common/command_buffer.rs@@ -1,6 +1,6 @@use std::time::Duration;-use super::{Backend, BufferRangeMut, BufferRangeRef};+use super::{Backend, Buffer, BufferRangeMut, BufferRangeRef};pub trait CommandBuffer {type Backend: Backend<CommandBuffer = Self>;@@ -70,15 +70,18 @@ impl AccessFlags {pub trait CommandBufferEncoding {type CommandBuffer: CommandBuffer<Encoding = Self>;- fn encode_copy(+ fn encode_copy<+ Src: Buffer<Backend = <Self::CommandBuffer as CommandBuffer>::Backend>,+ Dst: Buffer<Backend = <Self::CommandBuffer as CommandBuffer>::Backend>,+ >(&mut self,- src: BufferRangeRef<'_, <<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::DenseBuffer>,- dst: BufferRangeMut<'_, <<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::DenseBuffer>,+ src: BufferRangeRef<'_, Src>,+ dst: BufferRangeMut<'_, Dst>,);- fn encode_fill(+ fn encode_fill<Dst: Buffer<Backend = <Self::CommandBuffer as CommandBuffer>::Backend>>(&mut self,- dst: BufferRangeMut<'_, <<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::DenseBuffer>,+ dst: BufferRangeMut<'_, Dst>,value: u8,);diff --git a/crates/backend-uzu/src/backends/common/context.rs b/crates/backend-uzu/src/backends/common/context.rsindex 8c2d47326..9e429ea7e 100644--- a/crates/backend-uzu/src/backends/common/context.rs+++ b/crates/backend-uzu/src/backends/common/context.rs@@ -52,6 +52,8 @@ pub trait Context: Sized {fn stop_capture(&self) -> Result<(), <Self::Backend as Backend>::Error>;+ fn sparse_buffers_supported(&self) -> bool;+fn tf32_enabled() -> bool {env::var("UZU_TF32").map(|v| v == "1" || v.eq_ignore_ascii_case("true")).unwrap_or(false)}diff --git a/crates/backend-uzu/src/backends/common/encoder.rs b/crates/backend-uzu/src/backends/common/encoder.rsindex a791ab952..81c8ff052 100644--- a/crates/backend-uzu/src/backends/common/encoder.rs+++ b/crates/backend-uzu/src/backends/common/encoder.rs@@ -4,7 +4,7 @@ use std::{};use crate::backends::common::{- AccessFlags, Allocation, AllocationPool, AllocationType, AsBufferRangeMut, AsBufferRangeRef, Backend,+ AccessFlags, Allocation, AllocationPool, AllocationType, AsBufferRangeMut, AsBufferRangeRef, Backend, Buffer,BufferGpuAddressRangeExt, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable,CommandBufferInitial, CommandBufferPending, Context,hazard_tracker::{Access, HazardTracker},@@ -79,11 +79,14 @@ impl<'encoding, B: Backend> Encoder<'encoding, B> {)}- pub fn encode_copy(+ pub fn encode_copy<+ Src: AsBufferRangeRef<Buffer: Buffer<Backend = B>>,+ Dst: AsBufferRangeMut<Buffer: Buffer<Backend = B>>,+ >(&mut self,- src: &Allocation<B>,+ src: &Src,src_range: impl RangeBounds<usize>,- dst: &mut Allocation<B>,+ dst: &mut Dst,dst_range: impl RangeBounds<usize>,) {let src_buffer_range = src.as_buffer_range_ref();@@ -108,9 +111,9 @@ impl<'encoding, B: Backend> Encoder<'encoding, B> {self.command_buffer.encode_copy(src_buffer_range, dst_buffer_range);}- pub fn encode_fill(+ pub fn encode_fill<Dst: AsBufferRangeMut<Buffer: Buffer<Backend = B>>>(&mut self,- dst: &mut Allocation<B>,+ dst: &mut Dst,value: u8,) {let dst_buffer_range = dst.as_buffer_range_mut();diff --git a/crates/backend-uzu/src/backends/common/kernel/attention.rs b/crates/backend-uzu/src/backends/common/kernel/attention.rsindex 5fcf577c1..d162b91d1 100644--- a/crates/backend-uzu/src/backends/common/kernel/attention.rs+++ b/crates/backend-uzu/src/backends/common/kernel/attention.rs@@ -6,7 +6,7 @@ use std::{use crate::{DataType,backends::common::{- Allocation, Backend, Encoder, Kernels,+ Allocation, AsBufferRangeRef, Backend, Buffer, Encoder, Kernels,gpu_types::{AttnParams, ring::RingParams},kernel::AttentionGemmKernel,},@@ -14,10 +14,10 @@ use crate::{const BQ: usize = 32;-pub struct AttentionGemmArguments<'a, B: Backend> {+pub struct AttentionGemmArguments<'a, B: Backend, KVBuf: AsBufferRangeRef> {pub queries: &'a Allocation<B>,- pub keys: &'a Allocation<B>,- pub values: &'a Allocation<B>,+ pub keys: &'a KVBuf,+ pub values: &'a KVBuf,pub output: &'a mut Allocation<B>,pub trie: Option<&'a Allocation<B>>,pub sinks: Option<&'a Allocation<B>>,@@ -56,10 +56,10 @@ impl<B: Backend> AttentionGemmBlock<B> {}}- pub fn encode(+ pub fn encode<KVBuf: AsBufferRangeRef<Buffer: Buffer<Backend = B>>>(&self,encoder: &mut Encoder<B>,- args: AttentionGemmArguments<B>,+ args: AttentionGemmArguments<B, KVBuf>,) -> Result<(), B::Error> {let bk: usize = if args.head_dim < 128 {32diff --git a/crates/backend-uzu/src/backends/common/kernel/kv_cache_update.rs b/crates/backend-uzu/src/backends/common/kernel/kv_cache_update.rsindex 6adb42def..2da61b6a3 100644--- a/crates/backend-uzu/src/backends/common/kernel/kv_cache_update.rs+++ b/crates/backend-uzu/src/backends/common/kernel/kv_cache_update.rs@@ -4,13 +4,13 @@ use thiserror::Error;use crate::{DataType,- backends::common::{Allocation, Backend, Encoder, Kernels, gpu_types::Swap, kernel::KVCacheUpdateKernel},+ backends::common::{Backend, Buffer, Encoder, Kernels, gpu_types::Swap, kernel::KVCacheUpdateKernel},};-pub struct KVLayerData<'a, B: Backend> {- pub key_allocation: &'a mut Allocation<B>,+pub struct KVLayerData<'a, B: Backend, Buf: Buffer<Backend = B>> {+ pub key_buffer: &'a mut Buf,pub key_shape: [usize; 3],- pub value_allocation: &'a mut Allocation<B>,+ pub value_buffer: &'a mut Buf,pub value_shape: [usize; 3],}@@ -53,9 +53,9 @@ impl<B: Backend> KVCacheUpdate<B> {}/// Encode the KV cache update operation using a provided command buffer- pub fn encode(+ pub fn encode<Buf: Buffer<Backend = B>>(&self,- in_place_data: &mut [KVLayerData<B>],+ in_place_data: &mut [KVLayerData<B, Buf>],source_indices: &[usize],destination_indices: &[usize],encoder: &mut Encoder<B>,@@ -80,8 +80,8 @@ impl<B: Backend> KVCacheUpdate<B> {// non-inline is not supported yet (and is broken anyways due to a data race)for swaps_chunk in swaps.chunks(max_inline_swaps) {self.kernel.encode(- &mut *layer_data.key_allocation,- &mut *layer_data.value_allocation,+ &mut *layer_data.key_buffer,+ &mut *layer_data.value_buffer,swaps_chunk,swaps_chunk.len() as u32,num_heads as u32,diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul.rs b/crates/backend-uzu/src/backends/common/kernel/matmul.rsindex 4986dbe47..3578ac4d5 100644--- a/crates/backend-uzu/src/backends/common/kernel/matmul.rs+++ b/crates/backend-uzu/src/backends/common/kernel/matmul.rs@@ -2,7 +2,7 @@ use thiserror::Error;use crate::{DataType,- backends::common::{Allocation, Backend, Encoder, kernel::ManualKernels},+ backends::common::{Allocation, AsBufferRangeRef, Backend, Buffer, Encoder, kernel::ManualKernels},};#[derive(Debug, Error)]@@ -22,12 +22,12 @@ pub enum MatmulArgumentC<'a, B: Backend> {}// D = ab_scale * (A @ op(B)) + C, where op(B) = B^T when b_transpose else B.-pub struct MatmulArguments<'a, B: Backend> {+pub struct MatmulArguments<'a, B: Backend, TB: AsBufferRangeRef = Allocation<B>> {/// A: [M, K]pub a: &'a Allocation<B>,pub a_offset: usize,/// B: [N, K] when b_transpose, else [K, N]- pub b: &'a Allocation<B>,+ pub b: &'a TB,pub b_offset: usize,pub b_leading_dimension: Option<u32>,pub b_transpose: bool,@@ -53,9 +53,9 @@ pub trait MatmulKernel: Sized {data_type: DataType,) -> Result<Self, MatmulError<Self::Backend>>;- fn encode(+ fn encode<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Self::Backend>>>(&mut self,- arguments: MatmulArguments<Self::Backend>,+ arguments: MatmulArguments<Self::Backend, TB>,encoder: &mut Encoder<Self::Backend>,);}diff --git a/crates/backend-uzu/src/backends/common/mod.rs b/crates/backend-uzu/src/backends/common/mod.rsindex e25acdfc9..24074b4a0 100644--- a/crates/backend-uzu/src/backends/common/mod.rs+++ b/crates/backend-uzu/src/backends/common/mod.rs@@ -30,4 +30,4 @@ pub use dense_buffer::DenseBuffer;pub use encoder::{Completed, Encoder, Executable, Pending};pub use hazard_tracker::Access;pub use kernel::Kernels;-pub use sparse_buffer::SparseBuffer;+pub use sparse_buffer::{SparseBuffer, SparseBufferExt};diff --git a/crates/backend-uzu/src/backends/common/sparse_buffer.rs b/crates/backend-uzu/src/backends/common/sparse_buffer.rsindex ec2ae87f2..5b522c1bc 100644--- a/crates/backend-uzu/src/backends/common/sparse_buffer.rs+++ b/crates/backend-uzu/src/backends/common/sparse_buffer.rs@@ -14,4 +14,14 @@ pub trait SparseBuffer: Buffer<Backend: Backend<SparseBuffer = Self>> {context: &<Self::Backend as Backend>::Context,pages: &Range<usize>,) -> Result<(), <Self::Backend as Backend>::Error>;++ fn page_size_bytes(&self) -> usize;+}++pub trait SparseBufferExt: SparseBuffer {+ fn total_pages(&self) -> usize {+ self.size() / self.page_size_bytes()+ }}++impl<B: SparseBuffer> SparseBufferExt for B {}diff --git a/crates/backend-uzu/src/backends/cpu/command_buffer.rs b/crates/backend-uzu/src/backends/cpu/command_buffer.rsindex dd32ab9cd..7cb79d790 100644--- a/crates/backend-uzu/src/backends/cpu/command_buffer.rs+++ b/crates/backend-uzu/src/backends/cpu/command_buffer.rs@@ -6,10 +6,10 @@ use std::{use crate::{backends::{common::{- AccessFlags, Backend, BufferRangeMut, BufferRangeRef, CommandBuffer, CommandBufferCompleted,+ AccessFlags, Buffer, BufferRangeMut, BufferRangeRef, CommandBuffer, CommandBufferCompleted,CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,},- cpu::{Cpu, error::CpuError},+ cpu::{BufferDowncastExt, Cpu, error::CpuError},},utils::pointers::{SendPtr, SendPtrMut},};@@ -68,31 +68,34 @@ impl CpuCommandBufferEncoding {impl CommandBufferEncoding for CpuCommandBufferEncoding {type CommandBuffer = CpuCommandBuffer;- fn encode_copy(+ fn encode_copy<Src: Buffer<Backend = Cpu>, Dst: Buffer<Backend = Cpu>>(&mut self,- src: BufferRangeRef<'_, <Cpu as Backend>::DenseBuffer>,- dst: BufferRangeMut<'_, <Cpu as Backend>::DenseBuffer>,+ src: BufferRangeRef<'_, Src>,+ dst: BufferRangeMut<'_, Dst>,) {let src_range = src.range();let dst_range = dst.range();- let size = src_range.end - src_range.start;- assert_eq!(size, dst_range.end - dst_range.start);+ assert_eq!(src_range.len(), dst_range.len());++ let src_buffer = src.buffer().downcast();+ let src_ptr = SendPtr(unsafe { (&*src_buffer.get()).as_ptr().add(src_range.start) });++ let dst_buffer = dst.buffer().downcast();+ let dst_ptr = SendPtrMut(unsafe { (&mut *dst_buffer.get()).as_mut_ptr().add(dst_range.start) });- let src_ptr = SendPtr(unsafe { (&*src.buffer().get()).as_ptr().add(src_range.start) });- let dst_ptr = SendPtrMut(unsafe { (&mut *dst.buffer().get()).as_mut_ptr().add(dst_range.start) });self.push_command(move || unsafe {- std::ptr::copy(src_ptr.as_ptr(), dst_ptr.as_ptr(), size);+ std::ptr::copy(src_ptr.as_ptr(), dst_ptr.as_ptr(), src_range.len());});}- fn encode_fill(+ fn encode_fill<Dst: Buffer<Backend = Cpu>>(&mut self,- dst: BufferRangeMut<'_, <Cpu as Backend>::DenseBuffer>,+ dst: BufferRangeMut<'_, Dst>,value: u8,) {let range = dst.range();let size = range.end - range.start;- let dst = SendPtrMut(unsafe { (&mut *dst.buffer().get()).as_mut_ptr().add(range.start) });+ let dst = SendPtrMut(unsafe { (&mut *dst.buffer().downcast().get()).as_mut_ptr().add(range.start) });self.push_command(move || unsafe {dst.as_ptr().write_bytes(value, size);});diff --git a/crates/backend-uzu/src/backends/cpu/context.rs b/crates/backend-uzu/src/backends/cpu/context.rsindex 2d208e349..deeb3edfc 100644--- a/crates/backend-uzu/src/backends/cpu/context.rs+++ b/crates/backend-uzu/src/backends/cpu/context.rs@@ -93,4 +93,8 @@ impl Context for CpuContext {fn stop_capture(&self) -> Result<(), CpuError> {Err(CpuError::NotSupported)}++ fn sparse_buffers_supported(&self) -> bool {+ false+ }}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rsindex 3cc122c5d..9cac8dce3 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs@@ -4,10 +4,10 @@ use crate::{DataType,backends::{common::{- AsBufferRangeMut, AsBufferRangeRef, Encoder,+ AsBufferRangeMut, AsBufferRangeRef, Buffer, Encoder,kernel::matmul::{MatmulArgumentC, MatmulArguments, MatmulError, MatmulKernel},},- cpu::{Cpu, context::CpuContext},+ cpu::{BufferDowncastExt, Cpu, context::CpuContext},},utils::pointers::{SendPtr, SendPtrMut},};@@ -31,9 +31,9 @@ impl MatmulKernel for MatmulCpuKernel {})}- fn encode(+ fn encode<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Cpu>>>(&mut self,- arguments: MatmulArguments<Cpu>,+ arguments: MatmulArguments<Cpu, TB>,encoder: &mut Encoder<Cpu>,) {let MatmulArguments {@@ -70,7 +70,7 @@ impl MatmulKernel for MatmulCpuKernel {let d_offset = d_buffer_range.range().start;let a_ptr = SendPtr(unsafe { &*a_buffer_range.buffer().get() }.as_ptr().wrapping_byte_add(a_offset));- let b_ptr = SendPtr(unsafe { &*b_buffer_range.buffer().get() }.as_ptr().wrapping_byte_add(b_offset));+ let b_ptr = SendPtr(unsafe { &*b_buffer_range.buffer().downcast().get() }.as_ptr().wrapping_byte_add(b_offset));let d_ptr =SendPtrMut(unsafe { (&*d_buffer_range.buffer().get()).as_ptr().wrapping_byte_add(d_offset) as *mut u8 });diff --git a/crates/backend-uzu/src/backends/cpu/sparse/sparse_buffer.rs b/crates/backend-uzu/src/backends/cpu/sparse/sparse_buffer.rsindex 8958146a1..5b0a4da9e 100644--- a/crates/backend-uzu/src/backends/cpu/sparse/sparse_buffer.rs+++ b/crates/backend-uzu/src/backends/cpu/sparse/sparse_buffer.rs@@ -36,4 +36,8 @@ impl SparseBuffer for CpuSparseBuffer {) -> Result<(), <Self::Backend as Backend>::Error> {todo!()}++ fn page_size_bytes(&self) -> usize {+ todo!()+ }}diff --git a/crates/backend-uzu/src/backends/metal/command_buffer.rs b/crates/backend-uzu/src/backends/metal/command_buffer.rsindex ad276fca7..a55e6dc1a 100644--- a/crates/backend-uzu/src/backends/metal/command_buffer.rs+++ b/crates/backend-uzu/src/backends/metal/command_buffer.rs@@ -6,11 +6,11 @@ use metal::{};use objc2::{Message, rc::Retained, runtime::ProtocolObject};-use super::Metal;+use super::{BufferDowncastExt, Metal};use crate::{backends::{common::{- AccessFlags, Backend, BufferRangeMut, BufferRangeRef, CommandBuffer, CommandBufferCompleted,+ AccessFlags, Buffer, BufferRangeMut, BufferRangeRef, CommandBuffer, CommandBufferCompleted,CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,},metal::error::MetalError,@@ -128,29 +128,34 @@ impl Drop for MetalCommandBufferEncoding {impl CommandBufferEncoding for MetalCommandBufferEncoding {type CommandBuffer = MetalCommandBuffer;- fn encode_copy(+ fn encode_copy<Src: Buffer<Backend = Metal>, Dst: Buffer<Backend = Metal>>(&mut self,- src: BufferRangeRef<'_, <Metal as Backend>::DenseBuffer>,- dst: BufferRangeMut<'_, <Metal as Backend>::DenseBuffer>,+ src: BufferRangeRef<'_, Src>,+ dst: BufferRangeMut<'_, Dst>,) {let src_range = src.range();let dst_range = dst.range();- let size = src_range.end - src_range.start;- assert_eq!(size, dst_range.end - dst_range.start);-- self.ensure_blit().copy_buffer_to_buffer(src.buffer(), src_range.start, dst.buffer(), dst_range.start, size);+ assert_eq!(src_range.len(), dst_range.len());++ self.ensure_blit().copy_buffer_to_buffer(+ src.buffer().downcast(),+ src_range.start,+ dst.buffer().downcast(),+ dst_range.start,+ src_range.len(),+ );}- fn encode_fill(+ fn encode_fill<Dst: Buffer<Backend = Metal>>(&mut self,- dst: BufferRangeMut<'_, <Metal as Backend>::DenseBuffer>,+ dst: BufferRangeMut<'_, Dst>,value: u8,) {let range = dst.range();assert!(range.end > range.start);assert!(range.start % 4 == 0 && range.end % 4 == 0);- self.ensure_blit().fill_buffer_range_value(dst.buffer(), range, value);+ self.ensure_blit().fill_buffer_range_value(dst.buffer().downcast(), range, value);}fn encode_barrier(diff --git a/crates/backend-uzu/src/backends/metal/context.rs b/crates/backend-uzu/src/backends/metal/context.rsindex 82142170f..ef90189b1 100644--- a/crates/backend-uzu/src/backends/metal/context.rs+++ b/crates/backend-uzu/src/backends/metal/context.rs@@ -1,22 +1,22 @@-#[cfg(test)]-use std::cell::Ref;use std::{- cell::{RefCell, RefMut},+ cell::{Ref, RefCell, RefMut},collections::HashMap,path::Path,rc::{Rc, Weak},sync::atomic::{AtomicU64, Ordering},};+#[cfg(test)]+use metal::MTLSharedEvent;use metal::{- MTL4CommandQueue, MTLBuffer, MTLCaptureDescriptor, MTLCaptureDestination, MTLCaptureManager, MTLCommandQueue,- MTLCommandQueueExt, MTLComputePipelineState, MTLDevice, MTLDeviceExt, MTLEvent, MTLFunctionConstantValues,- MTLLibrary, MTLResourceOptions, MTLSparsePageSize,+ MTL4CommandQueue, MTL4CommandQueueExt, MTLBuffer, MTLCaptureDescriptor, MTLCaptureDestination, MTLCaptureManager,+ MTLCommandQueue, MTLCommandQueueExt, MTLComputePipelineState, MTLDevice, MTLDeviceExt, MTLEvent,+ MTLFunctionConstantValues, MTLLibrary, MTLResourceOptions, MTLSparsePageSize,};use objc2::{rc::Retained, runtime::ProtocolObject};use super::{- Metal,+ DeviceExt, Metal,device_capabilities::MetalDeviceCapabilities,error::MetalError,kernel,@@ -28,7 +28,7 @@ use crate::{metal::{command_buffer::MetalCommandBufferInitial,metal_extensions::SparsePageSizeExt,- sparse::{MetalSparseBuffer, MetalSparseHeapPool},+ sparse::{MetalSparseBuffer, MetalSparseHeapPool, MetalSparseMappingOpsBatch},},},utils::model_size::ModelSize,@@ -47,6 +47,8 @@ pub struct MetalContext {pipeline_cache: RefCell<HashMap<String, Retained<ProtocolObject<dyn MTLComputePipelineState>>>>,sparse_heap_pool: RefCell<MetalSparseHeapPool>,weak_self: Weak<MetalContext>,+ #[cfg(test)]+ timeline_shared_event: Retained<ProtocolObject<dyn MTLSharedEvent>>,}impl MetalContext {@@ -71,7 +73,6 @@ impl MetalContext {Ok(pipeline)}- #[cfg(test)]pub(super) fn sparse_heap_pool(&self) -> Ref<'_, MetalSparseHeapPool> {self.sparse_heap_pool.borrow()}@@ -80,6 +81,26 @@ impl MetalContext {self.sparse_heap_pool.borrow_mut()}+ pub(super) fn sparse_update_mappings(+ &self,+ mappings: &[MetalSparseMappingOpsBatch],+ ) {+ if mappings.is_empty() {+ return;+ }++ let wait_value = self.timeline_get_and_increment();+ self.command_queue4.wait_for_event_value(&self.timeline_event, wait_value);+ for op in mappings {+ self.command_queue4.update_buffer_mappings(&op.buffer, Some(op.heap.borrow().heap()), &op.mtl_operations);+ }+ self.command_queue4.signal_event_value(&self.timeline_event, wait_value + 1);++ // This line prevent tests from freezing, showing pink screen and shutting down computer+ #[cfg(test)]+ self.timeline_shared_event.wait_until_signaled_value_timeout_ms(wait_value, 10);+ }+pub(super) fn timeline_get_and_increment(&self) -> u64 {self.timeline_value.fetch_add(1, Ordering::Release)}@@ -111,6 +132,8 @@ impl Context for MetalContext {let heap_capacity = 64 * 4 * page_size.in_bytes();let sparse_pool = MetalSparseHeapPool::new(page_size, heap_capacity);let timeline_event = device.new_event().ok_or(MetalError::CannotCreateEvent)?;+ #[cfg(test)]+ let timeline_shared_event = device.new_shared_event().ok_or(MetalError::CannotCreateEvent)?;Ok(Rc::new_cyclic(|weak_self| Self {device,@@ -125,6 +148,8 @@ impl Context for MetalContext {pipeline_cache: RefCell::new(HashMap::new()),sparse_heap_pool: RefCell::new(sparse_pool),weak_self: weak_self.clone(),+ #[cfg(test)]+ timeline_shared_event,}))}@@ -233,4 +258,8 @@ impl Context for MetalContext {Ok(())}++ fn sparse_buffers_supported(&self) -> bool {+ self.device.supports_placement_sparse_resources()+ }}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/dispatch.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/dispatch.rsindex 57e38cbbb..7b24036a7 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/dispatch.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/dispatch.rs@@ -1,13 +1,17 @@use crate::{DataType,- backends::common::{- Allocation, Backend,- gpu_types::{- GemmParams, QuantizationMode,- gemm::{- GemmAlignment, GemmInputPrologueKind, GemmOutputTransformKind, GemmTilingConfig, GemmWeightPrologueKind,+ backends::{+ common::{+ Allocation, AsBufferRangeRef, Backend, Buffer,+ gpu_types::{+ GemmParams, QuantizationMode,+ gemm::{+ GemmAlignment, GemmInputPrologueKind, GemmOutputTransformKind, GemmTilingConfig,+ GemmWeightPrologueKind,+ },},},+ metal::Metal,},};@@ -27,19 +31,19 @@ impl GemmAlignment {}}-pub enum GemmWeights<'a, B: Backend> {+pub enum GemmWeights<'a, B: Backend, T: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>> = Allocation<B>> {FullPrecision {- weights: &'a Allocation<B>,+ weights: &'a T,},ScaleBias {- weights: &'a Allocation<B>,+ weights: &'a T,scales: &'a Allocation<B>,biases: &'a Allocation<B>,mode: QuantizationMode,group_size: u32,},ScaleZeroPoint {- weights: &'a Allocation<B>,+ weights: &'a T,scales: &'a Allocation<B>,zero_points: &'a Allocation<B>,mode: QuantizationMode,@@ -47,7 +51,7 @@ pub enum GemmWeights<'a, B: Backend> {},}-impl<B: Backend> GemmWeights<'_, B> {+impl<B: Backend, T: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>> GemmWeights<'_, B, T> {pub fn weight_prologue(&self) -> GemmWeightPrologueKind {match self {Self::FullPrecision {@@ -95,7 +99,7 @@ impl<B: Backend> GemmWeights<'_, B> {}}-pub struct GemmDispatch<'a, B: Backend> {+pub struct GemmDispatch<'a, B: Backend, T: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>> {pub tiling_config: GemmTilingConfig,pub input_prologue: GemmInputPrologueKind,pub use_mxu: bool,@@ -104,7 +108,7 @@ pub struct GemmDispatch<'a, B: Backend> {pub transpose_b: bool,pub a: &'a Allocation<B>,pub a_offset: usize,- pub b: GemmWeights<'a, B>,+ pub b: GemmWeights<'a, B, T>,pub b_offset: usize,pub d: &'a mut Allocation<B>,pub params: GemmParams,@@ -112,7 +116,7 @@ pub struct GemmDispatch<'a, B: Backend> {pub group_count_y: u32,}-impl<B: Backend> GemmDispatch<'_, B> {+impl<B: Backend, T: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>> GemmDispatch<'_, B, T> {pub(crate) fn specialization(&self) -> GemmSpecialization {GemmSpecialization {tiling_config: self.tiling_config,diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/fp.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/fp.rsindex 4f39ce56b..7ea4fb243 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/fp.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/fp.rs@@ -3,7 +3,7 @@ use crate::{DataType,backends::{common::{- Encoder,+ AsBufferRangeRef, Buffer, Encoder,gpu_types::{GemmParams,gemm::{GemmAlignment, GemmInputPrologueKind, GemmOutputTransformKind, GemmTilingConfig},@@ -17,13 +17,13 @@ use crate::{},};-pub(crate) fn encode(+pub(crate) fn encode<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(gemm: &mut GemmKernel,bias_add: &mut TensorAddBiasMetalKernel,data_type: DataType,context: &MetalContext,encoder: &mut Encoder<Metal>,- arguments: MatmulArguments<Metal>,+ arguments: MatmulArguments<Metal, TB>,use_mxu: bool,) -> Result<(), MatmulError<Metal>> {if use_mxu && !context.device.supports_mxu() {@@ -121,9 +121,9 @@ pub(crate) fn encode(Ok(())}-fn select_simdgroup_tile(+fn select_simdgroup_tile<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(data_type: DataType,- arguments: &MatmulArguments<Metal>,+ arguments: &MatmulArguments<Metal, TB>,) -> GemmTilingConfig {let (threadgroup_m, threadgroup_n, threadgroup_k) = match data_type {DataType::F32 => (32u32, 64u32, 16u32),@@ -144,7 +144,9 @@ fn select_simdgroup_tile(}}-fn select_mxu_tile(arguments: &MatmulArguments<Metal>) -> GemmTilingConfig {+fn select_mxu_tile<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(+ arguments: &MatmulArguments<Metal, TB>+) -> GemmTilingConfig {let (threadgroup_m, threadgroup_n, simdgroups_m, simdgroups_n) =if arguments.batch_dim >= 256 && arguments.output_dim >= 128 {(128u32, 128u32, 4u32, 4u32)@@ -164,7 +166,9 @@ fn select_mxu_tile(arguments: &MatmulArguments<Metal>) -> GemmTilingConfig {}}-fn output_transform_from(arguments: &MatmulArguments<Metal>) -> GemmOutputTransformKind {+fn output_transform_from<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(+ arguments: &MatmulArguments<Metal, TB>+) -> GemmOutputTransformKind {let scale = arguments.ab_scale != 1.0;let accumulate = matches!(arguments.c, MatmulArgumentC::Accumulate);match (scale, accumulate) {diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rsindex 44a925987..5f00d0280 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs@@ -5,7 +5,7 @@ use crate::{DataType,backends::{common::{- Encoder,+ AsBufferRangeRef, Buffer, Encoder,gpu_types::gemm::{GemmInputPrologueKind, GemmWeightPrologueKind},},metal::{Metal, context::MetalContext, error::MetalError, kernel::GemmMetalKernel},@@ -62,10 +62,10 @@ impl GemmKernel {}}- pub(crate) fn encode(+ pub(crate) fn encode<T: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(&mut self,context: &MetalContext,- dispatch: GemmDispatch<'_, Metal>,+ dispatch: GemmDispatch<'_, Metal, T>,encoder: &mut Encoder<Metal>,) -> Result<(), MetalError> {let specialization = dispatch.specialization();diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/fp.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/fp.rsindex 90441c3db..8d3baf734 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/fp.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/fp.rs@@ -1,16 +1,16 @@use super::{kernel::GemvKernel, spec::GemvSpecialization};use crate::backends::{common::{- Encoder,+ AsBufferRangeRef, Buffer, Encoder,kernel::matmul::{MatmulArgumentC, MatmulArguments, MatmulError},},metal::Metal,};-pub(crate) fn encode(+pub(crate) fn encode<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(kernel: &mut GemvKernel,encoder: &mut Encoder<Metal>,- arguments: MatmulArguments<Metal>,+ arguments: MatmulArguments<Metal, TB>,) -> Result<(), MatmulError<Metal>> {let MatmulArguments {a,diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rsindex 464bbd0d8..8f803c24a 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs@@ -9,7 +9,7 @@ use crate::{DataType,backends::{common::{- Encoder,+ AsBufferRangeRef, Buffer, Encoder,kernel::{TensorAddBiasKernel,matmul::{MatmulArguments, MatmulError, MatmulKernel},@@ -101,9 +101,9 @@ impl MatmulKernel for MatmulMetalKernel {})}- fn encode(+ fn encode<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(&mut self,- arguments: MatmulArguments<Metal>,+ arguments: MatmulArguments<Metal, TB>,encoder: &mut Encoder<Metal>,) {// gemv only handles the canonical B layout; non-canonical inputs go to GEMM.diff --git a/crates/backend-uzu/src/backends/metal/metal_extensions/device_extensions.rs b/crates/backend-uzu/src/backends/metal/metal_extensions/device_extensions.rsindex 8eb91bbb2..aca8e93a7 100644--- a/crates/backend-uzu/src/backends/metal/metal_extensions/device_extensions.rs+++ b/crates/backend-uzu/src/backends/metal/metal_extensions/device_extensions.rs@@ -157,6 +157,15 @@ pub trait DeviceExt: MTLDevice + Message + NSObjectProtocol + Sized {false}}++ /// Whether the GPU supports placement sparse resources.+ fn supports_placement_sparse_resources(&self) -> bool {+ if self.respondsToSelector(sel!(supportsPlacementSparse)) {+ unsafe { raw_msg_send(self, sel!(supportsPlacementSparse)) }+ } else {+ false+ }+ }}impl DeviceExt for ProtocolObject<dyn MTLDevice> {}diff --git a/crates/backend-uzu/src/backends/metal/sparse/mod.rs b/crates/backend-uzu/src/backends/metal/sparse/mod.rsindex dd4321d32..3a3b9ed52 100644--- a/crates/backend-uzu/src/backends/metal/sparse/mod.rs+++ b/crates/backend-uzu/src/backends/metal/sparse/mod.rs@@ -5,3 +5,4 @@ mod sparse_utils;pub(crate) use sparse_buffer::MetalSparseBuffer;pub(crate) use sparse_heap_pool::MetalSparseHeapPool;+pub(crate) use sparse_utils::MetalSparseMappingOpsBatch;diff --git a/crates/backend-uzu/src/backends/metal/sparse/sparse_buffer.rs b/crates/backend-uzu/src/backends/metal/sparse/sparse_buffer.rsindex 081c43e02..f3a3c3dbe 100644--- a/crates/backend-uzu/src/backends/metal/sparse/sparse_buffer.rs+++ b/crates/backend-uzu/src/backends/metal/sparse/sparse_buffer.rs@@ -11,8 +11,8 @@ use rangemap::RangeSet;use crate::{backends::{- common::{Backend, Buffer, SparseBuffer},- metal::{Metal, error::MetalError, metal_extensions::SparsePageSizeExt},+ common::{Backend, Buffer, SparseBuffer, SparseBufferExt},+ metal::{Metal, error::MetalError, metal_extensions::SparsePageSizeExt, sparse::MetalSparseMappingOpsBatch},},prelude::MetalContext,};@@ -21,6 +21,7 @@ pub struct MetalSparseBuffer {buffer: Retained<ProtocolObject<dyn MTLBuffer>>,mapped_pages: RangeSet<usize>,context: Rc<MetalContext>,+ page_size: MTLSparsePageSize,}impl MetalSparseBuffer {@@ -43,6 +44,7 @@ impl MetalSparseBuffer {buffer,mapped_pages: RangeSet::new(),context,+ page_size,})}@@ -66,9 +68,7 @@ impl Debug for MetalSparseBuffer {impl Drop for MetalSparseBuffer {fn drop(&mut self) {let context = self.context.clone();- for range in self.mapped_pages.iter() {- context.sparse_heap_pool_mut().unmap(&context, &self.buffer, &range).expect("Failed to unmap");- }+ self.unmap(&context, &(0..self.total_pages())).expect("Failed to unmap");}}@@ -90,11 +90,35 @@ impl SparseBuffer for MetalSparseBuffer {context: &<Self::Backend as Backend>::Context,pages: &Range<usize>,) -> Result<(), <Self::Backend as Backend>::Error> {- self.mapped_pages.gaps(pages).collect::<Vec<_>>().into_iter().try_for_each(|gap| {- context.sparse_heap_pool_mut().map(context, &self.buffer, &gap)?;- self.mapped_pages.insert(gap);- Ok(())- })+ if pages.len() == 0 {+ return Ok(());+ }++ // prepare operations+ let mut all_batches: Vec<MetalSparseMappingOpsBatch> = Vec::new();+ let gaps = self.mapped_pages.gaps(pages).collect::<Vec<_>>();+ let pages_to_map = gaps.iter().map(|gap| gap.len()).sum();+ context.sparse_heap_pool_mut().ensure_enough_free_pages(context, pages_to_map)?;++ for gap in gaps.iter() {+ let mut pool = context.sparse_heap_pool_mut();+ let result = pool.create_map_operations(context, &self.buffer, gap);+ match result {+ Ok(batches) => {+ pool.apply_map_operations(&batches);+ all_batches.extend(batches);+ },+ Err(err) => return Err(err),+ };+ }++ // execute operations+ context.sparse_update_mappings(&all_batches);+ for gap in gaps {+ self.mapped_pages.insert(gap)+ }++ Ok(())}fn unmap(@@ -102,16 +126,34 @@ impl SparseBuffer for MetalSparseBuffer {context: &<Self::Backend as Backend>::Context,pages: &Range<usize>,) -> Result<(), <Self::Backend as Backend>::Error> {- self.mapped_pages+ if pages.len() == 0 {+ return Ok(());+ }++ // prepare operations+ let mut all_batches: Vec<MetalSparseMappingOpsBatch> = Vec::new();+ let mapped_ranges = self+ .mapped_pages.overlapping(pages).map(|range| max(range.start, pages.start)..min(range.end, pages.end))- .collect::<Vec<_>>()- .into_iter()- .try_for_each(|range| {- context.sparse_heap_pool_mut().unmap(context, &self.buffer, &range)?;- self.mapped_pages.remove(range);- Ok(())- })+ .collect::<Vec<_>>();+ for mapped_range in mapped_ranges.iter() {+ let batches = context.sparse_heap_pool().create_unmap_operations(&self.buffer, mapped_range);+ context.sparse_heap_pool_mut().apply_map_operations(&batches);+ all_batches.extend(batches);+ }++ // execute operations+ context.sparse_update_mappings(&all_batches);+ for mapped_range in mapped_ranges {+ self.mapped_pages.remove(mapped_range)+ }++ Ok(())+ }++ fn page_size_bytes(&self) -> usize {+ self.page_size.in_bytes()}}diff --git a/crates/backend-uzu/src/backends/metal/sparse/sparse_heap.rs b/crates/backend-uzu/src/backends/metal/sparse/sparse_heap.rsindex 241ca0c32..73519dbe6 100644--- a/crates/backend-uzu/src/backends/metal/sparse/sparse_heap.rs+++ b/crates/backend-uzu/src/backends/metal/sparse/sparse_heap.rs@@ -1,8 +1,8 @@use std::{collections::HashMap, ops::Range};use metal::{- MTL4CommandQueue, MTL4CommandQueueExt, MTL4UpdateSparseBufferMappingOperation, MTLBuffer, MTLDeviceExt, MTLHeap,- MTLHeapDescriptor, MTLHeapType, MTLSparsePageSize, MTLSparseTextureMappingMode, MTLStorageMode,+ MTLBuffer, MTLDeviceExt, MTLHeap, MTLHeapDescriptor, MTLHeapType, MTLSparsePageSize, MTLSparseTextureMappingMode,+ MTLStorageMode,};use objc2::{rc::Retained, runtime::ProtocolObject};use rangemap::{RangeMap, RangeSet};@@ -11,12 +11,12 @@ use crate::{backends::metal::{error::MetalError,metal_extensions::SparsePageSizeExt,- sparse::sparse_utils::{MetalSparseHeapBufferMapping, MetalSparseHeapMappingParameters},+ sparse::{MetalSparseMappingOpsBatch, sparse_utils::MetalSparseHeapBufferMapping},},prelude::MetalContext,};-pub(super) struct MetalSparseHeap {+pub(crate) struct MetalSparseHeap {heap: Retained<ProtocolObject<dyn MTLHeap>>,// Mappings are tracked per buffer (keyed by gpu_address) so multiple// buffers can alias overlapping heap page ranges without losing track of@@ -55,6 +55,10 @@ impl MetalSparseHeap {})}+ pub fn heap(&self) -> &Retained<ProtocolObject<dyn MTLHeap>> {+ &self.heap+ }+pub fn free_pages(&self) -> &RangeSet<usize> {&self.free_pages}@@ -74,38 +78,15 @@ impl MetalSparseHeap {self.buffer_mappings.is_empty()}- /// Command queue calls doesn't have any synchronization.- /// It is the responsibility of the caller.- pub fn execute(+ pub fn apply_mapping_operations(&mut self,- buffer: &ProtocolObject<dyn MTLBuffer>,- cmd_queue: &ProtocolObject<dyn MTL4CommandQueue>,- operations: &[MetalSparseHeapMappingParameters],- map: bool,+ ops: &MetalSparseMappingOpsBatch,) {- if operations.is_empty() {- return;- }-- let mtl_operations: Vec<MTL4UpdateSparseBufferMappingOperation> = operations- .iter()- .map(|op| {- let mode = if map {- MTLSparseTextureMappingMode::Map- } else {- MTLSparseTextureMappingMode::Unmap- };- MTL4UpdateSparseBufferMappingOperation::new(mode, op.buffer_pages.clone(), op.heap_page_offset)- })- .collect();-- cmd_queue.update_buffer_mappings(buffer, Some(&self.heap), &mtl_operations);-- let buffer_address = buffer.gpu_address();+ let buffer_address = ops.buffer.gpu_address();let entry = self.buffer_mappings.entry(buffer_address).or_default();- mtl_operations.iter().for_each(|mtl_op| {+ for mtl_op in ops.mtl_operations.iter() {let heap_range = mtl_op.heap_offset..(mtl_op.heap_offset + mtl_op.buffer_range().len());- if map {+ if mtl_op.mode == MTLSparseTextureMappingMode::Map {let buffer_range = mtl_op.buffer_range();let buffer_mapping = MetalSparseHeapBufferMapping::new(mtl_op.heap_offset, buffer_range.start);entry.insert(heap_range.clone(), buffer_mapping);@@ -114,7 +95,7 @@ impl MetalSparseHeap {entry.remove(heap_range.clone());self.free_pages.insert(heap_range);}- });+ }if entry.is_empty() {self.buffer_mappings.remove(&buffer_address);diff --git a/crates/backend-uzu/src/backends/metal/sparse/sparse_heap_pool.rs b/crates/backend-uzu/src/backends/metal/sparse/sparse_heap_pool.rsindex fe63a06fd..864efbdbf 100644--- a/crates/backend-uzu/src/backends/metal/sparse/sparse_heap_pool.rs+++ b/crates/backend-uzu/src/backends/metal/sparse/sparse_heap_pool.rs@@ -1,22 +1,24 @@use std::{+ cell::RefCell,cmp::{max, min},ops::Range,+ rc::Rc,};-use metal::{MTLBuffer, MTLSparsePageSize};-use objc2::runtime::ProtocolObject;+use metal::{MTL4UpdateSparseBufferMappingOperation, MTLBuffer, MTLSparsePageSize, MTLSparseTextureMappingMode};+use objc2::{rc::Retained, runtime::ProtocolObject};use crate::{backends::metal::{error::MetalError,metal_extensions::SparsePageSizeExt,- sparse::{sparse_heap::MetalSparseHeap, sparse_utils::MetalSparseHeapMappingParameters},+ sparse::{MetalSparseMappingOpsBatch, sparse_heap::MetalSparseHeap},},prelude::MetalContext,};pub struct MetalSparseHeapPool {- heaps: Vec<MetalSparseHeap>,+ heaps: Vec<Rc<RefCell<MetalSparseHeap>>>,heap_capacity: usize,page_size: MTLSparsePageSize,}@@ -33,112 +35,123 @@ impl MetalSparseHeapPool {}}- pub fn map(+ pub fn ensure_enough_free_pages(&mut self,context: &MetalContext,- buffer: &ProtocolObject<dyn MTLBuffer>,- buffer_pages: &Range<usize>,+ pages: usize,) -> Result<(), MetalError> {- if buffer_pages.len() == 0 {+ let mut pages_to_alloc = pages;+ for heap in self.heaps.iter() {+ for free_pages_range in heap.borrow().free_pages().iter() {+ pages_to_alloc -= min(free_pages_range.len(), pages_to_alloc);+ }+ }+ if pages_to_alloc == 0 {return Ok(());}- let mut pages_to_map = buffer_pages.clone();- let heap_capacity_pages = self.heap_capacity_pages();-- // Try to find pages in existing heaps.- // While `pages_to_map` is not empty in each heap find unmapped pages, collect them into `mappings` and map- let mut existing_heaps_mappings: Vec<(usize, Box<[MetalSparseHeapMappingParameters]>)> = Vec::new();- for (i, heap) in self.heaps.iter_mut().enumerate() {- let mut mappings: Vec<MetalSparseHeapMappingParameters> = Vec::new();-- for free_pages in heap.free_pages().iter() {- let map_pages_count = min(free_pages.len(), pages_to_map.len());- let mapping = MetalSparseHeapMappingParameters {- buffer_pages: pages_to_map.start..(pages_to_map.start + map_pages_count),- heap_page_offset: free_pages.start,- };- mappings.push(mapping);+ let new_heaps_count = (pages_to_alloc * self.page_size.in_bytes()).div_ceil(self.heap_capacity);+ for _ in 0..new_heaps_count {+ let heap = MetalSparseHeap::new(context, self.heap_capacity, self.page_size)?;+ self.heaps.push(Rc::new(RefCell::new(heap)));+ }- pages_to_map.start += map_pages_count;- if pages_to_map.len() == 0 {+ Ok(())+ }++ pub fn create_map_operations(+ &mut self,+ context: &MetalContext,+ buffer: &Retained<ProtocolObject<dyn MTLBuffer>>,+ buffer_pages: &Range<usize>,+ ) -> Result<Vec<MetalSparseMappingOpsBatch>, MetalError> {+ self.ensure_enough_free_pages(context, buffer_pages.len())?;++ let mut pages_to_map = buffer_pages.clone();+ let mut batches: Vec<MetalSparseMappingOpsBatch> = Vec::new();+ for heap in self.heaps.iter() {+ let mut heap_mtl_operations: Vec<MTL4UpdateSparseBufferMappingOperation> = Vec::new();+ for heap_free_pages_range in heap.borrow().free_pages().iter() {+ let map_pages_count = min(heap_free_pages_range.len(), pages_to_map.len());+ if map_pages_count == 0 {break;}- }- heap.execute(buffer, &context.command_queue4, &mappings, true);- existing_heaps_mappings.push((i, mappings.into_boxed_slice()));+ let mtl_operation = MTL4UpdateSparseBufferMappingOperation::new(+ MTLSparseTextureMappingMode::Map,+ pages_to_map.start..(pages_to_map.start + map_pages_count),+ heap_free_pages_range.start,+ );+ heap_mtl_operations.push(mtl_operation);- if pages_to_map.len() == 0 {- break;+ pages_to_map.start += map_pages_count;}- }- // If `pages_to_map` still not empty, it's necessary to allocate new heaps- let new_heaps_required = pages_to_map.len().div_ceil(heap_capacity_pages);- for _ in 0..new_heaps_required {- let map_pages_count = min(heap_capacity_pages, pages_to_map.len());- let buffer_pages = pages_to_map.start..(pages_to_map.start + map_pages_count);- let op_params = MetalSparseHeapMappingParameters {- buffer_pages,- heap_page_offset: 0,- };-- match MetalSparseHeap::new(context, self.heap_capacity, self.page_size) {- Ok(mut heap) => {- let operations = [op_params];- heap.execute(buffer, &context.command_queue4, &operations, true);- self.heaps.push(heap);- pages_to_map.start += map_pages_count;- existing_heaps_mappings.push((self.heaps.len() - 1, Box::new(operations)));- },- Err(err) => {- // it's necessary to unmap previously mapped pages in existing heaps- for (heap_pos, mappings) in existing_heaps_mappings {- self.heaps[heap_pos].execute(buffer, &context.command_queue4, &mappings, false);- }- return Err(err);- },- };+ if !heap_mtl_operations.is_empty() {+ let batch = MetalSparseMappingOpsBatch {+ buffer: buffer.clone(),+ heap: heap.clone(),+ mtl_operations: heap_mtl_operations.into_boxed_slice(),+ };+ batches.push(batch);+ }}- Ok(())+ Ok(batches)}- pub fn unmap(- &mut self,- context: &MetalContext,- buffer: &ProtocolObject<dyn MTLBuffer>,+ pub fn create_unmap_operations(+ &self,+ buffer: &Retained<ProtocolObject<dyn MTLBuffer>>,buffer_pages: &Range<usize>,- ) -> Result<(), MetalError> {+ ) -> Vec<MetalSparseMappingOpsBatch> {+ let mut batches: Vec<MetalSparseMappingOpsBatch> = Vec::new();+let buffer_address = buffer.gpu_address();+ for heap in self.heaps.iter() {+ let mut heap_mtl_operations: Vec<MTL4UpdateSparseBufferMappingOperation> = Vec::new();++ for (heap_range, mapping) in heap.borrow().mappings_for(buffer_address) {+ let mapped_buffer_pages = mapping.buffer_pages_for(&heap_range);+ let unmap_start_page = max(buffer_pages.start, mapped_buffer_pages.start);+ let unmap_end_page = min(buffer_pages.end, mapped_buffer_pages.end);+ if unmap_start_page < unmap_end_page {+ let offset_within_range = unmap_start_page - mapped_buffer_pages.start;+ let mtl_operation = MTL4UpdateSparseBufferMappingOperation::new(+ MTLSparseTextureMappingMode::Unmap,+ unmap_start_page..unmap_end_page,+ heap_range.start + offset_within_range,+ );+ heap_mtl_operations.push(mtl_operation);+ };+ }- // Iterate over heaps, find mappings for `buffer_address`, unmap them- self.heaps.iter_mut().for_each(|heap| {- let unmappings: Vec<MetalSparseHeapMappingParameters> = heap- .mappings_for(buffer_address)- .filter_map(|(heap_range, mapping)| {- let mapped_buffer_pages = mapping.buffer_pages_for(&heap_range);- let unmap_start = max(buffer_pages.start, mapped_buffer_pages.start);- let unmap_end = min(buffer_pages.end, mapped_buffer_pages.end);- if unmap_start >= unmap_end {- return None;- };-- let offset_within_range = unmap_start - mapped_buffer_pages.start;- Some(MetalSparseHeapMappingParameters {- buffer_pages: unmap_start..unmap_end,- heap_page_offset: heap_range.start + offset_within_range,- })- })- .collect();- heap.execute(buffer, &context.command_queue4, &unmappings, false);- });-- // Remove empty heaps- self.heaps.retain(|heap| !heap.is_empty());+ if !heap_mtl_operations.is_empty() {+ let batch = MetalSparseMappingOpsBatch {+ buffer: buffer.clone(),+ heap: heap.clone(),+ mtl_operations: heap_mtl_operations.into_boxed_slice(),+ };+ batches.push(batch);+ }+ }- Ok(())+ batches+ }++ pub fn apply_map_operations(+ &mut self,+ batches: &[MetalSparseMappingOpsBatch],+ ) {+ for batch in batches.iter() {+ for sparse_heap in self.heaps.iter() {+ if sparse_heap.borrow().heap() == batch.heap.borrow().heap() {+ sparse_heap.borrow_mut().apply_mapping_operations(batch);+ }+ }+ }++ self.heaps.retain(|heap| !heap.borrow().is_empty());}#[cfg(test)]@@ -146,6 +159,7 @@ impl MetalSparseHeapPool {self.heap_capacity}+ #[cfg(test)]pub fn heap_capacity_pages(&self) -> usize {self.heap_capacity / self.page_size.in_bytes()}diff --git a/crates/backend-uzu/src/backends/metal/sparse/sparse_utils.rs b/crates/backend-uzu/src/backends/metal/sparse/sparse_utils.rsindex 103de7d1f..5e5fecd4b 100644--- a/crates/backend-uzu/src/backends/metal/sparse/sparse_utils.rs+++ b/crates/backend-uzu/src/backends/metal/sparse/sparse_utils.rs@@ -1,7 +1,12 @@-use std::ops::Range;+use std::{cell::RefCell, ops::Range, rc::Rc};++use metal::{MTL4UpdateSparseBufferMappingOperation, MTLBuffer};+use objc2::{rc::Retained, runtime::ProtocolObject};++use crate::backends::metal::sparse::sparse_heap::MetalSparseHeap;#[derive(Clone, PartialEq)]-pub(super) struct MetalSparseHeapBufferMapping {+pub(crate) struct MetalSparseHeapBufferMapping {// Anchors for an originally inserted contiguous mapping. They survive// rangemap splits unchanged, so the heap↔buffer correspondence can always// be recovered from any surviving heap subrange via `buffer_pages_for`.@@ -30,7 +35,8 @@ impl MetalSparseHeapBufferMapping {}}-pub(super) struct MetalSparseHeapMappingParameters {- pub(super) buffer_pages: Range<usize>,- pub(super) heap_page_offset: usize,+pub(crate) struct MetalSparseMappingOpsBatch {+ pub buffer: Retained<ProtocolObject<dyn MTLBuffer>>,+ pub heap: Rc<RefCell<MetalSparseHeap>>,+ pub mtl_operations: Box<[MTL4UpdateSparseBufferMappingOperation]>,}diff --git a/crates/backend-uzu/src/encodable_block/attention.rs b/crates/backend-uzu/src/encodable_block/attention.rsindex 945528bcb..7d9744439 100644--- a/crates/backend-uzu/src/encodable_block/attention.rs+++ b/crates/backend-uzu/src/encodable_block/attention.rs@@ -8,7 +8,7 @@ use crate::{DataType,array::size_for_shape,backends::common::{- Allocation, Backend, Encoder, Kernels,+ Allocation, AsBufferRangeRef, Backend, Buffer, Encoder, Kernels,gpu_types::ring::RingParams,kernel::{AttentionFallbackScatterScoresKernel, AttentionFallbackScatterValuesKernel, AttentionSinglePassKernel,@@ -21,7 +21,7 @@ use crate::{config::AttentionConfig,forward_pass::{cache_layers::LayerCacheAccess,- kv_cache_layer::KVCacheLayerState,+ kv_cache_layer::{KVCacheLayer, KVCacheLayerState},},};@@ -228,27 +228,26 @@ impl<B: Backend> Attention<B> {.expect("attention expects transformer cache")});- let (max_sequence_length, segment_prefix_length, ring_params) =- if let Some(layer) = cache_layer {- let max_sequence_length = layer.shape[0];- let ring_params = match layer.state {- KVCacheLayerState::Windowed {- ring_offset,- ring_length,- window_length,- } => {- let overflow = ring_length.saturating_sub(window_length);- Some(RingParams {- ring_offset: ((ring_offset + overflow) % window_length) as u32,- ring_length: ring_length.min(window_length) as u32,- })- },- _ => None,- };- (max_sequence_length, layer.prefix_segment_length(), ring_params)- } else {- (suffix_length, 0, None)+ let (max_sequence_length, segment_prefix_length, ring_params) = if let Some(layer) = cache_layer {+ let max_sequence_length = layer.shape()[0];+ let ring_params = match layer.state() {+ KVCacheLayerState::Windowed {+ ring_offset,+ ring_length,+ window_length,+ } => {+ let overflow = ring_length.saturating_sub(window_length);+ Some(RingParams {+ ring_offset: ((ring_offset + overflow) % window_length) as u32,+ ring_length: ring_length.min(window_length) as u32,+ })+ },+ _ => None,};+ (max_sequence_length, layer.prefix_segment_length(), ring_params)+ } else {+ (suffix_length, 0, None)+ };let is_kv_cache_ring = ring_params.is_some();@@ -312,62 +311,92 @@ impl<B: Backend> Attention<B> {is_kv_cache_ring,};- let (keys, values) = match args.cache_access {- Some(LayerCacheAccess::Shared {- source,- }) => {- let source = source.as_transformer().expect("kv_source must be a transformer cache");- (&source.keys, &source.values)- },- Some(LayerCacheAccess::Owned {- entry,- }) => {- let layer = entry.as_transformer_mut().expect("Attention layer expects transformer cache");- self.update_kv_cache_kernel.encode(- Some(&rotated_keys),- qkv,- &mut layer.keys,- &mut layer.values,- num_groups as u32,- num_heads as u32,- head_dim as u32,- suffix_length as u32,- segment_prefix_length as u32,- max_sequence_length as u32,+ macro_rules! encode_cached_attention {+ ($keys:expr, $values:expr) => {+ self.encode_attention_variant(+ variant,+ &kernel_key,+ queries,+ $keys,+ $values,+ trie_allocation,+ sinks_allocation,+ gqa_factor,+ sequence_length,+ k_head_stride,+ k_seq_stride,+ v_head_stride,+ v_seq_stride,+ ring_params,+ scale,+ num_heads,+ num_groups,+ suffix_length,+ segment_prefix_length,+ max_sequence_length,+ head_dim,encoder,- );- (&layer.keys, &layer.values)- },- None => (- &rotated_keys,- extracted_values.as_ref().expect("Missing extracted values for classifier attention"),- ),- };+ )?+ };+ }- let mut attention_output = self.encode_attention_variant(- variant,- &kernel_key,- queries,- keys,- values,- trie_allocation,- sinks_allocation,- gqa_factor,- sequence_length,- k_head_stride,- k_seq_stride,- v_head_stride,- v_seq_stride,- ring_params,- scale,- num_heads,- num_groups,- suffix_length,- segment_prefix_length,- max_sequence_length,- head_dim,- encoder,- )?;+ let mut attention_output = if let Some(access) = args.cache_access {+ match access {+ LayerCacheAccess::Owned {+ entry,+ } => {+ let layer = entry.as_transformer_mut().expect("Attention layer expects transformer cache");+ if let Some(layer) = layer.as_any_mut().downcast_mut::<KVCacheLayer<B, B::SparseBuffer>>() {+ self.update_kv_cache_kernel.encode(+ Some(&rotated_keys),+ qkv,+ &mut layer.keys,+ &mut layer.values,+ num_groups as u32,+ num_heads as u32,+ head_dim as u32,+ suffix_length as u32,+ segment_prefix_length as u32,+ max_sequence_length as u32,+ encoder,+ );+ encode_cached_attention!(&layer.keys, &layer.values)+ } else if let Some(layer) = layer.as_any_mut().downcast_mut::<KVCacheLayer<B, B::DenseBuffer>>() {+ self.update_kv_cache_kernel.encode(+ Some(&rotated_keys),+ qkv,+ &mut layer.keys,+ &mut layer.values,+ num_groups as u32,+ num_heads as u32,+ head_dim as u32,+ suffix_length as u32,+ segment_prefix_length as u32,+ max_sequence_length as u32,+ encoder,+ );+ encode_cached_attention!(&layer.keys, &layer.values)+ } else {+ panic!("Attention layer expects sparse or dense transformer cache")+ }+ },+ LayerCacheAccess::Shared {+ source,+ } => {+ let source = source.as_transformer().expect("kv_source must be a transformer cache");+ if let Some(source) = source.as_any().downcast_ref::<KVCacheLayer<B, B::SparseBuffer>>() {+ encode_cached_attention!(&source.keys, &source.values)+ } else if let Some(source) = source.as_any().downcast_ref::<KVCacheLayer<B, B::DenseBuffer>>() {+ encode_cached_attention!(&source.keys, &source.values)+ } else {+ panic!("kv_source must be a sparse or dense transformer cache")+ }+ },+ }+ } else {+ let values = extracted_values.as_ref().expect("Missing extracted values for classifier attention");+ encode_cached_attention!(&rotated_keys, values)+ };if let Some(gate_kernel) = &self.gate_kernel {let total_elements = (suffix_length * num_heads * head_dim) as u32;@@ -379,13 +408,13 @@ impl<B: Backend> Attention<B> {}#[allow(clippy::too_many_arguments)]- fn encode_attention_variant(+ fn encode_attention_variant<KVBuf: AsBufferRangeRef<Buffer: Buffer<Backend = B>>>(&self,variant: KernelVariant,kernel_key: &KernelKey,queries: &Allocation<B>,- keys: &Allocation<B>,- values: &Allocation<B>,+ keys: &KVBuf,+ values: &KVBuf,trie_allocation: Option<&Allocation<B>>,sinks_allocation: Option<&Allocation<B>>,gqa_factor: usize,@@ -526,12 +555,15 @@ impl<B: Backend> Attention<B> {}#[allow(clippy::too_many_arguments)]- fn encode_fallback(+ fn encode_fallback<+ Keys: AsBufferRangeRef<Buffer: Buffer<Backend = B>>,+ Values: AsBufferRangeRef<Buffer: Buffer<Backend = B>>,+ >(&self,is_kv_cache_ring: bool,queries: &Allocation<B>,- keys: &Allocation<B>,- values: &Allocation<B>,+ keys: &Keys,+ values: &Values,attention_output: &mut Allocation<B>,sinks_allocation: Option<&Allocation<B>>,gqa_factor: usize,diff --git a/crates/backend-uzu/src/forward_pass/cache_layers.rs b/crates/backend-uzu/src/forward_pass/cache_layers.rsindex 8eda52178..de84ab441 100644--- a/crates/backend-uzu/src/forward_pass/cache_layers.rs+++ b/crates/backend-uzu/src/forward_pass/cache_layers.rs@@ -6,7 +6,7 @@ use crate::{config::MixerConfig,forward_pass::{delta_net_layer::DeltaNetLayer,- kv_cache_layer::{KVCacheLayer, KVCacheLayerState, KVSlice},+ kv_cache_layer::{KVCacheLayerState, KVCacheLayerTrait, KVSlice},model_shape::ModelShape,short_conv_layer::ShortConvLayer,ssm_layer::SSMLayer,@@ -14,7 +14,7 @@ use crate::{};pub enum CacheLayer<B: Backend> {- Transformer(KVCacheLayer<B>),+ Transformer(Box<dyn KVCacheLayerTrait<B>>),StateSpace(SSMLayer<B>),ShortConv(ShortConvLayer<B>),DeltaNet(DeltaNetLayer<B>),@@ -28,16 +28,16 @@ pub enum CacheLayerSlice<B: Backend> {}impl<B: Backend> CacheLayer<B> {- pub fn as_transformer(&self) -> Option<&KVCacheLayer<B>> {+ pub fn as_transformer(&self) -> Option<&dyn KVCacheLayerTrait<B>> {match self {- CacheLayer::Transformer(layer) => Some(layer),+ CacheLayer::Transformer(layer) => Some(layer.as_ref()),_ => None,}}- pub fn as_transformer_mut(&mut self) -> Option<&mut KVCacheLayer<B>> {+ pub fn as_transformer_mut(&mut self) -> Option<&mut dyn KVCacheLayerTrait<B>> {match self {- CacheLayer::Transformer(layer) => Some(layer),+ CacheLayer::Transformer(layer) => Some(layer.as_mut()),_ => None,}}@@ -137,12 +137,12 @@ impl<B: Backend> CacheLayers<B> {if let Some(source_layer_index) = kv_source_layers.get(layer_index).copied().flatten() {let LayerCacheBinding::Owned {entry: source,- } = bindings[source_layer_index] else {+ } = bindings[source_layer_index]+ else {panic!("kv_source_layer {source_layer_index} (layer {layer_index}) must be a prior owner");};- let geometry = |m: &MixerConfig| {- m.as_attention().map(|a| (a.num_groups, a.head_dim, a.sliding_window_size))- };+ let geometry =+ |m: &MixerConfig| m.as_attention().map(|a| (a.num_groups, a.head_dim, a.sliding_window_size));assert_eq!(geometry(&model_shape.layer_mixers()[source_layer_index]),geometry(mixer),@@ -159,8 +159,7 @@ impl<B: Backend> CacheLayers<B> {let sliding_window = attn.sliding_window_size;let length = sliding_window.unwrap_or(max_prefix_length);let shape = [length + max_suffix_length, attn.num_groups, attn.head_dim];- let window_length =- sliding_window.filter(|&window_size| window_size < total_context_length);+ let window_length = sliding_window.filter(|&window_size| window_size < total_context_length);let state = if let Some(w) = window_length {KVCacheLayerState::Windowed {@@ -173,19 +172,11 @@ impl<B: Backend> CacheLayers<B> {prefix_len: 0,}};- let kv_bytes = size_for_shape(&shape, model_shape.kv_cache_data_type());-- CacheLayer::Transformer(KVCacheLayer {- state,- keys: context- .create_allocation(kv_bytes, AllocationType::Global)- .expect("Failed to create kv keys allocation"),- values: context- .create_allocation(kv_bytes, AllocationType::Global)- .expect("Failed to create kv values allocation"),- shape,- data_type: model_shape.kv_cache_data_type(),- })++ let kv_layer =+ <dyn KVCacheLayerTrait<B>>::new(context, &state, shape, model_shape.kv_cache_data_type())+ .expect("Failed to create KVCacheLayer");+ CacheLayer::Transformer(kv_layer)},MixerConfig::Mamba(c) => {let conv_shape = [c.conv_dim(), c.kernel_size.saturating_sub(1)];@@ -251,7 +242,9 @@ impl<B: Backend> CacheLayers<B> {},};bindings.push(LayerCacheBinding::Owned {- entry: CacheEntryIndex { index: entries.len() },+ entry: CacheEntryIndex {+ index: entries.len(),+ },});entries.push(layer);}@@ -262,8 +255,7 @@ impl<B: Backend> CacheLayers<B> {for layer in entries.iter_mut() {match layer {CacheLayer::Transformer(layer) => {- encoder.encode_fill(&mut layer.keys, 0);- encoder.encode_fill(&mut layer.values, 0);+ layer.encode_zero(&mut encoder);},CacheLayer::StateSpace(layer) => {if let Some(conv_state) = layer.conv_state.as_mut() {@@ -327,21 +319,7 @@ impl<B: Backend> CacheLayers<B> {let mut encoder: Option<Encoder<B>> = None;for layer in self.entries.iter_mut() {match layer {- CacheLayer::Transformer(layer) => match &mut layer.state {- KVCacheLayerState::Full {- prefix_len,- } => {- *prefix_len = 0;- },- KVCacheLayerState::Windowed {- ring_offset,- ring_length,- ..- } => {- *ring_offset = 0;- *ring_length = 0;- },- },+ CacheLayer::Transformer(layer) => layer.clear_state(),CacheLayer::StateSpace(layer) => {let encoder = encoder.get_or_insert_with(|| Encoder::new(context).expect("Failed to create cache clear encoder"));@@ -382,19 +360,46 @@ impl<B: Backend> CacheLayers<B> {&mut self,accepted_suffix_indices: &[usize],suffix_start: Option<usize>,+ context: &B::Context,encoder: &mut Encoder<B>,kv_cache_update: &KVCacheUpdate<B>,) {let short_conv_commit_index = accepted_suffix_indices.last().copied().unwrap_or(0);for layer in self.entries.iter_mut() {if let Some(layer) = layer.as_transformer_mut() {- layer.update_after_acceptance(accepted_suffix_indices, suffix_start, encoder, kv_cache_update);+ layer.update_after_acceptance(accepted_suffix_indices, suffix_start, context, encoder, kv_cache_update);} else if let Some(layer) = layer.as_short_conv_mut() {layer.commit_from_suffix_state_if_valid(short_conv_commit_index, encoder);}}}+ pub fn prepare_for_forward_pass(+ &mut self,+ context: &B::Context,+ active_row_count: usize,+ ) {+ if active_row_count == 0 {+ return;+ }++ for layer in self.entries.iter_mut() {+ let Some(layer) = layer.as_transformer_mut() else {+ continue;+ };+ let row_range = match layer.state() {+ KVCacheLayerState::Full {+ prefix_len,+ } => prefix_len..prefix_len + active_row_count,+ KVCacheLayerState::Windowed {+ window_length,+ ..+ } => 0..window_length + active_row_count,+ };+ layer.map_row_range(context, row_range).expect("Failed to map KV cache rows for forward pass");+ }+ }+pub fn register_accepted_tokens(&mut self,number_of_accepted_tokens: usize,@@ -412,10 +417,14 @@ impl<B: Backend> CacheLayers<B> {range: std::ops::Range<usize>,) -> Option<CacheLayersSlice<B>> {let mut layers = Vec::with_capacity(self.entries.len());+ let mut encoder: Option<Encoder<B>> = None;+for layer in self.entries.iter() {match layer {CacheLayer::Transformer(kv) => {- let Some(slice) = kv.slice(context, range.clone()) else {+ let mut encoder =+ encoder.get_or_insert_with(|| Encoder::new(context).expect("Failed to create Encoder"));+ let Some(slice) = kv.slice(context, &mut encoder, range.clone()) else {return None;};layers.push(CacheLayerSlice::Transformer(slice));@@ -426,6 +435,10 @@ impl<B: Backend> CacheLayers<B> {}}+ if let Some(encoder) = encoder {+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to end and wait encoder");+ }+Some(CacheLayersSlice {layers,})@@ -433,19 +446,27 @@ impl<B: Backend> CacheLayers<B> {pub fn apply_slice(&mut self,+ context: &B::Context,slice: &CacheLayersSlice<B>,range: Option<std::ops::Range<usize>>,) {+ let mut encoder: Option<Encoder<B>> = None;for (layer, snapshot) in self.entries.iter_mut().zip(slice.layers.iter()) {match (layer, snapshot) {(CacheLayer::Transformer(kv), CacheLayerSlice::Transformer(s)) => {- kv.apply_slice(&s, range.clone());+ let mut encoder =+ encoder.get_or_insert_with(|| Encoder::new(context).expect("Failed to create Encoder"));+ kv.apply_slice(context, &mut encoder, &s, range.clone());},(CacheLayer::StateSpace(_), CacheLayerSlice::StateSpace) => {},(CacheLayer::ShortConv(_), CacheLayerSlice::ShortConv) => {},_ => {},}}++ if let Some(encoder) = encoder {+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to end and wait encoder");+ }}pub fn copy_from(@@ -456,7 +477,7 @@ impl<B: Backend> CacheLayers<B> {self.copy_metadata_and_windowed_data_from(source, context);let mut encoder = Encoder::new(context).expect("Failed to create cache layer copy encoder");- self.encode_copy_data_from(source, &mut encoder);+ self.encode_copy_data_from(source, context, &mut encoder);encoder.end_encoding().submit().wait_until_completed().expect("Failed to copy cache layers");}@@ -467,15 +488,24 @@ impl<B: Backend> CacheLayers<B> {) {assert_eq!(source.entries.len(), self.entries.len(), "cache layer count mismatch");+ // Key/value arrays stay alive until after encoder.submit().wait_until_completed() returns, then explicitly drops them.+ // This prevents the slice buffers from being freed while the GPU is still reading from them during windowed cache cloning.+ let mut pending_slices: Vec<KVSlice<B>> = Vec::new();+ let mut encoder: Option<Encoder<B>> = None;+for (source_layer, destination_layer) in source.entries.iter().zip(self.entries.iter_mut()) {match (source_layer, destination_layer) {(CacheLayer::Transformer(source), CacheLayer::Transformer(destination)) => {let copy_rows = source.prefix_segment_length();- if copy_rows > 0 && matches!(source.state, KVCacheLayerState::Windowed { .. }) {- let slice = source.slice(context, 0..copy_rows).expect("Failed to slice KV cache layer");- destination.apply_slice(&slice, None);+ if copy_rows > 0 && matches!(source.state(), KVCacheLayerState::Windowed { .. }) {+ let encoder =+ encoder.get_or_insert_with(|| Encoder::new(context).expect("Failed to create Encoder"));+ let slice =+ source.slice(context, encoder, 0..copy_rows).expect("Failed to slice KV cache layer");+ destination.apply_slice(context, encoder, &slice, None);+ pending_slices.push(slice);}- destination.state = source.state.clone();+ destination.set_state(&source.state());},(CacheLayer::StateSpace(_), CacheLayer::StateSpace(_)) => {},(CacheLayer::ShortConv(_), CacheLayer::ShortConv(destination)) => {@@ -485,11 +515,17 @@ impl<B: Backend> CacheLayers<B> {_ => panic!("cache layer type mismatch while copying cache layers"),}}++ if let Some(encoder) = encoder {+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to end and wait encoder");+ }+ drop(pending_slices);}fn encode_copy_data_from(&mut self,source: &Self,+ context: &B::Context,encoder: &mut Encoder<B>,) {assert_eq!(source.entries.len(), self.entries.len(), "cache layer count mismatch");@@ -497,8 +533,13 @@ impl<B: Backend> CacheLayers<B> {for (source_layer, destination_layer) in source.entries.iter().zip(self.entries.iter_mut()) {match (source_layer, destination_layer) {(CacheLayer::Transformer(source), CacheLayer::Transformer(destination)) => {- if matches!(source.state, KVCacheLayerState::Full { .. }) {- source.encode_copy_prefix_rows_to(destination, source.prefix_segment_length(), encoder);+ if matches!(source.state(), KVCacheLayerState::Full { .. }) {+ source.encode_copy_prefix_rows_to(+ destination.as_mut(),+ source.prefix_segment_length(),+ context,+ encoder,+ );}},(CacheLayer::StateSpace(source), CacheLayer::StateSpace(destination)) => {@@ -533,9 +574,9 @@ impl<B: Backend> CacheLayers<B> {.iter().map(|layer| match layer {CacheLayer::Transformer(layer) => {- let shape = layer.shape;+ let shape = layer.shape();let [_, num_groups, head_dim] = shape;- let dtype = layer.data_type;+ let dtype = layer.data_type();let copy_rows = layer.prefix_segment_length();let new_total_len = copy_rows + self.max_suffix_length;@@ -544,21 +585,9 @@ impl<B: Backend> CacheLayers<B> {}let new_shape = [new_total_len, num_groups, head_dim];- let new_bytes = size_for_shape(&new_shape, dtype);- let new_keys = context- .create_allocation(new_bytes, AllocationType::Global)- .expect("Failed to create kv keys clone allocation");- let new_values = context- .create_allocation(new_bytes, AllocationType::Global)- .expect("Failed to create kv values clone allocation");-- CacheLayer::Transformer(KVCacheLayer {- state: layer.state.clone(),- keys: new_keys,- values: new_values,- shape: new_shape,- data_type: dtype,- })+ let kv_layer = <dyn KVCacheLayerTrait<B>>::new(context, &layer.state(), new_shape, dtype)+ .expect("Failed to create KVCacheLayer");+ CacheLayer::Transformer(kv_layer)},CacheLayer::StateSpace(layer) => {let conv_bytes = size_for_shape(&layer.conv_shape, layer.data_type);@@ -625,8 +654,7 @@ impl<B: Backend> CacheLayers<B> {for layer in entries.iter_mut() {match layer {CacheLayer::Transformer(layer) => {- zero_encoder.encode_fill(&mut layer.keys, 0);- zero_encoder.encode_fill(&mut layer.values, 0);+ layer.encode_zero(&mut zero_encoder);},CacheLayer::ShortConv(layer) => {zero_encoder.encode_fill(&mut layer.suffix_state, 0);@@ -646,3 +674,7 @@ impl<B: Backend> CacheLayers<B> {cloned}}++#[cfg(test)]+#[path = "../../tests/unit/forward_pass/cache_layers_test.rs"]+mod tests;diff --git a/crates/backend-uzu/src/forward_pass/kv_cache_layer.rs b/crates/backend-uzu/src/forward_pass/kv_cache_layer.rsindex 6a32cf07f..dbd1fb688 100644--- a/crates/backend-uzu/src/forward_pass/kv_cache_layer.rs+++ b/crates/backend-uzu/src/forward_pass/kv_cache_layer.rs@@ -1,8 +1,10 @@+use std::{any::Any, ops::Range};+use crate::{DataType,array::{Array, ArrayContextExt, size_for_shape},backends::common::{- Allocation, Backend, Encoder, allocation_as_bytes, allocation_as_bytes_mut,+ AsBufferRangeMut, AsBufferRangeRef, Backend, Buffer, Context, Encoder, SparseBuffer,kernel::kv_cache_update::{KVCacheUpdate, KVLayerData},},};@@ -39,39 +41,275 @@ pub enum KVCacheLayerState {pub const INVALID_POSITION: usize = i32::MAX as usize;-pub struct KVCacheLayer<B: Backend> {+pub trait KVCacheLayerTrait<B: Backend> {+ fn as_any(&self) -> &dyn Any;++ fn as_any_mut(&mut self) -> &mut dyn Any;++ fn shape(&self) -> [usize; 3];++ fn data_type(&self) -> DataType;++ fn encode_zero(+ &mut self,+ encoder: &mut Encoder<'_, B>,+ );++ fn state(&self) -> KVCacheLayerState;++ fn set_state(+ &mut self,+ state: &KVCacheLayerState,+ );++ fn clear_state(&mut self);++ fn encode_copy_prefix_rows_to(+ &self,+ destination: &mut dyn KVCacheLayerTrait<B>,+ row_count: usize,+ context: &B::Context,+ encoder: &mut Encoder<B>,+ );++ fn prefix_segment_length(&self) -> usize;++ fn update_after_acceptance(+ &mut self,+ accepted_suffix_indices: &[usize],+ suffix_start: Option<usize>,+ context: &B::Context,+ encoder: &mut Encoder<B>,+ kv_cache_update: &KVCacheUpdate<B>,+ );++ fn register_accepted_tokens(+ &mut self,+ number_of_accepted_tokens: usize,+ );++ fn slice(+ &self,+ context: &B::Context,+ encoder: &mut Encoder<B>,+ range: Range<usize>,+ ) -> Option<KVSlice<B>>;++ fn apply_slice(+ &mut self,+ context: &B::Context,+ encoder: &mut Encoder<B>,+ slice: &KVSlice<B>,+ range: Option<Range<usize>>,+ );++ fn map_row_range(+ &mut self,+ context: &B::Context,+ range: Range<usize>,+ ) -> Result<(), B::Error>;+}++impl<B: Backend> dyn KVCacheLayerTrait<B> {+ pub fn new(+ context: &B::Context,+ state: &KVCacheLayerState,+ shape: [usize; 3],+ data_type: DataType,+ ) -> Result<Box<dyn KVCacheLayerTrait<B>>, B::Error> {+ let buffer_size = size_for_shape(&shape, data_type);+ if context.sparse_buffers_supported() {+ let layer = KVCacheLayer {+ state: state.clone(),+ keys: context.create_sparse_buffer(buffer_size)?,+ values: context.create_sparse_buffer(buffer_size)?,+ shape,+ data_type,+ };+ Ok(Box::new(layer))+ } else {+ let layer = KVCacheLayer {+ state: state.clone(),+ keys: context.create_buffer(buffer_size)?,+ values: context.create_buffer(buffer_size)?,+ shape,+ data_type,+ };+ Ok(Box::new(layer))+ }+ }+}++pub struct KVCacheLayer<B: Backend, Buf: Buffer<Backend = B>> {pub state: KVCacheLayerState,/// [max_prefix_length + max_suffix_length, num_groups, head_dim]- pub keys: Allocation<B>,+ pub keys: Buf,/// [max_prefix_length + max_suffix_length, num_groups, head_dim]- pub values: Allocation<B>,+ pub values: Buf,pub shape: [usize; 3],pub data_type: DataType,}-fn copy_rows(- source_keys_bytes: &[u8],- source_values_bytes: &[u8],- destination_keys_bytes: &mut [u8],- destination_values_bytes: &mut [u8],- row_size: usize,- row_pairs: impl IntoIterator<Item = (usize, usize)>,-) {- for (source_row, destination_row) in row_pairs {- let source_offset = source_row * row_size;- let destination_offset = destination_row * row_size;- destination_keys_bytes[destination_offset..destination_offset + row_size]- .copy_from_slice(&source_keys_bytes[source_offset..source_offset + row_size]);- destination_values_bytes[destination_offset..destination_offset + row_size]- .copy_from_slice(&source_values_bytes[source_offset..source_offset + row_size]);+impl<B: Backend, Buf: Buffer<Backend = B>> KVCacheLayer<B, Buf> {+ fn scatter_if_required(+ &mut self,+ source_indices: &[usize],+ destination_indices: &[usize],+ encoder: &mut Encoder<B>,+ kv_cache_update: &KVCacheUpdate<B>,+ ) {+ if source_indices == destination_indices {+ return;+ }++ let mut layer_data = KVLayerData {+ key_buffer: &mut self.keys,+ key_shape: self.shape,+ value_buffer: &mut self.values,+ value_shape: self.shape,+ };++ let data = std::slice::from_mut(&mut layer_data);+ let _ = kv_cache_update.encode(data, source_indices, destination_indices, encoder);+ }++ fn copy_rows<+ SrcKeys: AsBufferRangeRef<Buffer: Buffer<Backend = B>>,+ DstKeys: AsBufferRangeMut<Buffer: Buffer<Backend = B>>,+ SrcValues: AsBufferRangeRef<Buffer: Buffer<Backend = B>>,+ DstValues: AsBufferRangeMut<Buffer: Buffer<Backend = B>>,+ >(+ encoder: &mut Encoder<B>,+ src_keys: &SrcKeys,+ dst_keys: &mut DstKeys,+ src_values: &SrcValues,+ dst_values: &mut DstValues,+ row_size: usize,+ row_pairs: impl IntoIterator<Item = (usize, usize)>,+ ) {+ for (src_row, dst_row) in row_pairs {+ let src_offset = src_row * row_size;+ let dst_offset = dst_row * row_size;+ encoder.encode_copy(+ src_keys,+ src_offset..src_offset + row_size,+ dst_keys,+ dst_offset..dst_offset + row_size,+ );+ encoder.encode_copy(+ src_values,+ src_offset..src_offset + row_size,+ dst_values,+ dst_offset..dst_offset + row_size,+ );+ }+ }++ fn page_range_for_row_range(+ shape: [usize; 3],+ data_type: DataType,+ page_size_bytes: usize,+ row_range: Range<usize>,+ ) -> Range<usize> {+ if row_range.is_empty() {+ return 0..0;+ }++ let [_, num_groups, head_dim] = shape;+ let row_size = size_for_shape(&[1, num_groups, head_dim], data_type);+ let byte_start = row_range.start * row_size;+ let byte_end = row_range.end * row_size;+ byte_start / page_size_bytes..byte_end.div_ceil(page_size_bytes)+ }++ fn map_sparse_row_range(+ &mut self,+ context: &B::Context,+ row_range: Range<usize>,+ ) -> Result<(), B::Error> {+ if row_range.is_empty() {+ return Ok(());+ }++ if let Some(layer) = self.as_any_mut().downcast_mut::<KVCacheLayer<B, B::SparseBuffer>>() {+ let key_pages = Self::page_range_for_row_range(+ layer.shape,+ layer.data_type,+ layer.keys.page_size_bytes(),+ row_range.clone(),+ );+ layer.keys.map(context, &key_pages)?;++ let value_pages =+ Self::page_range_for_row_range(layer.shape, layer.data_type, layer.values.page_size_bytes(), row_range);+ layer.values.map(context, &value_pages)?;+ }+ Ok(())}}-impl<B: Backend> KVCacheLayer<B> {- pub fn encode_copy_prefix_rows_to(+impl<B: Backend, Buf: Buffer<Backend = B>> KVCacheLayerTrait<B> for KVCacheLayer<B, Buf> {+ fn as_any(&self) -> &dyn Any {+ self+ }++ fn as_any_mut(&mut self) -> &mut dyn Any {+ self+ }++ fn shape(&self) -> [usize; 3] {+ self.shape+ }++ fn data_type(&self) -> DataType {+ self.data_type+ }++ fn encode_zero(+ &mut self,+ encoder: &mut Encoder<'_, B>,+ ) {+ if self.as_any().is::<KVCacheLayer<B, B::SparseBuffer>>() {+ return;+ }+ encoder.encode_fill(&mut self.keys, 0);+ encoder.encode_fill(&mut self.values, 0);+ }++ fn state(&self) -> KVCacheLayerState {+ self.state.clone()+ }++ fn set_state(+ &mut self,+ state: &KVCacheLayerState,+ ) {+ self.state = state.clone();+ }++ fn clear_state(&mut self) {+ match &mut self.state {+ KVCacheLayerState::Full {+ prefix_len,+ } => {+ *prefix_len = 0;+ },+ KVCacheLayerState::Windowed {+ ring_offset,+ ring_length,+ ..+ } => {+ *ring_offset = 0;+ *ring_length = 0;+ },+ }+ }++ fn encode_copy_prefix_rows_to(&self,- destination: &mut KVCacheLayer<B>,+ destination: &mut dyn KVCacheLayerTrait<B>,row_count: usize,+ context: &B::Context,encoder: &mut Encoder<B>,) {if row_count == 0 {@@ -79,21 +317,28 @@ impl<B: Backend> KVCacheLayer<B> {}let [source_seq, num_groups, head_dim] = self.shape;- let [destination_seq, destination_groups, destination_head_dim] = destination.shape;+ let [destination_seq, destination_groups, destination_head_dim] = destination.shape();assert_eq!(num_groups, destination_groups, "KV cache group count mismatch");assert_eq!(head_dim, destination_head_dim, "KV cache head dim mismatch");- assert_eq!(self.data_type, destination.data_type, "KV cache dtype mismatch");+ assert_eq!(self.data_type, destination.data_type(), "KV cache dtype mismatch");assert!(row_count <= source_seq, "source KV cache copy exceeds source sequence");assert!(row_count <= destination_seq, "source KV cache copy exceeds destination sequence");+ destination.map_row_range(context, 0..row_count).expect("Failed to map destination KV cache rows");let row_size = size_for_shape(&[1, num_groups, head_dim], self.data_type);let copy_size = row_count * row_size;-- encoder.encode_copy(&self.keys, 0..copy_size, &mut destination.keys, 0..copy_size);- encoder.encode_copy(&self.values, 0..copy_size, &mut destination.values, 0..copy_size);+ if let Some(dest) = destination.as_any_mut().downcast_mut::<KVCacheLayer<B, B::SparseBuffer>>() {+ encoder.encode_copy(&self.keys, 0..copy_size, &mut dest.keys, 0..copy_size);+ encoder.encode_copy(&self.values, 0..copy_size, &mut dest.values, 0..copy_size);+ } else if let Some(dest) = destination.as_any_mut().downcast_mut::<KVCacheLayer<B, B::DenseBuffer>>() {+ encoder.encode_copy(&self.keys, 0..copy_size, &mut dest.keys, 0..copy_size);+ encoder.encode_copy(&self.values, 0..copy_size, &mut dest.values, 0..copy_size);+ } else {+ panic!("Wrong destination type!")+ }}- pub fn prefix_segment_length(&self) -> usize {+ fn prefix_segment_length(&self) -> usize {match &self.state {KVCacheLayerState::Full {prefix_len,@@ -105,22 +350,11 @@ impl<B: Backend> KVCacheLayer<B> {}}- pub fn window_length(&self) -> Option<usize> {- match &self.state {- KVCacheLayerState::Full {- ..- } => None,- KVCacheLayerState::Windowed {- window_length,- ..- } => Some(*window_length),- }- }-- pub fn update_after_acceptance(+ fn update_after_acceptance(&mut self,accepted_suffix_indices: &[usize],suffix_start: Option<usize>,+ context: &B::Context,encoder: &mut Encoder<B>,kv_cache_update: &KVCacheUpdate<B>,) {@@ -132,14 +366,18 @@ impl<B: Backend> KVCacheLayer<B> {return;}+ let prefix_len_value = *prefix_len;+// Absolute positions of the *source* rows.let source_indices: Vec<usize> =- accepted_suffix_indices.iter().map(|i| i + suffix_start.unwrap_or(*prefix_len)).collect();+ accepted_suffix_indices.iter().map(|i| i + suffix_start.unwrap_or(prefix_len_value)).collect();// Absolute positions of the *destination* rows.let destination_indices: Vec<usize> =- (*prefix_len..*prefix_len + accepted_suffix_indices.len()).collect();+ (prefix_len_value..prefix_len_value + accepted_suffix_indices.len()).collect();+ self.map_sparse_row_range(context, prefix_len_value..prefix_len_value + accepted_suffix_indices.len())+ .expect("Failed to map accepted KV cache rows");self.scatter_if_required(&source_indices, &destination_indices, encoder, kv_cache_update);},@@ -162,34 +400,16 @@ impl<B: Backend> KVCacheLayer<B> {destination_indices.push((*ring_length + *ring_offset + i) % *window_length);}+ for destination_index in destination_indices.iter().copied() {+ self.map_sparse_row_range(context, destination_index..destination_index + 1)+ .expect("Failed to map accepted KV cache row");+ }self.scatter_if_required(&source_indices, &destination_indices, encoder, kv_cache_update);},}}- fn scatter_if_required(- &mut self,- source_indices: &[usize],- destination_indices: &[usize],- encoder: &mut Encoder<B>,- kv_cache_update: &KVCacheUpdate<B>,- ) {- if source_indices == destination_indices {- return;- }-- let mut layer_data = KVLayerData {- key_allocation: &mut self.keys,- key_shape: self.shape,- value_allocation: &mut self.values,- value_shape: self.shape,- };-- let _ =- kv_cache_update.encode(std::slice::from_mut(&mut layer_data), source_indices, destination_indices, encoder);- }-- pub fn register_accepted_tokens(+ fn register_accepted_tokens(&mut self,number_of_accepted_tokens: usize,) {@@ -211,10 +431,11 @@ impl<B: Backend> KVCacheLayer<B> {}}- pub fn slice(+ fn slice(&self,context: &B::Context,- range: std::ops::Range<usize>,+ encoder: &mut Encoder<B>,+ range: Range<usize>,) -> Option<KVSlice<B>> {match self.state {KVCacheLayerState::Full {@@ -235,12 +456,6 @@ impl<B: Backend> KVCacheLayer<B> {let dtype = self.data_type;let row_size = size_for_shape(&[1, num_groups, head_dim], dtype);- let slice_shape = [len, num_groups, head_dim];- let mut slice_keys = context.create_array_uninitialized(&slice_shape, dtype);- let mut slice_values = context.create_array_uninitialized(&slice_shape, dtype);- let source_keys_bytes = allocation_as_bytes(&self.keys);- let source_values_bytes = allocation_as_bytes(&self.values);-let slots: Vec<usize> = (range.start..range.end).enumerate().map(|(offset, _)| {@@ -252,17 +467,19 @@ impl<B: Backend> KVCacheLayer<B> {}}).collect();+ let row_pairs = slots.iter().enumerate().map(|(dst_row, &src_row)| (src_row, dst_row));- let destination_keys_bytes = slice_keys.as_bytes_mut();- let destination_values_bytes = slice_values.as_bytes_mut();-- copy_rows(- source_keys_bytes,- source_values_bytes,- destination_keys_bytes,- destination_values_bytes,+ let slice_shape = [len, num_groups, head_dim];+ let mut dst_keys = context.create_array_uninitialized(&slice_shape, dtype);+ let mut dst_values = context.create_array_uninitialized(&slice_shape, dtype);+ Self::copy_rows(+ encoder,+ &self.keys,+ dst_keys.allocation_mut(),+ &self.values,+ dst_values.allocation_mut(),row_size,- slots.iter().enumerate().map(|(destination_row, &source_row)| (source_row, destination_row)),+ row_pairs,);Some(KVSlice::Window {@@ -270,17 +487,19 @@ impl<B: Backend> KVCacheLayer<B> {base_ring_offset: ring_offset,base_ring_length: ring_length,slots,- keys: slice_keys,- values: slice_values,+ keys: dst_keys,+ values: dst_values,})},}}- pub fn apply_slice(+ fn apply_slice(&mut self,+ context: &B::Context,+ encoder: &mut Encoder<B>,slice: &KVSlice<B>,- range: Option<std::ops::Range<usize>>,+ range: Option<Range<usize>>,) {match (slice, &mut self.state) {(@@ -316,69 +535,73 @@ impl<B: Backend> KVCacheLayer<B> {*w_len = *window_length;let [_, num_groups, head_dim] = self.shape;let row_size = size_for_shape(&[1, num_groups, head_dim], self.data_type);- let source_keys_bytes = keys.as_bytes();- let source_values_bytes = values.as_bytes();- let destination_keys_bytes = allocation_as_bytes_mut(&mut self.keys);- let destination_values_bytes = allocation_as_bytes_mut(&mut self.values);- match range {++ let row_pairs: Option<Vec<(usize, usize)>> = match &range {None => {*ring_offset = *base_ring_offset;*ring_length = *base_ring_length;-- copy_rows(- source_keys_bytes,- source_values_bytes,- destination_keys_bytes,- destination_values_bytes,- row_size,- slots- .iter()- .enumerate()- .map(|(source_row, &destination_row)| (source_row, destination_row)),- );+ let pairs = slots.iter().enumerate().map(|(src_row, &dst_row)| (src_row, dst_row)).collect();+ Some(pairs)},Some(r) => {let len = r.end.saturating_sub(r.start);if len == 0 {- return;- }+ None+ } else {+ let accepted = r.start;+ let base_len = *base_ring_length;+ let w = *window_length;++ let (new_offset, new_len) = if base_len < w {+ let after = base_len.saturating_add(accepted);+ if after <= w {+ (*base_ring_offset, after)+ } else {+ let overflow = after - w;+ ((base_ring_offset + overflow) % w, w)+ }+ } else {+ ((base_ring_offset + accepted) % w, w)+ };- let accepted = r.start;- let base_len = *base_ring_length;- let w = *window_length;+ *ring_offset = new_offset;+ *ring_length = new_len;- let (new_offset, new_len) = if base_len < w {- let after = base_len.saturating_add(accepted);- if after <= w {- (*base_ring_offset, after)- } else {- let overflow = after - w;- ((base_ring_offset + overflow) % w, w)- }- } else {- ((base_ring_offset + accepted) % w, w)- };-- *ring_offset = new_offset;- *ring_length = new_len;-- copy_rows(- source_keys_bytes,- source_values_bytes,- destination_keys_bytes,- destination_values_bytes,- row_size,- slots[r.clone()]- .iter()- .enumerate()- .map(|(index, &destination_row)| (r.start + index, destination_row)),- );+ let pairs =+ slots[r.clone()].iter().enumerate().map(move |(i, &dst_row)| (r.start + i, dst_row));+ Some(pairs.collect())+ }},+ };+ let Some(row_pairs) = row_pairs else {+ return;+ };+ for (_, destination_row) in row_pairs.iter().copied() {+ self.map_sparse_row_range(context, destination_row..destination_row + 1)+ .expect("Failed to map sliced KV cache row");}++ Self::copy_rows(+ encoder,+ keys.allocation(),+ &mut self.keys,+ values.allocation(),+ &mut self.values,+ row_size,+ row_pairs,+ );},_ => {},}}++ fn map_row_range(+ &mut self,+ context: &B::Context,+ range: Range<usize>,+ ) -> Result<(), B::Error> {+ self.map_sparse_row_range(context, range)+ }}#[cfg(test)]diff --git a/crates/backend-uzu/src/language_model/language_model_generator.rs b/crates/backend-uzu/src/language_model/language_model_generator.rsindex f9e0b38f2..5a0716400 100644--- a/crates/backend-uzu/src/language_model/language_model_generator.rs+++ b/crates/backend-uzu/src/language_model/language_model_generator.rs@@ -535,6 +535,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {self.context.cache_layers.borrow_mut().update_after_acceptance(&[0],None,+ &self.context.context,&mut encoder,&self.context.kv_cache_update,);@@ -629,7 +630,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {range: Range<usize>,) {let slice = slice.downcast_ref::<CacheLayersSlice<B>>().unwrap();- self.context.cache_layers.borrow_mut().apply_slice(slice, Some(range));+ self.context.cache_layers.borrow_mut().apply_slice(&self.context.context, slice, Some(range));}fn build_llm_context(&self) -> Box<dyn Any> {@@ -744,6 +745,7 @@ impl<B: Backend> LanguageModelGenerator<B> {let mut logits = None;if is_prefilling {let mut cache_layers = context.cache_layers.borrow_mut();+ cache_layers.prepare_for_forward_pass(&context.context, batch_dim);let decoder_arguments = token_inputs.decoder_arguments(context.shared_buffers.as_ref(),Some(&mut *cache_layers),@@ -759,6 +761,7 @@ impl<B: Backend> LanguageModelGenerator<B> {.map_err(|e| Error::EncodeFailed(Box::new(e)))?;} else {let mut cache_layers = context.cache_layers.borrow_mut();+ cache_layers.prepare_for_forward_pass(&context.context, batch_dim);let decoder_arguments = token_inputs.decoder_arguments(context.shared_buffers.as_ref(),Some(&mut *cache_layers),@@ -857,14 +860,15 @@ impl<B: Backend> LanguageModelGenerator<B> {suffix_start: Option<usize>,wait_until_completed: bool,) -> Result<(), Error> {- let mut encoder = Encoder::<B>::new(self.context.context.as_ref())- .map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?;+ let mut encoder =+ Encoder::<B>::new(&self.context.context).map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?;{let mut cache_layers = self.context.cache_layers.borrow_mut();cache_layers.update_after_acceptance(accepted_token_indices,suffix_start,+ &self.context.context,&mut encoder,&self.context.kv_cache_update,);diff --git a/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs b/crates/backend-uzu/src/session/tts_session/decoder_runtime.rsindex 7f3bffb8e..76b73cc51 100644--- a/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs+++ b/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs@@ -363,6 +363,7 @@ impl<B: Backend> TokenDecoderRunner<B> {let mut encoder = Encoder::new(context.as_ref()).map_err(unable_to_create_context)?;let mut cache_layers = self.ctx.cache_layers.borrow_mut();+ cache_layers.prepare_for_forward_pass(&context, token_count);self.ctx.executables.encode_prefill(@@ -571,6 +572,7 @@ impl<B: Backend> TokenDecoderRunner<B> {{let mut logits = {let mut cache_layers = cache_layers_rc.borrow_mut();+ cache_layers.prepare_for_forward_pass(&self.ctx.context, 1);self.ctx.executables.encode_decode(@@ -608,7 +610,13 @@ impl<B: Backend> TokenDecoderRunner<B> {pending_sampling_outputs.push(sampling_output);}- self.ctx.cache_layers.borrow_mut().update_after_acceptance(&[0], None, encoder, &self.ctx.kv_cache_update);+ self.ctx.cache_layers.borrow_mut().update_after_acceptance(+ &[0],+ None,+ &self.ctx.context,+ encoder,+ &self.ctx.kv_cache_update,+ );self.ctx.cache_layers.borrow_mut().register_accepted_tokens(1);}Ok(())@@ -787,6 +795,7 @@ impl<B: Backend> TokenDecoderRunner<B> {}let mut logits = {let mut cache_layers = self.ctx.cache_layers.borrow_mut();+ cache_layers.prepare_for_forward_pass(&self.ctx.context, token_count);let decoder_arguments = DecoderArguments {token_positions: token_inputs.token_positions(),token_parents: token_inputs.token_parents(),@@ -841,6 +850,7 @@ impl<B: Backend> TokenDecoderRunner<B> {self.ctx.cache_layers.borrow_mut().update_after_acceptance(accepted_suffix_indices,None,+ &self.ctx.context,encoder,&self.ctx.kv_cache_update,);diff --git a/crates/backend-uzu/tests/common/helpers.rs b/crates/backend-uzu/tests/common/helpers.rsindex e30b9a93a..60c8bebde 100644--- a/crates/backend-uzu/tests/common/helpers.rs+++ b/crates/backend-uzu/tests/common/helpers.rs@@ -2,7 +2,7 @@ use std::{mem::size_of, rc::Rc};use backend_uzu::{ArrayElement, allocation_copy_from_slice,- backends::common::{Allocation, AllocationType, Backend, Context, Encoder},+ backends::common::{Allocation, AllocationType, Backend, Buffer, Context, Encoder, SparseBuffer, SparseBufferExt},};pub fn allocation_size_bytes<T>(elements_count: usize) -> usize {@@ -56,3 +56,67 @@ pub fn create_context<B: Backend>() -> Rc<<B as Backend>::Context> {pub fn submit_encoder<B: Backend>(encoder: Encoder<B>) {encoder.end_encoding().submit().wait_until_completed().unwrap();}++pub fn sparse_buffer_create<B: Backend>(+ context: &B::Context,+ capacity: usize,+) -> B::SparseBuffer {+ context.create_sparse_buffer(capacity).expect("Failed to create sparse buffer")+}++pub fn sparse_buffer_create_and_map<B: Backend>(+ context: &B::Context,+ capacity: usize,+) -> B::SparseBuffer {+ let mut buffer = sparse_buffer_create::<B>(context, capacity);+ buffer.map(context, &(0..buffer.total_pages())).expect("Failed to map sparse buffer");+ buffer+}++pub fn sparse_buffer_create_with<B: Backend, T: ArrayElement>(+ context: &B::Context,+ data: &[T],+) -> B::SparseBuffer {+ let capacity_bytes = allocation_size_bytes::<T>(data.len());+ let mut buffer = sparse_buffer_create_and_map::<B>(context, capacity_bytes);+ sparse_buffer_write::<B, T>(context, &mut buffer, data);+ buffer+}++pub fn sparse_buffer_read_allocation<B: Backend>(+ context: &B::Context,+ buffer: &B::SparseBuffer,+ size: usize,+) -> Allocation<B> {+ let mut allocation = alloc_allocation::<B, u8>(context, size);+ let range = 0..size;++ let mut encoder = Encoder::new(context).expect("Failed to create encoder");+ encoder.encode_copy(buffer, range.clone(), &mut allocation, range.clone());+ submit_encoder(encoder);++ allocation+}++pub fn sparse_buffer_read_vec<B: Backend, T: ArrayElement>(+ context: &B::Context,+ buffer: &B::SparseBuffer,+ elements_count: usize,+) -> Vec<T> {+ let allocation =+ sparse_buffer_read_allocation::<B>(context, buffer, elements_count * T::data_type().size_in_bytes());+ allocation_to_vec(&allocation)+}++pub fn sparse_buffer_write<B: Backend, T: ArrayElement>(+ context: &B::Context,+ buffer: &mut B::SparseBuffer,+ data: &[T],+) {+ let data_allocation = alloc_allocation_with_data::<B, T>(context, data);+ let data_range = 0..allocation_size_bytes::<T>(data.len());++ let mut encoder = Encoder::new(context).expect("Failed to create encoder");+ encoder.encode_copy(&data_allocation, data_range.clone(), buffer, data_range.clone());+ submit_encoder(encoder);+}diff --git a/crates/backend-uzu/tests/common/matmul/harness.rs b/crates/backend-uzu/tests/common/matmul/harness.rsindex 3a52529cf..4aeb83d5d 100644--- a/crates/backend-uzu/tests/common/matmul/harness.rs+++ b/crates/backend-uzu/tests/common/matmul/harness.rs@@ -4,7 +4,7 @@ use backend_uzu::{ArrayContextExt, ArrayElement,backends::{common::{- AllocationType, Backend, Context, Encoder,+ Allocation, AllocationType, AsBufferRangeRef, Backend, Buffer, Context, Encoder,kernel::{ManualKernels,matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},@@ -118,7 +118,7 @@ fn run<B: Backend, T: ArrayElement + Float>(let mut encoder = Encoder::new(context).expect("encoder");encode(kernel,- MatmulArguments {+ MatmulArguments::<'_, B> {a: &a_allocation,a_offset: 0,b: b_array.allocation(),diff --git a/crates/backend-uzu/tests/integration/tracer/trace_validator.rs b/crates/backend-uzu/tests/integration/tracer/trace_validator.rsindex 058590d23..ec16386df 100644--- a/crates/backend-uzu/tests/integration/tracer/trace_validator.rs+++ b/crates/backend-uzu/tests/integration/tracer/trace_validator.rs@@ -17,7 +17,9 @@ use backend_uzu::{classifier::Classifier,config::{ModelConfig, ModelMetadata, ModelType},encodable_block::{DecoderDecodeInput, Sampling},- forward_pass::{cache_layers::CacheLayers, token_inputs::TokenInputs, traces::ActivationTrace},+ forward_pass::{+ cache_layers::CacheLayers, kv_cache_layer::KVCacheLayer, token_inputs::TokenInputs, traces::ActivationTrace,+ },language_model::{language_model_generator_context::LanguageModelGeneratorContext,sampler::{ArgmaxSampler, LogitsSampler},@@ -33,6 +35,7 @@ use half::{bf16, f16};use ndarray::{IxDyn, s};use num_traits::NumCast;+use crate::common;// ============================================================================// Validation Types// ============================================================================@@ -313,26 +316,38 @@ impl<B: Backend> TraceValidator<B> {};if let Ok(expected) = traces_view.leaf_array(&format!("updated_kv_cache.{}.keys", index)) {+ let size = kv.shape().iter().product::<usize>() * data_type.size_in_bytes();+ let keys = if let Some(layer) = kv.as_any().downcast_ref::<KVCacheLayer<B, B::SparseBuffer>>() {+ common::helpers::sparse_buffer_read_allocation(ctx.context.as_ref(), &layer.keys, size)+ } else {+ panic!("Wrong keys type")+ };results.push(TracerValidationResult {name: format!("updated_kv_cache.{}.keys", index),metrics: Self::validate_allocation(data_type,&expected,- &kv.keys,- &kv.shape,+ &keys,+ &kv.shape(),Some(ArrayTransform::KVCacheSlice),),});}if let Ok(expected) = traces_view.leaf_array(&format!("updated_kv_cache.{}.values", index)) {+ let size = kv.shape().iter().product::<usize>() * data_type.size_in_bytes();+ let values = if let Some(layer) = kv.as_any().downcast_ref::<KVCacheLayer<B, B::SparseBuffer>>() {+ common::helpers::sparse_buffer_read_allocation(ctx.context.as_ref(), &layer.values, size)+ } else {+ panic!("Wrong values type")+ };results.push(TracerValidationResult {name: format!("updated_kv_cache.{}.values", index),metrics: Self::validate_allocation(data_type,&expected,- &kv.values,- &kv.shape,+ &values,+ &kv.shape(),Some(ArrayTransform::KVCacheSlice),),});diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/kv_cache_update_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/kv_cache_update_test.rsindex e0f99a8a2..9b7ea8237 100644--- a/crates/backend-uzu/tests/unit/backends/common/kernel/kv_cache_update_test.rs+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/kv_cache_update_test.rs@@ -30,7 +30,7 @@ fn apply_swaps_3d<T: Clone>(}}-fn test_random_pattern(context: &<Metal as Backend>::Context) {+fn test_random_pattern<B: Backend>(context: &B::Context) {println!("Testing with random pattern...");let max_sequence_length = 256usize;@@ -64,15 +64,15 @@ fn test_random_pattern(context: &<Metal as Backend>::Context) {apply_swaps_3d(&mut expected_keys, &swaps);apply_swaps_3d(&mut expected_values, &swaps);- let mut key_allocation = common::helpers::alloc_allocation_with_data(context, key_data.as_slice().unwrap());- let mut value_allocation = common::helpers::alloc_allocation_with_data(context, value_data.as_slice().unwrap());-- let mut encoder = Encoder::new(context).unwrap();+ let mut key_buffer = common::helpers::sparse_buffer_create_with::<B, f32>(context, key_data.as_slice().unwrap());+ let mut value_buffer =+ common::helpers::sparse_buffer_create_with::<B, f32>(context, value_data.as_slice().unwrap());+ let mut encoder = Encoder::<B>::new(context).unwrap();{- let mut kv_layers = [KVLayerData::<Metal> {- key_allocation: &mut key_allocation,+ let mut kv_layers = [KVLayerData::<B, B::SparseBuffer> {+ key_buffer: &mut key_buffer,key_shape: [seq_len, num_heads, head_dim],- value_allocation: &mut value_allocation,+ value_buffer: &mut value_buffer,value_shape: [seq_len, num_heads, head_dim],}];match kv_cache_update.encode(&mut kv_layers, &source_indices, &destination_indices, &mut encoder) {@@ -86,11 +86,15 @@ fn test_random_pattern(context: &<Metal as Backend>::Context) {encoder.end_encoding().submit().wait_until_completed().unwrap();- let key_values: Vec<f32> = common::helpers::allocation_to_vec(&key_allocation);+ let shape = [seq_len, num_heads, head_dim];+ let elements_count = shape.iter().product();++ let key_values: Vec<f32> = common::helpers::sparse_buffer_read_vec::<B, f32>(context, &key_buffer, elements_count);let key_result = Array::from_shape_vec((seq_len, num_heads, head_dim), key_values).expect("Failed to convert key result to ndarray");- let value_values: Vec<f32> = common::helpers::allocation_to_vec(&value_allocation);+ let value_values: Vec<f32> =+ common::helpers::sparse_buffer_read_vec::<B, f32>(context, &value_buffer, elements_count);let value_result = Array::from_shape_vec((seq_len, num_heads, head_dim), value_values).expect("Failed to convert value result to ndarray");@@ -116,7 +120,7 @@ fn test_kv_cache_update_kernel() {},};- test_random_pattern(&metal_context);+ test_random_pattern::<Metal>(&metal_context);}#[test]diff --git a/crates/backend-uzu/tests/unit/backends/metal/sparse/sparse_buffer_test.rs b/crates/backend-uzu/tests/unit/backends/metal/sparse/sparse_buffer_test.rsindex 22f5315bf..8466d1970 100644--- a/crates/backend-uzu/tests/unit/backends/metal/sparse/sparse_buffer_test.rs+++ b/crates/backend-uzu/tests/unit/backends/metal/sparse/sparse_buffer_test.rs@@ -1,12 +1,18 @@+#[path = "../../../../common/mod.rs"]+mod common;+use std::{ops::Range, rc::Rc};-use crate::backends::{- common::{Backend, Context, SparseBuffer},- metal::Metal,+use backend_uzu::{+ backends::{+ common::{Backend, SparseBuffer},+ metal::Metal,+ },+ prelude::MetalContext,};-fn create_context() -> Rc<<Metal as Backend>::Context> {- <Metal as Backend>::Context::new().expect("Failed to create Metal context")+fn create_context() -> Rc<MetalContext> {+ common::helpers::create_context::<Metal>()}/// Capacity sized to fit `heap_count` worth of heaps.@@ -33,7 +39,7 @@ fn test_mapping_succeeds() {let capacity = buffer_capacity(ctx.as_ref(), 4);let pages = pages_for_heaps(ctx.as_ref(), 0, 4);- let mut sparse_buffer = ctx.create_sparse_buffer(capacity).expect("Failed to create sparse buffer");+ let mut sparse_buffer = common::helpers::sparse_buffer_create::<Metal>(&ctx, capacity);sparse_buffer.map(ctx.as_ref(), &pages).expect("Failed to map sparse buffer");}@@ -43,7 +49,7 @@ fn test_unmapping_succeeds() {let capacity = buffer_capacity(ctx.as_ref(), 4);let pages = pages_for_heaps(ctx.as_ref(), 0, 4);- let mut sparse_buffer = ctx.create_sparse_buffer(capacity).expect("Failed to create sparse buffer");+ let mut sparse_buffer = common::helpers::sparse_buffer_create::<Metal>(&ctx, capacity);sparse_buffer.map(ctx.as_ref(), &pages).expect("Failed to map sparse buffer");sparse_buffer.unmap(ctx.as_ref(), &pages).expect("Failed to unmap sparse buffer");}@@ -55,7 +61,7 @@ fn test_partial_unmapping_succeeds() {let mapped = pages_for_heaps(ctx.as_ref(), 0, 4);let unmapped = pages_for_heaps(ctx.as_ref(), 1, 2);- let mut sparse_buffer = ctx.create_sparse_buffer(capacity).expect("Failed to create sparse buffer");+ let mut sparse_buffer = common::helpers::sparse_buffer_create::<Metal>(&ctx, capacity);sparse_buffer.map(ctx.as_ref(), &mapped).expect("Failed to map sparse buffer");sparse_buffer.unmap(ctx.as_ref(), &unmapped).expect("Failed to unmap sparse buffer");}@@ -67,7 +73,7 @@ fn test_mapping_uses_minimum_heaps() {let capacity = buffer_capacity(ctx.as_ref(), heap_count);let pages = pages_for_heaps(ctx.as_ref(), 0, heap_count);- let mut sparse_buffer = ctx.create_sparse_buffer(capacity).expect("Failed to create sparse buffer");+ let mut sparse_buffer = common::helpers::sparse_buffer_create::<Metal>(&ctx, capacity);sparse_buffer.map(ctx.as_ref(), &pages).expect("Failed to map sparse buffer");let heaps = ctx.sparse_heap_pool();@@ -83,7 +89,7 @@ fn test_remapping_reuses_freed_pages() {let unmapped_pages = pages_for_heaps(ctx.as_ref(), 1, 1);let extra_pages = pages_for_heaps(ctx.as_ref(), 4, 1);- let mut sparse_buffer = ctx.create_sparse_buffer(capacity).expect("Failed to create sparse buffer");+ let mut sparse_buffer = common::helpers::sparse_buffer_create::<Metal>(&ctx, capacity);sparse_buffer.map(ctx.as_ref(), &initial_pages).expect("Failed to map sparse buffer");let initial_heaps = ctx.sparse_heap_pool().heaps_count();@@ -103,7 +109,7 @@ fn test_full_unmap_then_remap_reuses_heaps() {let capacity = buffer_capacity(ctx.as_ref(), 4);let pages = pages_for_heaps(ctx.as_ref(), 0, 4);- let mut sparse_buffer = ctx.create_sparse_buffer(capacity).expect("Failed to create sparse buffer");+ let mut sparse_buffer = common::helpers::sparse_buffer_create::<Metal>(&ctx, capacity);sparse_buffer.map(ctx.as_ref(), &pages).expect("Failed to map sparse buffer");let initial_heaps = ctx.sparse_heap_pool().heaps_count();@@ -123,7 +129,7 @@ fn test_remapping_same_pages_is_noop() {let capacity = buffer_capacity(ctx.as_ref(), 4);let pages = pages_for_heaps(ctx.as_ref(), 0, 4);- let mut sparse_buffer = ctx.create_sparse_buffer(capacity).expect("Failed to create sparse buffer");+ let mut sparse_buffer = common::helpers::sparse_buffer_create::<Metal>(&ctx, capacity);sparse_buffer.map(ctx.as_ref(), &pages).expect("Failed to map sparse buffer");let initial_heaps = ctx.sparse_heap_pool().heaps_count();@@ -136,6 +142,23 @@ fn test_remapping_same_pages_is_noop() {);}+#[test]+fn test_mapping_multiple_gaps_reserves_pages_between_gaps() {+ let ctx = create_context();+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();+ let capacity = buffer_capacity(ctx.as_ref(), 2);++ let mut sparse_buffer = common::helpers::sparse_buffer_create::<Metal>(&ctx, capacity);+ sparse_buffer.map(ctx.as_ref(), &(1..2)).expect("Failed to map initial page");+ sparse_buffer.map(ctx.as_ref(), &(0..pages_per_heap + 1)).expect("Failed to map range with multiple gaps");++ assert_eq!(+ ctx.sparse_heap_pool().heaps_count(),+ 2,+ "mapping multiple gaps in one call must reserve heap pages between gaps",+ );+}+#[test]fn test_drop_releases_pool_heaps() {// Regression: a mapped buffer dropped without an explicit unmap must@@ -145,7 +168,7 @@ fn test_drop_releases_pool_heaps() {let pages = pages_for_heaps(ctx.as_ref(), 0, 2);{- let mut sparse_buffer = ctx.create_sparse_buffer(capacity).expect("Failed to create sparse buffer");+ let mut sparse_buffer = common::helpers::sparse_buffer_create::<Metal>(&ctx, capacity);sparse_buffer.map(ctx.as_ref(), &pages).expect("Failed to map sparse buffer");assert!(ctx.sparse_heap_pool().heaps_count() > 0);}@@ -165,20 +188,17 @@ fn test_drop_does_not_disturb_other_buffer_mappings() {let capacity = buffer_capacity(ctx.as_ref(), 1);let pages = pages_for_heaps(ctx.as_ref(), 0, 1);- let mut keeper = ctx.create_sparse_buffer(capacity).expect("Failed to create keeper buffer");+ let mut keeper = common::helpers::sparse_buffer_create::<Metal>(&ctx, capacity);keeper.map(ctx.as_ref(), &pages).expect("Failed to map keeper buffer");let baseline = ctx.sparse_heap_pool().heaps_count();{- let mut transient = ctx.create_sparse_buffer(capacity).expect("Failed to create transient buffer");+ let mut transient = common::helpers::sparse_buffer_create::<Metal>(&ctx, capacity);transient.map(ctx.as_ref(), &pages).expect("Failed to map transient buffer");}- assert_eq!(- ctx.sparse_heap_pool().heaps_count(),- baseline,- "dropping a transient buffer must leave keeper buffer's mappings intact",- );+ let current_heaps = ctx.sparse_heap_pool().heaps_count();+ assert_eq!(current_heaps, baseline, "dropping a transient buffer must leave keeper buffer's mappings intact",);}#[test]@@ -188,7 +208,7 @@ fn test_sequential_mappings_are_compact() {let first = pages_for_heaps(ctx.as_ref(), 0, 2);let second = pages_for_heaps(ctx.as_ref(), 2, 2);- let mut sparse_buffer = ctx.create_sparse_buffer(capacity).expect("Failed to create sparse buffer");+ let mut sparse_buffer = common::helpers::sparse_buffer_create::<Metal>(&ctx, capacity);sparse_buffer.map(ctx.as_ref(), &first).expect("Failed to map first range");sparse_buffer.map(ctx.as_ref(), &second).expect("Failed to map second range");diff --git a/crates/backend-uzu/tests/unit/backends/metal/sparse/sparse_heap_pool_test.rs b/crates/backend-uzu/tests/unit/backends/metal/sparse/sparse_heap_pool_test.rsindex 0615e5a21..77c928463 100644--- a/crates/backend-uzu/tests/unit/backends/metal/sparse/sparse_heap_pool_test.rs+++ b/crates/backend-uzu/tests/unit/backends/metal/sparse/sparse_heap_pool_test.rs@@ -1,33 +1,18 @@-use std::rc::Rc;+#[path = "../../../../common/mod.rs"]+mod common;-use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};-use objc2::{rc::Retained, runtime::ProtocolObject};+use std::rc::Rc;use crate::{backends::{- common::{Backend, Context},+ common::SparseBuffer,metal::{Metal, metal_extensions::SparsePageSizeExt},},prelude::MetalContext,};fn create_context() -> Rc<MetalContext> {- <Metal as Backend>::Context::new().expect("Failed to create Metal context")-}--fn create_sparse_buffer(- ctx: &MetalContext,- capacity_bytes: usize,-) -> Retained<ProtocolObject<dyn MTLBuffer>> {- let page_size = ctx.sparse_heap_pool().page_size();- let aligned = capacity_bytes.next_multiple_of(page_size.in_bytes());- ctx.device- .new_buffer_with_length_options_placement_sparse_page_size(- aligned,- MTLResourceOptions::STORAGE_MODE_PRIVATE,- page_size,- )- .expect("Failed to create sparse buffer")+ common::helpers::create_context::<Metal>()}#[test]@@ -40,9 +25,9 @@ fn test_new_pool_is_empty() {fn test_map_empty_range_allocates_no_heaps() {let ctx = create_context();let cap = ctx.sparse_heap_pool().heap_capacity_bytes();- let buffer = create_sparse_buffer(ctx.as_ref(), cap);+ let mut sparse_buffer = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(0..0)).expect("map empty range");+ sparse_buffer.map(&ctx, &(0..0)).expect("map empty range");assert_eq!(ctx.sparse_heap_pool().heaps_count(), 0);}@@ -52,9 +37,9 @@ fn test_map_single_heap_allocates_one_heap() {let ctx = create_context();let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();let cap = ctx.sparse_heap_pool().heap_capacity_bytes();- let buffer = create_sparse_buffer(ctx.as_ref(), cap);+ let mut sparse_buffer = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(0..pages_per_heap)).expect("map");+ sparse_buffer.map(&ctx, &(0..pages_per_heap)).expect("map");assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);}@@ -64,9 +49,9 @@ fn test_map_multi_heap_allocates_minimum_heaps() {let ctx = create_context();let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();let cap = 4 * ctx.sparse_heap_pool().heap_capacity_bytes();- let buffer = create_sparse_buffer(ctx.as_ref(), cap);+ let mut sparse_buffer = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(0..4 * pages_per_heap)).expect("map");+ sparse_buffer.map(&ctx, &(0..4 * pages_per_heap)).expect("map");assert_eq!(ctx.sparse_heap_pool().heaps_count(), 4);}@@ -76,9 +61,9 @@ fn test_map_partial_heap_rounds_up_heap_count() {let ctx = create_context();let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();let cap = 2 * ctx.sparse_heap_pool().heap_capacity_bytes();- let buffer = create_sparse_buffer(ctx.as_ref(), cap);+ let mut sparse_buffer = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(0..pages_per_heap + 1)).expect("map");+ sparse_buffer.map(&ctx, &(0..pages_per_heap + 1)).expect("map");assert_eq!(ctx.sparse_heap_pool().heaps_count(), 2);}@@ -89,10 +74,10 @@ fn test_sequential_mappings_pack_into_one_heap() {let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();let half = pages_per_heap / 2;let cap = ctx.sparse_heap_pool().heap_capacity_bytes();- let buffer = create_sparse_buffer(ctx.as_ref(), cap);+ let mut sparse_buffer = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(0..half)).expect("map first");- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(half..pages_per_heap)).expect("map second");+ sparse_buffer.map(&ctx, &(0..half)).expect("map first");+ sparse_buffer.map(&ctx, &(half..pages_per_heap)).expect("map second");assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);}@@ -103,10 +88,10 @@ fn test_map_overflows_existing_heap_into_new_heap() {let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();let half = pages_per_heap / 2;let cap = 2 * ctx.sparse_heap_pool().heap_capacity_bytes();- let buffer = create_sparse_buffer(ctx.as_ref(), cap);+ let mut sparse_buffer = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(0..half)).expect("map first");- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(half..half + pages_per_heap)).expect("map overflow");+ sparse_buffer.map(&ctx, &(0..half)).expect("map first");+ sparse_buffer.map(&ctx, &(half..half + pages_per_heap)).expect("map overflow");assert_eq!(ctx.sparse_heap_pool().heaps_count(), 2);}@@ -117,11 +102,11 @@ fn test_two_buffers_share_one_heap() {let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();let half = pages_per_heap / 2;let cap = ctx.sparse_heap_pool().heap_capacity_bytes();- let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);- let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);+ let mut sparse_buffer_a = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);+ let mut sparse_buffer_b = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..half)).expect("map a");- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..half)).expect("map b");+ sparse_buffer_a.map(&ctx, &(0..half)).expect("map a");+ sparse_buffer_b.map(&ctx, &(0..half)).expect("map b");assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);}@@ -131,11 +116,11 @@ fn test_full_unmap_removes_heaps() {let ctx = create_context();let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();let cap = 2 * ctx.sparse_heap_pool().heap_capacity_bytes();- let buffer = create_sparse_buffer(ctx.as_ref(), cap);+ let mut sparse_buffer = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);let pages = 0..2 * pages_per_heap;- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &pages).expect("map");- ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer, &pages).expect("unmap");+ sparse_buffer.map(&ctx, &pages).expect("map");+ sparse_buffer.unmap(&ctx, &pages).expect("unmap");assert_eq!(ctx.sparse_heap_pool().heaps_count(), 0);}@@ -145,10 +130,10 @@ fn test_partial_unmap_keeps_heap() {let ctx = create_context();let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();let cap = ctx.sparse_heap_pool().heap_capacity_bytes();- let buffer = create_sparse_buffer(ctx.as_ref(), cap);+ let mut sparse_buffer = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(0..pages_per_heap)).expect("map");- ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer, &(0..pages_per_heap / 2)).expect("partial unmap");+ sparse_buffer.map(&ctx, &(0..pages_per_heap)).expect("map");+ sparse_buffer.unmap(&ctx, &(0..pages_per_heap / 2)).expect("partial unmap");assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);}@@ -158,12 +143,12 @@ fn test_unmap_with_other_buffer_keeps_mappings() {let ctx = create_context();let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();let cap = ctx.sparse_heap_pool().heap_capacity_bytes();- let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);- let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);+ let mut sparse_buffer_a = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);+ let mut sparse_buffer_b = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);let pages = 0..pages_per_heap;- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &pages).expect("map a");- ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer_b, &pages).expect("unmap b");+ sparse_buffer_a.map(&ctx, &pages).expect("map a");+ sparse_buffer_b.unmap(&ctx, &pages).expect("unmap b");assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);}@@ -174,12 +159,12 @@ fn test_unmap_one_buffer_keeps_heap_for_other() {let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();let half = pages_per_heap / 2;let cap = ctx.sparse_heap_pool().heap_capacity_bytes();- let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);- let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);+ let mut sparse_buffer_a = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);+ let mut sparse_buffer_b = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..half)).expect("map a");- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..half)).expect("map b");- ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer_a, &(0..half)).expect("unmap a");+ sparse_buffer_a.map(&ctx, &(0..half)).expect("map a");+ sparse_buffer_b.map(&ctx, &(0..half)).expect("map b");+ sparse_buffer_a.unmap(&ctx, &(0..half)).expect("unmap a");assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);}@@ -189,13 +174,13 @@ fn test_remap_after_unmap_does_not_grow_pool() {let ctx = create_context();let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();let cap = 2 * ctx.sparse_heap_pool().heap_capacity_bytes();- let buffer = create_sparse_buffer(ctx.as_ref(), cap);+ let mut sparse_buffer = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);let pages = 0..2 * pages_per_heap;- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &pages).expect("map");+ sparse_buffer.map(&ctx, &pages).expect("map");let initial = ctx.sparse_heap_pool().heaps_count();- ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer, &pages).expect("unmap");- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &pages).expect("remap");+ sparse_buffer.unmap(&ctx, &pages).expect("unmap");+ sparse_buffer.map(&ctx, &pages).expect("remap");assert_eq!(ctx.sparse_heap_pool().heaps_count(), initial);}@@ -206,11 +191,11 @@ fn test_remap_into_freed_gap_reuses_heap() {let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();let half = pages_per_heap / 2;let cap = ctx.sparse_heap_pool().heap_capacity_bytes();- let buffer = create_sparse_buffer(ctx.as_ref(), cap);+ let mut sparse_buffer = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(0..pages_per_heap)).expect("map full");- ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer, &(0..half)).expect("unmap half");- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(0..half)).expect("remap into gap");+ sparse_buffer.map(&ctx, &(0..pages_per_heap)).expect("map full");+ sparse_buffer.unmap(&ctx, &(0..half)).expect("unmap half");+ sparse_buffer.map(&ctx, &(0..half)).expect("remap into gap");assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);}@@ -224,11 +209,11 @@ fn test_sequential_partial_unmaps_release_full_heap() {let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();let half = pages_per_heap / 2;let cap = ctx.sparse_heap_pool().heap_capacity_bytes();- let buffer = create_sparse_buffer(ctx.as_ref(), cap);+ let mut sparse_buffer = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(0..pages_per_heap)).expect("map");- ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer, &(0..half)).expect("unmap prefix");- ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer, &(half..pages_per_heap)).expect("unmap suffix");+ sparse_buffer.map(&ctx, &(0..pages_per_heap)).expect("map");+ sparse_buffer.unmap(&ctx, &(0..half)).expect("unmap prefix");+ sparse_buffer.unmap(&ctx, &(half..pages_per_heap)).expect("unmap suffix");assert_eq!(ctx.sparse_heap_pool().heaps_count(), 0);}@@ -239,13 +224,13 @@ fn test_three_buffers_share_one_heap() {let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();let third = pages_per_heap / 3;let cap = ctx.sparse_heap_pool().heap_capacity_bytes();- let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);- let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);- let buffer_c = create_sparse_buffer(ctx.as_ref(), cap);+ let mut sparse_buffer_a = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);+ let mut sparse_buffer_b = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);+ let mut sparse_buffer_c = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..third)).expect("map a");- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..third)).expect("map b");- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_c, &(0..third)).expect("map c");+ sparse_buffer_a.map(&ctx, &(0..third)).expect("map a");+ sparse_buffer_b.map(&ctx, &(0..third)).expect("map b");+ sparse_buffer_c.map(&ctx, &(0..third)).expect("map c");assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);}@@ -256,11 +241,11 @@ fn test_two_buffers_exactly_fill_one_heap() {let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();let half = pages_per_heap / 2;let cap = ctx.sparse_heap_pool().heap_capacity_bytes();- let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);- let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);+ let mut sparse_buffer_a = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);+ let mut sparse_buffer_b = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..half)).expect("map a");- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..pages_per_heap - half)).expect("map b fills heap");+ sparse_buffer_a.map(&ctx, &(0..half)).expect("map a");+ sparse_buffer_b.map(&ctx, &(0..pages_per_heap - half)).expect("map b fills heap");assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);}@@ -271,13 +256,13 @@ fn test_third_buffer_spills_when_heap_filled_by_others() {let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();let half = pages_per_heap / 2;let cap = ctx.sparse_heap_pool().heap_capacity_bytes();- let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);- let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);- let buffer_c = create_sparse_buffer(ctx.as_ref(), cap);+ let mut sparse_buffer_a = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);+ let mut sparse_buffer_b = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);+ let mut sparse_buffer_c = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..half)).expect("map a");- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..pages_per_heap - half)).expect("map b fills heap");- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_c, &(0..1)).expect("map c spills");+ sparse_buffer_a.map(&ctx, &(0..half)).expect("map a");+ sparse_buffer_b.map(&ctx, &(0..pages_per_heap - half)).expect("map b fills heap");+ sparse_buffer_c.map(&ctx, &(0..1)).expect("map c spills");assert_eq!(ctx.sparse_heap_pool().heaps_count(), 2);}@@ -288,14 +273,14 @@ fn test_unmap_buffer_frees_pages_for_another_in_same_heap() {let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();let half = pages_per_heap / 2;let cap = ctx.sparse_heap_pool().heap_capacity_bytes();- let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);- let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);- let buffer_c = create_sparse_buffer(ctx.as_ref(), cap);+ let mut sparse_buffer_a = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);+ let mut sparse_buffer_b = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);+ let mut sparse_buffer_c = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..half)).expect("map a");- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..pages_per_heap - half)).expect("map b fills heap");- ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer_a, &(0..half)).expect("unmap a");- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_c, &(0..half)).expect("map c into gap");+ sparse_buffer_a.map(&ctx, &(0..half)).expect("map a");+ sparse_buffer_b.map(&ctx, &(0..pages_per_heap - half)).expect("map b fills heap");+ sparse_buffer_a.unmap(&ctx, &(0..half)).expect("unmap a");+ sparse_buffer_c.map(&ctx, &(0..half)).expect("map c into gap");assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);}@@ -306,14 +291,14 @@ fn test_unmap_middle_of_three_buffers_keeps_heap() {let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();let third = pages_per_heap / 3;let cap = ctx.sparse_heap_pool().heap_capacity_bytes();- let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);- let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);- let buffer_c = create_sparse_buffer(ctx.as_ref(), cap);+ let mut sparse_buffer_a = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);+ let mut sparse_buffer_b = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);+ let mut sparse_buffer_c = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..third)).expect("map a");- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..third)).expect("map b");- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_c, &(0..third)).expect("map c");- ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer_b, &(0..third)).expect("unmap b");+ sparse_buffer_a.map(&ctx, &(0..third)).expect("map a");+ sparse_buffer_b.map(&ctx, &(0..third)).expect("map b");+ sparse_buffer_c.map(&ctx, &(0..third)).expect("map c");+ sparse_buffer_b.unmap(&ctx, &(0..third)).expect("unmap b");assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);}@@ -324,16 +309,16 @@ fn test_unmap_all_shared_buffers_releases_heap() {let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();let third = pages_per_heap / 3;let cap = ctx.sparse_heap_pool().heap_capacity_bytes();- let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);- let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);- let buffer_c = create_sparse_buffer(ctx.as_ref(), cap);+ let mut sparse_buffer_a = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);+ let mut sparse_buffer_b = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);+ let mut sparse_buffer_c = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..third)).expect("map a");- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..third)).expect("map b");- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_c, &(0..third)).expect("map c");- ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer_a, &(0..third)).expect("unmap a");- ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer_b, &(0..third)).expect("unmap b");- ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer_c, &(0..third)).expect("unmap c");+ sparse_buffer_a.map(&ctx, &(0..third)).expect("map a");+ sparse_buffer_b.map(&ctx, &(0..third)).expect("map b");+ sparse_buffer_c.map(&ctx, &(0..third)).expect("map c");+ sparse_buffer_a.unmap(&ctx, &(0..third)).expect("unmap a");+ sparse_buffer_b.unmap(&ctx, &(0..third)).expect("unmap b");+ sparse_buffer_c.unmap(&ctx, &(0..third)).expect("unmap c");assert_eq!(ctx.sparse_heap_pool().heaps_count(), 0);}@@ -344,11 +329,11 @@ fn test_second_buffer_spans_existing_and_new_heap() {let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();let half = pages_per_heap / 2;let cap = 2 * ctx.sparse_heap_pool().heap_capacity_bytes();- let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);- let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);+ let mut sparse_buffer_a = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);+ let mut sparse_buffer_b = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..half)).expect("map a");- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..pages_per_heap)).expect("map b spans heaps");+ sparse_buffer_a.map(&ctx, &(0..half)).expect("map a");+ sparse_buffer_b.map(&ctx, &(0..pages_per_heap)).expect("map b spans heaps");assert_eq!(ctx.sparse_heap_pool().heaps_count(), 2);}@@ -359,12 +344,12 @@ fn test_unmap_spanning_buffer_keeps_heap_with_other_buffer() {let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();let half = pages_per_heap / 2;let cap = 2 * ctx.sparse_heap_pool().heap_capacity_bytes();- let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);- let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);+ let mut sparse_buffer_a = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);+ let mut sparse_buffer_b = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..half)).expect("map a");- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..pages_per_heap)).expect("map b spans heaps");- ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer_b, &(0..pages_per_heap)).expect("unmap b");+ sparse_buffer_a.map(&ctx, &(0..half)).expect("map a");+ sparse_buffer_b.map(&ctx, &(0..pages_per_heap)).expect("map b spans heaps");+ sparse_buffer_b.unmap(&ctx, &(0..pages_per_heap)).expect("unmap b");assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);}@@ -375,16 +360,16 @@ fn test_remap_two_buffers_after_unmap_does_not_grow_pool() {let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();let half = pages_per_heap / 2;let cap = ctx.sparse_heap_pool().heap_capacity_bytes();- let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);- let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);+ let mut sparse_buffer_a = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);+ let mut sparse_buffer_b = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..half)).expect("map a");- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..half)).expect("map b");+ sparse_buffer_a.map(&ctx, &(0..half)).expect("map a");+ sparse_buffer_b.map(&ctx, &(0..half)).expect("map b");let initial = ctx.sparse_heap_pool().heaps_count();- ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer_a, &(0..half)).expect("unmap a");- ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer_b, &(0..half)).expect("unmap b");- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..half)).expect("remap a");- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..half)).expect("remap b");+ sparse_buffer_a.unmap(&ctx, &(0..half)).expect("unmap a");+ sparse_buffer_b.unmap(&ctx, &(0..half)).expect("unmap b");+ sparse_buffer_a.map(&ctx, &(0..half)).expect("remap a");+ sparse_buffer_b.map(&ctx, &(0..half)).expect("remap b");assert_eq!(ctx.sparse_heap_pool().heaps_count(), initial);}diff --git a/crates/backend-uzu/tests/unit/forward_pass/cache_layers_test.rs b/crates/backend-uzu/tests/unit/forward_pass/cache_layers_test.rsnew file mode 100644index 000000000..a4684da3a--- /dev/null+++ b/crates/backend-uzu/tests/unit/forward_pass/cache_layers_test.rs@@ -0,0 +1,204 @@+#![cfg(metal_backend)]++use std::{mem::size_of, ops::Range};++use super::{CacheEntryIndex, CacheLayer, CacheLayers, LayerCacheBinding};+use crate::{+ DataType, allocation_copy_from_slice, allocation_to_vec,+ backends::{+ common::{Allocation, AllocationType, Backend, Context, Encoder, SparseBuffer},+ metal::Metal,+ },+ forward_pass::kv_cache_layer::{KVCacheLayer, KVCacheLayerState, KVCacheLayerTrait},+};++fn submit_encoder(encoder: Encoder<Metal>) {+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to submit encoder");+}++fn allocation_from_slice(+ context: &<Metal as Backend>::Context,+ data: &[f32],+) -> Allocation<Metal> {+ let mut allocation =+ context.create_allocation(size_of_val(data), AllocationType::Global).expect("Failed to create allocation");+ allocation_copy_from_slice(&mut allocation, data).expect("Failed to write allocation");+ allocation+}++fn copy_allocation_to_sparse_row(+ context: &<Metal as Backend>::Context,+ allocation: &Allocation<Metal>,+ buffer: &mut <Metal as Backend>::SparseBuffer,+ row_size: usize,+ row_index: usize,+) {+ let row_range = row_byte_range(row_size, row_index);+ let mut encoder = Encoder::<Metal>::new(context).expect("Failed to create encoder");+ encoder.encode_copy(allocation, 0..row_size, buffer, row_range);+ submit_encoder(encoder);+}++fn read_sparse_row(+ context: &<Metal as Backend>::Context,+ buffer: &<Metal as Backend>::SparseBuffer,+ row_size: usize,+ row_index: usize,+) -> Vec<f32> {+ let mut allocation =+ context.create_allocation(row_size, AllocationType::Global).expect("Failed to create allocation");+ let mut encoder = Encoder::<Metal>::new(context).expect("Failed to create encoder");+ encoder.encode_copy(buffer, row_byte_range(row_size, row_index), &mut allocation, 0..row_size);+ submit_encoder(encoder);+ allocation_to_vec(&allocation)+}++fn row_byte_range(+ row_size: usize,+ row_index: usize,+) -> Range<usize> {+ row_index * row_size..(row_index + 1) * row_size+}++fn row_data(+ elements_per_row: usize,+ base_value: f32,+) -> Vec<f32> {+ (0..elements_per_row).map(|element_index| base_value + element_index as f32).collect()+}++fn make_cache_layers(+ context: &<Metal as Backend>::Context,+ state: KVCacheLayerState,+ elements_per_row: usize,+) -> CacheLayers<Metal> {+ let window_length = match state {+ KVCacheLayerState::Windowed {+ window_length,+ ..+ } => window_length,+ KVCacheLayerState::Full {+ ..+ } => panic!("test expects windowed state"),+ };+ let shape = [window_length, elements_per_row, 1];+ let buffer_size = window_length * elements_per_row * size_of::<f32>();+ let layer = KVCacheLayer {+ state,+ keys: context.create_sparse_buffer(buffer_size).expect("Failed to create key buffer"),+ values: context.create_sparse_buffer(buffer_size).expect("Failed to create value buffer"),+ shape,+ data_type: DataType::F32,+ };++ CacheLayers {+ max_suffix_length: 0,+ max_prefix_length: window_length,+ entries: vec![CacheLayer::Transformer(Box::new(layer))].into_boxed_slice(),+ bindings: vec![LayerCacheBinding::Owned {+ entry: CacheEntryIndex {+ index: 0,+ },+ }]+ .into_boxed_slice(),+ }+}++#[test]+fn test_cache_layers_copy_from_clones_only_live_window_rows() {+ let Some(context) = <Metal as Backend>::Context::new().ok() else {+ return;+ };++ let page_size = context.create_sparse_buffer(1).expect("Failed to create sparse buffer").page_size_bytes();+ let elements_per_row = page_size / size_of::<f32>();+ let row_size = page_size;++ let state = KVCacheLayerState::Windowed {+ ring_offset: 1,+ ring_length: 2,+ window_length: 4,+ };+ let mut source = make_cache_layers(&context, state.clone(), elements_per_row);+ let mut destination = make_cache_layers(+ &context,+ KVCacheLayerState::Windowed {+ ring_offset: 0,+ ring_length: 0,+ window_length: 4,+ },+ elements_per_row,+ );++ let CacheLayer::Transformer(source_layer) = &mut source.entries[0] else {+ panic!("expected transformer layer");+ };+ let source_layer = source_layer+ .as_any_mut()+ .downcast_mut::<KVCacheLayer<Metal, <Metal as Backend>::SparseBuffer>>()+ .expect("expected sparse KV layer");++ source_layer.map_row_range(&context, 1..2).expect("Failed to map source row 1");+ source_layer.map_row_range(&context, 2..3).expect("Failed to map source row 2");++ let row_1_keys = row_data(elements_per_row, 101.0);+ let row_2_keys = row_data(elements_per_row, 102.0);+ let row_1_values = row_data(elements_per_row, 201.0);+ let row_2_values = row_data(elements_per_row, 202.0);++ copy_allocation_to_sparse_row(+ &context,+ &allocation_from_slice(&context, &row_1_keys),+ &mut source_layer.keys,+ row_size,+ 1,+ );+ copy_allocation_to_sparse_row(+ &context,+ &allocation_from_slice(&context, &row_2_keys),+ &mut source_layer.keys,+ row_size,+ 2,+ );+ copy_allocation_to_sparse_row(+ &context,+ &allocation_from_slice(&context, &row_1_values),+ &mut source_layer.values,+ row_size,+ 1,+ );+ copy_allocation_to_sparse_row(+ &context,+ &allocation_from_slice(&context, &row_2_values),+ &mut source_layer.values,+ row_size,+ 2,+ );++ destination.copy_from(&source, &context);++ let CacheLayer::Transformer(destination_layer) = &destination.entries[0] else {+ panic!("expected transformer layer");+ };+ let destination_layer = destination_layer+ .as_any()+ .downcast_ref::<KVCacheLayer<Metal, <Metal as Backend>::SparseBuffer>>()+ .expect("expected sparse KV layer");++ assert_eq!(read_sparse_row(&context, &destination_layer.keys, row_size, 1), row_1_keys);+ assert_eq!(read_sparse_row(&context, &destination_layer.keys, row_size, 2), row_2_keys);+ assert_eq!(read_sparse_row(&context, &destination_layer.values, row_size, 1), row_1_values);+ assert_eq!(read_sparse_row(&context, &destination_layer.values, row_size, 2), row_2_values);++ let KVCacheLayerState::Windowed {+ ring_offset,+ ring_length,+ window_length,+ } = destination_layer.state+ else {+ panic!("expected windowed state");+ };+ assert_eq!(ring_offset, 1);+ assert_eq!(ring_length, 2);+ assert_eq!(window_length, 4);+}diff --git a/crates/backend-uzu/tests/unit/forward_pass/kv_cache_state_test.rs b/crates/backend-uzu/tests/unit/forward_pass/kv_cache_state_test.rsindex 95a66311b..03cceef5f 100644--- a/crates/backend-uzu/tests/unit/forward_pass/kv_cache_state_test.rs+++ b/crates/backend-uzu/tests/unit/forward_pass/kv_cache_state_test.rs@@ -1,9 +1,11 @@#![cfg(metal_backend)]+use backend_uzu::forward_pass::kv_cache_layer::KVCacheLayerTrait;+use crate::{DataType,backends::{- common::{Allocation, Backend, Context, Encoder, kernel::kv_cache_update::KVCacheUpdate},+ common::{Backend, Context, Encoder, kernel::kv_cache_update::KVCacheUpdate},metal::Metal,},forward_pass::kv_cache_layer::{KVCacheLayer, KVCacheLayerState},@@ -12,8 +14,6 @@ use crate::{#[path = "../../common/mod.rs"]mod common;-use common::helpers::{alloc_allocation_with_data, allocation_to_vec, write_allocation};-#[derive(Debug)]struct Scenario {name: &'static str,@@ -29,12 +29,12 @@ struct Scenario {expected_prefix_segment_length: usize,}-fn make_test_layer(- context: &<Metal as Backend>::Context,+fn make_test_layer<B: Backend>(+ context: &B::Context,state: KVCacheLayerState,prefix_capacity: usize,suffix_capacity: usize,-) -> KVCacheLayer<Metal> {+) -> (KVCacheLayer<B, B::SparseBuffer>, [usize; 3]) {let total_len = match &state {KVCacheLayerState::Full {..@@ -47,41 +47,46 @@ fn make_test_layer(let shape = [total_len.max(1), 1, 1];let zeroes = vec![0.0_f32; shape.iter().product()];- let keys = alloc_allocation_with_data(context, &zeroes);- let values = alloc_allocation_with_data(context, &zeroes);-- KVCacheLayer {+ let keys = common::helpers::sparse_buffer_create_with::<B, f32>(context, &zeroes);+ let values = common::helpers::sparse_buffer_create_with::<B, f32>(context, &zeroes);+ let layer = KVCacheLayer {state,keys,values,shape,data_type: DataType::F32,- }+ };+ (layer, shape)}-fn overwrite_allocation(- allocation: &mut Allocation<Metal>,+fn overwrite_sparse_buffer<B: Backend>(+ context: &B::Context,+ buffer: &mut B::SparseBuffer,+ elements_count: usize,updates: &[(usize, f32)],) {- let mut data: Vec<f32> = allocation_to_vec(allocation);+ let mut data: Vec<f32> = common::helpers::sparse_buffer_read_vec::<B, f32>(context, buffer, elements_count);for (index, value) in updates {data[*index] = *value;}- write_allocation(allocation, &data);+ common::helpers::sparse_buffer_write::<B, f32>(context, buffer, &data);}-fn fill_arrays(layer: &mut KVCacheLayer<Metal>) -> (Vec<f32>, Vec<f32>) {+fn fill_arrays<B: Backend>(+ context: &B::Context,+ layer: &mut KVCacheLayer<B, B::SparseBuffer>,+) -> (Vec<f32>, Vec<f32>) {let initial_keys = {let len = layer.shape.iter().product();let data: Vec<f32> = (0..len).map(|idx| 1_000.0 + idx as f32).collect();- write_allocation(&mut layer.keys, &data);+ common::helpers::sparse_buffer_write::<B, f32>(context, &mut layer.keys, &data);data};let initial_values = {let len = layer.shape.iter().product();let data: Vec<f32> = (0..len).map(|idx| 2_000.0 + idx as f32).collect();- write_allocation(&mut layer.values, &data);+ common::helpers::sparse_buffer_write::<B, f32>(context, &mut layer.values, &data);data};@@ -128,16 +133,16 @@ fn expected_after_update(expected}-fn run_scenario(- context: &<Metal as Backend>::Context,+fn run_scenario<B: Backend>(+ context: &B::Context,scenario: &Scenario,) {- let mut layer =+ let (mut layer, _) =make_test_layer(context, scenario.state.clone(), scenario.prefix_capacity, scenario.suffix_capacity);let state_before_update = layer.state.clone();- let (initial_keys, initial_values) = fill_arrays(&mut layer);+ let (initial_keys, initial_values) = fill_arrays::<B>(context, &mut layer);let expected_keys = expected_after_update(&state_before_update, &scenario.accepted_suffix_indices, &initial_keys);let expected_values =@@ -164,6 +169,7 @@ fn run_scenario(layer.update_after_acceptance(&scenario.accepted_suffix_indices,scenario.suffix_start,+ context,&mut encoder,&kv_cache_update,);@@ -172,10 +178,12 @@ fn run_scenario(layer.register_accepted_tokens(scenario.number_of_accepted_tokens);- let actual_keys: Vec<f32> = allocation_to_vec(&layer.keys);+ let actual_keys: Vec<f32> =+ common::helpers::sparse_buffer_read_vec::<B, f32>(context, &layer.keys, expected_keys.len());assert_eq!(actual_keys, expected_keys, "{}: key buffer mismatch", scenario.name);- let actual_values: Vec<f32> = allocation_to_vec(&layer.values);+ let actual_values: Vec<f32> =+ common::helpers::sparse_buffer_read_vec::<B, f32>(context, &layer.values, expected_values.len());assert_eq!(actual_values, expected_values, "{}: value buffer mismatch", scenario.name);match &layer.state {@@ -284,7 +292,7 @@ fn kv_cache_state_scenarios() {];for scenario in &scenarios {- run_scenario(&context, scenario);+ run_scenario::<Metal>(&context, scenario);}}@@ -294,7 +302,7 @@ fn kv_cache_slice_apply_contiguous_window() {return;};- let mut layer = make_test_layer(+ let (mut layer, shape) = make_test_layer::<Metal>(&context,KVCacheLayerState::Windowed {ring_offset: 1,@@ -304,17 +312,25 @@ fn kv_cache_slice_apply_contiguous_window() {4,0,);- let (initial_keys, initial_values) = fill_arrays(&mut layer);+ let (initial_keys, initial_values) = fill_arrays::<Metal>(&context, &mut layer);- let slice = layer.slice(&context, 0..2).expect("slice should exist");- // Mutate the captured slots.- overwrite_allocation(&mut layer.keys, &[(0, -1.0), (1, -2.0)]);- overwrite_allocation(&mut layer.values, &[(0, -3.0), (1, -4.0)]);+ let mut encoder = Encoder::<Metal>::new(&context).expect("Failed to create encoder");+ let slice = layer.slice(&context, &mut encoder, 0..2).expect("slice should exist");+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to end and wait encoder");- layer.apply_slice(&slice, None);-- let keys_after: Vec<f32> = allocation_to_vec(&layer.keys);- let values_after: Vec<f32> = allocation_to_vec(&layer.values);+ // Mutate the captured slots.+ let elements_count = shape.iter().product();+ overwrite_sparse_buffer::<Metal>(&context, &mut layer.keys, elements_count, &[(0, -1.0), (1, -2.0)]);+ overwrite_sparse_buffer::<Metal>(&context, &mut layer.values, elements_count, &[(0, -3.0), (1, -4.0)]);++ let mut encoder = Encoder::<Metal>::new(&context).expect("Failed to create encoder");+ layer.apply_slice(&context, &mut encoder, &slice, None);+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to end and wait encoder");++ let keys_after: Vec<f32> =+ common::helpers::sparse_buffer_read_vec::<Metal, f32>(&context, &layer.keys, elements_count);+ let values_after: Vec<f32> =+ common::helpers::sparse_buffer_read_vec::<Metal, f32>(&context, &layer.values, elements_count);assert_eq!(keys_after[0..4], initial_keys[0..4], "keys restored for contiguous slice");assert_eq!(values_after[0..4], initial_values[0..4], "values restored for contiguous slice");}@@ -325,7 +341,7 @@ fn kv_cache_slice_apply_wrap_window() {return;};- let mut layer = make_test_layer(+ let (mut layer, shape) = make_test_layer::<Metal>(&context,KVCacheLayerState::Windowed {ring_offset: 3,@@ -335,17 +351,25 @@ fn kv_cache_slice_apply_wrap_window() {4,0,);- let (initial_keys, initial_values) = fill_arrays(&mut layer);+ let (initial_keys, initial_values) = fill_arrays::<Metal>(&context, &mut layer);- let slice = layer.slice(&context, 2..4).expect("slice should exist");- // Captured slots are expected to wrap; mutate them.- overwrite_allocation(&mut layer.keys, &[(2, -11.0), (3, -12.0)]);- overwrite_allocation(&mut layer.values, &[(2, -13.0), (3, -14.0)]);+ let mut encoder = Encoder::<Metal>::new(&context).expect("Failed to create encoder");+ let slice = layer.slice(&context, &mut encoder, 2..4).expect("slice should exist");+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to end and wait encoder");- layer.apply_slice(&slice, None);-- let keys_after: Vec<f32> = allocation_to_vec(&layer.keys);- let values_after: Vec<f32> = allocation_to_vec(&layer.values);+ // Captured slots are expected to wrap; mutate them.+ let elements_count = shape.iter().product();+ overwrite_sparse_buffer::<Metal>(&context, &mut layer.keys, elements_count, &[(2, -11.0), (3, -12.0)]);+ overwrite_sparse_buffer::<Metal>(&context, &mut layer.values, elements_count, &[(2, -13.0), (3, -14.0)]);++ let mut encoder = Encoder::<Metal>::new(&context).expect("Failed to create encoder");+ layer.apply_slice(&context, &mut encoder, &slice, None);+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to end and wait encoder");++ let keys_after: Vec<f32> =+ common::helpers::sparse_buffer_read_vec::<Metal, f32>(&context, &layer.keys, elements_count);+ let values_after: Vec<f32> =+ common::helpers::sparse_buffer_read_vec::<Metal, f32>(&context, &layer.values, elements_count);assert_eq!(keys_after[0..4], initial_keys[0..4], "keys restored for wrapped slice");assert_eq!(values_after[0..4], initial_values[0..4], "values restored for wrapped slice");}@@ -356,7 +380,7 @@ fn kv_cache_slice_apply_full_restores_metadata() {return;};- let mut layer = make_test_layer(+ let (mut layer, _) = make_test_layer::<Metal>(&context,KVCacheLayerState::Full {prefix_len: 3,@@ -365,7 +389,9 @@ fn kv_cache_slice_apply_full_restores_metadata() {0,);- let slice = layer.slice(&context, 0..2).expect("full slice exists");+ let mut encoder = Encoder::<Metal>::new(&context).expect("Failed to create encoder");+ let slice = layer.slice(&context, &mut encoder, 0..2).expect("full slice exists");+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to end and wait encoder");// Mutate metadata.if let KVCacheLayerState::Full {@@ -375,7 +401,9 @@ fn kv_cache_slice_apply_full_restores_metadata() {*prefix_len = 1;}- layer.apply_slice(&slice, None);+ let mut encoder = Encoder::<Metal>::new(&context).expect("Failed to create encoder");+ layer.apply_slice(&context, &mut encoder, &slice, None);+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to end and wait encoder");if let KVCacheLayerState::Full {prefix_len,diff --git a/crates/backend-uzu/tests/unit/kernel/attention/attention_gemm_test.rs b/crates/backend-uzu/tests/unit/kernel/attention/attention_gemm_test.rsindex b1dff7fde..be3989145 100644--- a/crates/backend-uzu/tests/unit/kernel/attention/attention_gemm_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/attention/attention_gemm_test.rs@@ -4,7 +4,7 @@ use backend_uzu::{ArrayElement, DataType,backends::{common::{- Backend, Context, Encoder,+ Allocation, Backend, Context, Encoder,kernel::attention::{AttentionGemmArguments, AttentionGemmBlock},},cpu::Cpu,@@ -96,7 +96,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {let segment_prefix_length = input.sequence_length - input.suffix_length;- let args = AttentionGemmArguments::<B> {+ let args = AttentionGemmArguments::<B, Allocation<B>> {queries: &queries_allocation,keys: &keys_allocation,values: &values_allocation,