Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

KV cache sparse buffers (#434)

Aleksandr Golokoz28fed454ee07
diff --git a/Cargo.lock b/Cargo.lock
index 3fd5ea8d9..91095f9e7 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3831,9 +3831,9 @@ dependencies = [
[[package]]
name = "mtl-rs"
-version = "0.1.11"
+version = "0.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "88e5c70772dfcf1f3f89d873fbd7e039d9d291b192b84df2b5f462e3d3597fba"
+checksum = "b5da07ba3cea3adad7b479c3d1b097875e8c56e9711830b72c8c2b5f5cf0ce8f"
dependencies = [
"bitflags 2.11.1",
"block2",
diff --git a/Cargo.toml b/Cargo.toml
index 60c112317..63e88a91e 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -69,7 +69,7 @@ tokenizers = { version = "0.22", features = [
objc2 = "0.6.4"
objc2-foundation = "0.3.2"
objc2-security = "0.3.2"
-metal = { package = "mtl-rs", version = "0.1" }
+metal = { package = "mtl-rs", version = "0.1.12" }
block2 = "0.6"
bytemuck = "1.23"
bytesize = "2"
diff --git a/crates/backend-uzu/src/backends/common/command_buffer.rs b/crates/backend-uzu/src/backends/common/command_buffer.rs
index 2702dbdb7..a4190b047 100644
--- a/crates/backend-uzu/src/backends/common/command_buffer.rs
+++ b/crates/backend-uzu/src/backends/common/command_buffer.rs
@@ -1,6 +1,6 @@
use std::time::Duration;
-use super::{Backend, BufferRangeMut, BufferRangeRef};
+use super::{Backend, Buffer, BufferRangeMut, BufferRangeRef};
pub trait CommandBuffer {
type Backend: Backend<CommandBuffer = Self>;
@@ -70,15 +70,18 @@ impl AccessFlags {
pub trait CommandBufferEncoding {
type CommandBuffer: CommandBuffer<Encoding = Self>;
- fn encode_copy(
+ fn encode_copy<
+ Src: Buffer<Backend = <Self::CommandBuffer as CommandBuffer>::Backend>,
+ Dst: Buffer<Backend = <Self::CommandBuffer as CommandBuffer>::Backend>,
+ >(
&mut self,
- src: BufferRangeRef<'_, <<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::DenseBuffer>,
- dst: BufferRangeMut<'_, <<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::DenseBuffer>,
+ src: BufferRangeRef<'_, Src>,
+ dst: BufferRangeMut<'_, Dst>,
);
- fn encode_fill(
+ fn encode_fill<Dst: Buffer<Backend = <Self::CommandBuffer as CommandBuffer>::Backend>>(
&mut self,
- dst: BufferRangeMut<'_, <<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::DenseBuffer>,
+ dst: BufferRangeMut<'_, Dst>,
value: u8,
);
diff --git a/crates/backend-uzu/src/backends/common/context.rs b/crates/backend-uzu/src/backends/common/context.rs
index 8c2d47326..9e429ea7e 100644
--- a/crates/backend-uzu/src/backends/common/context.rs
+++ b/crates/backend-uzu/src/backends/common/context.rs
@@ -52,6 +52,8 @@ pub trait Context: Sized {
fn stop_capture(&self) -> Result<(), <Self::Backend as Backend>::Error>;
+ fn sparse_buffers_supported(&self) -> bool;
+
fn tf32_enabled() -> bool {
env::var("UZU_TF32").map(|v| v == "1" || v.eq_ignore_ascii_case("true")).unwrap_or(false)
}
diff --git a/crates/backend-uzu/src/backends/common/encoder.rs b/crates/backend-uzu/src/backends/common/encoder.rs
index a791ab952..81c8ff052 100644
--- a/crates/backend-uzu/src/backends/common/encoder.rs
+++ b/crates/backend-uzu/src/backends/common/encoder.rs
@@ -4,7 +4,7 @@ use std::{
};
use crate::backends::common::{
- AccessFlags, Allocation, AllocationPool, AllocationType, AsBufferRangeMut, AsBufferRangeRef, Backend,
+ AccessFlags, Allocation, AllocationPool, AllocationType, AsBufferRangeMut, AsBufferRangeRef, Backend, Buffer,
BufferGpuAddressRangeExt, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable,
CommandBufferInitial, CommandBufferPending, Context,
hazard_tracker::{Access, HazardTracker},
@@ -79,11 +79,14 @@ impl<'encoding, B: Backend> Encoder<'encoding, B> {
)
}
- pub fn encode_copy(
+ pub fn encode_copy<
+ Src: AsBufferRangeRef<Buffer: Buffer<Backend = B>>,
+ Dst: AsBufferRangeMut<Buffer: Buffer<Backend = B>>,
+ >(
&mut self,
- src: &Allocation<B>,
+ src: &Src,
src_range: impl RangeBounds<usize>,
- dst: &mut Allocation<B>,
+ dst: &mut Dst,
dst_range: impl RangeBounds<usize>,
) {
let src_buffer_range = src.as_buffer_range_ref();
@@ -108,9 +111,9 @@ impl<'encoding, B: Backend> Encoder<'encoding, B> {
self.command_buffer.encode_copy(src_buffer_range, dst_buffer_range);
}
- pub fn encode_fill(
+ pub fn encode_fill<Dst: AsBufferRangeMut<Buffer: Buffer<Backend = B>>>(
&mut self,
- dst: &mut Allocation<B>,
+ dst: &mut Dst,
value: u8,
) {
let dst_buffer_range = dst.as_buffer_range_mut();
diff --git a/crates/backend-uzu/src/backends/common/kernel/attention.rs b/crates/backend-uzu/src/backends/common/kernel/attention.rs
index 5fcf577c1..d162b91d1 100644
--- a/crates/backend-uzu/src/backends/common/kernel/attention.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/attention.rs
@@ -6,7 +6,7 @@ use std::{
use crate::{
DataType,
backends::common::{
- Allocation, Backend, Encoder, Kernels,
+ Allocation, AsBufferRangeRef, Backend, Buffer, Encoder, Kernels,
gpu_types::{AttnParams, ring::RingParams},
kernel::AttentionGemmKernel,
},
@@ -14,10 +14,10 @@ use crate::{
const BQ: usize = 32;
-pub struct AttentionGemmArguments<'a, B: Backend> {
+pub struct AttentionGemmArguments<'a, B: Backend, KVBuf: AsBufferRangeRef> {
pub queries: &'a Allocation<B>,
- pub keys: &'a Allocation<B>,
- pub values: &'a Allocation<B>,
+ pub keys: &'a KVBuf,
+ pub values: &'a KVBuf,
pub output: &'a mut Allocation<B>,
pub trie: Option<&'a Allocation<B>>,
pub sinks: Option<&'a Allocation<B>>,
@@ -56,10 +56,10 @@ impl<B: Backend> AttentionGemmBlock<B> {
}
}
- pub fn encode(
+ pub fn encode<KVBuf: AsBufferRangeRef<Buffer: Buffer<Backend = B>>>(
&self,
encoder: &mut Encoder<B>,
- args: AttentionGemmArguments<B>,
+ args: AttentionGemmArguments<B, KVBuf>,
) -> Result<(), B::Error> {
let bk: usize = if args.head_dim < 128 {
32
diff --git a/crates/backend-uzu/src/backends/common/kernel/kv_cache_update.rs b/crates/backend-uzu/src/backends/common/kernel/kv_cache_update.rs
index 6adb42def..2da61b6a3 100644
--- a/crates/backend-uzu/src/backends/common/kernel/kv_cache_update.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/kv_cache_update.rs
@@ -4,13 +4,13 @@ use thiserror::Error;
use crate::{
DataType,
- backends::common::{Allocation, Backend, Encoder, Kernels, gpu_types::Swap, kernel::KVCacheUpdateKernel},
+ backends::common::{Backend, Buffer, Encoder, Kernels, gpu_types::Swap, kernel::KVCacheUpdateKernel},
};
-pub struct KVLayerData<'a, B: Backend> {
- pub key_allocation: &'a mut Allocation<B>,
+pub struct KVLayerData<'a, B: Backend, Buf: Buffer<Backend = B>> {
+ pub key_buffer: &'a mut Buf,
pub key_shape: [usize; 3],
- pub value_allocation: &'a mut Allocation<B>,
+ pub value_buffer: &'a mut Buf,
pub value_shape: [usize; 3],
}
@@ -53,9 +53,9 @@ impl<B: Backend> KVCacheUpdate<B> {
}
/// Encode the KV cache update operation using a provided command buffer
- pub fn encode(
+ pub fn encode<Buf: Buffer<Backend = B>>(
&self,
- in_place_data: &mut [KVLayerData<B>],
+ in_place_data: &mut [KVLayerData<B, Buf>],
source_indices: &[usize],
destination_indices: &[usize],
encoder: &mut Encoder<B>,
@@ -80,8 +80,8 @@ impl<B: Backend> KVCacheUpdate<B> {
// non-inline is not supported yet (and is broken anyways due to a data race)
for swaps_chunk in swaps.chunks(max_inline_swaps) {
self.kernel.encode(
- &mut *layer_data.key_allocation,
- &mut *layer_data.value_allocation,
+ &mut *layer_data.key_buffer,
+ &mut *layer_data.value_buffer,
swaps_chunk,
swaps_chunk.len() as u32,
num_heads as u32,
diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul.rs b/crates/backend-uzu/src/backends/common/kernel/matmul.rs
index 4986dbe47..3578ac4d5 100644
--- a/crates/backend-uzu/src/backends/common/kernel/matmul.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/matmul.rs
@@ -2,7 +2,7 @@ use thiserror::Error;
use crate::{
DataType,
- backends::common::{Allocation, Backend, Encoder, kernel::ManualKernels},
+ backends::common::{Allocation, AsBufferRangeRef, Backend, Buffer, Encoder, kernel::ManualKernels},
};
#[derive(Debug, Error)]
@@ -22,12 +22,12 @@ pub enum MatmulArgumentC<'a, B: Backend> {
}
// D = ab_scale * (A @ op(B)) + C, where op(B) = B^T when b_transpose else B.
-pub struct MatmulArguments<'a, B: Backend> {
+pub struct MatmulArguments<'a, B: Backend, TB: AsBufferRangeRef = Allocation<B>> {
/// A: [M, K]
pub a: &'a Allocation<B>,
pub a_offset: usize,
/// B: [N, K] when b_transpose, else [K, N]
- pub b: &'a Allocation<B>,
+ pub b: &'a TB,
pub b_offset: usize,
pub b_leading_dimension: Option<u32>,
pub b_transpose: bool,
@@ -53,9 +53,9 @@ pub trait MatmulKernel: Sized {
data_type: DataType,
) -> Result<Self, MatmulError<Self::Backend>>;
- fn encode(
+ fn encode<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Self::Backend>>>(
&mut self,
- arguments: MatmulArguments<Self::Backend>,
+ arguments: MatmulArguments<Self::Backend, TB>,
encoder: &mut Encoder<Self::Backend>,
);
}
diff --git a/crates/backend-uzu/src/backends/common/mod.rs b/crates/backend-uzu/src/backends/common/mod.rs
index e25acdfc9..24074b4a0 100644
--- a/crates/backend-uzu/src/backends/common/mod.rs
+++ b/crates/backend-uzu/src/backends/common/mod.rs
@@ -30,4 +30,4 @@ pub use dense_buffer::DenseBuffer;
pub use encoder::{Completed, Encoder, Executable, Pending};
pub use hazard_tracker::Access;
pub use kernel::Kernels;
-pub use sparse_buffer::SparseBuffer;
+pub use sparse_buffer::{SparseBuffer, SparseBufferExt};
diff --git a/crates/backend-uzu/src/backends/common/sparse_buffer.rs b/crates/backend-uzu/src/backends/common/sparse_buffer.rs
index ec2ae87f2..5b522c1bc 100644
--- a/crates/backend-uzu/src/backends/common/sparse_buffer.rs
+++ b/crates/backend-uzu/src/backends/common/sparse_buffer.rs
@@ -14,4 +14,14 @@ pub trait SparseBuffer: Buffer<Backend: Backend<SparseBuffer = Self>> {
context: &<Self::Backend as Backend>::Context,
pages: &Range<usize>,
) -> Result<(), <Self::Backend as Backend>::Error>;
+
+ fn page_size_bytes(&self) -> usize;
+}
+
+pub trait SparseBufferExt: SparseBuffer {
+ fn total_pages(&self) -> usize {
+ self.size() / self.page_size_bytes()
+ }
}
+
+impl<B: SparseBuffer> SparseBufferExt for B {}
diff --git a/crates/backend-uzu/src/backends/cpu/command_buffer.rs b/crates/backend-uzu/src/backends/cpu/command_buffer.rs
index dd32ab9cd..7cb79d790 100644
--- a/crates/backend-uzu/src/backends/cpu/command_buffer.rs
+++ b/crates/backend-uzu/src/backends/cpu/command_buffer.rs
@@ -6,10 +6,10 @@ use std::{
use crate::{
backends::{
common::{
- AccessFlags, Backend, BufferRangeMut, BufferRangeRef, CommandBuffer, CommandBufferCompleted,
+ AccessFlags, Buffer, BufferRangeMut, BufferRangeRef, CommandBuffer, CommandBufferCompleted,
CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
},
- cpu::{Cpu, error::CpuError},
+ cpu::{BufferDowncastExt, Cpu, error::CpuError},
},
utils::pointers::{SendPtr, SendPtrMut},
};
@@ -68,31 +68,34 @@ impl CpuCommandBufferEncoding {
impl CommandBufferEncoding for CpuCommandBufferEncoding {
type CommandBuffer = CpuCommandBuffer;
- fn encode_copy(
+ fn encode_copy<Src: Buffer<Backend = Cpu>, Dst: Buffer<Backend = Cpu>>(
&mut self,
- src: BufferRangeRef<'_, <Cpu as Backend>::DenseBuffer>,
- dst: BufferRangeMut<'_, <Cpu as Backend>::DenseBuffer>,
+ src: BufferRangeRef<'_, Src>,
+ dst: BufferRangeMut<'_, Dst>,
) {
let src_range = src.range();
let dst_range = dst.range();
- let size = src_range.end - src_range.start;
- assert_eq!(size, dst_range.end - dst_range.start);
+ assert_eq!(src_range.len(), dst_range.len());
+
+ let src_buffer = src.buffer().downcast();
+ let src_ptr = SendPtr(unsafe { (&*src_buffer.get()).as_ptr().add(src_range.start) });
+
+ let dst_buffer = dst.buffer().downcast();
+ let dst_ptr = SendPtrMut(unsafe { (&mut *dst_buffer.get()).as_mut_ptr().add(dst_range.start) });
- let src_ptr = SendPtr(unsafe { (&*src.buffer().get()).as_ptr().add(src_range.start) });
- let dst_ptr = SendPtrMut(unsafe { (&mut *dst.buffer().get()).as_mut_ptr().add(dst_range.start) });
self.push_command(move || unsafe {
- std::ptr::copy(src_ptr.as_ptr(), dst_ptr.as_ptr(), size);
+ std::ptr::copy(src_ptr.as_ptr(), dst_ptr.as_ptr(), src_range.len());
});
}
- fn encode_fill(
+ fn encode_fill<Dst: Buffer<Backend = Cpu>>(
&mut self,
- dst: BufferRangeMut<'_, <Cpu as Backend>::DenseBuffer>,
+ dst: BufferRangeMut<'_, Dst>,
value: u8,
) {
let range = dst.range();
let size = range.end - range.start;
- let dst = SendPtrMut(unsafe { (&mut *dst.buffer().get()).as_mut_ptr().add(range.start) });
+ let dst = SendPtrMut(unsafe { (&mut *dst.buffer().downcast().get()).as_mut_ptr().add(range.start) });
self.push_command(move || unsafe {
dst.as_ptr().write_bytes(value, size);
});
diff --git a/crates/backend-uzu/src/backends/cpu/context.rs b/crates/backend-uzu/src/backends/cpu/context.rs
index 2d208e349..deeb3edfc 100644
--- a/crates/backend-uzu/src/backends/cpu/context.rs
+++ b/crates/backend-uzu/src/backends/cpu/context.rs
@@ -93,4 +93,8 @@ impl Context for CpuContext {
fn stop_capture(&self) -> Result<(), CpuError> {
Err(CpuError::NotSupported)
}
+
+ fn sparse_buffers_supported(&self) -> bool {
+ false
+ }
}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs
index 3cc122c5d..9cac8dce3 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs
@@ -4,10 +4,10 @@ use crate::{
DataType,
backends::{
common::{
- AsBufferRangeMut, AsBufferRangeRef, Encoder,
+ AsBufferRangeMut, AsBufferRangeRef, Buffer, Encoder,
kernel::matmul::{MatmulArgumentC, MatmulArguments, MatmulError, MatmulKernel},
},
- cpu::{Cpu, context::CpuContext},
+ cpu::{BufferDowncastExt, Cpu, context::CpuContext},
},
utils::pointers::{SendPtr, SendPtrMut},
};
@@ -31,9 +31,9 @@ impl MatmulKernel for MatmulCpuKernel {
})
}
- fn encode(
+ fn encode<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Cpu>>>(
&mut self,
- arguments: MatmulArguments<Cpu>,
+ arguments: MatmulArguments<Cpu, TB>,
encoder: &mut Encoder<Cpu>,
) {
let MatmulArguments {
@@ -70,7 +70,7 @@ impl MatmulKernel for MatmulCpuKernel {
let d_offset = d_buffer_range.range().start;
let a_ptr = SendPtr(unsafe { &*a_buffer_range.buffer().get() }.as_ptr().wrapping_byte_add(a_offset));
- let b_ptr = SendPtr(unsafe { &*b_buffer_range.buffer().get() }.as_ptr().wrapping_byte_add(b_offset));
+ let b_ptr = SendPtr(unsafe { &*b_buffer_range.buffer().downcast().get() }.as_ptr().wrapping_byte_add(b_offset));
let d_ptr =
SendPtrMut(unsafe { (&*d_buffer_range.buffer().get()).as_ptr().wrapping_byte_add(d_offset) as *mut u8 });
diff --git a/crates/backend-uzu/src/backends/cpu/sparse/sparse_buffer.rs b/crates/backend-uzu/src/backends/cpu/sparse/sparse_buffer.rs
index 8958146a1..5b0a4da9e 100644
--- a/crates/backend-uzu/src/backends/cpu/sparse/sparse_buffer.rs
+++ b/crates/backend-uzu/src/backends/cpu/sparse/sparse_buffer.rs
@@ -36,4 +36,8 @@ impl SparseBuffer for CpuSparseBuffer {
) -> Result<(), <Self::Backend as Backend>::Error> {
todo!()
}
+
+ fn page_size_bytes(&self) -> usize {
+ todo!()
+ }
}
diff --git a/crates/backend-uzu/src/backends/metal/command_buffer.rs b/crates/backend-uzu/src/backends/metal/command_buffer.rs
index ad276fca7..a55e6dc1a 100644
--- a/crates/backend-uzu/src/backends/metal/command_buffer.rs
+++ b/crates/backend-uzu/src/backends/metal/command_buffer.rs
@@ -6,11 +6,11 @@ use metal::{
};
use objc2::{Message, rc::Retained, runtime::ProtocolObject};
-use super::Metal;
+use super::{BufferDowncastExt, Metal};
use crate::{
backends::{
common::{
- AccessFlags, Backend, BufferRangeMut, BufferRangeRef, CommandBuffer, CommandBufferCompleted,
+ AccessFlags, Buffer, BufferRangeMut, BufferRangeRef, CommandBuffer, CommandBufferCompleted,
CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
},
metal::error::MetalError,
@@ -128,29 +128,34 @@ impl Drop for MetalCommandBufferEncoding {
impl CommandBufferEncoding for MetalCommandBufferEncoding {
type CommandBuffer = MetalCommandBuffer;
- fn encode_copy(
+ fn encode_copy<Src: Buffer<Backend = Metal>, Dst: Buffer<Backend = Metal>>(
&mut self,
- src: BufferRangeRef<'_, <Metal as Backend>::DenseBuffer>,
- dst: BufferRangeMut<'_, <Metal as Backend>::DenseBuffer>,
+ src: BufferRangeRef<'_, Src>,
+ dst: BufferRangeMut<'_, Dst>,
) {
let src_range = src.range();
let dst_range = dst.range();
- let size = src_range.end - src_range.start;
- assert_eq!(size, dst_range.end - dst_range.start);
-
- self.ensure_blit().copy_buffer_to_buffer(src.buffer(), src_range.start, dst.buffer(), dst_range.start, size);
+ assert_eq!(src_range.len(), dst_range.len());
+
+ self.ensure_blit().copy_buffer_to_buffer(
+ src.buffer().downcast(),
+ src_range.start,
+ dst.buffer().downcast(),
+ dst_range.start,
+ src_range.len(),
+ );
}
- fn encode_fill(
+ fn encode_fill<Dst: Buffer<Backend = Metal>>(
&mut self,
- dst: BufferRangeMut<'_, <Metal as Backend>::DenseBuffer>,
+ dst: BufferRangeMut<'_, Dst>,
value: u8,
) {
let range = dst.range();
assert!(range.end > range.start);
assert!(range.start % 4 == 0 && range.end % 4 == 0);
- self.ensure_blit().fill_buffer_range_value(dst.buffer(), range, value);
+ self.ensure_blit().fill_buffer_range_value(dst.buffer().downcast(), range, value);
}
fn encode_barrier(
diff --git a/crates/backend-uzu/src/backends/metal/context.rs b/crates/backend-uzu/src/backends/metal/context.rs
index 82142170f..ef90189b1 100644
--- a/crates/backend-uzu/src/backends/metal/context.rs
+++ b/crates/backend-uzu/src/backends/metal/context.rs
@@ -1,22 +1,22 @@
-#[cfg(test)]
-use std::cell::Ref;
use std::{
- cell::{RefCell, RefMut},
+ cell::{Ref, RefCell, RefMut},
collections::HashMap,
path::Path,
rc::{Rc, Weak},
sync::atomic::{AtomicU64, Ordering},
};
+#[cfg(test)]
+use metal::MTLSharedEvent;
use metal::{
- MTL4CommandQueue, MTLBuffer, MTLCaptureDescriptor, MTLCaptureDestination, MTLCaptureManager, MTLCommandQueue,
- MTLCommandQueueExt, MTLComputePipelineState, MTLDevice, MTLDeviceExt, MTLEvent, MTLFunctionConstantValues,
- MTLLibrary, MTLResourceOptions, MTLSparsePageSize,
+ MTL4CommandQueue, MTL4CommandQueueExt, MTLBuffer, MTLCaptureDescriptor, MTLCaptureDestination, MTLCaptureManager,
+ MTLCommandQueue, MTLCommandQueueExt, MTLComputePipelineState, MTLDevice, MTLDeviceExt, MTLEvent,
+ MTLFunctionConstantValues, MTLLibrary, MTLResourceOptions, MTLSparsePageSize,
};
use objc2::{rc::Retained, runtime::ProtocolObject};
use super::{
- Metal,
+ DeviceExt, Metal,
device_capabilities::MetalDeviceCapabilities,
error::MetalError,
kernel,
@@ -28,7 +28,7 @@ use crate::{
metal::{
command_buffer::MetalCommandBufferInitial,
metal_extensions::SparsePageSizeExt,
- sparse::{MetalSparseBuffer, MetalSparseHeapPool},
+ sparse::{MetalSparseBuffer, MetalSparseHeapPool, MetalSparseMappingOpsBatch},
},
},
utils::model_size::ModelSize,
@@ -47,6 +47,8 @@ pub struct MetalContext {
pipeline_cache: RefCell<HashMap<String, Retained<ProtocolObject<dyn MTLComputePipelineState>>>>,
sparse_heap_pool: RefCell<MetalSparseHeapPool>,
weak_self: Weak<MetalContext>,
+ #[cfg(test)]
+ timeline_shared_event: Retained<ProtocolObject<dyn MTLSharedEvent>>,
}
impl MetalContext {
@@ -71,7 +73,6 @@ impl MetalContext {
Ok(pipeline)
}
- #[cfg(test)]
pub(super) fn sparse_heap_pool(&self) -> Ref<'_, MetalSparseHeapPool> {
self.sparse_heap_pool.borrow()
}
@@ -80,6 +81,26 @@ impl MetalContext {
self.sparse_heap_pool.borrow_mut()
}
+ pub(super) fn sparse_update_mappings(
+ &self,
+ mappings: &[MetalSparseMappingOpsBatch],
+ ) {
+ if mappings.is_empty() {
+ return;
+ }
+
+ let wait_value = self.timeline_get_and_increment();
+ self.command_queue4.wait_for_event_value(&self.timeline_event, wait_value);
+ for op in mappings {
+ self.command_queue4.update_buffer_mappings(&op.buffer, Some(op.heap.borrow().heap()), &op.mtl_operations);
+ }
+ self.command_queue4.signal_event_value(&self.timeline_event, wait_value + 1);
+
+ // This line prevent tests from freezing, showing pink screen and shutting down computer
+ #[cfg(test)]
+ self.timeline_shared_event.wait_until_signaled_value_timeout_ms(wait_value, 10);
+ }
+
pub(super) fn timeline_get_and_increment(&self) -> u64 {
self.timeline_value.fetch_add(1, Ordering::Release)
}
@@ -111,6 +132,8 @@ impl Context for MetalContext {
let heap_capacity = 64 * 4 * page_size.in_bytes();
let sparse_pool = MetalSparseHeapPool::new(page_size, heap_capacity);
let timeline_event = device.new_event().ok_or(MetalError::CannotCreateEvent)?;
+ #[cfg(test)]
+ let timeline_shared_event = device.new_shared_event().ok_or(MetalError::CannotCreateEvent)?;
Ok(Rc::new_cyclic(|weak_self| Self {
device,
@@ -125,6 +148,8 @@ impl Context for MetalContext {
pipeline_cache: RefCell::new(HashMap::new()),
sparse_heap_pool: RefCell::new(sparse_pool),
weak_self: weak_self.clone(),
+ #[cfg(test)]
+ timeline_shared_event,
}))
}
@@ -233,4 +258,8 @@ impl Context for MetalContext {
Ok(())
}
+
+ fn sparse_buffers_supported(&self) -> bool {
+ self.device.supports_placement_sparse_resources()
+ }
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/dispatch.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/dispatch.rs
index 57e38cbbb..7b24036a7 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/dispatch.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/dispatch.rs
@@ -1,13 +1,17 @@
use crate::{
DataType,
- backends::common::{
- Allocation, Backend,
- gpu_types::{
- GemmParams, QuantizationMode,
- gemm::{
- GemmAlignment, GemmInputPrologueKind, GemmOutputTransformKind, GemmTilingConfig, GemmWeightPrologueKind,
+ backends::{
+ common::{
+ Allocation, AsBufferRangeRef, Backend, Buffer,
+ gpu_types::{
+ GemmParams, QuantizationMode,
+ gemm::{
+ GemmAlignment, GemmInputPrologueKind, GemmOutputTransformKind, GemmTilingConfig,
+ GemmWeightPrologueKind,
+ },
},
},
+ metal::Metal,
},
};
@@ -27,19 +31,19 @@ impl GemmAlignment {
}
}
-pub enum GemmWeights<'a, B: Backend> {
+pub enum GemmWeights<'a, B: Backend, T: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>> = Allocation<B>> {
FullPrecision {
- weights: &'a Allocation<B>,
+ weights: &'a T,
},
ScaleBias {
- weights: &'a Allocation<B>,
+ weights: &'a T,
scales: &'a Allocation<B>,
biases: &'a Allocation<B>,
mode: QuantizationMode,
group_size: u32,
},
ScaleZeroPoint {
- weights: &'a Allocation<B>,
+ weights: &'a T,
scales: &'a Allocation<B>,
zero_points: &'a Allocation<B>,
mode: QuantizationMode,
@@ -47,7 +51,7 @@ pub enum GemmWeights<'a, B: Backend> {
},
}
-impl<B: Backend> GemmWeights<'_, B> {
+impl<B: Backend, T: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>> GemmWeights<'_, B, T> {
pub fn weight_prologue(&self) -> GemmWeightPrologueKind {
match self {
Self::FullPrecision {
@@ -95,7 +99,7 @@ impl<B: Backend> GemmWeights<'_, B> {
}
}
-pub struct GemmDispatch<'a, B: Backend> {
+pub struct GemmDispatch<'a, B: Backend, T: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>> {
pub tiling_config: GemmTilingConfig,
pub input_prologue: GemmInputPrologueKind,
pub use_mxu: bool,
@@ -104,7 +108,7 @@ pub struct GemmDispatch<'a, B: Backend> {
pub transpose_b: bool,
pub a: &'a Allocation<B>,
pub a_offset: usize,
- pub b: GemmWeights<'a, B>,
+ pub b: GemmWeights<'a, B, T>,
pub b_offset: usize,
pub d: &'a mut Allocation<B>,
pub params: GemmParams,
@@ -112,7 +116,7 @@ pub struct GemmDispatch<'a, B: Backend> {
pub group_count_y: u32,
}
-impl<B: Backend> GemmDispatch<'_, B> {
+impl<B: Backend, T: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>> GemmDispatch<'_, B, T> {
pub(crate) fn specialization(&self) -> GemmSpecialization {
GemmSpecialization {
tiling_config: self.tiling_config,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/fp.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/fp.rs
index 4f39ce56b..7ea4fb243 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/fp.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/fp.rs
@@ -3,7 +3,7 @@ use crate::{
DataType,
backends::{
common::{
- Encoder,
+ AsBufferRangeRef, Buffer, Encoder,
gpu_types::{
GemmParams,
gemm::{GemmAlignment, GemmInputPrologueKind, GemmOutputTransformKind, GemmTilingConfig},
@@ -17,13 +17,13 @@ use crate::{
},
};
-pub(crate) fn encode(
+pub(crate) fn encode<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(
gemm: &mut GemmKernel,
bias_add: &mut TensorAddBiasMetalKernel,
data_type: DataType,
context: &MetalContext,
encoder: &mut Encoder<Metal>,
- arguments: MatmulArguments<Metal>,
+ arguments: MatmulArguments<Metal, TB>,
use_mxu: bool,
) -> Result<(), MatmulError<Metal>> {
if use_mxu && !context.device.supports_mxu() {
@@ -121,9 +121,9 @@ pub(crate) fn encode(
Ok(())
}
-fn select_simdgroup_tile(
+fn select_simdgroup_tile<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(
data_type: DataType,
- arguments: &MatmulArguments<Metal>,
+ arguments: &MatmulArguments<Metal, TB>,
) -> GemmTilingConfig {
let (threadgroup_m, threadgroup_n, threadgroup_k) = match data_type {
DataType::F32 => (32u32, 64u32, 16u32),
@@ -144,7 +144,9 @@ fn select_simdgroup_tile(
}
}
-fn select_mxu_tile(arguments: &MatmulArguments<Metal>) -> GemmTilingConfig {
+fn select_mxu_tile<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(
+ arguments: &MatmulArguments<Metal, TB>
+) -> GemmTilingConfig {
let (threadgroup_m, threadgroup_n, simdgroups_m, simdgroups_n) =
if arguments.batch_dim >= 256 && arguments.output_dim >= 128 {
(128u32, 128u32, 4u32, 4u32)
@@ -164,7 +166,9 @@ fn select_mxu_tile(arguments: &MatmulArguments<Metal>) -> GemmTilingConfig {
}
}
-fn output_transform_from(arguments: &MatmulArguments<Metal>) -> GemmOutputTransformKind {
+fn output_transform_from<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(
+ arguments: &MatmulArguments<Metal, TB>
+) -> GemmOutputTransformKind {
let scale = arguments.ab_scale != 1.0;
let accumulate = matches!(arguments.c, MatmulArgumentC::Accumulate);
match (scale, accumulate) {
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
index 44a925987..5f00d0280 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
@@ -5,7 +5,7 @@ use crate::{
DataType,
backends::{
common::{
- Encoder,
+ AsBufferRangeRef, Buffer, Encoder,
gpu_types::gemm::{GemmInputPrologueKind, GemmWeightPrologueKind},
},
metal::{Metal, context::MetalContext, error::MetalError, kernel::GemmMetalKernel},
@@ -62,10 +62,10 @@ impl GemmKernel {
}
}
- pub(crate) fn encode(
+ pub(crate) fn encode<T: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(
&mut self,
context: &MetalContext,
- dispatch: GemmDispatch<'_, Metal>,
+ dispatch: GemmDispatch<'_, Metal, T>,
encoder: &mut Encoder<Metal>,
) -> Result<(), MetalError> {
let specialization = dispatch.specialization();
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/fp.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/fp.rs
index 90441c3db..8d3baf734 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/fp.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/fp.rs
@@ -1,16 +1,16 @@
use super::{kernel::GemvKernel, spec::GemvSpecialization};
use crate::backends::{
common::{
- Encoder,
+ AsBufferRangeRef, Buffer, Encoder,
kernel::matmul::{MatmulArgumentC, MatmulArguments, MatmulError},
},
metal::Metal,
};
-pub(crate) fn encode(
+pub(crate) fn encode<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(
kernel: &mut GemvKernel,
encoder: &mut Encoder<Metal>,
- arguments: MatmulArguments<Metal>,
+ arguments: MatmulArguments<Metal, TB>,
) -> Result<(), MatmulError<Metal>> {
let MatmulArguments {
a,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
index 464bbd0d8..8f803c24a 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
@@ -9,7 +9,7 @@ use crate::{
DataType,
backends::{
common::{
- Encoder,
+ AsBufferRangeRef, Buffer, Encoder,
kernel::{
TensorAddBiasKernel,
matmul::{MatmulArguments, MatmulError, MatmulKernel},
@@ -101,9 +101,9 @@ impl MatmulKernel for MatmulMetalKernel {
})
}
- fn encode(
+ fn encode<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(
&mut self,
- arguments: MatmulArguments<Metal>,
+ arguments: MatmulArguments<Metal, TB>,
encoder: &mut Encoder<Metal>,
) {
// gemv only handles the canonical B layout; non-canonical inputs go to GEMM.
diff --git a/crates/backend-uzu/src/backends/metal/metal_extensions/device_extensions.rs b/crates/backend-uzu/src/backends/metal/metal_extensions/device_extensions.rs
index 8eb91bbb2..aca8e93a7 100644
--- a/crates/backend-uzu/src/backends/metal/metal_extensions/device_extensions.rs
+++ b/crates/backend-uzu/src/backends/metal/metal_extensions/device_extensions.rs
@@ -157,6 +157,15 @@ pub trait DeviceExt: MTLDevice + Message + NSObjectProtocol + Sized {
false
}
}
+
+ /// Whether the GPU supports placement sparse resources.
+ fn supports_placement_sparse_resources(&self) -> bool {
+ if self.respondsToSelector(sel!(supportsPlacementSparse)) {
+ unsafe { raw_msg_send(self, sel!(supportsPlacementSparse)) }
+ } else {
+ false
+ }
+ }
}
impl DeviceExt for ProtocolObject<dyn MTLDevice> {}
diff --git a/crates/backend-uzu/src/backends/metal/sparse/mod.rs b/crates/backend-uzu/src/backends/metal/sparse/mod.rs
index dd4321d32..3a3b9ed52 100644
--- a/crates/backend-uzu/src/backends/metal/sparse/mod.rs
+++ b/crates/backend-uzu/src/backends/metal/sparse/mod.rs
@@ -5,3 +5,4 @@ mod sparse_utils;
pub(crate) use sparse_buffer::MetalSparseBuffer;
pub(crate) use sparse_heap_pool::MetalSparseHeapPool;
+pub(crate) use sparse_utils::MetalSparseMappingOpsBatch;
diff --git a/crates/backend-uzu/src/backends/metal/sparse/sparse_buffer.rs b/crates/backend-uzu/src/backends/metal/sparse/sparse_buffer.rs
index 081c43e02..f3a3c3dbe 100644
--- a/crates/backend-uzu/src/backends/metal/sparse/sparse_buffer.rs
+++ b/crates/backend-uzu/src/backends/metal/sparse/sparse_buffer.rs
@@ -11,8 +11,8 @@ use rangemap::RangeSet;
use crate::{
backends::{
- common::{Backend, Buffer, SparseBuffer},
- metal::{Metal, error::MetalError, metal_extensions::SparsePageSizeExt},
+ common::{Backend, Buffer, SparseBuffer, SparseBufferExt},
+ metal::{Metal, error::MetalError, metal_extensions::SparsePageSizeExt, sparse::MetalSparseMappingOpsBatch},
},
prelude::MetalContext,
};
@@ -21,6 +21,7 @@ pub struct MetalSparseBuffer {
buffer: Retained<ProtocolObject<dyn MTLBuffer>>,
mapped_pages: RangeSet<usize>,
context: Rc<MetalContext>,
+ page_size: MTLSparsePageSize,
}
impl MetalSparseBuffer {
@@ -43,6 +44,7 @@ impl MetalSparseBuffer {
buffer,
mapped_pages: RangeSet::new(),
context,
+ page_size,
})
}
@@ -66,9 +68,7 @@ impl Debug for MetalSparseBuffer {
impl Drop for MetalSparseBuffer {
fn drop(&mut self) {
let context = self.context.clone();
- for range in self.mapped_pages.iter() {
- context.sparse_heap_pool_mut().unmap(&context, &self.buffer, &range).expect("Failed to unmap");
- }
+ self.unmap(&context, &(0..self.total_pages())).expect("Failed to unmap");
}
}
@@ -90,11 +90,35 @@ impl SparseBuffer for MetalSparseBuffer {
context: &<Self::Backend as Backend>::Context,
pages: &Range<usize>,
) -> Result<(), <Self::Backend as Backend>::Error> {
- self.mapped_pages.gaps(pages).collect::<Vec<_>>().into_iter().try_for_each(|gap| {
- context.sparse_heap_pool_mut().map(context, &self.buffer, &gap)?;
- self.mapped_pages.insert(gap);
- Ok(())
- })
+ if pages.len() == 0 {
+ return Ok(());
+ }
+
+ // prepare operations
+ let mut all_batches: Vec<MetalSparseMappingOpsBatch> = Vec::new();
+ let gaps = self.mapped_pages.gaps(pages).collect::<Vec<_>>();
+ let pages_to_map = gaps.iter().map(|gap| gap.len()).sum();
+ context.sparse_heap_pool_mut().ensure_enough_free_pages(context, pages_to_map)?;
+
+ for gap in gaps.iter() {
+ let mut pool = context.sparse_heap_pool_mut();
+ let result = pool.create_map_operations(context, &self.buffer, gap);
+ match result {
+ Ok(batches) => {
+ pool.apply_map_operations(&batches);
+ all_batches.extend(batches);
+ },
+ Err(err) => return Err(err),
+ };
+ }
+
+ // execute operations
+ context.sparse_update_mappings(&all_batches);
+ for gap in gaps {
+ self.mapped_pages.insert(gap)
+ }
+
+ Ok(())
}
fn unmap(
@@ -102,16 +126,34 @@ impl SparseBuffer for MetalSparseBuffer {
context: &<Self::Backend as Backend>::Context,
pages: &Range<usize>,
) -> Result<(), <Self::Backend as Backend>::Error> {
- self.mapped_pages
+ if pages.len() == 0 {
+ return Ok(());
+ }
+
+ // prepare operations
+ let mut all_batches: Vec<MetalSparseMappingOpsBatch> = Vec::new();
+ let mapped_ranges = self
+ .mapped_pages
.overlapping(pages)
.map(|range| max(range.start, pages.start)..min(range.end, pages.end))
- .collect::<Vec<_>>()
- .into_iter()
- .try_for_each(|range| {
- context.sparse_heap_pool_mut().unmap(context, &self.buffer, &range)?;
- self.mapped_pages.remove(range);
- Ok(())
- })
+ .collect::<Vec<_>>();
+ for mapped_range in mapped_ranges.iter() {
+ let batches = context.sparse_heap_pool().create_unmap_operations(&self.buffer, mapped_range);
+ context.sparse_heap_pool_mut().apply_map_operations(&batches);
+ all_batches.extend(batches);
+ }
+
+ // execute operations
+ context.sparse_update_mappings(&all_batches);
+ for mapped_range in mapped_ranges {
+ self.mapped_pages.remove(mapped_range)
+ }
+
+ Ok(())
+ }
+
+ fn page_size_bytes(&self) -> usize {
+ self.page_size.in_bytes()
}
}
diff --git a/crates/backend-uzu/src/backends/metal/sparse/sparse_heap.rs b/crates/backend-uzu/src/backends/metal/sparse/sparse_heap.rs
index 241ca0c32..73519dbe6 100644
--- a/crates/backend-uzu/src/backends/metal/sparse/sparse_heap.rs
+++ b/crates/backend-uzu/src/backends/metal/sparse/sparse_heap.rs
@@ -1,8 +1,8 @@
use std::{collections::HashMap, ops::Range};
use metal::{
- MTL4CommandQueue, MTL4CommandQueueExt, MTL4UpdateSparseBufferMappingOperation, MTLBuffer, MTLDeviceExt, MTLHeap,
- MTLHeapDescriptor, MTLHeapType, MTLSparsePageSize, MTLSparseTextureMappingMode, MTLStorageMode,
+ MTLBuffer, MTLDeviceExt, MTLHeap, MTLHeapDescriptor, MTLHeapType, MTLSparsePageSize, MTLSparseTextureMappingMode,
+ MTLStorageMode,
};
use objc2::{rc::Retained, runtime::ProtocolObject};
use rangemap::{RangeMap, RangeSet};
@@ -11,12 +11,12 @@ use crate::{
backends::metal::{
error::MetalError,
metal_extensions::SparsePageSizeExt,
- sparse::sparse_utils::{MetalSparseHeapBufferMapping, MetalSparseHeapMappingParameters},
+ sparse::{MetalSparseMappingOpsBatch, sparse_utils::MetalSparseHeapBufferMapping},
},
prelude::MetalContext,
};
-pub(super) struct MetalSparseHeap {
+pub(crate) struct MetalSparseHeap {
heap: Retained<ProtocolObject<dyn MTLHeap>>,
// Mappings are tracked per buffer (keyed by gpu_address) so multiple
// buffers can alias overlapping heap page ranges without losing track of
@@ -55,6 +55,10 @@ impl MetalSparseHeap {
})
}
+ pub fn heap(&self) -> &Retained<ProtocolObject<dyn MTLHeap>> {
+ &self.heap
+ }
+
pub fn free_pages(&self) -> &RangeSet<usize> {
&self.free_pages
}
@@ -74,38 +78,15 @@ impl MetalSparseHeap {
self.buffer_mappings.is_empty()
}
- /// Command queue calls doesn't have any synchronization.
- /// It is the responsibility of the caller.
- pub fn execute(
+ pub fn apply_mapping_operations(
&mut self,
- buffer: &ProtocolObject<dyn MTLBuffer>,
- cmd_queue: &ProtocolObject<dyn MTL4CommandQueue>,
- operations: &[MetalSparseHeapMappingParameters],
- map: bool,
+ ops: &MetalSparseMappingOpsBatch,
) {
- if operations.is_empty() {
- return;
- }
-
- let mtl_operations: Vec<MTL4UpdateSparseBufferMappingOperation> = operations
- .iter()
- .map(|op| {
- let mode = if map {
- MTLSparseTextureMappingMode::Map
- } else {
- MTLSparseTextureMappingMode::Unmap
- };
- MTL4UpdateSparseBufferMappingOperation::new(mode, op.buffer_pages.clone(), op.heap_page_offset)
- })
- .collect();
-
- cmd_queue.update_buffer_mappings(buffer, Some(&self.heap), &mtl_operations);
-
- let buffer_address = buffer.gpu_address();
+ let buffer_address = ops.buffer.gpu_address();
let entry = self.buffer_mappings.entry(buffer_address).or_default();
- mtl_operations.iter().for_each(|mtl_op| {
+ for mtl_op in ops.mtl_operations.iter() {
let heap_range = mtl_op.heap_offset..(mtl_op.heap_offset + mtl_op.buffer_range().len());
- if map {
+ if mtl_op.mode == MTLSparseTextureMappingMode::Map {
let buffer_range = mtl_op.buffer_range();
let buffer_mapping = MetalSparseHeapBufferMapping::new(mtl_op.heap_offset, buffer_range.start);
entry.insert(heap_range.clone(), buffer_mapping);
@@ -114,7 +95,7 @@ impl MetalSparseHeap {
entry.remove(heap_range.clone());
self.free_pages.insert(heap_range);
}
- });
+ }
if entry.is_empty() {
self.buffer_mappings.remove(&buffer_address);
diff --git a/crates/backend-uzu/src/backends/metal/sparse/sparse_heap_pool.rs b/crates/backend-uzu/src/backends/metal/sparse/sparse_heap_pool.rs
index fe63a06fd..864efbdbf 100644
--- a/crates/backend-uzu/src/backends/metal/sparse/sparse_heap_pool.rs
+++ b/crates/backend-uzu/src/backends/metal/sparse/sparse_heap_pool.rs
@@ -1,22 +1,24 @@
use std::{
+ cell::RefCell,
cmp::{max, min},
ops::Range,
+ rc::Rc,
};
-use metal::{MTLBuffer, MTLSparsePageSize};
-use objc2::runtime::ProtocolObject;
+use metal::{MTL4UpdateSparseBufferMappingOperation, MTLBuffer, MTLSparsePageSize, MTLSparseTextureMappingMode};
+use objc2::{rc::Retained, runtime::ProtocolObject};
use crate::{
backends::metal::{
error::MetalError,
metal_extensions::SparsePageSizeExt,
- sparse::{sparse_heap::MetalSparseHeap, sparse_utils::MetalSparseHeapMappingParameters},
+ sparse::{MetalSparseMappingOpsBatch, sparse_heap::MetalSparseHeap},
},
prelude::MetalContext,
};
pub struct MetalSparseHeapPool {
- heaps: Vec<MetalSparseHeap>,
+ heaps: Vec<Rc<RefCell<MetalSparseHeap>>>,
heap_capacity: usize,
page_size: MTLSparsePageSize,
}
@@ -33,112 +35,123 @@ impl MetalSparseHeapPool {
}
}
- pub fn map(
+ pub fn ensure_enough_free_pages(
&mut self,
context: &MetalContext,
- buffer: &ProtocolObject<dyn MTLBuffer>,
- buffer_pages: &Range<usize>,
+ pages: usize,
) -> Result<(), MetalError> {
- if buffer_pages.len() == 0 {
+ let mut pages_to_alloc = pages;
+ for heap in self.heaps.iter() {
+ for free_pages_range in heap.borrow().free_pages().iter() {
+ pages_to_alloc -= min(free_pages_range.len(), pages_to_alloc);
+ }
+ }
+ if pages_to_alloc == 0 {
return Ok(());
}
- let mut pages_to_map = buffer_pages.clone();
- let heap_capacity_pages = self.heap_capacity_pages();
-
- // Try to find pages in existing heaps.
- // While `pages_to_map` is not empty in each heap find unmapped pages, collect them into `mappings` and map
- let mut existing_heaps_mappings: Vec<(usize, Box<[MetalSparseHeapMappingParameters]>)> = Vec::new();
- for (i, heap) in self.heaps.iter_mut().enumerate() {
- let mut mappings: Vec<MetalSparseHeapMappingParameters> = Vec::new();
-
- for free_pages in heap.free_pages().iter() {
- let map_pages_count = min(free_pages.len(), pages_to_map.len());
- let mapping = MetalSparseHeapMappingParameters {
- buffer_pages: pages_to_map.start..(pages_to_map.start + map_pages_count),
- heap_page_offset: free_pages.start,
- };
- mappings.push(mapping);
+ let new_heaps_count = (pages_to_alloc * self.page_size.in_bytes()).div_ceil(self.heap_capacity);
+ for _ in 0..new_heaps_count {
+ let heap = MetalSparseHeap::new(context, self.heap_capacity, self.page_size)?;
+ self.heaps.push(Rc::new(RefCell::new(heap)));
+ }
- pages_to_map.start += map_pages_count;
- if pages_to_map.len() == 0 {
+ Ok(())
+ }
+
+ pub fn create_map_operations(
+ &mut self,
+ context: &MetalContext,
+ buffer: &Retained<ProtocolObject<dyn MTLBuffer>>,
+ buffer_pages: &Range<usize>,
+ ) -> Result<Vec<MetalSparseMappingOpsBatch>, MetalError> {
+ self.ensure_enough_free_pages(context, buffer_pages.len())?;
+
+ let mut pages_to_map = buffer_pages.clone();
+ let mut batches: Vec<MetalSparseMappingOpsBatch> = Vec::new();
+ for heap in self.heaps.iter() {
+ let mut heap_mtl_operations: Vec<MTL4UpdateSparseBufferMappingOperation> = Vec::new();
+ for heap_free_pages_range in heap.borrow().free_pages().iter() {
+ let map_pages_count = min(heap_free_pages_range.len(), pages_to_map.len());
+ if map_pages_count == 0 {
break;
}
- }
- heap.execute(buffer, &context.command_queue4, &mappings, true);
- existing_heaps_mappings.push((i, mappings.into_boxed_slice()));
+ let mtl_operation = MTL4UpdateSparseBufferMappingOperation::new(
+ MTLSparseTextureMappingMode::Map,
+ pages_to_map.start..(pages_to_map.start + map_pages_count),
+ heap_free_pages_range.start,
+ );
+ heap_mtl_operations.push(mtl_operation);
- if pages_to_map.len() == 0 {
- break;
+ pages_to_map.start += map_pages_count;
}
- }
- // If `pages_to_map` still not empty, it's necessary to allocate new heaps
- let new_heaps_required = pages_to_map.len().div_ceil(heap_capacity_pages);
- for _ in 0..new_heaps_required {
- let map_pages_count = min(heap_capacity_pages, pages_to_map.len());
- let buffer_pages = pages_to_map.start..(pages_to_map.start + map_pages_count);
- let op_params = MetalSparseHeapMappingParameters {
- buffer_pages,
- heap_page_offset: 0,
- };
-
- match MetalSparseHeap::new(context, self.heap_capacity, self.page_size) {
- Ok(mut heap) => {
- let operations = [op_params];
- heap.execute(buffer, &context.command_queue4, &operations, true);
- self.heaps.push(heap);
- pages_to_map.start += map_pages_count;
- existing_heaps_mappings.push((self.heaps.len() - 1, Box::new(operations)));
- },
- Err(err) => {
- // it's necessary to unmap previously mapped pages in existing heaps
- for (heap_pos, mappings) in existing_heaps_mappings {
- self.heaps[heap_pos].execute(buffer, &context.command_queue4, &mappings, false);
- }
- return Err(err);
- },
- };
+ if !heap_mtl_operations.is_empty() {
+ let batch = MetalSparseMappingOpsBatch {
+ buffer: buffer.clone(),
+ heap: heap.clone(),
+ mtl_operations: heap_mtl_operations.into_boxed_slice(),
+ };
+ batches.push(batch);
+ }
}
- Ok(())
+ Ok(batches)
}
- pub fn unmap(
- &mut self,
- context: &MetalContext,
- buffer: &ProtocolObject<dyn MTLBuffer>,
+ pub fn create_unmap_operations(
+ &self,
+ buffer: &Retained<ProtocolObject<dyn MTLBuffer>>,
buffer_pages: &Range<usize>,
- ) -> Result<(), MetalError> {
+ ) -> Vec<MetalSparseMappingOpsBatch> {
+ let mut batches: Vec<MetalSparseMappingOpsBatch> = Vec::new();
+
let buffer_address = buffer.gpu_address();
+ for heap in self.heaps.iter() {
+ let mut heap_mtl_operations: Vec<MTL4UpdateSparseBufferMappingOperation> = Vec::new();
+
+ for (heap_range, mapping) in heap.borrow().mappings_for(buffer_address) {
+ let mapped_buffer_pages = mapping.buffer_pages_for(&heap_range);
+ let unmap_start_page = max(buffer_pages.start, mapped_buffer_pages.start);
+ let unmap_end_page = min(buffer_pages.end, mapped_buffer_pages.end);
+ if unmap_start_page < unmap_end_page {
+ let offset_within_range = unmap_start_page - mapped_buffer_pages.start;
+ let mtl_operation = MTL4UpdateSparseBufferMappingOperation::new(
+ MTLSparseTextureMappingMode::Unmap,
+ unmap_start_page..unmap_end_page,
+ heap_range.start + offset_within_range,
+ );
+ heap_mtl_operations.push(mtl_operation);
+ };
+ }
- // Iterate over heaps, find mappings for `buffer_address`, unmap them
- self.heaps.iter_mut().for_each(|heap| {
- let unmappings: Vec<MetalSparseHeapMappingParameters> = heap
- .mappings_for(buffer_address)
- .filter_map(|(heap_range, mapping)| {
- let mapped_buffer_pages = mapping.buffer_pages_for(&heap_range);
- let unmap_start = max(buffer_pages.start, mapped_buffer_pages.start);
- let unmap_end = min(buffer_pages.end, mapped_buffer_pages.end);
- if unmap_start >= unmap_end {
- return None;
- };
-
- let offset_within_range = unmap_start - mapped_buffer_pages.start;
- Some(MetalSparseHeapMappingParameters {
- buffer_pages: unmap_start..unmap_end,
- heap_page_offset: heap_range.start + offset_within_range,
- })
- })
- .collect();
- heap.execute(buffer, &context.command_queue4, &unmappings, false);
- });
-
- // Remove empty heaps
- self.heaps.retain(|heap| !heap.is_empty());
+ if !heap_mtl_operations.is_empty() {
+ let batch = MetalSparseMappingOpsBatch {
+ buffer: buffer.clone(),
+ heap: heap.clone(),
+ mtl_operations: heap_mtl_operations.into_boxed_slice(),
+ };
+ batches.push(batch);
+ }
+ }
- Ok(())
+ batches
+ }
+
+ pub fn apply_map_operations(
+ &mut self,
+ batches: &[MetalSparseMappingOpsBatch],
+ ) {
+ for batch in batches.iter() {
+ for sparse_heap in self.heaps.iter() {
+ if sparse_heap.borrow().heap() == batch.heap.borrow().heap() {
+ sparse_heap.borrow_mut().apply_mapping_operations(batch);
+ }
+ }
+ }
+
+ self.heaps.retain(|heap| !heap.borrow().is_empty());
}
#[cfg(test)]
@@ -146,6 +159,7 @@ impl MetalSparseHeapPool {
self.heap_capacity
}
+ #[cfg(test)]
pub fn heap_capacity_pages(&self) -> usize {
self.heap_capacity / self.page_size.in_bytes()
}
diff --git a/crates/backend-uzu/src/backends/metal/sparse/sparse_utils.rs b/crates/backend-uzu/src/backends/metal/sparse/sparse_utils.rs
index 103de7d1f..5e5fecd4b 100644
--- a/crates/backend-uzu/src/backends/metal/sparse/sparse_utils.rs
+++ b/crates/backend-uzu/src/backends/metal/sparse/sparse_utils.rs
@@ -1,7 +1,12 @@
-use std::ops::Range;
+use std::{cell::RefCell, ops::Range, rc::Rc};
+
+use metal::{MTL4UpdateSparseBufferMappingOperation, MTLBuffer};
+use objc2::{rc::Retained, runtime::ProtocolObject};
+
+use crate::backends::metal::sparse::sparse_heap::MetalSparseHeap;
#[derive(Clone, PartialEq)]
-pub(super) struct MetalSparseHeapBufferMapping {
+pub(crate) struct MetalSparseHeapBufferMapping {
// Anchors for an originally inserted contiguous mapping. They survive
// rangemap splits unchanged, so the heap↔buffer correspondence can always
// be recovered from any surviving heap subrange via `buffer_pages_for`.
@@ -30,7 +35,8 @@ impl MetalSparseHeapBufferMapping {
}
}
-pub(super) struct MetalSparseHeapMappingParameters {
- pub(super) buffer_pages: Range<usize>,
- pub(super) heap_page_offset: usize,
+pub(crate) struct MetalSparseMappingOpsBatch {
+ pub buffer: Retained<ProtocolObject<dyn MTLBuffer>>,
+ pub heap: Rc<RefCell<MetalSparseHeap>>,
+ pub mtl_operations: Box<[MTL4UpdateSparseBufferMappingOperation]>,
}
diff --git a/crates/backend-uzu/src/encodable_block/attention.rs b/crates/backend-uzu/src/encodable_block/attention.rs
index 945528bcb..7d9744439 100644
--- a/crates/backend-uzu/src/encodable_block/attention.rs
+++ b/crates/backend-uzu/src/encodable_block/attention.rs
@@ -8,7 +8,7 @@ use crate::{
DataType,
array::size_for_shape,
backends::common::{
- Allocation, Backend, Encoder, Kernels,
+ Allocation, AsBufferRangeRef, Backend, Buffer, Encoder, Kernels,
gpu_types::ring::RingParams,
kernel::{
AttentionFallbackScatterScoresKernel, AttentionFallbackScatterValuesKernel, AttentionSinglePassKernel,
@@ -21,7 +21,7 @@ use crate::{
config::AttentionConfig,
forward_pass::{
cache_layers::LayerCacheAccess,
- kv_cache_layer::KVCacheLayerState,
+ kv_cache_layer::{KVCacheLayer, KVCacheLayerState},
},
};
@@ -228,27 +228,26 @@ impl<B: Backend> Attention<B> {
.expect("attention expects transformer cache")
});
- let (max_sequence_length, segment_prefix_length, ring_params) =
- if let Some(layer) = cache_layer {
- let max_sequence_length = layer.shape[0];
- let ring_params = match layer.state {
- KVCacheLayerState::Windowed {
- ring_offset,
- ring_length,
- window_length,
- } => {
- let overflow = ring_length.saturating_sub(window_length);
- Some(RingParams {
- ring_offset: ((ring_offset + overflow) % window_length) as u32,
- ring_length: ring_length.min(window_length) as u32,
- })
- },
- _ => None,
- };
- (max_sequence_length, layer.prefix_segment_length(), ring_params)
- } else {
- (suffix_length, 0, None)
+ let (max_sequence_length, segment_prefix_length, ring_params) = if let Some(layer) = cache_layer {
+ let max_sequence_length = layer.shape()[0];
+ let ring_params = match layer.state() {
+ KVCacheLayerState::Windowed {
+ ring_offset,
+ ring_length,
+ window_length,
+ } => {
+ let overflow = ring_length.saturating_sub(window_length);
+ Some(RingParams {
+ ring_offset: ((ring_offset + overflow) % window_length) as u32,
+ ring_length: ring_length.min(window_length) as u32,
+ })
+ },
+ _ => None,
};
+ (max_sequence_length, layer.prefix_segment_length(), ring_params)
+ } else {
+ (suffix_length, 0, None)
+ };
let is_kv_cache_ring = ring_params.is_some();
@@ -312,62 +311,92 @@ impl<B: Backend> Attention<B> {
is_kv_cache_ring,
};
- let (keys, values) = match args.cache_access {
- Some(LayerCacheAccess::Shared {
- source,
- }) => {
- let source = source.as_transformer().expect("kv_source must be a transformer cache");
- (&source.keys, &source.values)
- },
- Some(LayerCacheAccess::Owned {
- entry,
- }) => {
- let layer = entry.as_transformer_mut().expect("Attention layer expects transformer cache");
- self.update_kv_cache_kernel.encode(
- Some(&rotated_keys),
- qkv,
- &mut layer.keys,
- &mut layer.values,
- num_groups as u32,
- num_heads as u32,
- head_dim as u32,
- suffix_length as u32,
- segment_prefix_length as u32,
- max_sequence_length as u32,
+ macro_rules! encode_cached_attention {
+ ($keys:expr, $values:expr) => {
+ self.encode_attention_variant(
+ variant,
+ &kernel_key,
+ queries,
+ $keys,
+ $values,
+ trie_allocation,
+ sinks_allocation,
+ gqa_factor,
+ sequence_length,
+ k_head_stride,
+ k_seq_stride,
+ v_head_stride,
+ v_seq_stride,
+ ring_params,
+ scale,
+ num_heads,
+ num_groups,
+ suffix_length,
+ segment_prefix_length,
+ max_sequence_length,
+ head_dim,
encoder,
- );
- (&layer.keys, &layer.values)
- },
- None => (
- &rotated_keys,
- extracted_values.as_ref().expect("Missing extracted values for classifier attention"),
- ),
- };
+ )?
+ };
+ }
- let mut attention_output = self.encode_attention_variant(
- variant,
- &kernel_key,
- queries,
- keys,
- values,
- trie_allocation,
- sinks_allocation,
- gqa_factor,
- sequence_length,
- k_head_stride,
- k_seq_stride,
- v_head_stride,
- v_seq_stride,
- ring_params,
- scale,
- num_heads,
- num_groups,
- suffix_length,
- segment_prefix_length,
- max_sequence_length,
- head_dim,
- encoder,
- )?;
+ let mut attention_output = if let Some(access) = args.cache_access {
+ match access {
+ LayerCacheAccess::Owned {
+ entry,
+ } => {
+ let layer = entry.as_transformer_mut().expect("Attention layer expects transformer cache");
+ if let Some(layer) = layer.as_any_mut().downcast_mut::<KVCacheLayer<B, B::SparseBuffer>>() {
+ self.update_kv_cache_kernel.encode(
+ Some(&rotated_keys),
+ qkv,
+ &mut layer.keys,
+ &mut layer.values,
+ num_groups as u32,
+ num_heads as u32,
+ head_dim as u32,
+ suffix_length as u32,
+ segment_prefix_length as u32,
+ max_sequence_length as u32,
+ encoder,
+ );
+ encode_cached_attention!(&layer.keys, &layer.values)
+ } else if let Some(layer) = layer.as_any_mut().downcast_mut::<KVCacheLayer<B, B::DenseBuffer>>() {
+ self.update_kv_cache_kernel.encode(
+ Some(&rotated_keys),
+ qkv,
+ &mut layer.keys,
+ &mut layer.values,
+ num_groups as u32,
+ num_heads as u32,
+ head_dim as u32,
+ suffix_length as u32,
+ segment_prefix_length as u32,
+ max_sequence_length as u32,
+ encoder,
+ );
+ encode_cached_attention!(&layer.keys, &layer.values)
+ } else {
+ panic!("Attention layer expects sparse or dense transformer cache")
+ }
+ },
+ LayerCacheAccess::Shared {
+ source,
+ } => {
+ let source = source.as_transformer().expect("kv_source must be a transformer cache");
+ if let Some(source) = source.as_any().downcast_ref::<KVCacheLayer<B, B::SparseBuffer>>() {
+ encode_cached_attention!(&source.keys, &source.values)
+ } else if let Some(source) = source.as_any().downcast_ref::<KVCacheLayer<B, B::DenseBuffer>>() {
+ encode_cached_attention!(&source.keys, &source.values)
+ } else {
+ panic!("kv_source must be a sparse or dense transformer cache")
+ }
+ },
+ }
+ } else {
+ let values = extracted_values.as_ref().expect("Missing extracted values for classifier attention");
+ encode_cached_attention!(&rotated_keys, values)
+ };
if let Some(gate_kernel) = &self.gate_kernel {
let total_elements = (suffix_length * num_heads * head_dim) as u32;
@@ -379,13 +408,13 @@ impl<B: Backend> Attention<B> {
}
#[allow(clippy::too_many_arguments)]
- fn encode_attention_variant(
+ fn encode_attention_variant<KVBuf: AsBufferRangeRef<Buffer: Buffer<Backend = B>>>(
&self,
variant: KernelVariant,
kernel_key: &KernelKey,
queries: &Allocation<B>,
- keys: &Allocation<B>,
- values: &Allocation<B>,
+ keys: &KVBuf,
+ values: &KVBuf,
trie_allocation: Option<&Allocation<B>>,
sinks_allocation: Option<&Allocation<B>>,
gqa_factor: usize,
@@ -526,12 +555,15 @@ impl<B: Backend> Attention<B> {
}
#[allow(clippy::too_many_arguments)]
- fn encode_fallback(
+ fn encode_fallback<
+ Keys: AsBufferRangeRef<Buffer: Buffer<Backend = B>>,
+ Values: AsBufferRangeRef<Buffer: Buffer<Backend = B>>,
+ >(
&self,
is_kv_cache_ring: bool,
queries: &Allocation<B>,
- keys: &Allocation<B>,
- values: &Allocation<B>,
+ keys: &Keys,
+ values: &Values,
attention_output: &mut Allocation<B>,
sinks_allocation: Option<&Allocation<B>>,
gqa_factor: usize,
diff --git a/crates/backend-uzu/src/forward_pass/cache_layers.rs b/crates/backend-uzu/src/forward_pass/cache_layers.rs
index 8eda52178..de84ab441 100644
--- a/crates/backend-uzu/src/forward_pass/cache_layers.rs
+++ b/crates/backend-uzu/src/forward_pass/cache_layers.rs
@@ -6,7 +6,7 @@ use crate::{
config::MixerConfig,
forward_pass::{
delta_net_layer::DeltaNetLayer,
- kv_cache_layer::{KVCacheLayer, KVCacheLayerState, KVSlice},
+ kv_cache_layer::{KVCacheLayerState, KVCacheLayerTrait, KVSlice},
model_shape::ModelShape,
short_conv_layer::ShortConvLayer,
ssm_layer::SSMLayer,
@@ -14,7 +14,7 @@ use crate::{
};
pub enum CacheLayer<B: Backend> {
- Transformer(KVCacheLayer<B>),
+ Transformer(Box<dyn KVCacheLayerTrait<B>>),
StateSpace(SSMLayer<B>),
ShortConv(ShortConvLayer<B>),
DeltaNet(DeltaNetLayer<B>),
@@ -28,16 +28,16 @@ pub enum CacheLayerSlice<B: Backend> {
}
impl<B: Backend> CacheLayer<B> {
- pub fn as_transformer(&self) -> Option<&KVCacheLayer<B>> {
+ pub fn as_transformer(&self) -> Option<&dyn KVCacheLayerTrait<B>> {
match self {
- CacheLayer::Transformer(layer) => Some(layer),
+ CacheLayer::Transformer(layer) => Some(layer.as_ref()),
_ => None,
}
}
- pub fn as_transformer_mut(&mut self) -> Option<&mut KVCacheLayer<B>> {
+ pub fn as_transformer_mut(&mut self) -> Option<&mut dyn KVCacheLayerTrait<B>> {
match self {
- CacheLayer::Transformer(layer) => Some(layer),
+ CacheLayer::Transformer(layer) => Some(layer.as_mut()),
_ => None,
}
}
@@ -137,12 +137,12 @@ impl<B: Backend> CacheLayers<B> {
if let Some(source_layer_index) = kv_source_layers.get(layer_index).copied().flatten() {
let LayerCacheBinding::Owned {
entry: source,
- } = bindings[source_layer_index] else {
+ } = bindings[source_layer_index]
+ else {
panic!("kv_source_layer {source_layer_index} (layer {layer_index}) must be a prior owner");
};
- let geometry = |m: &MixerConfig| {
- m.as_attention().map(|a| (a.num_groups, a.head_dim, a.sliding_window_size))
- };
+ let geometry =
+ |m: &MixerConfig| m.as_attention().map(|a| (a.num_groups, a.head_dim, a.sliding_window_size));
assert_eq!(
geometry(&model_shape.layer_mixers()[source_layer_index]),
geometry(mixer),
@@ -159,8 +159,7 @@ impl<B: Backend> CacheLayers<B> {
let sliding_window = attn.sliding_window_size;
let length = sliding_window.unwrap_or(max_prefix_length);
let shape = [length + max_suffix_length, attn.num_groups, attn.head_dim];
- let window_length =
- sliding_window.filter(|&window_size| window_size < total_context_length);
+ let window_length = sliding_window.filter(|&window_size| window_size < total_context_length);
let state = if let Some(w) = window_length {
KVCacheLayerState::Windowed {
@@ -173,19 +172,11 @@ impl<B: Backend> CacheLayers<B> {
prefix_len: 0,
}
};
- let kv_bytes = size_for_shape(&shape, model_shape.kv_cache_data_type());
-
- CacheLayer::Transformer(KVCacheLayer {
- state,
- keys: context
- .create_allocation(kv_bytes, AllocationType::Global)
- .expect("Failed to create kv keys allocation"),
- values: context
- .create_allocation(kv_bytes, AllocationType::Global)
- .expect("Failed to create kv values allocation"),
- shape,
- data_type: model_shape.kv_cache_data_type(),
- })
+
+ let kv_layer =
+ <dyn KVCacheLayerTrait<B>>::new(context, &state, shape, model_shape.kv_cache_data_type())
+ .expect("Failed to create KVCacheLayer");
+ CacheLayer::Transformer(kv_layer)
},
MixerConfig::Mamba(c) => {
let conv_shape = [c.conv_dim(), c.kernel_size.saturating_sub(1)];
@@ -251,7 +242,9 @@ impl<B: Backend> CacheLayers<B> {
},
};
bindings.push(LayerCacheBinding::Owned {
- entry: CacheEntryIndex { index: entries.len() },
+ entry: CacheEntryIndex {
+ index: entries.len(),
+ },
});
entries.push(layer);
}
@@ -262,8 +255,7 @@ impl<B: Backend> CacheLayers<B> {
for layer in entries.iter_mut() {
match layer {
CacheLayer::Transformer(layer) => {
- encoder.encode_fill(&mut layer.keys, 0);
- encoder.encode_fill(&mut layer.values, 0);
+ layer.encode_zero(&mut encoder);
},
CacheLayer::StateSpace(layer) => {
if let Some(conv_state) = layer.conv_state.as_mut() {
@@ -327,21 +319,7 @@ impl<B: Backend> CacheLayers<B> {
let mut encoder: Option<Encoder<B>> = None;
for layer in self.entries.iter_mut() {
match layer {
- CacheLayer::Transformer(layer) => match &mut layer.state {
- KVCacheLayerState::Full {
- prefix_len,
- } => {
- *prefix_len = 0;
- },
- KVCacheLayerState::Windowed {
- ring_offset,
- ring_length,
- ..
- } => {
- *ring_offset = 0;
- *ring_length = 0;
- },
- },
+ CacheLayer::Transformer(layer) => layer.clear_state(),
CacheLayer::StateSpace(layer) => {
let encoder = encoder
.get_or_insert_with(|| Encoder::new(context).expect("Failed to create cache clear encoder"));
@@ -382,19 +360,46 @@ impl<B: Backend> CacheLayers<B> {
&mut self,
accepted_suffix_indices: &[usize],
suffix_start: Option<usize>,
+ context: &B::Context,
encoder: &mut Encoder<B>,
kv_cache_update: &KVCacheUpdate<B>,
) {
let short_conv_commit_index = accepted_suffix_indices.last().copied().unwrap_or(0);
for layer in self.entries.iter_mut() {
if let Some(layer) = layer.as_transformer_mut() {
- layer.update_after_acceptance(accepted_suffix_indices, suffix_start, encoder, kv_cache_update);
+ layer.update_after_acceptance(accepted_suffix_indices, suffix_start, context, encoder, kv_cache_update);
} else if let Some(layer) = layer.as_short_conv_mut() {
layer.commit_from_suffix_state_if_valid(short_conv_commit_index, encoder);
}
}
}
+ pub fn prepare_for_forward_pass(
+ &mut self,
+ context: &B::Context,
+ active_row_count: usize,
+ ) {
+ if active_row_count == 0 {
+ return;
+ }
+
+ for layer in self.entries.iter_mut() {
+ let Some(layer) = layer.as_transformer_mut() else {
+ continue;
+ };
+ let row_range = match layer.state() {
+ KVCacheLayerState::Full {
+ prefix_len,
+ } => prefix_len..prefix_len + active_row_count,
+ KVCacheLayerState::Windowed {
+ window_length,
+ ..
+ } => 0..window_length + active_row_count,
+ };
+ layer.map_row_range(context, row_range).expect("Failed to map KV cache rows for forward pass");
+ }
+ }
+
pub fn register_accepted_tokens(
&mut self,
number_of_accepted_tokens: usize,
@@ -412,10 +417,14 @@ impl<B: Backend> CacheLayers<B> {
range: std::ops::Range<usize>,
) -> Option<CacheLayersSlice<B>> {
let mut layers = Vec::with_capacity(self.entries.len());
+ let mut encoder: Option<Encoder<B>> = None;
+
for layer in self.entries.iter() {
match layer {
CacheLayer::Transformer(kv) => {
- let Some(slice) = kv.slice(context, range.clone()) else {
+ let mut encoder =
+ encoder.get_or_insert_with(|| Encoder::new(context).expect("Failed to create Encoder"));
+ let Some(slice) = kv.slice(context, &mut encoder, range.clone()) else {
return None;
};
layers.push(CacheLayerSlice::Transformer(slice));
@@ -426,6 +435,10 @@ impl<B: Backend> CacheLayers<B> {
}
}
+ if let Some(encoder) = encoder {
+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to end and wait encoder");
+ }
+
Some(CacheLayersSlice {
layers,
})
@@ -433,19 +446,27 @@ impl<B: Backend> CacheLayers<B> {
pub fn apply_slice(
&mut self,
+ context: &B::Context,
slice: &CacheLayersSlice<B>,
range: Option<std::ops::Range<usize>>,
) {
+ let mut encoder: Option<Encoder<B>> = None;
for (layer, snapshot) in self.entries.iter_mut().zip(slice.layers.iter()) {
match (layer, snapshot) {
(CacheLayer::Transformer(kv), CacheLayerSlice::Transformer(s)) => {
- kv.apply_slice(&s, range.clone());
+ let mut encoder =
+ encoder.get_or_insert_with(|| Encoder::new(context).expect("Failed to create Encoder"));
+ kv.apply_slice(context, &mut encoder, &s, range.clone());
},
(CacheLayer::StateSpace(_), CacheLayerSlice::StateSpace) => {},
(CacheLayer::ShortConv(_), CacheLayerSlice::ShortConv) => {},
_ => {},
}
}
+
+ if let Some(encoder) = encoder {
+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to end and wait encoder");
+ }
}
pub fn copy_from(
@@ -456,7 +477,7 @@ impl<B: Backend> CacheLayers<B> {
self.copy_metadata_and_windowed_data_from(source, context);
let mut encoder = Encoder::new(context).expect("Failed to create cache layer copy encoder");
- self.encode_copy_data_from(source, &mut encoder);
+ self.encode_copy_data_from(source, context, &mut encoder);
encoder.end_encoding().submit().wait_until_completed().expect("Failed to copy cache layers");
}
@@ -467,15 +488,24 @@ impl<B: Backend> CacheLayers<B> {
) {
assert_eq!(source.entries.len(), self.entries.len(), "cache layer count mismatch");
+ // Key/value arrays stay alive until after encoder.submit().wait_until_completed() returns, then explicitly drops them.
+ // This prevents the slice buffers from being freed while the GPU is still reading from them during windowed cache cloning.
+ let mut pending_slices: Vec<KVSlice<B>> = Vec::new();
+ let mut encoder: Option<Encoder<B>> = None;
+
for (source_layer, destination_layer) in source.entries.iter().zip(self.entries.iter_mut()) {
match (source_layer, destination_layer) {
(CacheLayer::Transformer(source), CacheLayer::Transformer(destination)) => {
let copy_rows = source.prefix_segment_length();
- if copy_rows > 0 && matches!(source.state, KVCacheLayerState::Windowed { .. }) {
- let slice = source.slice(context, 0..copy_rows).expect("Failed to slice KV cache layer");
- destination.apply_slice(&slice, None);
+ if copy_rows > 0 && matches!(source.state(), KVCacheLayerState::Windowed { .. }) {
+ let encoder =
+ encoder.get_or_insert_with(|| Encoder::new(context).expect("Failed to create Encoder"));
+ let slice =
+ source.slice(context, encoder, 0..copy_rows).expect("Failed to slice KV cache layer");
+ destination.apply_slice(context, encoder, &slice, None);
+ pending_slices.push(slice);
}
- destination.state = source.state.clone();
+ destination.set_state(&source.state());
},
(CacheLayer::StateSpace(_), CacheLayer::StateSpace(_)) => {},
(CacheLayer::ShortConv(_), CacheLayer::ShortConv(destination)) => {
@@ -485,11 +515,17 @@ impl<B: Backend> CacheLayers<B> {
_ => panic!("cache layer type mismatch while copying cache layers"),
}
}
+
+ if let Some(encoder) = encoder {
+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to end and wait encoder");
+ }
+ drop(pending_slices);
}
fn encode_copy_data_from(
&mut self,
source: &Self,
+ context: &B::Context,
encoder: &mut Encoder<B>,
) {
assert_eq!(source.entries.len(), self.entries.len(), "cache layer count mismatch");
@@ -497,8 +533,13 @@ impl<B: Backend> CacheLayers<B> {
for (source_layer, destination_layer) in source.entries.iter().zip(self.entries.iter_mut()) {
match (source_layer, destination_layer) {
(CacheLayer::Transformer(source), CacheLayer::Transformer(destination)) => {
- if matches!(source.state, KVCacheLayerState::Full { .. }) {
- source.encode_copy_prefix_rows_to(destination, source.prefix_segment_length(), encoder);
+ if matches!(source.state(), KVCacheLayerState::Full { .. }) {
+ source.encode_copy_prefix_rows_to(
+ destination.as_mut(),
+ source.prefix_segment_length(),
+ context,
+ encoder,
+ );
}
},
(CacheLayer::StateSpace(source), CacheLayer::StateSpace(destination)) => {
@@ -533,9 +574,9 @@ impl<B: Backend> CacheLayers<B> {
.iter()
.map(|layer| match layer {
CacheLayer::Transformer(layer) => {
- let shape = layer.shape;
+ let shape = layer.shape();
let [_, num_groups, head_dim] = shape;
- let dtype = layer.data_type;
+ let dtype = layer.data_type();
let copy_rows = layer.prefix_segment_length();
let new_total_len = copy_rows + self.max_suffix_length;
@@ -544,21 +585,9 @@ impl<B: Backend> CacheLayers<B> {
}
let new_shape = [new_total_len, num_groups, head_dim];
- let new_bytes = size_for_shape(&new_shape, dtype);
- let new_keys = context
- .create_allocation(new_bytes, AllocationType::Global)
- .expect("Failed to create kv keys clone allocation");
- let new_values = context
- .create_allocation(new_bytes, AllocationType::Global)
- .expect("Failed to create kv values clone allocation");
-
- CacheLayer::Transformer(KVCacheLayer {
- state: layer.state.clone(),
- keys: new_keys,
- values: new_values,
- shape: new_shape,
- data_type: dtype,
- })
+ let kv_layer = <dyn KVCacheLayerTrait<B>>::new(context, &layer.state(), new_shape, dtype)
+ .expect("Failed to create KVCacheLayer");
+ CacheLayer::Transformer(kv_layer)
},
CacheLayer::StateSpace(layer) => {
let conv_bytes = size_for_shape(&layer.conv_shape, layer.data_type);
@@ -625,8 +654,7 @@ impl<B: Backend> CacheLayers<B> {
for layer in entries.iter_mut() {
match layer {
CacheLayer::Transformer(layer) => {
- zero_encoder.encode_fill(&mut layer.keys, 0);
- zero_encoder.encode_fill(&mut layer.values, 0);
+ layer.encode_zero(&mut zero_encoder);
},
CacheLayer::ShortConv(layer) => {
zero_encoder.encode_fill(&mut layer.suffix_state, 0);
@@ -646,3 +674,7 @@ impl<B: Backend> CacheLayers<B> {
cloned
}
}
+
+#[cfg(test)]
+#[path = "../../tests/unit/forward_pass/cache_layers_test.rs"]
+mod tests;
diff --git a/crates/backend-uzu/src/forward_pass/kv_cache_layer.rs b/crates/backend-uzu/src/forward_pass/kv_cache_layer.rs
index 6a32cf07f..dbd1fb688 100644
--- a/crates/backend-uzu/src/forward_pass/kv_cache_layer.rs
+++ b/crates/backend-uzu/src/forward_pass/kv_cache_layer.rs
@@ -1,8 +1,10 @@
+use std::{any::Any, ops::Range};
+
use crate::{
DataType,
array::{Array, ArrayContextExt, size_for_shape},
backends::common::{
- Allocation, Backend, Encoder, allocation_as_bytes, allocation_as_bytes_mut,
+ AsBufferRangeMut, AsBufferRangeRef, Backend, Buffer, Context, Encoder, SparseBuffer,
kernel::kv_cache_update::{KVCacheUpdate, KVLayerData},
},
};
@@ -39,39 +41,275 @@ pub enum KVCacheLayerState {
pub const INVALID_POSITION: usize = i32::MAX as usize;
-pub struct KVCacheLayer<B: Backend> {
+pub trait KVCacheLayerTrait<B: Backend> {
+ fn as_any(&self) -> &dyn Any;
+
+ fn as_any_mut(&mut self) -> &mut dyn Any;
+
+ fn shape(&self) -> [usize; 3];
+
+ fn data_type(&self) -> DataType;
+
+ fn encode_zero(
+ &mut self,
+ encoder: &mut Encoder<'_, B>,
+ );
+
+ fn state(&self) -> KVCacheLayerState;
+
+ fn set_state(
+ &mut self,
+ state: &KVCacheLayerState,
+ );
+
+ fn clear_state(&mut self);
+
+ fn encode_copy_prefix_rows_to(
+ &self,
+ destination: &mut dyn KVCacheLayerTrait<B>,
+ row_count: usize,
+ context: &B::Context,
+ encoder: &mut Encoder<B>,
+ );
+
+ fn prefix_segment_length(&self) -> usize;
+
+ fn update_after_acceptance(
+ &mut self,
+ accepted_suffix_indices: &[usize],
+ suffix_start: Option<usize>,
+ context: &B::Context,
+ encoder: &mut Encoder<B>,
+ kv_cache_update: &KVCacheUpdate<B>,
+ );
+
+ fn register_accepted_tokens(
+ &mut self,
+ number_of_accepted_tokens: usize,
+ );
+
+ fn slice(
+ &self,
+ context: &B::Context,
+ encoder: &mut Encoder<B>,
+ range: Range<usize>,
+ ) -> Option<KVSlice<B>>;
+
+ fn apply_slice(
+ &mut self,
+ context: &B::Context,
+ encoder: &mut Encoder<B>,
+ slice: &KVSlice<B>,
+ range: Option<Range<usize>>,
+ );
+
+ fn map_row_range(
+ &mut self,
+ context: &B::Context,
+ range: Range<usize>,
+ ) -> Result<(), B::Error>;
+}
+
+impl<B: Backend> dyn KVCacheLayerTrait<B> {
+ pub fn new(
+ context: &B::Context,
+ state: &KVCacheLayerState,
+ shape: [usize; 3],
+ data_type: DataType,
+ ) -> Result<Box<dyn KVCacheLayerTrait<B>>, B::Error> {
+ let buffer_size = size_for_shape(&shape, data_type);
+ if context.sparse_buffers_supported() {
+ let layer = KVCacheLayer {
+ state: state.clone(),
+ keys: context.create_sparse_buffer(buffer_size)?,
+ values: context.create_sparse_buffer(buffer_size)?,
+ shape,
+ data_type,
+ };
+ Ok(Box::new(layer))
+ } else {
+ let layer = KVCacheLayer {
+ state: state.clone(),
+ keys: context.create_buffer(buffer_size)?,
+ values: context.create_buffer(buffer_size)?,
+ shape,
+ data_type,
+ };
+ Ok(Box::new(layer))
+ }
+ }
+}
+
+pub struct KVCacheLayer<B: Backend, Buf: Buffer<Backend = B>> {
pub state: KVCacheLayerState,
/// [max_prefix_length + max_suffix_length, num_groups, head_dim]
- pub keys: Allocation<B>,
+ pub keys: Buf,
/// [max_prefix_length + max_suffix_length, num_groups, head_dim]
- pub values: Allocation<B>,
+ pub values: Buf,
pub shape: [usize; 3],
pub data_type: DataType,
}
-fn copy_rows(
- source_keys_bytes: &[u8],
- source_values_bytes: &[u8],
- destination_keys_bytes: &mut [u8],
- destination_values_bytes: &mut [u8],
- row_size: usize,
- row_pairs: impl IntoIterator<Item = (usize, usize)>,
-) {
- for (source_row, destination_row) in row_pairs {
- let source_offset = source_row * row_size;
- let destination_offset = destination_row * row_size;
- destination_keys_bytes[destination_offset..destination_offset + row_size]
- .copy_from_slice(&source_keys_bytes[source_offset..source_offset + row_size]);
- destination_values_bytes[destination_offset..destination_offset + row_size]
- .copy_from_slice(&source_values_bytes[source_offset..source_offset + row_size]);
+impl<B: Backend, Buf: Buffer<Backend = B>> KVCacheLayer<B, Buf> {
+ fn scatter_if_required(
+ &mut self,
+ source_indices: &[usize],
+ destination_indices: &[usize],
+ encoder: &mut Encoder<B>,
+ kv_cache_update: &KVCacheUpdate<B>,
+ ) {
+ if source_indices == destination_indices {
+ return;
+ }
+
+ let mut layer_data = KVLayerData {
+ key_buffer: &mut self.keys,
+ key_shape: self.shape,
+ value_buffer: &mut self.values,
+ value_shape: self.shape,
+ };
+
+ let data = std::slice::from_mut(&mut layer_data);
+ let _ = kv_cache_update.encode(data, source_indices, destination_indices, encoder);
+ }
+
+ fn copy_rows<
+ SrcKeys: AsBufferRangeRef<Buffer: Buffer<Backend = B>>,
+ DstKeys: AsBufferRangeMut<Buffer: Buffer<Backend = B>>,
+ SrcValues: AsBufferRangeRef<Buffer: Buffer<Backend = B>>,
+ DstValues: AsBufferRangeMut<Buffer: Buffer<Backend = B>>,
+ >(
+ encoder: &mut Encoder<B>,
+ src_keys: &SrcKeys,
+ dst_keys: &mut DstKeys,
+ src_values: &SrcValues,
+ dst_values: &mut DstValues,
+ row_size: usize,
+ row_pairs: impl IntoIterator<Item = (usize, usize)>,
+ ) {
+ for (src_row, dst_row) in row_pairs {
+ let src_offset = src_row * row_size;
+ let dst_offset = dst_row * row_size;
+ encoder.encode_copy(
+ src_keys,
+ src_offset..src_offset + row_size,
+ dst_keys,
+ dst_offset..dst_offset + row_size,
+ );
+ encoder.encode_copy(
+ src_values,
+ src_offset..src_offset + row_size,
+ dst_values,
+ dst_offset..dst_offset + row_size,
+ );
+ }
+ }
+
+ fn page_range_for_row_range(
+ shape: [usize; 3],
+ data_type: DataType,
+ page_size_bytes: usize,
+ row_range: Range<usize>,
+ ) -> Range<usize> {
+ if row_range.is_empty() {
+ return 0..0;
+ }
+
+ let [_, num_groups, head_dim] = shape;
+ let row_size = size_for_shape(&[1, num_groups, head_dim], data_type);
+ let byte_start = row_range.start * row_size;
+ let byte_end = row_range.end * row_size;
+ byte_start / page_size_bytes..byte_end.div_ceil(page_size_bytes)
+ }
+
+ fn map_sparse_row_range(
+ &mut self,
+ context: &B::Context,
+ row_range: Range<usize>,
+ ) -> Result<(), B::Error> {
+ if row_range.is_empty() {
+ return Ok(());
+ }
+
+ if let Some(layer) = self.as_any_mut().downcast_mut::<KVCacheLayer<B, B::SparseBuffer>>() {
+ let key_pages = Self::page_range_for_row_range(
+ layer.shape,
+ layer.data_type,
+ layer.keys.page_size_bytes(),
+ row_range.clone(),
+ );
+ layer.keys.map(context, &key_pages)?;
+
+ let value_pages =
+ Self::page_range_for_row_range(layer.shape, layer.data_type, layer.values.page_size_bytes(), row_range);
+ layer.values.map(context, &value_pages)?;
+ }
+ Ok(())
}
}
-impl<B: Backend> KVCacheLayer<B> {
- pub fn encode_copy_prefix_rows_to(
+impl<B: Backend, Buf: Buffer<Backend = B>> KVCacheLayerTrait<B> for KVCacheLayer<B, Buf> {
+ fn as_any(&self) -> &dyn Any {
+ self
+ }
+
+ fn as_any_mut(&mut self) -> &mut dyn Any {
+ self
+ }
+
+ fn shape(&self) -> [usize; 3] {
+ self.shape
+ }
+
+ fn data_type(&self) -> DataType {
+ self.data_type
+ }
+
+ fn encode_zero(
+ &mut self,
+ encoder: &mut Encoder<'_, B>,
+ ) {
+ if self.as_any().is::<KVCacheLayer<B, B::SparseBuffer>>() {
+ return;
+ }
+ encoder.encode_fill(&mut self.keys, 0);
+ encoder.encode_fill(&mut self.values, 0);
+ }
+
+ fn state(&self) -> KVCacheLayerState {
+ self.state.clone()
+ }
+
+ fn set_state(
+ &mut self,
+ state: &KVCacheLayerState,
+ ) {
+ self.state = state.clone();
+ }
+
+ fn clear_state(&mut self) {
+ match &mut self.state {
+ KVCacheLayerState::Full {
+ prefix_len,
+ } => {
+ *prefix_len = 0;
+ },
+ KVCacheLayerState::Windowed {
+ ring_offset,
+ ring_length,
+ ..
+ } => {
+ *ring_offset = 0;
+ *ring_length = 0;
+ },
+ }
+ }
+
+ fn encode_copy_prefix_rows_to(
&self,
- destination: &mut KVCacheLayer<B>,
+ destination: &mut dyn KVCacheLayerTrait<B>,
row_count: usize,
+ context: &B::Context,
encoder: &mut Encoder<B>,
) {
if row_count == 0 {
@@ -79,21 +317,28 @@ impl<B: Backend> KVCacheLayer<B> {
}
let [source_seq, num_groups, head_dim] = self.shape;
- let [destination_seq, destination_groups, destination_head_dim] = destination.shape;
+ let [destination_seq, destination_groups, destination_head_dim] = destination.shape();
assert_eq!(num_groups, destination_groups, "KV cache group count mismatch");
assert_eq!(head_dim, destination_head_dim, "KV cache head dim mismatch");
- assert_eq!(self.data_type, destination.data_type, "KV cache dtype mismatch");
+ assert_eq!(self.data_type, destination.data_type(), "KV cache dtype mismatch");
assert!(row_count <= source_seq, "source KV cache copy exceeds source sequence");
assert!(row_count <= destination_seq, "source KV cache copy exceeds destination sequence");
+ destination.map_row_range(context, 0..row_count).expect("Failed to map destination KV cache rows");
let row_size = size_for_shape(&[1, num_groups, head_dim], self.data_type);
let copy_size = row_count * row_size;
-
- encoder.encode_copy(&self.keys, 0..copy_size, &mut destination.keys, 0..copy_size);
- encoder.encode_copy(&self.values, 0..copy_size, &mut destination.values, 0..copy_size);
+ if let Some(dest) = destination.as_any_mut().downcast_mut::<KVCacheLayer<B, B::SparseBuffer>>() {
+ encoder.encode_copy(&self.keys, 0..copy_size, &mut dest.keys, 0..copy_size);
+ encoder.encode_copy(&self.values, 0..copy_size, &mut dest.values, 0..copy_size);
+ } else if let Some(dest) = destination.as_any_mut().downcast_mut::<KVCacheLayer<B, B::DenseBuffer>>() {
+ encoder.encode_copy(&self.keys, 0..copy_size, &mut dest.keys, 0..copy_size);
+ encoder.encode_copy(&self.values, 0..copy_size, &mut dest.values, 0..copy_size);
+ } else {
+ panic!("Wrong destination type!")
+ }
}
- pub fn prefix_segment_length(&self) -> usize {
+ fn prefix_segment_length(&self) -> usize {
match &self.state {
KVCacheLayerState::Full {
prefix_len,
@@ -105,22 +350,11 @@ impl<B: Backend> KVCacheLayer<B> {
}
}
- pub fn window_length(&self) -> Option<usize> {
- match &self.state {
- KVCacheLayerState::Full {
- ..
- } => None,
- KVCacheLayerState::Windowed {
- window_length,
- ..
- } => Some(*window_length),
- }
- }
-
- pub fn update_after_acceptance(
+ fn update_after_acceptance(
&mut self,
accepted_suffix_indices: &[usize],
suffix_start: Option<usize>,
+ context: &B::Context,
encoder: &mut Encoder<B>,
kv_cache_update: &KVCacheUpdate<B>,
) {
@@ -132,14 +366,18 @@ impl<B: Backend> KVCacheLayer<B> {
return;
}
+ let prefix_len_value = *prefix_len;
+
// Absolute positions of the *source* rows.
let source_indices: Vec<usize> =
- accepted_suffix_indices.iter().map(|i| i + suffix_start.unwrap_or(*prefix_len)).collect();
+ accepted_suffix_indices.iter().map(|i| i + suffix_start.unwrap_or(prefix_len_value)).collect();
// Absolute positions of the *destination* rows.
let destination_indices: Vec<usize> =
- (*prefix_len..*prefix_len + accepted_suffix_indices.len()).collect();
+ (prefix_len_value..prefix_len_value + accepted_suffix_indices.len()).collect();
+ self.map_sparse_row_range(context, prefix_len_value..prefix_len_value + accepted_suffix_indices.len())
+ .expect("Failed to map accepted KV cache rows");
self.scatter_if_required(&source_indices, &destination_indices, encoder, kv_cache_update);
},
@@ -162,34 +400,16 @@ impl<B: Backend> KVCacheLayer<B> {
destination_indices.push((*ring_length + *ring_offset + i) % *window_length);
}
+ for destination_index in destination_indices.iter().copied() {
+ self.map_sparse_row_range(context, destination_index..destination_index + 1)
+ .expect("Failed to map accepted KV cache row");
+ }
self.scatter_if_required(&source_indices, &destination_indices, encoder, kv_cache_update);
},
}
}
- fn scatter_if_required(
- &mut self,
- source_indices: &[usize],
- destination_indices: &[usize],
- encoder: &mut Encoder<B>,
- kv_cache_update: &KVCacheUpdate<B>,
- ) {
- if source_indices == destination_indices {
- return;
- }
-
- let mut layer_data = KVLayerData {
- key_allocation: &mut self.keys,
- key_shape: self.shape,
- value_allocation: &mut self.values,
- value_shape: self.shape,
- };
-
- let _ =
- kv_cache_update.encode(std::slice::from_mut(&mut layer_data), source_indices, destination_indices, encoder);
- }
-
- pub fn register_accepted_tokens(
+ fn register_accepted_tokens(
&mut self,
number_of_accepted_tokens: usize,
) {
@@ -211,10 +431,11 @@ impl<B: Backend> KVCacheLayer<B> {
}
}
- pub fn slice(
+ fn slice(
&self,
context: &B::Context,
- range: std::ops::Range<usize>,
+ encoder: &mut Encoder<B>,
+ range: Range<usize>,
) -> Option<KVSlice<B>> {
match self.state {
KVCacheLayerState::Full {
@@ -235,12 +456,6 @@ impl<B: Backend> KVCacheLayer<B> {
let dtype = self.data_type;
let row_size = size_for_shape(&[1, num_groups, head_dim], dtype);
- let slice_shape = [len, num_groups, head_dim];
- let mut slice_keys = context.create_array_uninitialized(&slice_shape, dtype);
- let mut slice_values = context.create_array_uninitialized(&slice_shape, dtype);
- let source_keys_bytes = allocation_as_bytes(&self.keys);
- let source_values_bytes = allocation_as_bytes(&self.values);
-
let slots: Vec<usize> = (range.start..range.end)
.enumerate()
.map(|(offset, _)| {
@@ -252,17 +467,19 @@ impl<B: Backend> KVCacheLayer<B> {
}
})
.collect();
+ let row_pairs = slots.iter().enumerate().map(|(dst_row, &src_row)| (src_row, dst_row));
- let destination_keys_bytes = slice_keys.as_bytes_mut();
- let destination_values_bytes = slice_values.as_bytes_mut();
-
- copy_rows(
- source_keys_bytes,
- source_values_bytes,
- destination_keys_bytes,
- destination_values_bytes,
+ let slice_shape = [len, num_groups, head_dim];
+ let mut dst_keys = context.create_array_uninitialized(&slice_shape, dtype);
+ let mut dst_values = context.create_array_uninitialized(&slice_shape, dtype);
+ Self::copy_rows(
+ encoder,
+ &self.keys,
+ dst_keys.allocation_mut(),
+ &self.values,
+ dst_values.allocation_mut(),
row_size,
- slots.iter().enumerate().map(|(destination_row, &source_row)| (source_row, destination_row)),
+ row_pairs,
);
Some(KVSlice::Window {
@@ -270,17 +487,19 @@ impl<B: Backend> KVCacheLayer<B> {
base_ring_offset: ring_offset,
base_ring_length: ring_length,
slots,
- keys: slice_keys,
- values: slice_values,
+ keys: dst_keys,
+ values: dst_values,
})
},
}
}
- pub fn apply_slice(
+ fn apply_slice(
&mut self,
+ context: &B::Context,
+ encoder: &mut Encoder<B>,
slice: &KVSlice<B>,
- range: Option<std::ops::Range<usize>>,
+ range: Option<Range<usize>>,
) {
match (slice, &mut self.state) {
(
@@ -316,69 +535,73 @@ impl<B: Backend> KVCacheLayer<B> {
*w_len = *window_length;
let [_, num_groups, head_dim] = self.shape;
let row_size = size_for_shape(&[1, num_groups, head_dim], self.data_type);
- let source_keys_bytes = keys.as_bytes();
- let source_values_bytes = values.as_bytes();
- let destination_keys_bytes = allocation_as_bytes_mut(&mut self.keys);
- let destination_values_bytes = allocation_as_bytes_mut(&mut self.values);
- match range {
+
+ let row_pairs: Option<Vec<(usize, usize)>> = match &range {
None => {
*ring_offset = *base_ring_offset;
*ring_length = *base_ring_length;
-
- copy_rows(
- source_keys_bytes,
- source_values_bytes,
- destination_keys_bytes,
- destination_values_bytes,
- row_size,
- slots
- .iter()
- .enumerate()
- .map(|(source_row, &destination_row)| (source_row, destination_row)),
- );
+ let pairs = slots.iter().enumerate().map(|(src_row, &dst_row)| (src_row, dst_row)).collect();
+ Some(pairs)
},
Some(r) => {
let len = r.end.saturating_sub(r.start);
if len == 0 {
- return;
- }
+ None
+ } else {
+ let accepted = r.start;
+ let base_len = *base_ring_length;
+ let w = *window_length;
+
+ let (new_offset, new_len) = if base_len < w {
+ let after = base_len.saturating_add(accepted);
+ if after <= w {
+ (*base_ring_offset, after)
+ } else {
+ let overflow = after - w;
+ ((base_ring_offset + overflow) % w, w)
+ }
+ } else {
+ ((base_ring_offset + accepted) % w, w)
+ };
- let accepted = r.start;
- let base_len = *base_ring_length;
- let w = *window_length;
+ *ring_offset = new_offset;
+ *ring_length = new_len;
- let (new_offset, new_len) = if base_len < w {
- let after = base_len.saturating_add(accepted);
- if after <= w {
- (*base_ring_offset, after)
- } else {
- let overflow = after - w;
- ((base_ring_offset + overflow) % w, w)
- }
- } else {
- ((base_ring_offset + accepted) % w, w)
- };
-
- *ring_offset = new_offset;
- *ring_length = new_len;
-
- copy_rows(
- source_keys_bytes,
- source_values_bytes,
- destination_keys_bytes,
- destination_values_bytes,
- row_size,
- slots[r.clone()]
- .iter()
- .enumerate()
- .map(|(index, &destination_row)| (r.start + index, destination_row)),
- );
+ let pairs =
+ slots[r.clone()].iter().enumerate().map(move |(i, &dst_row)| (r.start + i, dst_row));
+ Some(pairs.collect())
+ }
},
+ };
+ let Some(row_pairs) = row_pairs else {
+ return;
+ };
+ for (_, destination_row) in row_pairs.iter().copied() {
+ self.map_sparse_row_range(context, destination_row..destination_row + 1)
+ .expect("Failed to map sliced KV cache row");
}
+
+ Self::copy_rows(
+ encoder,
+ keys.allocation(),
+ &mut self.keys,
+ values.allocation(),
+ &mut self.values,
+ row_size,
+ row_pairs,
+ );
},
_ => {},
}
}
+
+ fn map_row_range(
+ &mut self,
+ context: &B::Context,
+ range: Range<usize>,
+ ) -> Result<(), B::Error> {
+ self.map_sparse_row_range(context, range)
+ }
}
#[cfg(test)]
diff --git a/crates/backend-uzu/src/language_model/language_model_generator.rs b/crates/backend-uzu/src/language_model/language_model_generator.rs
index f9e0b38f2..5a0716400 100644
--- a/crates/backend-uzu/src/language_model/language_model_generator.rs
+++ b/crates/backend-uzu/src/language_model/language_model_generator.rs
@@ -535,6 +535,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
self.context.cache_layers.borrow_mut().update_after_acceptance(
&[0],
None,
+ &self.context.context,
&mut encoder,
&self.context.kv_cache_update,
);
@@ -629,7 +630,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
range: Range<usize>,
) {
let slice = slice.downcast_ref::<CacheLayersSlice<B>>().unwrap();
- self.context.cache_layers.borrow_mut().apply_slice(slice, Some(range));
+ self.context.cache_layers.borrow_mut().apply_slice(&self.context.context, slice, Some(range));
}
fn build_llm_context(&self) -> Box<dyn Any> {
@@ -744,6 +745,7 @@ impl<B: Backend> LanguageModelGenerator<B> {
let mut logits = None;
if is_prefilling {
let mut cache_layers = context.cache_layers.borrow_mut();
+ cache_layers.prepare_for_forward_pass(&context.context, batch_dim);
let decoder_arguments = token_inputs.decoder_arguments(
context.shared_buffers.as_ref(),
Some(&mut *cache_layers),
@@ -759,6 +761,7 @@ impl<B: Backend> LanguageModelGenerator<B> {
.map_err(|e| Error::EncodeFailed(Box::new(e)))?;
} else {
let mut cache_layers = context.cache_layers.borrow_mut();
+ cache_layers.prepare_for_forward_pass(&context.context, batch_dim);
let decoder_arguments = token_inputs.decoder_arguments(
context.shared_buffers.as_ref(),
Some(&mut *cache_layers),
@@ -857,14 +860,15 @@ impl<B: Backend> LanguageModelGenerator<B> {
suffix_start: Option<usize>,
wait_until_completed: bool,
) -> Result<(), Error> {
- let mut encoder = Encoder::<B>::new(self.context.context.as_ref())
- .map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?;
+ let mut encoder =
+ Encoder::<B>::new(&self.context.context).map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?;
{
let mut cache_layers = self.context.cache_layers.borrow_mut();
cache_layers.update_after_acceptance(
accepted_token_indices,
suffix_start,
+ &self.context.context,
&mut encoder,
&self.context.kv_cache_update,
);
diff --git a/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs b/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs
index 7f3bffb8e..76b73cc51 100644
--- a/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs
+++ b/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs
@@ -363,6 +363,7 @@ impl<B: Backend> TokenDecoderRunner<B> {
let mut encoder = Encoder::new(context.as_ref()).map_err(unable_to_create_context)?;
let mut cache_layers = self.ctx.cache_layers.borrow_mut();
+ cache_layers.prepare_for_forward_pass(&context, token_count);
self.ctx
.executables
.encode_prefill(
@@ -571,6 +572,7 @@ impl<B: Backend> TokenDecoderRunner<B> {
{
let mut logits = {
let mut cache_layers = cache_layers_rc.borrow_mut();
+ cache_layers.prepare_for_forward_pass(&self.ctx.context, 1);
self.ctx
.executables
.encode_decode(
@@ -608,7 +610,13 @@ impl<B: Backend> TokenDecoderRunner<B> {
pending_sampling_outputs.push(sampling_output);
}
- self.ctx.cache_layers.borrow_mut().update_after_acceptance(&[0], None, encoder, &self.ctx.kv_cache_update);
+ self.ctx.cache_layers.borrow_mut().update_after_acceptance(
+ &[0],
+ None,
+ &self.ctx.context,
+ encoder,
+ &self.ctx.kv_cache_update,
+ );
self.ctx.cache_layers.borrow_mut().register_accepted_tokens(1);
}
Ok(())
@@ -787,6 +795,7 @@ impl<B: Backend> TokenDecoderRunner<B> {
}
let mut logits = {
let mut cache_layers = self.ctx.cache_layers.borrow_mut();
+ cache_layers.prepare_for_forward_pass(&self.ctx.context, token_count);
let decoder_arguments = DecoderArguments {
token_positions: token_inputs.token_positions(),
token_parents: token_inputs.token_parents(),
@@ -841,6 +850,7 @@ impl<B: Backend> TokenDecoderRunner<B> {
self.ctx.cache_layers.borrow_mut().update_after_acceptance(
accepted_suffix_indices,
None,
+ &self.ctx.context,
encoder,
&self.ctx.kv_cache_update,
);
diff --git a/crates/backend-uzu/tests/common/helpers.rs b/crates/backend-uzu/tests/common/helpers.rs
index e30b9a93a..60c8bebde 100644
--- a/crates/backend-uzu/tests/common/helpers.rs
+++ b/crates/backend-uzu/tests/common/helpers.rs
@@ -2,7 +2,7 @@ use std::{mem::size_of, rc::Rc};
use backend_uzu::{
ArrayElement, allocation_copy_from_slice,
- backends::common::{Allocation, AllocationType, Backend, Context, Encoder},
+ backends::common::{Allocation, AllocationType, Backend, Buffer, Context, Encoder, SparseBuffer, SparseBufferExt},
};
pub fn allocation_size_bytes<T>(elements_count: usize) -> usize {
@@ -56,3 +56,67 @@ pub fn create_context<B: Backend>() -> Rc<<B as Backend>::Context> {
pub fn submit_encoder<B: Backend>(encoder: Encoder<B>) {
encoder.end_encoding().submit().wait_until_completed().unwrap();
}
+
+pub fn sparse_buffer_create<B: Backend>(
+ context: &B::Context,
+ capacity: usize,
+) -> B::SparseBuffer {
+ context.create_sparse_buffer(capacity).expect("Failed to create sparse buffer")
+}
+
+pub fn sparse_buffer_create_and_map<B: Backend>(
+ context: &B::Context,
+ capacity: usize,
+) -> B::SparseBuffer {
+ let mut buffer = sparse_buffer_create::<B>(context, capacity);
+ buffer.map(context, &(0..buffer.total_pages())).expect("Failed to map sparse buffer");
+ buffer
+}
+
+pub fn sparse_buffer_create_with<B: Backend, T: ArrayElement>(
+ context: &B::Context,
+ data: &[T],
+) -> B::SparseBuffer {
+ let capacity_bytes = allocation_size_bytes::<T>(data.len());
+ let mut buffer = sparse_buffer_create_and_map::<B>(context, capacity_bytes);
+ sparse_buffer_write::<B, T>(context, &mut buffer, data);
+ buffer
+}
+
+pub fn sparse_buffer_read_allocation<B: Backend>(
+ context: &B::Context,
+ buffer: &B::SparseBuffer,
+ size: usize,
+) -> Allocation<B> {
+ let mut allocation = alloc_allocation::<B, u8>(context, size);
+ let range = 0..size;
+
+ let mut encoder = Encoder::new(context).expect("Failed to create encoder");
+ encoder.encode_copy(buffer, range.clone(), &mut allocation, range.clone());
+ submit_encoder(encoder);
+
+ allocation
+}
+
+pub fn sparse_buffer_read_vec<B: Backend, T: ArrayElement>(
+ context: &B::Context,
+ buffer: &B::SparseBuffer,
+ elements_count: usize,
+) -> Vec<T> {
+ let allocation =
+ sparse_buffer_read_allocation::<B>(context, buffer, elements_count * T::data_type().size_in_bytes());
+ allocation_to_vec(&allocation)
+}
+
+pub fn sparse_buffer_write<B: Backend, T: ArrayElement>(
+ context: &B::Context,
+ buffer: &mut B::SparseBuffer,
+ data: &[T],
+) {
+ let data_allocation = alloc_allocation_with_data::<B, T>(context, data);
+ let data_range = 0..allocation_size_bytes::<T>(data.len());
+
+ let mut encoder = Encoder::new(context).expect("Failed to create encoder");
+ encoder.encode_copy(&data_allocation, data_range.clone(), buffer, data_range.clone());
+ submit_encoder(encoder);
+}
diff --git a/crates/backend-uzu/tests/common/matmul/harness.rs b/crates/backend-uzu/tests/common/matmul/harness.rs
index 3a52529cf..4aeb83d5d 100644
--- a/crates/backend-uzu/tests/common/matmul/harness.rs
+++ b/crates/backend-uzu/tests/common/matmul/harness.rs
@@ -4,7 +4,7 @@ use backend_uzu::{
ArrayContextExt, ArrayElement,
backends::{
common::{
- AllocationType, Backend, Context, Encoder,
+ Allocation, AllocationType, AsBufferRangeRef, Backend, Buffer, Context, Encoder,
kernel::{
ManualKernels,
matmul::{MatmulArgumentC, MatmulArguments, MatmulKernel},
@@ -118,7 +118,7 @@ fn run<B: Backend, T: ArrayElement + Float>(
let mut encoder = Encoder::new(context).expect("encoder");
encode(
kernel,
- MatmulArguments {
+ MatmulArguments::<'_, B> {
a: &a_allocation,
a_offset: 0,
b: b_array.allocation(),
diff --git a/crates/backend-uzu/tests/integration/tracer/trace_validator.rs b/crates/backend-uzu/tests/integration/tracer/trace_validator.rs
index 058590d23..ec16386df 100644
--- a/crates/backend-uzu/tests/integration/tracer/trace_validator.rs
+++ b/crates/backend-uzu/tests/integration/tracer/trace_validator.rs
@@ -17,7 +17,9 @@ use backend_uzu::{
classifier::Classifier,
config::{ModelConfig, ModelMetadata, ModelType},
encodable_block::{DecoderDecodeInput, Sampling},
- forward_pass::{cache_layers::CacheLayers, token_inputs::TokenInputs, traces::ActivationTrace},
+ forward_pass::{
+ cache_layers::CacheLayers, kv_cache_layer::KVCacheLayer, token_inputs::TokenInputs, traces::ActivationTrace,
+ },
language_model::{
language_model_generator_context::LanguageModelGeneratorContext,
sampler::{ArgmaxSampler, LogitsSampler},
@@ -33,6 +35,7 @@ use half::{bf16, f16};
use ndarray::{IxDyn, s};
use num_traits::NumCast;
+use crate::common;
// ============================================================================
// Validation Types
// ============================================================================
@@ -313,26 +316,38 @@ impl<B: Backend> TraceValidator<B> {
};
if let Ok(expected) = traces_view.leaf_array(&format!("updated_kv_cache.{}.keys", index)) {
+ let size = kv.shape().iter().product::<usize>() * data_type.size_in_bytes();
+ let keys = if let Some(layer) = kv.as_any().downcast_ref::<KVCacheLayer<B, B::SparseBuffer>>() {
+ common::helpers::sparse_buffer_read_allocation(ctx.context.as_ref(), &layer.keys, size)
+ } else {
+ panic!("Wrong keys type")
+ };
results.push(TracerValidationResult {
name: format!("updated_kv_cache.{}.keys", index),
metrics: Self::validate_allocation(
data_type,
&expected,
- &kv.keys,
- &kv.shape,
+ &keys,
+ &kv.shape(),
Some(ArrayTransform::KVCacheSlice),
),
});
}
if let Ok(expected) = traces_view.leaf_array(&format!("updated_kv_cache.{}.values", index)) {
+ let size = kv.shape().iter().product::<usize>() * data_type.size_in_bytes();
+ let values = if let Some(layer) = kv.as_any().downcast_ref::<KVCacheLayer<B, B::SparseBuffer>>() {
+ common::helpers::sparse_buffer_read_allocation(ctx.context.as_ref(), &layer.values, size)
+ } else {
+ panic!("Wrong values type")
+ };
results.push(TracerValidationResult {
name: format!("updated_kv_cache.{}.values", index),
metrics: Self::validate_allocation(
data_type,
&expected,
- &kv.values,
- &kv.shape,
+ &values,
+ &kv.shape(),
Some(ArrayTransform::KVCacheSlice),
),
});
diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/kv_cache_update_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/kv_cache_update_test.rs
index e0f99a8a2..9b7ea8237 100644
--- a/crates/backend-uzu/tests/unit/backends/common/kernel/kv_cache_update_test.rs
+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/kv_cache_update_test.rs
@@ -30,7 +30,7 @@ fn apply_swaps_3d<T: Clone>(
}
}
-fn test_random_pattern(context: &<Metal as Backend>::Context) {
+fn test_random_pattern<B: Backend>(context: &B::Context) {
println!("Testing with random pattern...");
let max_sequence_length = 256usize;
@@ -64,15 +64,15 @@ fn test_random_pattern(context: &<Metal as Backend>::Context) {
apply_swaps_3d(&mut expected_keys, &swaps);
apply_swaps_3d(&mut expected_values, &swaps);
- let mut key_allocation = common::helpers::alloc_allocation_with_data(context, key_data.as_slice().unwrap());
- let mut value_allocation = common::helpers::alloc_allocation_with_data(context, value_data.as_slice().unwrap());
-
- let mut encoder = Encoder::new(context).unwrap();
+ let mut key_buffer = common::helpers::sparse_buffer_create_with::<B, f32>(context, key_data.as_slice().unwrap());
+ let mut value_buffer =
+ common::helpers::sparse_buffer_create_with::<B, f32>(context, value_data.as_slice().unwrap());
+ let mut encoder = Encoder::<B>::new(context).unwrap();
{
- let mut kv_layers = [KVLayerData::<Metal> {
- key_allocation: &mut key_allocation,
+ let mut kv_layers = [KVLayerData::<B, B::SparseBuffer> {
+ key_buffer: &mut key_buffer,
key_shape: [seq_len, num_heads, head_dim],
- value_allocation: &mut value_allocation,
+ value_buffer: &mut value_buffer,
value_shape: [seq_len, num_heads, head_dim],
}];
match kv_cache_update.encode(&mut kv_layers, &source_indices, &destination_indices, &mut encoder) {
@@ -86,11 +86,15 @@ fn test_random_pattern(context: &<Metal as Backend>::Context) {
encoder.end_encoding().submit().wait_until_completed().unwrap();
- let key_values: Vec<f32> = common::helpers::allocation_to_vec(&key_allocation);
+ let shape = [seq_len, num_heads, head_dim];
+ let elements_count = shape.iter().product();
+
+ let key_values: Vec<f32> = common::helpers::sparse_buffer_read_vec::<B, f32>(context, &key_buffer, elements_count);
let key_result = Array::from_shape_vec((seq_len, num_heads, head_dim), key_values)
.expect("Failed to convert key result to ndarray");
- let value_values: Vec<f32> = common::helpers::allocation_to_vec(&value_allocation);
+ let value_values: Vec<f32> =
+ common::helpers::sparse_buffer_read_vec::<B, f32>(context, &value_buffer, elements_count);
let value_result = Array::from_shape_vec((seq_len, num_heads, head_dim), value_values)
.expect("Failed to convert value result to ndarray");
@@ -116,7 +120,7 @@ fn test_kv_cache_update_kernel() {
},
};
- test_random_pattern(&metal_context);
+ test_random_pattern::<Metal>(&metal_context);
}
#[test]
diff --git a/crates/backend-uzu/tests/unit/backends/metal/sparse/sparse_buffer_test.rs b/crates/backend-uzu/tests/unit/backends/metal/sparse/sparse_buffer_test.rs
index 22f5315bf..8466d1970 100644
--- a/crates/backend-uzu/tests/unit/backends/metal/sparse/sparse_buffer_test.rs
+++ b/crates/backend-uzu/tests/unit/backends/metal/sparse/sparse_buffer_test.rs
@@ -1,12 +1,18 @@
+#[path = "../../../../common/mod.rs"]
+mod common;
+
use std::{ops::Range, rc::Rc};
-use crate::backends::{
- common::{Backend, Context, SparseBuffer},
- metal::Metal,
+use backend_uzu::{
+ backends::{
+ common::{Backend, SparseBuffer},
+ metal::Metal,
+ },
+ prelude::MetalContext,
};
-fn create_context() -> Rc<<Metal as Backend>::Context> {
- <Metal as Backend>::Context::new().expect("Failed to create Metal context")
+fn create_context() -> Rc<MetalContext> {
+ common::helpers::create_context::<Metal>()
}
/// Capacity sized to fit `heap_count` worth of heaps.
@@ -33,7 +39,7 @@ fn test_mapping_succeeds() {
let capacity = buffer_capacity(ctx.as_ref(), 4);
let pages = pages_for_heaps(ctx.as_ref(), 0, 4);
- let mut sparse_buffer = ctx.create_sparse_buffer(capacity).expect("Failed to create sparse buffer");
+ let mut sparse_buffer = common::helpers::sparse_buffer_create::<Metal>(&ctx, capacity);
sparse_buffer.map(ctx.as_ref(), &pages).expect("Failed to map sparse buffer");
}
@@ -43,7 +49,7 @@ fn test_unmapping_succeeds() {
let capacity = buffer_capacity(ctx.as_ref(), 4);
let pages = pages_for_heaps(ctx.as_ref(), 0, 4);
- let mut sparse_buffer = ctx.create_sparse_buffer(capacity).expect("Failed to create sparse buffer");
+ let mut sparse_buffer = common::helpers::sparse_buffer_create::<Metal>(&ctx, capacity);
sparse_buffer.map(ctx.as_ref(), &pages).expect("Failed to map sparse buffer");
sparse_buffer.unmap(ctx.as_ref(), &pages).expect("Failed to unmap sparse buffer");
}
@@ -55,7 +61,7 @@ fn test_partial_unmapping_succeeds() {
let mapped = pages_for_heaps(ctx.as_ref(), 0, 4);
let unmapped = pages_for_heaps(ctx.as_ref(), 1, 2);
- let mut sparse_buffer = ctx.create_sparse_buffer(capacity).expect("Failed to create sparse buffer");
+ let mut sparse_buffer = common::helpers::sparse_buffer_create::<Metal>(&ctx, capacity);
sparse_buffer.map(ctx.as_ref(), &mapped).expect("Failed to map sparse buffer");
sparse_buffer.unmap(ctx.as_ref(), &unmapped).expect("Failed to unmap sparse buffer");
}
@@ -67,7 +73,7 @@ fn test_mapping_uses_minimum_heaps() {
let capacity = buffer_capacity(ctx.as_ref(), heap_count);
let pages = pages_for_heaps(ctx.as_ref(), 0, heap_count);
- let mut sparse_buffer = ctx.create_sparse_buffer(capacity).expect("Failed to create sparse buffer");
+ let mut sparse_buffer = common::helpers::sparse_buffer_create::<Metal>(&ctx, capacity);
sparse_buffer.map(ctx.as_ref(), &pages).expect("Failed to map sparse buffer");
let heaps = ctx.sparse_heap_pool();
@@ -83,7 +89,7 @@ fn test_remapping_reuses_freed_pages() {
let unmapped_pages = pages_for_heaps(ctx.as_ref(), 1, 1);
let extra_pages = pages_for_heaps(ctx.as_ref(), 4, 1);
- let mut sparse_buffer = ctx.create_sparse_buffer(capacity).expect("Failed to create sparse buffer");
+ let mut sparse_buffer = common::helpers::sparse_buffer_create::<Metal>(&ctx, capacity);
sparse_buffer.map(ctx.as_ref(), &initial_pages).expect("Failed to map sparse buffer");
let initial_heaps = ctx.sparse_heap_pool().heaps_count();
@@ -103,7 +109,7 @@ fn test_full_unmap_then_remap_reuses_heaps() {
let capacity = buffer_capacity(ctx.as_ref(), 4);
let pages = pages_for_heaps(ctx.as_ref(), 0, 4);
- let mut sparse_buffer = ctx.create_sparse_buffer(capacity).expect("Failed to create sparse buffer");
+ let mut sparse_buffer = common::helpers::sparse_buffer_create::<Metal>(&ctx, capacity);
sparse_buffer.map(ctx.as_ref(), &pages).expect("Failed to map sparse buffer");
let initial_heaps = ctx.sparse_heap_pool().heaps_count();
@@ -123,7 +129,7 @@ fn test_remapping_same_pages_is_noop() {
let capacity = buffer_capacity(ctx.as_ref(), 4);
let pages = pages_for_heaps(ctx.as_ref(), 0, 4);
- let mut sparse_buffer = ctx.create_sparse_buffer(capacity).expect("Failed to create sparse buffer");
+ let mut sparse_buffer = common::helpers::sparse_buffer_create::<Metal>(&ctx, capacity);
sparse_buffer.map(ctx.as_ref(), &pages).expect("Failed to map sparse buffer");
let initial_heaps = ctx.sparse_heap_pool().heaps_count();
@@ -136,6 +142,23 @@ fn test_remapping_same_pages_is_noop() {
);
}
+#[test]
+fn test_mapping_multiple_gaps_reserves_pages_between_gaps() {
+ let ctx = create_context();
+ let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();
+ let capacity = buffer_capacity(ctx.as_ref(), 2);
+
+ let mut sparse_buffer = common::helpers::sparse_buffer_create::<Metal>(&ctx, capacity);
+ sparse_buffer.map(ctx.as_ref(), &(1..2)).expect("Failed to map initial page");
+ sparse_buffer.map(ctx.as_ref(), &(0..pages_per_heap + 1)).expect("Failed to map range with multiple gaps");
+
+ assert_eq!(
+ ctx.sparse_heap_pool().heaps_count(),
+ 2,
+ "mapping multiple gaps in one call must reserve heap pages between gaps",
+ );
+}
+
#[test]
fn test_drop_releases_pool_heaps() {
// Regression: a mapped buffer dropped without an explicit unmap must
@@ -145,7 +168,7 @@ fn test_drop_releases_pool_heaps() {
let pages = pages_for_heaps(ctx.as_ref(), 0, 2);
{
- let mut sparse_buffer = ctx.create_sparse_buffer(capacity).expect("Failed to create sparse buffer");
+ let mut sparse_buffer = common::helpers::sparse_buffer_create::<Metal>(&ctx, capacity);
sparse_buffer.map(ctx.as_ref(), &pages).expect("Failed to map sparse buffer");
assert!(ctx.sparse_heap_pool().heaps_count() > 0);
}
@@ -165,20 +188,17 @@ fn test_drop_does_not_disturb_other_buffer_mappings() {
let capacity = buffer_capacity(ctx.as_ref(), 1);
let pages = pages_for_heaps(ctx.as_ref(), 0, 1);
- let mut keeper = ctx.create_sparse_buffer(capacity).expect("Failed to create keeper buffer");
+ let mut keeper = common::helpers::sparse_buffer_create::<Metal>(&ctx, capacity);
keeper.map(ctx.as_ref(), &pages).expect("Failed to map keeper buffer");
let baseline = ctx.sparse_heap_pool().heaps_count();
{
- let mut transient = ctx.create_sparse_buffer(capacity).expect("Failed to create transient buffer");
+ let mut transient = common::helpers::sparse_buffer_create::<Metal>(&ctx, capacity);
transient.map(ctx.as_ref(), &pages).expect("Failed to map transient buffer");
}
- assert_eq!(
- ctx.sparse_heap_pool().heaps_count(),
- baseline,
- "dropping a transient buffer must leave keeper buffer's mappings intact",
- );
+ let current_heaps = ctx.sparse_heap_pool().heaps_count();
+ assert_eq!(current_heaps, baseline, "dropping a transient buffer must leave keeper buffer's mappings intact",);
}
#[test]
@@ -188,7 +208,7 @@ fn test_sequential_mappings_are_compact() {
let first = pages_for_heaps(ctx.as_ref(), 0, 2);
let second = pages_for_heaps(ctx.as_ref(), 2, 2);
- let mut sparse_buffer = ctx.create_sparse_buffer(capacity).expect("Failed to create sparse buffer");
+ let mut sparse_buffer = common::helpers::sparse_buffer_create::<Metal>(&ctx, capacity);
sparse_buffer.map(ctx.as_ref(), &first).expect("Failed to map first range");
sparse_buffer.map(ctx.as_ref(), &second).expect("Failed to map second range");
diff --git a/crates/backend-uzu/tests/unit/backends/metal/sparse/sparse_heap_pool_test.rs b/crates/backend-uzu/tests/unit/backends/metal/sparse/sparse_heap_pool_test.rs
index 0615e5a21..77c928463 100644
--- a/crates/backend-uzu/tests/unit/backends/metal/sparse/sparse_heap_pool_test.rs
+++ b/crates/backend-uzu/tests/unit/backends/metal/sparse/sparse_heap_pool_test.rs
@@ -1,33 +1,18 @@
-use std::rc::Rc;
+#[path = "../../../../common/mod.rs"]
+mod common;
-use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};
-use objc2::{rc::Retained, runtime::ProtocolObject};
+use std::rc::Rc;
use crate::{
backends::{
- common::{Backend, Context},
+ common::SparseBuffer,
metal::{Metal, metal_extensions::SparsePageSizeExt},
},
prelude::MetalContext,
};
fn create_context() -> Rc<MetalContext> {
- <Metal as Backend>::Context::new().expect("Failed to create Metal context")
-}
-
-fn create_sparse_buffer(
- ctx: &MetalContext,
- capacity_bytes: usize,
-) -> Retained<ProtocolObject<dyn MTLBuffer>> {
- let page_size = ctx.sparse_heap_pool().page_size();
- let aligned = capacity_bytes.next_multiple_of(page_size.in_bytes());
- ctx.device
- .new_buffer_with_length_options_placement_sparse_page_size(
- aligned,
- MTLResourceOptions::STORAGE_MODE_PRIVATE,
- page_size,
- )
- .expect("Failed to create sparse buffer")
+ common::helpers::create_context::<Metal>()
}
#[test]
@@ -40,9 +25,9 @@ fn test_new_pool_is_empty() {
fn test_map_empty_range_allocates_no_heaps() {
let ctx = create_context();
let cap = ctx.sparse_heap_pool().heap_capacity_bytes();
- let buffer = create_sparse_buffer(ctx.as_ref(), cap);
+ let mut sparse_buffer = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);
- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(0..0)).expect("map empty range");
+ sparse_buffer.map(&ctx, &(0..0)).expect("map empty range");
assert_eq!(ctx.sparse_heap_pool().heaps_count(), 0);
}
@@ -52,9 +37,9 @@ fn test_map_single_heap_allocates_one_heap() {
let ctx = create_context();
let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();
let cap = ctx.sparse_heap_pool().heap_capacity_bytes();
- let buffer = create_sparse_buffer(ctx.as_ref(), cap);
+ let mut sparse_buffer = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);
- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(0..pages_per_heap)).expect("map");
+ sparse_buffer.map(&ctx, &(0..pages_per_heap)).expect("map");
assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);
}
@@ -64,9 +49,9 @@ fn test_map_multi_heap_allocates_minimum_heaps() {
let ctx = create_context();
let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();
let cap = 4 * ctx.sparse_heap_pool().heap_capacity_bytes();
- let buffer = create_sparse_buffer(ctx.as_ref(), cap);
+ let mut sparse_buffer = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);
- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(0..4 * pages_per_heap)).expect("map");
+ sparse_buffer.map(&ctx, &(0..4 * pages_per_heap)).expect("map");
assert_eq!(ctx.sparse_heap_pool().heaps_count(), 4);
}
@@ -76,9 +61,9 @@ fn test_map_partial_heap_rounds_up_heap_count() {
let ctx = create_context();
let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();
let cap = 2 * ctx.sparse_heap_pool().heap_capacity_bytes();
- let buffer = create_sparse_buffer(ctx.as_ref(), cap);
+ let mut sparse_buffer = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);
- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(0..pages_per_heap + 1)).expect("map");
+ sparse_buffer.map(&ctx, &(0..pages_per_heap + 1)).expect("map");
assert_eq!(ctx.sparse_heap_pool().heaps_count(), 2);
}
@@ -89,10 +74,10 @@ fn test_sequential_mappings_pack_into_one_heap() {
let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();
let half = pages_per_heap / 2;
let cap = ctx.sparse_heap_pool().heap_capacity_bytes();
- let buffer = create_sparse_buffer(ctx.as_ref(), cap);
+ let mut sparse_buffer = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);
- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(0..half)).expect("map first");
- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(half..pages_per_heap)).expect("map second");
+ sparse_buffer.map(&ctx, &(0..half)).expect("map first");
+ sparse_buffer.map(&ctx, &(half..pages_per_heap)).expect("map second");
assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);
}
@@ -103,10 +88,10 @@ fn test_map_overflows_existing_heap_into_new_heap() {
let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();
let half = pages_per_heap / 2;
let cap = 2 * ctx.sparse_heap_pool().heap_capacity_bytes();
- let buffer = create_sparse_buffer(ctx.as_ref(), cap);
+ let mut sparse_buffer = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);
- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(0..half)).expect("map first");
- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(half..half + pages_per_heap)).expect("map overflow");
+ sparse_buffer.map(&ctx, &(0..half)).expect("map first");
+ sparse_buffer.map(&ctx, &(half..half + pages_per_heap)).expect("map overflow");
assert_eq!(ctx.sparse_heap_pool().heaps_count(), 2);
}
@@ -117,11 +102,11 @@ fn test_two_buffers_share_one_heap() {
let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();
let half = pages_per_heap / 2;
let cap = ctx.sparse_heap_pool().heap_capacity_bytes();
- let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);
- let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);
+ let mut sparse_buffer_a = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);
+ let mut sparse_buffer_b = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);
- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..half)).expect("map a");
- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..half)).expect("map b");
+ sparse_buffer_a.map(&ctx, &(0..half)).expect("map a");
+ sparse_buffer_b.map(&ctx, &(0..half)).expect("map b");
assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);
}
@@ -131,11 +116,11 @@ fn test_full_unmap_removes_heaps() {
let ctx = create_context();
let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();
let cap = 2 * ctx.sparse_heap_pool().heap_capacity_bytes();
- let buffer = create_sparse_buffer(ctx.as_ref(), cap);
+ let mut sparse_buffer = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);
let pages = 0..2 * pages_per_heap;
- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &pages).expect("map");
- ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer, &pages).expect("unmap");
+ sparse_buffer.map(&ctx, &pages).expect("map");
+ sparse_buffer.unmap(&ctx, &pages).expect("unmap");
assert_eq!(ctx.sparse_heap_pool().heaps_count(), 0);
}
@@ -145,10 +130,10 @@ fn test_partial_unmap_keeps_heap() {
let ctx = create_context();
let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();
let cap = ctx.sparse_heap_pool().heap_capacity_bytes();
- let buffer = create_sparse_buffer(ctx.as_ref(), cap);
+ let mut sparse_buffer = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);
- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(0..pages_per_heap)).expect("map");
- ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer, &(0..pages_per_heap / 2)).expect("partial unmap");
+ sparse_buffer.map(&ctx, &(0..pages_per_heap)).expect("map");
+ sparse_buffer.unmap(&ctx, &(0..pages_per_heap / 2)).expect("partial unmap");
assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);
}
@@ -158,12 +143,12 @@ fn test_unmap_with_other_buffer_keeps_mappings() {
let ctx = create_context();
let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();
let cap = ctx.sparse_heap_pool().heap_capacity_bytes();
- let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);
- let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);
+ let mut sparse_buffer_a = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);
+ let mut sparse_buffer_b = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);
let pages = 0..pages_per_heap;
- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &pages).expect("map a");
- ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer_b, &pages).expect("unmap b");
+ sparse_buffer_a.map(&ctx, &pages).expect("map a");
+ sparse_buffer_b.unmap(&ctx, &pages).expect("unmap b");
assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);
}
@@ -174,12 +159,12 @@ fn test_unmap_one_buffer_keeps_heap_for_other() {
let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();
let half = pages_per_heap / 2;
let cap = ctx.sparse_heap_pool().heap_capacity_bytes();
- let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);
- let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);
+ let mut sparse_buffer_a = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);
+ let mut sparse_buffer_b = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);
- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..half)).expect("map a");
- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..half)).expect("map b");
- ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer_a, &(0..half)).expect("unmap a");
+ sparse_buffer_a.map(&ctx, &(0..half)).expect("map a");
+ sparse_buffer_b.map(&ctx, &(0..half)).expect("map b");
+ sparse_buffer_a.unmap(&ctx, &(0..half)).expect("unmap a");
assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);
}
@@ -189,13 +174,13 @@ fn test_remap_after_unmap_does_not_grow_pool() {
let ctx = create_context();
let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();
let cap = 2 * ctx.sparse_heap_pool().heap_capacity_bytes();
- let buffer = create_sparse_buffer(ctx.as_ref(), cap);
+ let mut sparse_buffer = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);
let pages = 0..2 * pages_per_heap;
- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &pages).expect("map");
+ sparse_buffer.map(&ctx, &pages).expect("map");
let initial = ctx.sparse_heap_pool().heaps_count();
- ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer, &pages).expect("unmap");
- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &pages).expect("remap");
+ sparse_buffer.unmap(&ctx, &pages).expect("unmap");
+ sparse_buffer.map(&ctx, &pages).expect("remap");
assert_eq!(ctx.sparse_heap_pool().heaps_count(), initial);
}
@@ -206,11 +191,11 @@ fn test_remap_into_freed_gap_reuses_heap() {
let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();
let half = pages_per_heap / 2;
let cap = ctx.sparse_heap_pool().heap_capacity_bytes();
- let buffer = create_sparse_buffer(ctx.as_ref(), cap);
+ let mut sparse_buffer = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);
- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(0..pages_per_heap)).expect("map full");
- ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer, &(0..half)).expect("unmap half");
- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(0..half)).expect("remap into gap");
+ sparse_buffer.map(&ctx, &(0..pages_per_heap)).expect("map full");
+ sparse_buffer.unmap(&ctx, &(0..half)).expect("unmap half");
+ sparse_buffer.map(&ctx, &(0..half)).expect("remap into gap");
assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);
}
@@ -224,11 +209,11 @@ fn test_sequential_partial_unmaps_release_full_heap() {
let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();
let half = pages_per_heap / 2;
let cap = ctx.sparse_heap_pool().heap_capacity_bytes();
- let buffer = create_sparse_buffer(ctx.as_ref(), cap);
+ let mut sparse_buffer = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);
- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer, &(0..pages_per_heap)).expect("map");
- ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer, &(0..half)).expect("unmap prefix");
- ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer, &(half..pages_per_heap)).expect("unmap suffix");
+ sparse_buffer.map(&ctx, &(0..pages_per_heap)).expect("map");
+ sparse_buffer.unmap(&ctx, &(0..half)).expect("unmap prefix");
+ sparse_buffer.unmap(&ctx, &(half..pages_per_heap)).expect("unmap suffix");
assert_eq!(ctx.sparse_heap_pool().heaps_count(), 0);
}
@@ -239,13 +224,13 @@ fn test_three_buffers_share_one_heap() {
let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();
let third = pages_per_heap / 3;
let cap = ctx.sparse_heap_pool().heap_capacity_bytes();
- let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);
- let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);
- let buffer_c = create_sparse_buffer(ctx.as_ref(), cap);
+ let mut sparse_buffer_a = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);
+ let mut sparse_buffer_b = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);
+ let mut sparse_buffer_c = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);
- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..third)).expect("map a");
- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..third)).expect("map b");
- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_c, &(0..third)).expect("map c");
+ sparse_buffer_a.map(&ctx, &(0..third)).expect("map a");
+ sparse_buffer_b.map(&ctx, &(0..third)).expect("map b");
+ sparse_buffer_c.map(&ctx, &(0..third)).expect("map c");
assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);
}
@@ -256,11 +241,11 @@ fn test_two_buffers_exactly_fill_one_heap() {
let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();
let half = pages_per_heap / 2;
let cap = ctx.sparse_heap_pool().heap_capacity_bytes();
- let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);
- let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);
+ let mut sparse_buffer_a = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);
+ let mut sparse_buffer_b = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);
- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..half)).expect("map a");
- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..pages_per_heap - half)).expect("map b fills heap");
+ sparse_buffer_a.map(&ctx, &(0..half)).expect("map a");
+ sparse_buffer_b.map(&ctx, &(0..pages_per_heap - half)).expect("map b fills heap");
assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);
}
@@ -271,13 +256,13 @@ fn test_third_buffer_spills_when_heap_filled_by_others() {
let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();
let half = pages_per_heap / 2;
let cap = ctx.sparse_heap_pool().heap_capacity_bytes();
- let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);
- let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);
- let buffer_c = create_sparse_buffer(ctx.as_ref(), cap);
+ let mut sparse_buffer_a = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);
+ let mut sparse_buffer_b = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);
+ let mut sparse_buffer_c = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);
- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..half)).expect("map a");
- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..pages_per_heap - half)).expect("map b fills heap");
- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_c, &(0..1)).expect("map c spills");
+ sparse_buffer_a.map(&ctx, &(0..half)).expect("map a");
+ sparse_buffer_b.map(&ctx, &(0..pages_per_heap - half)).expect("map b fills heap");
+ sparse_buffer_c.map(&ctx, &(0..1)).expect("map c spills");
assert_eq!(ctx.sparse_heap_pool().heaps_count(), 2);
}
@@ -288,14 +273,14 @@ fn test_unmap_buffer_frees_pages_for_another_in_same_heap() {
let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();
let half = pages_per_heap / 2;
let cap = ctx.sparse_heap_pool().heap_capacity_bytes();
- let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);
- let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);
- let buffer_c = create_sparse_buffer(ctx.as_ref(), cap);
+ let mut sparse_buffer_a = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);
+ let mut sparse_buffer_b = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);
+ let mut sparse_buffer_c = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);
- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..half)).expect("map a");
- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..pages_per_heap - half)).expect("map b fills heap");
- ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer_a, &(0..half)).expect("unmap a");
- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_c, &(0..half)).expect("map c into gap");
+ sparse_buffer_a.map(&ctx, &(0..half)).expect("map a");
+ sparse_buffer_b.map(&ctx, &(0..pages_per_heap - half)).expect("map b fills heap");
+ sparse_buffer_a.unmap(&ctx, &(0..half)).expect("unmap a");
+ sparse_buffer_c.map(&ctx, &(0..half)).expect("map c into gap");
assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);
}
@@ -306,14 +291,14 @@ fn test_unmap_middle_of_three_buffers_keeps_heap() {
let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();
let third = pages_per_heap / 3;
let cap = ctx.sparse_heap_pool().heap_capacity_bytes();
- let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);
- let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);
- let buffer_c = create_sparse_buffer(ctx.as_ref(), cap);
+ let mut sparse_buffer_a = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);
+ let mut sparse_buffer_b = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);
+ let mut sparse_buffer_c = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);
- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..third)).expect("map a");
- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..third)).expect("map b");
- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_c, &(0..third)).expect("map c");
- ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer_b, &(0..third)).expect("unmap b");
+ sparse_buffer_a.map(&ctx, &(0..third)).expect("map a");
+ sparse_buffer_b.map(&ctx, &(0..third)).expect("map b");
+ sparse_buffer_c.map(&ctx, &(0..third)).expect("map c");
+ sparse_buffer_b.unmap(&ctx, &(0..third)).expect("unmap b");
assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);
}
@@ -324,16 +309,16 @@ fn test_unmap_all_shared_buffers_releases_heap() {
let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();
let third = pages_per_heap / 3;
let cap = ctx.sparse_heap_pool().heap_capacity_bytes();
- let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);
- let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);
- let buffer_c = create_sparse_buffer(ctx.as_ref(), cap);
+ let mut sparse_buffer_a = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);
+ let mut sparse_buffer_b = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);
+ let mut sparse_buffer_c = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);
- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..third)).expect("map a");
- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..third)).expect("map b");
- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_c, &(0..third)).expect("map c");
- ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer_a, &(0..third)).expect("unmap a");
- ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer_b, &(0..third)).expect("unmap b");
- ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer_c, &(0..third)).expect("unmap c");
+ sparse_buffer_a.map(&ctx, &(0..third)).expect("map a");
+ sparse_buffer_b.map(&ctx, &(0..third)).expect("map b");
+ sparse_buffer_c.map(&ctx, &(0..third)).expect("map c");
+ sparse_buffer_a.unmap(&ctx, &(0..third)).expect("unmap a");
+ sparse_buffer_b.unmap(&ctx, &(0..third)).expect("unmap b");
+ sparse_buffer_c.unmap(&ctx, &(0..third)).expect("unmap c");
assert_eq!(ctx.sparse_heap_pool().heaps_count(), 0);
}
@@ -344,11 +329,11 @@ fn test_second_buffer_spans_existing_and_new_heap() {
let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();
let half = pages_per_heap / 2;
let cap = 2 * ctx.sparse_heap_pool().heap_capacity_bytes();
- let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);
- let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);
+ let mut sparse_buffer_a = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);
+ let mut sparse_buffer_b = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);
- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..half)).expect("map a");
- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..pages_per_heap)).expect("map b spans heaps");
+ sparse_buffer_a.map(&ctx, &(0..half)).expect("map a");
+ sparse_buffer_b.map(&ctx, &(0..pages_per_heap)).expect("map b spans heaps");
assert_eq!(ctx.sparse_heap_pool().heaps_count(), 2);
}
@@ -359,12 +344,12 @@ fn test_unmap_spanning_buffer_keeps_heap_with_other_buffer() {
let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();
let half = pages_per_heap / 2;
let cap = 2 * ctx.sparse_heap_pool().heap_capacity_bytes();
- let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);
- let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);
+ let mut sparse_buffer_a = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);
+ let mut sparse_buffer_b = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);
- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..half)).expect("map a");
- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..pages_per_heap)).expect("map b spans heaps");
- ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer_b, &(0..pages_per_heap)).expect("unmap b");
+ sparse_buffer_a.map(&ctx, &(0..half)).expect("map a");
+ sparse_buffer_b.map(&ctx, &(0..pages_per_heap)).expect("map b spans heaps");
+ sparse_buffer_b.unmap(&ctx, &(0..pages_per_heap)).expect("unmap b");
assert_eq!(ctx.sparse_heap_pool().heaps_count(), 1);
}
@@ -375,16 +360,16 @@ fn test_remap_two_buffers_after_unmap_does_not_grow_pool() {
let pages_per_heap = ctx.sparse_heap_pool().heap_capacity_pages();
let half = pages_per_heap / 2;
let cap = ctx.sparse_heap_pool().heap_capacity_bytes();
- let buffer_a = create_sparse_buffer(ctx.as_ref(), cap);
- let buffer_b = create_sparse_buffer(ctx.as_ref(), cap);
+ let mut sparse_buffer_a = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);
+ let mut sparse_buffer_b = common::helpers::sparse_buffer_create::<Metal>(ctx.as_ref(), cap);
- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..half)).expect("map a");
- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..half)).expect("map b");
+ sparse_buffer_a.map(&ctx, &(0..half)).expect("map a");
+ sparse_buffer_b.map(&ctx, &(0..half)).expect("map b");
let initial = ctx.sparse_heap_pool().heaps_count();
- ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer_a, &(0..half)).expect("unmap a");
- ctx.sparse_heap_pool_mut().unmap(ctx.as_ref(), &buffer_b, &(0..half)).expect("unmap b");
- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_a, &(0..half)).expect("remap a");
- ctx.sparse_heap_pool_mut().map(ctx.as_ref(), &buffer_b, &(0..half)).expect("remap b");
+ sparse_buffer_a.unmap(&ctx, &(0..half)).expect("unmap a");
+ sparse_buffer_b.unmap(&ctx, &(0..half)).expect("unmap b");
+ sparse_buffer_a.map(&ctx, &(0..half)).expect("remap a");
+ sparse_buffer_b.map(&ctx, &(0..half)).expect("remap b");
assert_eq!(ctx.sparse_heap_pool().heaps_count(), initial);
}
diff --git a/crates/backend-uzu/tests/unit/forward_pass/cache_layers_test.rs b/crates/backend-uzu/tests/unit/forward_pass/cache_layers_test.rs
new file mode 100644
index 000000000..a4684da3a
--- /dev/null
+++ b/crates/backend-uzu/tests/unit/forward_pass/cache_layers_test.rs
@@ -0,0 +1,204 @@
+#![cfg(metal_backend)]
+
+use std::{mem::size_of, ops::Range};
+
+use super::{CacheEntryIndex, CacheLayer, CacheLayers, LayerCacheBinding};
+use crate::{
+ DataType, allocation_copy_from_slice, allocation_to_vec,
+ backends::{
+ common::{Allocation, AllocationType, Backend, Context, Encoder, SparseBuffer},
+ metal::Metal,
+ },
+ forward_pass::kv_cache_layer::{KVCacheLayer, KVCacheLayerState, KVCacheLayerTrait},
+};
+
+fn submit_encoder(encoder: Encoder<Metal>) {
+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to submit encoder");
+}
+
+fn allocation_from_slice(
+ context: &<Metal as Backend>::Context,
+ data: &[f32],
+) -> Allocation<Metal> {
+ let mut allocation =
+ context.create_allocation(size_of_val(data), AllocationType::Global).expect("Failed to create allocation");
+ allocation_copy_from_slice(&mut allocation, data).expect("Failed to write allocation");
+ allocation
+}
+
+fn copy_allocation_to_sparse_row(
+ context: &<Metal as Backend>::Context,
+ allocation: &Allocation<Metal>,
+ buffer: &mut <Metal as Backend>::SparseBuffer,
+ row_size: usize,
+ row_index: usize,
+) {
+ let row_range = row_byte_range(row_size, row_index);
+ let mut encoder = Encoder::<Metal>::new(context).expect("Failed to create encoder");
+ encoder.encode_copy(allocation, 0..row_size, buffer, row_range);
+ submit_encoder(encoder);
+}
+
+fn read_sparse_row(
+ context: &<Metal as Backend>::Context,
+ buffer: &<Metal as Backend>::SparseBuffer,
+ row_size: usize,
+ row_index: usize,
+) -> Vec<f32> {
+ let mut allocation =
+ context.create_allocation(row_size, AllocationType::Global).expect("Failed to create allocation");
+ let mut encoder = Encoder::<Metal>::new(context).expect("Failed to create encoder");
+ encoder.encode_copy(buffer, row_byte_range(row_size, row_index), &mut allocation, 0..row_size);
+ submit_encoder(encoder);
+ allocation_to_vec(&allocation)
+}
+
+fn row_byte_range(
+ row_size: usize,
+ row_index: usize,
+) -> Range<usize> {
+ row_index * row_size..(row_index + 1) * row_size
+}
+
+fn row_data(
+ elements_per_row: usize,
+ base_value: f32,
+) -> Vec<f32> {
+ (0..elements_per_row).map(|element_index| base_value + element_index as f32).collect()
+}
+
+fn make_cache_layers(
+ context: &<Metal as Backend>::Context,
+ state: KVCacheLayerState,
+ elements_per_row: usize,
+) -> CacheLayers<Metal> {
+ let window_length = match state {
+ KVCacheLayerState::Windowed {
+ window_length,
+ ..
+ } => window_length,
+ KVCacheLayerState::Full {
+ ..
+ } => panic!("test expects windowed state"),
+ };
+ let shape = [window_length, elements_per_row, 1];
+ let buffer_size = window_length * elements_per_row * size_of::<f32>();
+ let layer = KVCacheLayer {
+ state,
+ keys: context.create_sparse_buffer(buffer_size).expect("Failed to create key buffer"),
+ values: context.create_sparse_buffer(buffer_size).expect("Failed to create value buffer"),
+ shape,
+ data_type: DataType::F32,
+ };
+
+ CacheLayers {
+ max_suffix_length: 0,
+ max_prefix_length: window_length,
+ entries: vec![CacheLayer::Transformer(Box::new(layer))].into_boxed_slice(),
+ bindings: vec![LayerCacheBinding::Owned {
+ entry: CacheEntryIndex {
+ index: 0,
+ },
+ }]
+ .into_boxed_slice(),
+ }
+}
+
+#[test]
+fn test_cache_layers_copy_from_clones_only_live_window_rows() {
+ let Some(context) = <Metal as Backend>::Context::new().ok() else {
+ return;
+ };
+
+ let page_size = context.create_sparse_buffer(1).expect("Failed to create sparse buffer").page_size_bytes();
+ let elements_per_row = page_size / size_of::<f32>();
+ let row_size = page_size;
+
+ let state = KVCacheLayerState::Windowed {
+ ring_offset: 1,
+ ring_length: 2,
+ window_length: 4,
+ };
+ let mut source = make_cache_layers(&context, state.clone(), elements_per_row);
+ let mut destination = make_cache_layers(
+ &context,
+ KVCacheLayerState::Windowed {
+ ring_offset: 0,
+ ring_length: 0,
+ window_length: 4,
+ },
+ elements_per_row,
+ );
+
+ let CacheLayer::Transformer(source_layer) = &mut source.entries[0] else {
+ panic!("expected transformer layer");
+ };
+ let source_layer = source_layer
+ .as_any_mut()
+ .downcast_mut::<KVCacheLayer<Metal, <Metal as Backend>::SparseBuffer>>()
+ .expect("expected sparse KV layer");
+
+ source_layer.map_row_range(&context, 1..2).expect("Failed to map source row 1");
+ source_layer.map_row_range(&context, 2..3).expect("Failed to map source row 2");
+
+ let row_1_keys = row_data(elements_per_row, 101.0);
+ let row_2_keys = row_data(elements_per_row, 102.0);
+ let row_1_values = row_data(elements_per_row, 201.0);
+ let row_2_values = row_data(elements_per_row, 202.0);
+
+ copy_allocation_to_sparse_row(
+ &context,
+ &allocation_from_slice(&context, &row_1_keys),
+ &mut source_layer.keys,
+ row_size,
+ 1,
+ );
+ copy_allocation_to_sparse_row(
+ &context,
+ &allocation_from_slice(&context, &row_2_keys),
+ &mut source_layer.keys,
+ row_size,
+ 2,
+ );
+ copy_allocation_to_sparse_row(
+ &context,
+ &allocation_from_slice(&context, &row_1_values),
+ &mut source_layer.values,
+ row_size,
+ 1,
+ );
+ copy_allocation_to_sparse_row(
+ &context,
+ &allocation_from_slice(&context, &row_2_values),
+ &mut source_layer.values,
+ row_size,
+ 2,
+ );
+
+ destination.copy_from(&source, &context);
+
+ let CacheLayer::Transformer(destination_layer) = &destination.entries[0] else {
+ panic!("expected transformer layer");
+ };
+ let destination_layer = destination_layer
+ .as_any()
+ .downcast_ref::<KVCacheLayer<Metal, <Metal as Backend>::SparseBuffer>>()
+ .expect("expected sparse KV layer");
+
+ assert_eq!(read_sparse_row(&context, &destination_layer.keys, row_size, 1), row_1_keys);
+ assert_eq!(read_sparse_row(&context, &destination_layer.keys, row_size, 2), row_2_keys);
+ assert_eq!(read_sparse_row(&context, &destination_layer.values, row_size, 1), row_1_values);
+ assert_eq!(read_sparse_row(&context, &destination_layer.values, row_size, 2), row_2_values);
+
+ let KVCacheLayerState::Windowed {
+ ring_offset,
+ ring_length,
+ window_length,
+ } = destination_layer.state
+ else {
+ panic!("expected windowed state");
+ };
+ assert_eq!(ring_offset, 1);
+ assert_eq!(ring_length, 2);
+ assert_eq!(window_length, 4);
+}
diff --git a/crates/backend-uzu/tests/unit/forward_pass/kv_cache_state_test.rs b/crates/backend-uzu/tests/unit/forward_pass/kv_cache_state_test.rs
index 95a66311b..03cceef5f 100644
--- a/crates/backend-uzu/tests/unit/forward_pass/kv_cache_state_test.rs
+++ b/crates/backend-uzu/tests/unit/forward_pass/kv_cache_state_test.rs
@@ -1,9 +1,11 @@
#![cfg(metal_backend)]
+use backend_uzu::forward_pass::kv_cache_layer::KVCacheLayerTrait;
+
use crate::{
DataType,
backends::{
- common::{Allocation, Backend, Context, Encoder, kernel::kv_cache_update::KVCacheUpdate},
+ common::{Backend, Context, Encoder, kernel::kv_cache_update::KVCacheUpdate},
metal::Metal,
},
forward_pass::kv_cache_layer::{KVCacheLayer, KVCacheLayerState},
@@ -12,8 +14,6 @@ use crate::{
#[path = "../../common/mod.rs"]
mod common;
-use common::helpers::{alloc_allocation_with_data, allocation_to_vec, write_allocation};
-
#[derive(Debug)]
struct Scenario {
name: &'static str,
@@ -29,12 +29,12 @@ struct Scenario {
expected_prefix_segment_length: usize,
}
-fn make_test_layer(
- context: &<Metal as Backend>::Context,
+fn make_test_layer<B: Backend>(
+ context: &B::Context,
state: KVCacheLayerState,
prefix_capacity: usize,
suffix_capacity: usize,
-) -> KVCacheLayer<Metal> {
+) -> (KVCacheLayer<B, B::SparseBuffer>, [usize; 3]) {
let total_len = match &state {
KVCacheLayerState::Full {
..
@@ -47,41 +47,46 @@ fn make_test_layer(
let shape = [total_len.max(1), 1, 1];
let zeroes = vec![0.0_f32; shape.iter().product()];
- let keys = alloc_allocation_with_data(context, &zeroes);
- let values = alloc_allocation_with_data(context, &zeroes);
-
- KVCacheLayer {
+ let keys = common::helpers::sparse_buffer_create_with::<B, f32>(context, &zeroes);
+ let values = common::helpers::sparse_buffer_create_with::<B, f32>(context, &zeroes);
+ let layer = KVCacheLayer {
state,
keys,
values,
shape,
data_type: DataType::F32,
- }
+ };
+ (layer, shape)
}
-fn overwrite_allocation(
- allocation: &mut Allocation<Metal>,
+fn overwrite_sparse_buffer<B: Backend>(
+ context: &B::Context,
+ buffer: &mut B::SparseBuffer,
+ elements_count: usize,
updates: &[(usize, f32)],
) {
- let mut data: Vec<f32> = allocation_to_vec(allocation);
+ let mut data: Vec<f32> = common::helpers::sparse_buffer_read_vec::<B, f32>(context, buffer, elements_count);
for (index, value) in updates {
data[*index] = *value;
}
- write_allocation(allocation, &data);
+ common::helpers::sparse_buffer_write::<B, f32>(context, buffer, &data);
}
-fn fill_arrays(layer: &mut KVCacheLayer<Metal>) -> (Vec<f32>, Vec<f32>) {
+fn fill_arrays<B: Backend>(
+ context: &B::Context,
+ layer: &mut KVCacheLayer<B, B::SparseBuffer>,
+) -> (Vec<f32>, Vec<f32>) {
let initial_keys = {
let len = layer.shape.iter().product();
let data: Vec<f32> = (0..len).map(|idx| 1_000.0 + idx as f32).collect();
- write_allocation(&mut layer.keys, &data);
+ common::helpers::sparse_buffer_write::<B, f32>(context, &mut layer.keys, &data);
data
};
let initial_values = {
let len = layer.shape.iter().product();
let data: Vec<f32> = (0..len).map(|idx| 2_000.0 + idx as f32).collect();
- write_allocation(&mut layer.values, &data);
+ common::helpers::sparse_buffer_write::<B, f32>(context, &mut layer.values, &data);
data
};
@@ -128,16 +133,16 @@ fn expected_after_update(
expected
}
-fn run_scenario(
- context: &<Metal as Backend>::Context,
+fn run_scenario<B: Backend>(
+ context: &B::Context,
scenario: &Scenario,
) {
- let mut layer =
+ let (mut layer, _) =
make_test_layer(context, scenario.state.clone(), scenario.prefix_capacity, scenario.suffix_capacity);
let state_before_update = layer.state.clone();
- let (initial_keys, initial_values) = fill_arrays(&mut layer);
+ let (initial_keys, initial_values) = fill_arrays::<B>(context, &mut layer);
let expected_keys = expected_after_update(&state_before_update, &scenario.accepted_suffix_indices, &initial_keys);
let expected_values =
@@ -164,6 +169,7 @@ fn run_scenario(
layer.update_after_acceptance(
&scenario.accepted_suffix_indices,
scenario.suffix_start,
+ context,
&mut encoder,
&kv_cache_update,
);
@@ -172,10 +178,12 @@ fn run_scenario(
layer.register_accepted_tokens(scenario.number_of_accepted_tokens);
- let actual_keys: Vec<f32> = allocation_to_vec(&layer.keys);
+ let actual_keys: Vec<f32> =
+ common::helpers::sparse_buffer_read_vec::<B, f32>(context, &layer.keys, expected_keys.len());
assert_eq!(actual_keys, expected_keys, "{}: key buffer mismatch", scenario.name);
- let actual_values: Vec<f32> = allocation_to_vec(&layer.values);
+ let actual_values: Vec<f32> =
+ common::helpers::sparse_buffer_read_vec::<B, f32>(context, &layer.values, expected_values.len());
assert_eq!(actual_values, expected_values, "{}: value buffer mismatch", scenario.name);
match &layer.state {
@@ -284,7 +292,7 @@ fn kv_cache_state_scenarios() {
];
for scenario in &scenarios {
- run_scenario(&context, scenario);
+ run_scenario::<Metal>(&context, scenario);
}
}
@@ -294,7 +302,7 @@ fn kv_cache_slice_apply_contiguous_window() {
return;
};
- let mut layer = make_test_layer(
+ let (mut layer, shape) = make_test_layer::<Metal>(
&context,
KVCacheLayerState::Windowed {
ring_offset: 1,
@@ -304,17 +312,25 @@ fn kv_cache_slice_apply_contiguous_window() {
4,
0,
);
- let (initial_keys, initial_values) = fill_arrays(&mut layer);
+ let (initial_keys, initial_values) = fill_arrays::<Metal>(&context, &mut layer);
- let slice = layer.slice(&context, 0..2).expect("slice should exist");
- // Mutate the captured slots.
- overwrite_allocation(&mut layer.keys, &[(0, -1.0), (1, -2.0)]);
- overwrite_allocation(&mut layer.values, &[(0, -3.0), (1, -4.0)]);
+ let mut encoder = Encoder::<Metal>::new(&context).expect("Failed to create encoder");
+ let slice = layer.slice(&context, &mut encoder, 0..2).expect("slice should exist");
+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to end and wait encoder");
- layer.apply_slice(&slice, None);
-
- let keys_after: Vec<f32> = allocation_to_vec(&layer.keys);
- let values_after: Vec<f32> = allocation_to_vec(&layer.values);
+ // Mutate the captured slots.
+ let elements_count = shape.iter().product();
+ overwrite_sparse_buffer::<Metal>(&context, &mut layer.keys, elements_count, &[(0, -1.0), (1, -2.0)]);
+ overwrite_sparse_buffer::<Metal>(&context, &mut layer.values, elements_count, &[(0, -3.0), (1, -4.0)]);
+
+ let mut encoder = Encoder::<Metal>::new(&context).expect("Failed to create encoder");
+ layer.apply_slice(&context, &mut encoder, &slice, None);
+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to end and wait encoder");
+
+ let keys_after: Vec<f32> =
+ common::helpers::sparse_buffer_read_vec::<Metal, f32>(&context, &layer.keys, elements_count);
+ let values_after: Vec<f32> =
+ common::helpers::sparse_buffer_read_vec::<Metal, f32>(&context, &layer.values, elements_count);
assert_eq!(keys_after[0..4], initial_keys[0..4], "keys restored for contiguous slice");
assert_eq!(values_after[0..4], initial_values[0..4], "values restored for contiguous slice");
}
@@ -325,7 +341,7 @@ fn kv_cache_slice_apply_wrap_window() {
return;
};
- let mut layer = make_test_layer(
+ let (mut layer, shape) = make_test_layer::<Metal>(
&context,
KVCacheLayerState::Windowed {
ring_offset: 3,
@@ -335,17 +351,25 @@ fn kv_cache_slice_apply_wrap_window() {
4,
0,
);
- let (initial_keys, initial_values) = fill_arrays(&mut layer);
+ let (initial_keys, initial_values) = fill_arrays::<Metal>(&context, &mut layer);
- let slice = layer.slice(&context, 2..4).expect("slice should exist");
- // Captured slots are expected to wrap; mutate them.
- overwrite_allocation(&mut layer.keys, &[(2, -11.0), (3, -12.0)]);
- overwrite_allocation(&mut layer.values, &[(2, -13.0), (3, -14.0)]);
+ let mut encoder = Encoder::<Metal>::new(&context).expect("Failed to create encoder");
+ let slice = layer.slice(&context, &mut encoder, 2..4).expect("slice should exist");
+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to end and wait encoder");
- layer.apply_slice(&slice, None);
-
- let keys_after: Vec<f32> = allocation_to_vec(&layer.keys);
- let values_after: Vec<f32> = allocation_to_vec(&layer.values);
+ // Captured slots are expected to wrap; mutate them.
+ let elements_count = shape.iter().product();
+ overwrite_sparse_buffer::<Metal>(&context, &mut layer.keys, elements_count, &[(2, -11.0), (3, -12.0)]);
+ overwrite_sparse_buffer::<Metal>(&context, &mut layer.values, elements_count, &[(2, -13.0), (3, -14.0)]);
+
+ let mut encoder = Encoder::<Metal>::new(&context).expect("Failed to create encoder");
+ layer.apply_slice(&context, &mut encoder, &slice, None);
+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to end and wait encoder");
+
+ let keys_after: Vec<f32> =
+ common::helpers::sparse_buffer_read_vec::<Metal, f32>(&context, &layer.keys, elements_count);
+ let values_after: Vec<f32> =
+ common::helpers::sparse_buffer_read_vec::<Metal, f32>(&context, &layer.values, elements_count);
assert_eq!(keys_after[0..4], initial_keys[0..4], "keys restored for wrapped slice");
assert_eq!(values_after[0..4], initial_values[0..4], "values restored for wrapped slice");
}
@@ -356,7 +380,7 @@ fn kv_cache_slice_apply_full_restores_metadata() {
return;
};
- let mut layer = make_test_layer(
+ let (mut layer, _) = make_test_layer::<Metal>(
&context,
KVCacheLayerState::Full {
prefix_len: 3,
@@ -365,7 +389,9 @@ fn kv_cache_slice_apply_full_restores_metadata() {
0,
);
- let slice = layer.slice(&context, 0..2).expect("full slice exists");
+ let mut encoder = Encoder::<Metal>::new(&context).expect("Failed to create encoder");
+ let slice = layer.slice(&context, &mut encoder, 0..2).expect("full slice exists");
+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to end and wait encoder");
// Mutate metadata.
if let KVCacheLayerState::Full {
@@ -375,7 +401,9 @@ fn kv_cache_slice_apply_full_restores_metadata() {
*prefix_len = 1;
}
- layer.apply_slice(&slice, None);
+ let mut encoder = Encoder::<Metal>::new(&context).expect("Failed to create encoder");
+ layer.apply_slice(&context, &mut encoder, &slice, None);
+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to end and wait encoder");
if let KVCacheLayerState::Full {
prefix_len,
diff --git a/crates/backend-uzu/tests/unit/kernel/attention/attention_gemm_test.rs b/crates/backend-uzu/tests/unit/kernel/attention/attention_gemm_test.rs
index b1dff7fde..be3989145 100644
--- a/crates/backend-uzu/tests/unit/kernel/attention/attention_gemm_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/attention/attention_gemm_test.rs
@@ -4,7 +4,7 @@ use backend_uzu::{
ArrayElement, DataType,
backends::{
common::{
- Backend, Context, Encoder,
+ Allocation, Backend, Context, Encoder,
kernel::attention::{AttentionGemmArguments, AttentionGemmBlock},
},
cpu::Cpu,
@@ -96,7 +96,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
let segment_prefix_length = input.sequence_length - input.suffix_length;
- let args = AttentionGemmArguments::<B> {
+ let args = AttentionGemmArguments::<B, Allocation<B>> {
queries: &queries_allocation,
keys: &keys_allocation,
values: &values_allocation,