A high-performance inference engine for AI models
Sliding window refactoring (#38)
LuckyIYIdfb71dc5287e
diff --git a/crates/uzu/src/backends/metal/forward_pass/buffers.rs b/crates/uzu/src/backends/metal/forward_pass/buffers.rsindex 2716e0341..3e7c3e51f 100644--- a/crates/uzu/src/backends/metal/forward_pass/buffers.rs+++ b/crates/uzu/src/backends/metal/forward_pass/buffers.rs@@ -2,10 +2,7 @@ use std::collections::HashMap;use metal::Buffer as MTLBuffer;-use super::{- super::MTLContext,- model_shape::ModelShape,-};+use super::{super::MTLContext, model_shape::ModelShape};use crate::{DataType,array::array_size_in_bytes,diff --git a/crates/uzu/src/backends/metal/forward_pass/kv_cache.rs b/crates/uzu/src/backends/metal/forward_pass/kv_cache.rsindex 4320a5fbd..790662b66 100644--- a/crates/uzu/src/backends/metal/forward_pass/kv_cache.rs+++ b/crates/uzu/src/backends/metal/forward_pass/kv_cache.rs@@ -1,6 +1,6 @@use std::{cell::RefCell, collections::HashMap};-use mpsgraph::CommandBuffer as MPSCommandBuffer;+use metal::CommandBuffer as MTLCommandBuffer;use super::{super::{MTLContext, MetalArray},@@ -43,19 +43,19 @@ pub struct KVCacheLayer {}impl KVCacheLayer {- pub fn effective_prefix_length(&self) -> usize {+ pub fn prefix_segment_length(&self) -> usize {match &self.state {KVCacheLayerState::Full {prefix_len,} => *prefix_len,KVCacheLayerState::Windowed {- ring_length,+ window_length,..- } => *ring_length,+ } => *window_length,}}- pub fn projected_effective_prefix_length(+ pub fn projected_segment_prefix_length(&self,projection_step: usize,) -> usize {@@ -64,10 +64,9 @@ impl KVCacheLayer {prefix_len,} => *prefix_len + projection_step,KVCacheLayerState::Windowed {- ring_length,window_length,..- } => std::cmp::min(*ring_length + projection_step, *window_length),+ } => *window_length,}}@@ -95,12 +94,12 @@ impl KVCacheLayer {context: &MTLContext,external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,) {- let effective_prefix_len = self.effective_prefix_length();+ let prefix_segment_length = self.prefix_segment_length();context.fill_attention_bias(dst,suffix_length,- effective_prefix_len,+ prefix_segment_length,|row_index, column_index| {if let Some(bias_fn) = external_bias_fn {bias_fn(row_index, column_index)@@ -127,23 +126,16 @@ impl KVCacheLayer {return true;}- let key_position = if column_index >= self.effective_prefix_length() {- suffix_token_positions- [column_index - self.effective_prefix_length()]+ let key_position = if column_index >= self.prefix_segment_length() {+ suffix_token_positions[column_index - self.prefix_segment_length()]} else {match &self.state {KVCacheLayerState::Full {..} => column_index,KVCacheLayerState::Windowed {- ring_offset,- window_length,..- } => {- let physical_index =- (*ring_offset + column_index) % *window_length;- self.prefix_token_positions[physical_index]- },+ } => self.prefix_token_positions[column_index],}};@@ -159,7 +151,7 @@ impl KVCacheLayer {KVCacheLayerState::Windowed {window_length,..- } => query_position > key_position + window_length,+ } => query_position >= key_position + window_length,_ => false,}}@@ -167,32 +159,26 @@ impl KVCacheLayer {pub fn update_after_acceptance(&mut self,accepted_suffix_indices: &[usize],- command_buffer: &MPSCommandBuffer,+ command_buffer: &MTLCommandBuffer,kv_cache_update: &KVCacheUpdate,) {- let effective_prefix_length = self.effective_prefix_length();- let effective_indices: Vec<usize> = if accepted_suffix_indices- .is_empty()- && matches!(self.state, KVCacheLayerState::Windowed { .. })- {- vec![0] // This represents the single new token at suffix position 0- } else {- accepted_suffix_indices.to_vec()- };-match &mut self.state {KVCacheLayerState::Full {prefix_len,} => {+ if accepted_suffix_indices.is_empty() {+ return;+ }+// Absolute positions of the *source* rows (still in suffix part).- let source_indices: Vec<usize> = effective_indices+ let source_indices: Vec<usize> = accepted_suffix_indices.iter()- .map(|i| i + effective_prefix_length)+ .map(|i| i + *prefix_len).collect();// Absolute positions of the *destination* rows in the prefix.let destination_indices: Vec<usize> = (*prefix_len- ..*prefix_len + effective_indices.len())+ ..*prefix_len + accepted_suffix_indices.len()).collect();self.scatter_if_required(@@ -208,21 +194,23 @@ impl KVCacheLayer {ring_length,window_length,} => {- if ring_length < window_length {- return;- }+ let suffix_indices: Vec<usize> =+ if accepted_suffix_indices.is_empty() {+ vec![0]+ } else {+ accepted_suffix_indices.to_vec()+ };- let source_indices: Vec<usize> = effective_indices- .iter()- .map(|i| i + effective_prefix_length)- .collect();+ let source_indices: Vec<usize> =+ suffix_indices.iter().map(|i| i + *window_length).collect();- // Consecutive slots starting at current ring_offset.let mut destination_indices =- Vec::with_capacity(effective_indices.len());- for i in 0..effective_indices.len() {- destination_indices- .push((*ring_offset + i) % *window_length);+ Vec::with_capacity(suffix_indices.len());++ for i in 0..suffix_indices.len() {+ destination_indices.push(+ (*ring_length + *ring_offset + i) % *window_length,+ );}self.scatter_if_required(@@ -239,14 +227,13 @@ impl KVCacheLayer {&self,source_indices: &[usize],destination_indices: &[usize],- command_buffer: &MPSCommandBuffer,+ command_buffer: &MTLCommandBuffer,kv_cache_update: &KVCacheUpdate,) {if source_indices == destination_indices {return;}- let root_cb = command_buffer.root_command_buffer().to_owned();let key_buffer = {let mut k = self.keys.borrow_mut();unsafe { k.mtl_buffer() }.clone()@@ -270,7 +257,7 @@ impl KVCacheLayer {&[layer_data],source_indices,destination_indices,- &root_cb,+ &command_buffer,);}@@ -282,7 +269,6 @@ impl KVCacheLayer {KVCacheLayerState::Full {prefix_len,} => {- // For full layers, just extend the positions and update prefix_lenself.prefix_token_positions.extend_from_slice(token_positions);*prefix_len = self.prefix_token_positions.len();},@@ -293,15 +279,14 @@ impl KVCacheLayer {} => {for &token_pos in token_positions {if *ring_length < *window_length {- // Still filling the window- self.prefix_token_positions.push(token_pos);+ let dst =+ (*ring_offset + *ring_length) % *window_length;++ self.prefix_token_positions[dst] = token_pos;*ring_length += 1;} else {- // Window is full, overwrite at ring_offset position- let buffer_index = *ring_offset % *window_length;- self.prefix_token_positions[buffer_index] = token_pos;+ self.prefix_token_positions[*ring_offset] = token_pos;- // NOW increment ring_offset after updating the bookkeeping*ring_offset = (*ring_offset + 1) % *window_length;}}@@ -345,16 +330,24 @@ impl KVCache {};KVCacheLayer {- state,+ state: state.clone(),keys: RefCell::new(context.array(&shape, model_shape.kv_cache_data_type()),),values: RefCell::new(context.array(&shape, model_shape.kv_cache_data_type()),),- prefix_token_positions: Vec::with_capacity(- max_prefix_length,- ),+ prefix_token_positions: match &state {+ KVCacheLayerState::Full {+ ..+ } => Vec::with_capacity(max_prefix_length),+ KVCacheLayerState::Windowed {+ window_length,+ ..+ } => (0..*window_length)+ .map(|_| INVALID_POSITION)+ .collect(),+ },max_suffix_length: max_suffix_length,}})@@ -384,7 +377,7 @@ impl KVCache {*ring_length = 0;},}- layer.prefix_token_positions.clear();+ layer.prefix_token_positions.fill(INVALID_POSITION);}}@@ -430,7 +423,7 @@ impl KVCache {pub fn update_after_acceptance(&mut self,accepted_suffix_indices: &[usize],- command_buffer: &MPSCommandBuffer,+ command_buffer: &MTLCommandBuffer,kv_cache_update: &KVCacheUpdate,) {for layer in self.data.iter_mut() {@@ -470,7 +463,7 @@ impl KVCache {let mut new_keys = context.array(&new_shape, dtype);let mut new_values = context.array(&new_shape, dtype);- let mut copy_rows = layer.effective_prefix_length();+ let mut copy_rows = layer.prefix_segment_length();if let Some(window_length) = layer.window_length() {copy_rows = std::cmp::min(copy_rows, window_length);}diff --git a/crates/uzu/src/backends/metal/forward_pass/state.rs b/crates/uzu/src/backends/metal/forward_pass/state.rsindex a01b0f608..7efacb4d5 100644--- a/crates/uzu/src/backends/metal/forward_pass/state.rs+++ b/crates/uzu/src/backends/metal/forward_pass/state.rs@@ -790,9 +790,7 @@ impl ForwardPassState {// --------------------// Attention Bias// --------------------- let prefix_length = kv_cache.borrow().max_prefix_length();- let attention_bias_shape =- [suffix_length, suffix_length + prefix_length];+let act_dtype = model_shape.activation_data_type();let mut attention_bias_map: HashMap<Option<usize>, MetalArray> =@@ -800,6 +798,11 @@ impl ForwardPassState {.attention_window_size_to_bias.iter().map(|(window_size, buffer)| {+ let prefix_length = window_size+ .unwrap_or(kv_cache.borrow().max_prefix_length());+ let attention_bias_shape =+ [suffix_length, suffix_length + prefix_length];+let array = unsafe {MetalArray::new(buffer.clone(),diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention.metal b/crates/uzu/src/backends/metal/kernel/attention/attention.metalindex 988114b99..47fffab20 100644--- a/crates/uzu/src/backends/metal/kernel/attention/attention.metal+++ b/crates/uzu/src/backends/metal/kernel/attention/attention.metal@@ -652,7 +652,7 @@ void update_kv_cache(const constant int& num_heads,const constant int& head_dim,const constant int& suffix_length,- const constant int& prefix_length,+ const constant int& prefix_segment_length,const constant int& max_sequence_length,uint3 position // x: group_idx, y: token_idx, z: dim_idx) {@@ -669,7 +669,7 @@ void update_kv_cache(TensorView3D<T> valueCacheTensorView = TensorView3D<T>(value_cache, num_groups, max_sequence_length, head_dim);// Copy rotated key to cache- keyCacheTensorView(groupIndex, prefix_length + tokenIndex, dimIndex) = rotatedKeysTensorView(groupIndex, tokenIndex, dimIndex);+ keyCacheTensorView(groupIndex, prefix_segment_length + tokenIndex, dimIndex) = rotatedKeysTensorView(groupIndex, tokenIndex, dimIndex);// Update value cache (only first thread in each token processes values to avoid redundant work)if (dimIndex == 0) {@@ -683,7 +683,7 @@ void update_kv_cache(// Values start at offset: total_query_dim + total_key_value_dimfor (uint d = 0; d < head_dim; d++) {const uint valueOffset = totalQueryDim + totalKeyValueDim + groupIndex * head_dim + d;- valueCacheTensorView(groupIndex, prefix_length + tokenIndex, d) = qkvTensorView(tokenIndex, valueOffset);+ valueCacheTensorView(groupIndex, prefix_segment_length + tokenIndex, d) = qkvTensorView(tokenIndex, valueOffset);}}}@@ -697,13 +697,13 @@ void update_kv_cache(const constant int& num_heads [[ buffer(5) ]], \const constant int& head_dim [[ buffer(6) ]], \const constant int& suffix_length [[ buffer(7) ]], \- const constant int& prefix_length [[ buffer(8) ]], \+ const constant int& prefix_segment_length [[ buffer(8) ]], \const constant int& max_sequence_length [[ buffer(9) ]], \uint3 position [[ thread_position_in_grid ]]) \#define innerArguments \(rotated_keys, qkv, key_cache, value_cache, num_groups, num_heads, \- head_dim, suffix_length, prefix_length, max_sequence_length, position) \+ head_dim, suffix_length, prefix_segment_length, max_sequence_length, position) \generateKernels(update_kv_cache)diff --git a/crates/uzu/src/backends/metal/kernel/attention/mod.rs b/crates/uzu/src/backends/metal/kernel/attention/mod.rsindex 63b8d8cb7..309d6165e 100644--- a/crates/uzu/src/backends/metal/kernel/attention/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/attention/mod.rs@@ -104,7 +104,7 @@ pub struct KVCacheUpdateArguments<'a> {pub num_heads: usize,pub head_dim: usize,pub suffix_length: usize,- pub prefix_length: usize,+ pub segment_prefix_length: usize,pub max_sequence_length: usize,}@@ -523,7 +523,7 @@ impl AttentionKernel {compute_encoder.set_bytes(8,size_of::<i32>() as u64,- &(args.prefix_length as i32) as *const i32 as *const _,+ &(args.segment_prefix_length as i32) as *const i32 as *const _,);compute_encoder.set_bytes(9,@@ -615,12 +615,13 @@ impl EncodableWithState for AttentionKernelEncodable {)};- let prefix_length = state.kv_cache.borrow().data[self.layer_index]- .projected_effective_prefix_length(+ let segment_prefix_length = state.kv_cache.borrow().data+ [self.layer_index]+ .projected_segment_prefix_length(parameters.projection_step.unwrap_or(0),);- let sequence_length = prefix_length + suffix_length;+ let sequence_length = segment_prefix_length + suffix_length;let window_length =state.kv_cache.borrow().data[self.layer_index].window_length();@@ -718,7 +719,7 @@ impl EncodableWithState for AttentionKernelEncodable {num_heads,head_dim,suffix_length,- prefix_length,+ segment_prefix_length,max_sequence_length,},) {diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts.rs b/crates/uzu/src/backends/metal/kernel/moe/experts.rsindex 3769c7eca..a13a7eaed 100644--- a/crates/uzu/src/backends/metal/kernel/moe/experts.rs+++ b/crates/uzu/src/backends/metal/kernel/moe/experts.rs@@ -408,7 +408,7 @@ impl MoeExpertsTwoPassPrefillKernel {let d_model_u32 = args.d_model as u32;let d_ff_u32 = args.d_ff as u32;let e_u32 = args.e as u32;- const COL_TILE_FF: u32 = 32; // Must match PASSA_BN in Metal kernel+ const COL_TILE_FF: u32 = 32; // Must match PASSA_BN in Metal kernelconst COL_TILE_MODEL: u32 = 64;let n_tiles_ff = if d_ff_u32 == 0 {0diff --git a/crates/uzu/src/device/device_context.rs b/crates/uzu/src/device/device_context.rsindex 0f12f6ee0..2b39e3273 100644--- a/crates/uzu/src/device/device_context.rs+++ b/crates/uzu/src/device/device_context.rs@@ -188,18 +188,19 @@ pub trait DeviceContext {&self,dst: &mut Self::DeviceArray,suffix_length: usize,- prefix_length: usize,+ prefix_segment_length: usize,mut should_be_neg_inf: F,) whereF: FnMut(usize, usize) -> bool,{- let total_elems = suffix_length * (suffix_length + prefix_length);+ let total_elems =+ suffix_length * (suffix_length + prefix_segment_length);match dst.data_type() {DataType::F16 => {// TODO: Use lambda to avoid crimeslet mut buf = Vec::<f16>::with_capacity(total_elems);for row in 0..suffix_length {- for col in 0..suffix_length + prefix_length {+ for col in 0..suffix_length + prefix_segment_length {buf.push(if should_be_neg_inf(row, col) {f16::NEG_INFINITY} else {@@ -212,7 +213,7 @@ pub trait DeviceContext {DataType::BF16 => {let mut buf = Vec::<bf16>::with_capacity(total_elems);for row in 0..suffix_length {- for col in 0..suffix_length + prefix_length {+ for col in 0..suffix_length + prefix_segment_length {buf.push(if should_be_neg_inf(row, col) {bf16::NEG_INFINITY} else {@@ -225,7 +226,7 @@ pub trait DeviceContext {DataType::F32 => {let mut buf = Vec::<f32>::with_capacity(total_elems);for row in 0..suffix_length {- for col in 0..suffix_length + prefix_length {+ for col in 0..suffix_length + prefix_segment_length {buf.push(if should_be_neg_inf(row, col) {f32::NEG_INFINITY} else {@@ -238,7 +239,7 @@ pub trait DeviceContext {DataType::F64 => {let mut buf = Vec::<f64>::with_capacity(total_elems);for row in 0..suffix_length {- for col in 0..suffix_length + prefix_length {+ for col in 0..suffix_length + prefix_segment_length {buf.push(if should_be_neg_inf(row, col) {f64::NEG_INFINITY} else {diff --git a/crates/uzu/src/generator/context.rs b/crates/uzu/src/generator/context.rsindex 8f1033ae6..4426fc7a3 100644--- a/crates/uzu/src/generator/context.rs+++ b/crates/uzu/src/generator/context.rs@@ -1,6 +1,5 @@use std::{cell::RefCell, fs::File, io::BufReader, path::Path, rc::Rc};-use metal::SharedEvent;use mpsgraph::CommandBuffer as MPSCommandBuffer;use objc2::rc::Retained;@@ -25,8 +24,6 @@ use crate::{pub struct GeneratorContext {pub mtl_context: Rc<MTLContext>,pub command_buffer: Retained<MPSCommandBuffer>,- pub kv_update_event: SharedEvent,- pub kv_update_signal: u64,pub kv_cache: Rc<RefCell<KVCache>>,pub shared_buffers: Rc<RefCell<SharedBuffers>>,@@ -141,14 +138,9 @@ impl GeneratorContext {).map_err(|_| Error::UnableToCreateMetalContext)?;- let kv_update_event = mtl_context.device.new_shared_event();- let kv_update_signal = 1;-let context = Self {mtl_context,command_buffer,- kv_update_event,- kv_update_signal,kv_cache,shared_buffers,scratch_buffers,diff --git a/crates/uzu/src/generator/generator.rs b/crates/uzu/src/generator/generator.rsindex 2e0e4bfd5..468f3cc72 100644--- a/crates/uzu/src/generator/generator.rs+++ b/crates/uzu/src/generator/generator.rs@@ -31,7 +31,6 @@ pub struct Generator {pub context: GeneratorContext,encoded_tasks: HashMap<String, GeneratorEncodedTask>,registered_prefix_len: usize,- pending_kv_update: Option<u64>,gpu_capture: GpuCaptureManager,}@@ -43,7 +42,6 @@ impl Generator {let gpu_capture = GpuCaptureManager::new();let context = GeneratorContext::new(model_path, &decoding_config)?;-let prefill_step_size =decoding_config.prefill_step_size.resolve(&context.model_config);let generate_suffix_length = decoding_config.generate_suffix_length();@@ -54,7 +52,6 @@ impl Generator {context,encoded_tasks: HashMap::new(),registered_prefix_len: 0,- pending_kv_update: None,gpu_capture,};@@ -152,7 +149,7 @@ impl Generator {expected_number_of_new_tokens: prefill_step_size,};- let (mut state, run_time) = self.run_model(+ let (state, run_time) = self.run_model(task,false,self.allow_pre_encode(),@@ -181,18 +178,17 @@ impl Generator {}if !positions_for_step.is_empty() {- self.context- .kv_cache- .borrow_mut()- .register_accepted_tokens(&positions_for_step);let accept_indices_for_step: Vec<usize> =(0..positions_for_step.len()).collect();if !accept_indices_for_step.is_empty() {- self.update_kv_cache(- &mut state,- &accept_indices_for_step,- );+ self.update_kv_cache(&accept_indices_for_step, true);}++ self.context+ .kv_cache+ .borrow_mut()+ .register_accepted_tokens(&positions_for_step);+if let Some(&last_idx) = positions_for_step.last() {self.registered_prefix_len = last_idx + 1;}@@ -206,9 +202,12 @@ impl Generator {let mut final_state = last_state.ok_or(Error::PrefillFailed)?;let sampled_tokens = self.sample(&mut final_state)?;- let mut accepted_token_indices: Vec<usize> = Vec::new();- let mut accepted_tokens: Vec<u64> = Vec::new();+ let sampling_row = (tokens_length+ - prefill_step_size * number_of_prefill_steps.saturating_sub(1))+ .saturating_sub(1);let mut current_token_index: isize = -1;+ let mut accepted_token_indices: Vec<usize> = vec![sampling_row];+ let mut accepted_tokens: Vec<u64> = Vec::new();loop {let current_index_in_window =((current_token_index + tokens_length as isize)@@ -230,7 +229,7 @@ impl Generator {}}- self.update_kv_cache(&mut final_state, &accepted_token_indices);+ self.update_kv_cache(&accepted_token_indices, false);self.tokens.extend(accepted_tokens.clone());@@ -310,7 +309,7 @@ impl Generator {}}- self.update_kv_cache(&mut state, &accepted_token_indices);+ self.update_kv_cache(&accepted_token_indices, false);self.tokens.extend(accepted_tokens.clone());self.sync_prefix();@@ -332,7 +331,6 @@ impl Generator {self.tokens.clear();self.registered_prefix_len = 0;self.encoded_tasks.clear();- self.pending_kv_update = None;self.gpu_capture.reset();}@@ -383,7 +381,7 @@ impl Generator {}if let Some(_) = self.encoded_tasks.remove(&encoded_task_key) {- //Nothing+ // Nothing} else {self.context.reset_command_buffer();@@ -398,13 +396,6 @@ impl Generator {let root_command_buffer =self.context.command_buffer.root_command_buffer().to_owned();- if let Some(signal_value) = self.pending_kv_update.take() {- root_command_buffer.encode_wait_for_event(- &self.context.kv_update_event,- signal_value,- );- }-if !warmup {self.context.gpu_sampler.encode(&mut state,@@ -465,30 +456,29 @@ impl Generator {fn update_kv_cache(&mut self,- _state: &mut ForwardPassState,accepted_token_indices: &[usize],+ wait_until_completed: bool,) {let command_buffer = CommandBuffer::from_command_queue(&self.context.mtl_context.command_queue,);- let root_command_buffer = command_buffer.command_buffer().to_owned();+ let root_command_buffer =+ command_buffer.root_command_buffer().to_owned();{let mut kv_cache = self.context.kv_cache.borrow_mut();kv_cache.update_after_acceptance(accepted_token_indices,- &command_buffer,+ &root_command_buffer,&self.context.kv_cache_update,);}- let signal_value = self.context.kv_update_signal;- root_command_buffer- .encode_signal_event(&self.context.kv_update_event, signal_value);- self.context.kv_update_signal += 1;- self.pending_kv_update = Some(signal_value);-command_buffer.commit();++ if wait_until_completed {+ command_buffer.root_command_buffer().wait_until_completed();+ }}fn allow_pre_encode(&self) -> bool {diff --git a/crates/uzu/src/session/parameter/prefill_step_size.rs b/crates/uzu/src/session/parameter/prefill_step_size.rsindex 9f75489da..62d2e5d38 100644--- a/crates/uzu/src/session/parameter/prefill_step_size.rs+++ b/crates/uzu/src/session/parameter/prefill_step_size.rs@@ -43,7 +43,7 @@ impl ConfigResolvableValue<LanguageModelConfig, usize> for PrefillStepSize {if cfg!(target_os = "ios") {64} else {- 64+ 256}},PrefillStepSize::Maximal => maximal_value,diff --git a/crates/uzu/src/session/session.rs b/crates/uzu/src/session/session.rsindex 08596b5e4..c0d689726 100644--- a/crates/uzu/src/session/session.rs+++ b/crates/uzu/src/session/session.rs@@ -341,7 +341,7 @@ impl Session {for (ctx_layer, gen_layer) inctx.kv_cache.data.iter().zip(generator_cache.data.iter_mut()){- let copy_rows = ctx_layer.effective_prefix_length();+ let copy_rows = ctx_layer.prefix_segment_length();if copy_rows > 0 {gen_layer.keys.borrow_mut().copy_slice(&ctx_layer.keys.borrow(),diff --git a/crates/uzu/tests/kv_cache_state_test.rs b/crates/uzu/tests/kv_cache_state_test.rsnew file mode 100644index 000000000..7c0dcbc1a--- /dev/null+++ b/crates/uzu/tests/kv_cache_state_test.rs@@ -0,0 +1,431 @@+#![cfg(any(target_os = "macos", target_os = "ios"))]++use metal::Device;+use mpsgraph::CommandBuffer;+use uzu::{+ Array, DataType, DeviceContext,+ backends::metal::{+ KVCacheUpdate, KernelDataType, MTLContext,+ forward_pass::{+ KVCacheLayer, KVCacheLayerState, kv_cache::INVALID_POSITION,+ },+ },+};++#[derive(Debug)]+struct Scenario {+ name: &'static str,+ state: KVCacheLayerState,+ prefix_capacity: usize,+ suffix_capacity: usize,+ initial_prefix_positions: Vec<usize>,+ accepted_suffix_indices: Vec<usize>,+ accepted_token_positions: Vec<usize>,+ suffix_token_positions: Vec<usize>,+ expected_prefix_positions: Vec<usize>,+ expected_mask: Vec<Vec<bool>>,+ expected_ring_offset: Option<usize>,+ expected_ring_length: Option<usize>,+ expected_prefix_len: Option<usize>,+ expected_prefix_segment_length: usize,+}++fn create_test_context() -> Option<MTLContext> {+ let device = Device::system_default()?;+ let command_queue = device.new_command_queue();+ match MTLContext::new(device, command_queue) {+ Ok(ctx) => Some(ctx),+ Err(e) => {+ eprintln!(+ "Skipping KV cache tests: failed to create Metal context: {:?}",+ e+ );+ None+ },+ }+}++fn make_test_layer(+ context: &MTLContext,+ state: KVCacheLayerState,+ prefix_capacity: usize,+ suffix_capacity: usize,+) -> KVCacheLayer {+ let total_len = match &state {+ KVCacheLayerState::Full {+ ..+ } => prefix_capacity + suffix_capacity,+ KVCacheLayerState::Windowed {+ window_length,+ ..+ } => window_length + suffix_capacity,+ };+ let shape = [1, total_len.max(1), 1];++ let keys = std::cell::RefCell::new(context.array(&shape, DataType::F32));+ let values = std::cell::RefCell::new(context.array(&shape, DataType::F32));++ let prefix_token_positions = match &state {+ KVCacheLayerState::Full {+ ..+ } => Vec::with_capacity(prefix_capacity),+ KVCacheLayerState::Windowed {+ window_length,+ ..+ } => vec![INVALID_POSITION; *window_length],+ };++ KVCacheLayer {+ state,+ keys,+ values,+ prefix_token_positions,+ max_suffix_length: suffix_capacity,+ }+}++fn fill_arrays(layer: &mut KVCacheLayer) -> (Vec<f32>, Vec<f32>) {+ let initial_keys = {+ let mut keys_ref = layer.keys.borrow_mut();+ let slice = keys_ref.as_slice_mut::<f32>().unwrap();+ for (idx, value) in slice.iter_mut().enumerate() {+ *value = 1_000.0 + idx as f32;+ }+ slice.to_vec()+ };++ let initial_values = {+ let mut values_ref = layer.values.borrow_mut();+ let slice = values_ref.as_slice_mut::<f32>().unwrap();+ for (idx, value) in slice.iter_mut().enumerate() {+ *value = 2_000.0 + idx as f32;+ }+ slice.to_vec()+ };++ (initial_keys, initial_values)+}++fn expected_after_update(+ state: &KVCacheLayerState,+ accepted_indices: &[usize],+ initial: &[f32],+) -> Vec<f32> {+ let mut expected = initial.to_vec();+ let effective_indices: Vec<usize> = if accepted_indices.is_empty()+ && matches!(state, KVCacheLayerState::Windowed { .. })+ {+ vec![0]+ } else {+ accepted_indices.to_vec()+ };+ match state {+ KVCacheLayerState::Full {+ prefix_len,+ } => {+ let prefix_start = *prefix_len;+ for (offset, suffix_idx) in effective_indices.iter().enumerate() {+ let src = prefix_start + *suffix_idx;+ let dst = prefix_start + offset;+ if src != dst {+ expected.swap(dst, src);+ }+ }+ },+ KVCacheLayerState::Windowed {+ ring_offset,+ ring_length,+ window_length,+ } => {+ for (offset, suffix_idx) in effective_indices.iter().enumerate() {+ let src = window_length + *suffix_idx;+ let dst = (ring_length + ring_offset + offset) % *window_length;+ expected.swap(dst, src);+ }+ },+ }+ expected+}++fn collect_mask(+ layer: &KVCacheLayer,+ suffix_positions: &[usize],+) -> Vec<Vec<bool>> {+ let suffix_length = suffix_positions.len();+ let prefix_segment_length = layer.prefix_segment_length();++ (0..suffix_length)+ .map(|row| {+ (0..prefix_segment_length + suffix_length)+ .map(|col| {+ layer.bias_should_be_neg_inf(row, col, suffix_positions)+ })+ .collect()+ })+ .collect()+}++fn run_scenario(+ context: &MTLContext,+ scenario: &Scenario,+) {+ let mut layer = make_test_layer(+ context,+ scenario.state.clone(),+ scenario.prefix_capacity,+ scenario.suffix_capacity,+ );++ layer.prefix_token_positions = scenario.initial_prefix_positions.clone();++ let state_before_update = layer.state.clone();++ let (initial_keys, initial_values) = fill_arrays(&mut layer);++ let expected_keys = expected_after_update(+ &state_before_update,+ &scenario.accepted_suffix_indices,+ &initial_keys,+ );+ let expected_values = expected_after_update(+ &state_before_update,+ &scenario.accepted_suffix_indices,+ &initial_values,+ );++ let total_sequence_length = match &layer.state {+ KVCacheLayerState::Full {+ ..+ } => scenario.prefix_capacity + scenario.suffix_capacity,+ KVCacheLayerState::Windowed {+ window_length,+ ..+ } => window_length + scenario.suffix_capacity,+ };++ let mask = collect_mask(&layer, &scenario.suffix_token_positions);+ assert_eq!(+ mask, scenario.expected_mask,+ "{}: bias mask mismatch",+ scenario.name+ );++ let kv_cache_update = match KVCacheUpdate::new(+ context,+ KernelDataType::Float32,+ total_sequence_length,+ ) {+ Ok(update) => update,+ Err(e) => {+ panic!(+ "Failed to create KV cache update for scenario {}: {:?}",+ scenario.name, e+ );+ },+ };++ let command_buffer =+ CommandBuffer::from_command_queue(&context.command_queue);++ let root_command_buffer = command_buffer.root_command_buffer().to_owned();+ layer.update_after_acceptance(+ &scenario.accepted_suffix_indices,+ &root_command_buffer,+ &kv_cache_update,+ );++ command_buffer.commit();+ root_command_buffer.wait_until_completed();++ layer.register_accepted_tokens(&scenario.accepted_token_positions);++ let actual_keys = {+ let keys_ref = layer.keys.borrow();+ keys_ref.as_slice::<f32>().unwrap().to_vec()+ };+ assert_eq!(+ actual_keys, expected_keys,+ "{}: key buffer mismatch",+ scenario.name+ );++ let actual_values = {+ let values_ref = layer.values.borrow();+ values_ref.as_slice::<f32>().unwrap().to_vec()+ };+ assert_eq!(+ actual_values, expected_values,+ "{}: value buffer mismatch",+ scenario.name+ );++ assert_eq!(+ layer.prefix_token_positions, scenario.expected_prefix_positions,+ "{}: prefix positions mismatch",+ scenario.name+ );++ match &layer.state {+ KVCacheLayerState::Full {+ prefix_len,+ } => {+ if let Some(expected_len) = scenario.expected_prefix_len {+ assert_eq!(+ *prefix_len, expected_len,+ "{}: prefix length mismatch",+ scenario.name+ );+ }+ },+ KVCacheLayerState::Windowed {+ ring_offset,+ ring_length,+ ..+ } => {+ if let Some(expected_offset) = scenario.expected_ring_offset {+ assert_eq!(+ *ring_offset, expected_offset,+ "{}: ring offset mismatch",+ scenario.name+ );+ }+ if let Some(expected_length) = scenario.expected_ring_length {+ assert_eq!(+ *ring_length, expected_length,+ "{}: ring length mismatch",+ scenario.name+ );+ }+ },+ }++ assert_eq!(+ layer.prefix_segment_length(),+ scenario.expected_prefix_segment_length,+ "{}: prefix segment length mismatch",+ scenario.name+ );+}++#[test]+fn kv_cache_state_and_mask_scenarios() {+ let Some(context) = create_test_context() else {+ return;+ };++ let scenarios = vec![+ Scenario {+ name: "windowed_after_crossing_window",+ state: KVCacheLayerState::Windowed {+ ring_offset: 1,+ ring_length: 6,+ window_length: 6,+ },+ prefix_capacity: 6,+ suffix_capacity: 3,+ initial_prefix_positions: vec![6, 1, 2, 3, 4, 5],+ accepted_suffix_indices: vec![0, 1, 2],+ accepted_token_positions: vec![7, 8, 9],+ suffix_token_positions: vec![7, 8, 9],+ expected_prefix_positions: vec![6, 7, 8, 9, 4, 5],+ expected_mask: vec![+ vec![+ false, true, false, false, false, false, false, true, true,+ ],+ vec![+ false, true, true, false, false, false, false, false, true,+ ],+ vec![+ false, true, true, true, false, false, false, false, false,+ ],+ ],+ expected_ring_offset: Some(4),+ expected_ring_length: Some(6),+ expected_prefix_len: None,+ expected_prefix_segment_length: 6,+ },+ Scenario {+ name: "windowed_during_crossing_window",+ state: KVCacheLayerState::Windowed {+ ring_offset: 0,+ ring_length: 4,+ window_length: 6,+ },+ prefix_capacity: 6,+ suffix_capacity: 3,+ initial_prefix_positions: vec![+ 0,+ 1,+ 2,+ 3,+ INVALID_POSITION,+ INVALID_POSITION,+ ],+ accepted_suffix_indices: vec![0, 1, 2],+ accepted_token_positions: vec![4, 5, 6],+ suffix_token_positions: vec![4, 5, 6],+ expected_prefix_positions: vec![6, 1, 2, 3, 4, 5],+ expected_mask: vec![+ vec![false, false, false, false, true, true, false, true, true],+ vec![+ false, false, false, false, true, true, false, false, true,+ ],+ vec![+ true, false, false, false, true, true, false, false, false,+ ],+ ],+ expected_ring_offset: Some(1),+ expected_ring_length: Some(6),+ expected_prefix_len: None,+ expected_prefix_segment_length: 6,+ },+ Scenario {+ name: "windowed_single_accept_second_wrap",+ state: KVCacheLayerState::Windowed {+ ring_offset: 5,+ ring_length: 6,+ window_length: 6,+ },+ prefix_capacity: 6,+ suffix_capacity: 1,+ initial_prefix_positions: vec![11, 12, 13, 14, 15, 10],+ accepted_suffix_indices: vec![0],+ accepted_token_positions: vec![16],+ suffix_token_positions: vec![16],+ expected_prefix_positions: vec![11, 12, 13, 14, 15, 16],+ expected_mask: vec![vec![+ false, false, false, false, false, true, false,+ ]],+ expected_ring_offset: Some(0),+ expected_ring_length: Some(6),+ expected_prefix_len: None,+ expected_prefix_segment_length: 6,+ },+ Scenario {+ name: "full_mode_pure_causal",+ state: KVCacheLayerState::Full {+ prefix_len: 4,+ },+ prefix_capacity: 6,+ suffix_capacity: 3,+ initial_prefix_positions: vec![0, 1, 2, 3],+ accepted_suffix_indices: vec![0, 1, 2],+ accepted_token_positions: vec![4, 5, 6],+ suffix_token_positions: vec![4, 5, 6],+ expected_prefix_positions: vec![0, 1, 2, 3, 4, 5, 6],+ expected_mask: vec![+ vec![false, false, false, false, false, true, true],+ vec![false, false, false, false, false, false, true],+ vec![false, false, false, false, false, false, false],+ ],+ expected_ring_offset: None,+ expected_ring_length: None,+ expected_prefix_len: Some(7),+ expected_prefix_segment_length: 7,+ },+ ];++ for scenario in &scenarios {+ run_scenario(&context, scenario);+ }+}