Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Sliding window refactoring (#38)

LuckyIYIdfb71dc5287e
diff --git a/crates/uzu/src/backends/metal/forward_pass/buffers.rs b/crates/uzu/src/backends/metal/forward_pass/buffers.rs
index 2716e0341..3e7c3e51f 100644
--- a/crates/uzu/src/backends/metal/forward_pass/buffers.rs
+++ b/crates/uzu/src/backends/metal/forward_pass/buffers.rs
@@ -2,10 +2,7 @@ use std::collections::HashMap;
use metal::Buffer as MTLBuffer;
-use super::{
- super::MTLContext,
- model_shape::ModelShape,
-};
+use super::{super::MTLContext, model_shape::ModelShape};
use crate::{
DataType,
array::array_size_in_bytes,
diff --git a/crates/uzu/src/backends/metal/forward_pass/kv_cache.rs b/crates/uzu/src/backends/metal/forward_pass/kv_cache.rs
index 4320a5fbd..790662b66 100644
--- a/crates/uzu/src/backends/metal/forward_pass/kv_cache.rs
+++ b/crates/uzu/src/backends/metal/forward_pass/kv_cache.rs
@@ -1,6 +1,6 @@
use std::{cell::RefCell, collections::HashMap};
-use mpsgraph::CommandBuffer as MPSCommandBuffer;
+use metal::CommandBuffer as MTLCommandBuffer;
use super::{
super::{MTLContext, MetalArray},
@@ -43,19 +43,19 @@ pub struct KVCacheLayer {
}
impl KVCacheLayer {
- pub fn effective_prefix_length(&self) -> usize {
+ pub fn prefix_segment_length(&self) -> usize {
match &self.state {
KVCacheLayerState::Full {
prefix_len,
} => *prefix_len,
KVCacheLayerState::Windowed {
- ring_length,
+ window_length,
..
- } => *ring_length,
+ } => *window_length,
}
}
- pub fn projected_effective_prefix_length(
+ pub fn projected_segment_prefix_length(
&self,
projection_step: usize,
) -> usize {
@@ -64,10 +64,9 @@ impl KVCacheLayer {
prefix_len,
} => *prefix_len + projection_step,
KVCacheLayerState::Windowed {
- ring_length,
window_length,
..
- } => std::cmp::min(*ring_length + projection_step, *window_length),
+ } => *window_length,
}
}
@@ -95,12 +94,12 @@ impl KVCacheLayer {
context: &MTLContext,
external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,
) {
- let effective_prefix_len = self.effective_prefix_length();
+ let prefix_segment_length = self.prefix_segment_length();
context.fill_attention_bias(
dst,
suffix_length,
- effective_prefix_len,
+ prefix_segment_length,
|row_index, column_index| {
if let Some(bias_fn) = external_bias_fn {
bias_fn(row_index, column_index)
@@ -127,23 +126,16 @@ impl KVCacheLayer {
return true;
}
- let key_position = if column_index >= self.effective_prefix_length() {
- suffix_token_positions
- [column_index - self.effective_prefix_length()]
+ let key_position = if column_index >= self.prefix_segment_length() {
+ suffix_token_positions[column_index - self.prefix_segment_length()]
} else {
match &self.state {
KVCacheLayerState::Full {
..
} => column_index,
KVCacheLayerState::Windowed {
- ring_offset,
- window_length,
..
- } => {
- let physical_index =
- (*ring_offset + column_index) % *window_length;
- self.prefix_token_positions[physical_index]
- },
+ } => self.prefix_token_positions[column_index],
}
};
@@ -159,7 +151,7 @@ impl KVCacheLayer {
KVCacheLayerState::Windowed {
window_length,
..
- } => query_position > key_position + window_length,
+ } => query_position >= key_position + window_length,
_ => false,
}
}
@@ -167,32 +159,26 @@ impl KVCacheLayer {
pub fn update_after_acceptance(
&mut self,
accepted_suffix_indices: &[usize],
- command_buffer: &MPSCommandBuffer,
+ command_buffer: &MTLCommandBuffer,
kv_cache_update: &KVCacheUpdate,
) {
- let effective_prefix_length = self.effective_prefix_length();
- let effective_indices: Vec<usize> = if accepted_suffix_indices
- .is_empty()
- && matches!(self.state, KVCacheLayerState::Windowed { .. })
- {
- vec![0] // This represents the single new token at suffix position 0
- } else {
- accepted_suffix_indices.to_vec()
- };
-
match &mut self.state {
KVCacheLayerState::Full {
prefix_len,
} => {
+ if accepted_suffix_indices.is_empty() {
+ return;
+ }
+
// Absolute positions of the *source* rows (still in suffix part).
- let source_indices: Vec<usize> = effective_indices
+ let source_indices: Vec<usize> = accepted_suffix_indices
.iter()
- .map(|i| i + effective_prefix_length)
+ .map(|i| i + *prefix_len)
.collect();
// Absolute positions of the *destination* rows in the prefix.
let destination_indices: Vec<usize> = (*prefix_len
- ..*prefix_len + effective_indices.len())
+ ..*prefix_len + accepted_suffix_indices.len())
.collect();
self.scatter_if_required(
@@ -208,21 +194,23 @@ impl KVCacheLayer {
ring_length,
window_length,
} => {
- if ring_length < window_length {
- return;
- }
+ let suffix_indices: Vec<usize> =
+ if accepted_suffix_indices.is_empty() {
+ vec![0]
+ } else {
+ accepted_suffix_indices.to_vec()
+ };
- let source_indices: Vec<usize> = effective_indices
- .iter()
- .map(|i| i + effective_prefix_length)
- .collect();
+ let source_indices: Vec<usize> =
+ suffix_indices.iter().map(|i| i + *window_length).collect();
- // Consecutive slots starting at current ring_offset.
let mut destination_indices =
- Vec::with_capacity(effective_indices.len());
- for i in 0..effective_indices.len() {
- destination_indices
- .push((*ring_offset + i) % *window_length);
+ Vec::with_capacity(suffix_indices.len());
+
+ for i in 0..suffix_indices.len() {
+ destination_indices.push(
+ (*ring_length + *ring_offset + i) % *window_length,
+ );
}
self.scatter_if_required(
@@ -239,14 +227,13 @@ impl KVCacheLayer {
&self,
source_indices: &[usize],
destination_indices: &[usize],
- command_buffer: &MPSCommandBuffer,
+ command_buffer: &MTLCommandBuffer,
kv_cache_update: &KVCacheUpdate,
) {
if source_indices == destination_indices {
return;
}
- let root_cb = command_buffer.root_command_buffer().to_owned();
let key_buffer = {
let mut k = self.keys.borrow_mut();
unsafe { k.mtl_buffer() }.clone()
@@ -270,7 +257,7 @@ impl KVCacheLayer {
&[layer_data],
source_indices,
destination_indices,
- &root_cb,
+ &command_buffer,
);
}
@@ -282,7 +269,6 @@ impl KVCacheLayer {
KVCacheLayerState::Full {
prefix_len,
} => {
- // For full layers, just extend the positions and update prefix_len
self.prefix_token_positions.extend_from_slice(token_positions);
*prefix_len = self.prefix_token_positions.len();
},
@@ -293,15 +279,14 @@ impl KVCacheLayer {
} => {
for &token_pos in token_positions {
if *ring_length < *window_length {
- // Still filling the window
- self.prefix_token_positions.push(token_pos);
+ let dst =
+ (*ring_offset + *ring_length) % *window_length;
+
+ self.prefix_token_positions[dst] = token_pos;
*ring_length += 1;
} else {
- // Window is full, overwrite at ring_offset position
- let buffer_index = *ring_offset % *window_length;
- self.prefix_token_positions[buffer_index] = token_pos;
+ self.prefix_token_positions[*ring_offset] = token_pos;
- // NOW increment ring_offset after updating the bookkeeping
*ring_offset = (*ring_offset + 1) % *window_length;
}
}
@@ -345,16 +330,24 @@ impl KVCache {
};
KVCacheLayer {
- state,
+ state: state.clone(),
keys: RefCell::new(
context.array(&shape, model_shape.kv_cache_data_type()),
),
values: RefCell::new(
context.array(&shape, model_shape.kv_cache_data_type()),
),
- prefix_token_positions: Vec::with_capacity(
- max_prefix_length,
- ),
+ prefix_token_positions: match &state {
+ KVCacheLayerState::Full {
+ ..
+ } => Vec::with_capacity(max_prefix_length),
+ KVCacheLayerState::Windowed {
+ window_length,
+ ..
+ } => (0..*window_length)
+ .map(|_| INVALID_POSITION)
+ .collect(),
+ },
max_suffix_length: max_suffix_length,
}
})
@@ -384,7 +377,7 @@ impl KVCache {
*ring_length = 0;
},
}
- layer.prefix_token_positions.clear();
+ layer.prefix_token_positions.fill(INVALID_POSITION);
}
}
@@ -430,7 +423,7 @@ impl KVCache {
pub fn update_after_acceptance(
&mut self,
accepted_suffix_indices: &[usize],
- command_buffer: &MPSCommandBuffer,
+ command_buffer: &MTLCommandBuffer,
kv_cache_update: &KVCacheUpdate,
) {
for layer in self.data.iter_mut() {
@@ -470,7 +463,7 @@ impl KVCache {
let mut new_keys = context.array(&new_shape, dtype);
let mut new_values = context.array(&new_shape, dtype);
- let mut copy_rows = layer.effective_prefix_length();
+ let mut copy_rows = layer.prefix_segment_length();
if let Some(window_length) = layer.window_length() {
copy_rows = std::cmp::min(copy_rows, window_length);
}
diff --git a/crates/uzu/src/backends/metal/forward_pass/state.rs b/crates/uzu/src/backends/metal/forward_pass/state.rs
index a01b0f608..7efacb4d5 100644
--- a/crates/uzu/src/backends/metal/forward_pass/state.rs
+++ b/crates/uzu/src/backends/metal/forward_pass/state.rs
@@ -790,9 +790,7 @@ impl ForwardPassState {
// --------------------
// Attention Bias
// --------------------
- let prefix_length = kv_cache.borrow().max_prefix_length();
- let attention_bias_shape =
- [suffix_length, suffix_length + prefix_length];
+
let act_dtype = model_shape.activation_data_type();
let mut attention_bias_map: HashMap<Option<usize>, MetalArray> =
@@ -800,6 +798,11 @@ impl ForwardPassState {
.attention_window_size_to_bias
.iter()
.map(|(window_size, buffer)| {
+ let prefix_length = window_size
+ .unwrap_or(kv_cache.borrow().max_prefix_length());
+ let attention_bias_shape =
+ [suffix_length, suffix_length + prefix_length];
+
let array = unsafe {
MetalArray::new(
buffer.clone(),
diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention.metal b/crates/uzu/src/backends/metal/kernel/attention/attention.metal
index 988114b99..47fffab20 100644
--- a/crates/uzu/src/backends/metal/kernel/attention/attention.metal
+++ b/crates/uzu/src/backends/metal/kernel/attention/attention.metal
@@ -652,7 +652,7 @@ void update_kv_cache(
const constant int& num_heads,
const constant int& head_dim,
const constant int& suffix_length,
- const constant int& prefix_length,
+ const constant int& prefix_segment_length,
const constant int& max_sequence_length,
uint3 position // x: group_idx, y: token_idx, z: dim_idx
) {
@@ -669,7 +669,7 @@ void update_kv_cache(
TensorView3D<T> valueCacheTensorView = TensorView3D<T>(value_cache, num_groups, max_sequence_length, head_dim);
// Copy rotated key to cache
- keyCacheTensorView(groupIndex, prefix_length + tokenIndex, dimIndex) = rotatedKeysTensorView(groupIndex, tokenIndex, dimIndex);
+ keyCacheTensorView(groupIndex, prefix_segment_length + tokenIndex, dimIndex) = rotatedKeysTensorView(groupIndex, tokenIndex, dimIndex);
// Update value cache (only first thread in each token processes values to avoid redundant work)
if (dimIndex == 0) {
@@ -683,7 +683,7 @@ void update_kv_cache(
// Values start at offset: total_query_dim + total_key_value_dim
for (uint d = 0; d < head_dim; d++) {
const uint valueOffset = totalQueryDim + totalKeyValueDim + groupIndex * head_dim + d;
- valueCacheTensorView(groupIndex, prefix_length + tokenIndex, d) = qkvTensorView(tokenIndex, valueOffset);
+ valueCacheTensorView(groupIndex, prefix_segment_length + tokenIndex, d) = qkvTensorView(tokenIndex, valueOffset);
}
}
}
@@ -697,13 +697,13 @@ void update_kv_cache(
const constant int& num_heads [[ buffer(5) ]], \
const constant int& head_dim [[ buffer(6) ]], \
const constant int& suffix_length [[ buffer(7) ]], \
- const constant int& prefix_length [[ buffer(8) ]], \
+ const constant int& prefix_segment_length [[ buffer(8) ]], \
const constant int& max_sequence_length [[ buffer(9) ]], \
uint3 position [[ thread_position_in_grid ]]) \
#define innerArguments \
(rotated_keys, qkv, key_cache, value_cache, num_groups, num_heads, \
- head_dim, suffix_length, prefix_length, max_sequence_length, position) \
+ head_dim, suffix_length, prefix_segment_length, max_sequence_length, position) \
generateKernels(update_kv_cache)
diff --git a/crates/uzu/src/backends/metal/kernel/attention/mod.rs b/crates/uzu/src/backends/metal/kernel/attention/mod.rs
index 63b8d8cb7..309d6165e 100644
--- a/crates/uzu/src/backends/metal/kernel/attention/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/attention/mod.rs
@@ -104,7 +104,7 @@ pub struct KVCacheUpdateArguments<'a> {
pub num_heads: usize,
pub head_dim: usize,
pub suffix_length: usize,
- pub prefix_length: usize,
+ pub segment_prefix_length: usize,
pub max_sequence_length: usize,
}
@@ -523,7 +523,7 @@ impl AttentionKernel {
compute_encoder.set_bytes(
8,
size_of::<i32>() as u64,
- &(args.prefix_length as i32) as *const i32 as *const _,
+ &(args.segment_prefix_length as i32) as *const i32 as *const _,
);
compute_encoder.set_bytes(
9,
@@ -615,12 +615,13 @@ impl EncodableWithState for AttentionKernelEncodable {
)
};
- let prefix_length = state.kv_cache.borrow().data[self.layer_index]
- .projected_effective_prefix_length(
+ let segment_prefix_length = state.kv_cache.borrow().data
+ [self.layer_index]
+ .projected_segment_prefix_length(
parameters.projection_step.unwrap_or(0),
);
- let sequence_length = prefix_length + suffix_length;
+ let sequence_length = segment_prefix_length + suffix_length;
let window_length =
state.kv_cache.borrow().data[self.layer_index].window_length();
@@ -718,7 +719,7 @@ impl EncodableWithState for AttentionKernelEncodable {
num_heads,
head_dim,
suffix_length,
- prefix_length,
+ segment_prefix_length,
max_sequence_length,
},
) {
diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts.rs b/crates/uzu/src/backends/metal/kernel/moe/experts.rs
index 3769c7eca..a13a7eaed 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/experts.rs
+++ b/crates/uzu/src/backends/metal/kernel/moe/experts.rs
@@ -408,7 +408,7 @@ impl MoeExpertsTwoPassPrefillKernel {
let d_model_u32 = args.d_model as u32;
let d_ff_u32 = args.d_ff as u32;
let e_u32 = args.e as u32;
- const COL_TILE_FF: u32 = 32; // Must match PASSA_BN in Metal kernel
+ const COL_TILE_FF: u32 = 32; // Must match PASSA_BN in Metal kernel
const COL_TILE_MODEL: u32 = 64;
let n_tiles_ff = if d_ff_u32 == 0 {
0
diff --git a/crates/uzu/src/device/device_context.rs b/crates/uzu/src/device/device_context.rs
index 0f12f6ee0..2b39e3273 100644
--- a/crates/uzu/src/device/device_context.rs
+++ b/crates/uzu/src/device/device_context.rs
@@ -188,18 +188,19 @@ pub trait DeviceContext {
&self,
dst: &mut Self::DeviceArray,
suffix_length: usize,
- prefix_length: usize,
+ prefix_segment_length: usize,
mut should_be_neg_inf: F,
) where
F: FnMut(usize, usize) -> bool,
{
- let total_elems = suffix_length * (suffix_length + prefix_length);
+ let total_elems =
+ suffix_length * (suffix_length + prefix_segment_length);
match dst.data_type() {
DataType::F16 => {
// TODO: Use lambda to avoid crimes
let mut buf = Vec::<f16>::with_capacity(total_elems);
for row in 0..suffix_length {
- for col in 0..suffix_length + prefix_length {
+ for col in 0..suffix_length + prefix_segment_length {
buf.push(if should_be_neg_inf(row, col) {
f16::NEG_INFINITY
} else {
@@ -212,7 +213,7 @@ pub trait DeviceContext {
DataType::BF16 => {
let mut buf = Vec::<bf16>::with_capacity(total_elems);
for row in 0..suffix_length {
- for col in 0..suffix_length + prefix_length {
+ for col in 0..suffix_length + prefix_segment_length {
buf.push(if should_be_neg_inf(row, col) {
bf16::NEG_INFINITY
} else {
@@ -225,7 +226,7 @@ pub trait DeviceContext {
DataType::F32 => {
let mut buf = Vec::<f32>::with_capacity(total_elems);
for row in 0..suffix_length {
- for col in 0..suffix_length + prefix_length {
+ for col in 0..suffix_length + prefix_segment_length {
buf.push(if should_be_neg_inf(row, col) {
f32::NEG_INFINITY
} else {
@@ -238,7 +239,7 @@ pub trait DeviceContext {
DataType::F64 => {
let mut buf = Vec::<f64>::with_capacity(total_elems);
for row in 0..suffix_length {
- for col in 0..suffix_length + prefix_length {
+ for col in 0..suffix_length + prefix_segment_length {
buf.push(if should_be_neg_inf(row, col) {
f64::NEG_INFINITY
} else {
diff --git a/crates/uzu/src/generator/context.rs b/crates/uzu/src/generator/context.rs
index 8f1033ae6..4426fc7a3 100644
--- a/crates/uzu/src/generator/context.rs
+++ b/crates/uzu/src/generator/context.rs
@@ -1,6 +1,5 @@
use std::{cell::RefCell, fs::File, io::BufReader, path::Path, rc::Rc};
-use metal::SharedEvent;
use mpsgraph::CommandBuffer as MPSCommandBuffer;
use objc2::rc::Retained;
@@ -25,8 +24,6 @@ use crate::{
pub struct GeneratorContext {
pub mtl_context: Rc<MTLContext>,
pub command_buffer: Retained<MPSCommandBuffer>,
- pub kv_update_event: SharedEvent,
- pub kv_update_signal: u64,
pub kv_cache: Rc<RefCell<KVCache>>,
pub shared_buffers: Rc<RefCell<SharedBuffers>>,
@@ -141,14 +138,9 @@ impl GeneratorContext {
)
.map_err(|_| Error::UnableToCreateMetalContext)?;
- let kv_update_event = mtl_context.device.new_shared_event();
- let kv_update_signal = 1;
-
let context = Self {
mtl_context,
command_buffer,
- kv_update_event,
- kv_update_signal,
kv_cache,
shared_buffers,
scratch_buffers,
diff --git a/crates/uzu/src/generator/generator.rs b/crates/uzu/src/generator/generator.rs
index 2e0e4bfd5..468f3cc72 100644
--- a/crates/uzu/src/generator/generator.rs
+++ b/crates/uzu/src/generator/generator.rs
@@ -31,7 +31,6 @@ pub struct Generator {
pub context: GeneratorContext,
encoded_tasks: HashMap<String, GeneratorEncodedTask>,
registered_prefix_len: usize,
- pending_kv_update: Option<u64>,
gpu_capture: GpuCaptureManager,
}
@@ -43,7 +42,6 @@ impl Generator {
let gpu_capture = GpuCaptureManager::new();
let context = GeneratorContext::new(model_path, &decoding_config)?;
-
let prefill_step_size =
decoding_config.prefill_step_size.resolve(&context.model_config);
let generate_suffix_length = decoding_config.generate_suffix_length();
@@ -54,7 +52,6 @@ impl Generator {
context,
encoded_tasks: HashMap::new(),
registered_prefix_len: 0,
- pending_kv_update: None,
gpu_capture,
};
@@ -152,7 +149,7 @@ impl Generator {
expected_number_of_new_tokens: prefill_step_size,
};
- let (mut state, run_time) = self.run_model(
+ let (state, run_time) = self.run_model(
task,
false,
self.allow_pre_encode(),
@@ -181,18 +178,17 @@ impl Generator {
}
if !positions_for_step.is_empty() {
- self.context
- .kv_cache
- .borrow_mut()
- .register_accepted_tokens(&positions_for_step);
let accept_indices_for_step: Vec<usize> =
(0..positions_for_step.len()).collect();
if !accept_indices_for_step.is_empty() {
- self.update_kv_cache(
- &mut state,
- &accept_indices_for_step,
- );
+ self.update_kv_cache(&accept_indices_for_step, true);
}
+
+ self.context
+ .kv_cache
+ .borrow_mut()
+ .register_accepted_tokens(&positions_for_step);
+
if let Some(&last_idx) = positions_for_step.last() {
self.registered_prefix_len = last_idx + 1;
}
@@ -206,9 +202,12 @@ impl Generator {
let mut final_state = last_state.ok_or(Error::PrefillFailed)?;
let sampled_tokens = self.sample(&mut final_state)?;
- let mut accepted_token_indices: Vec<usize> = Vec::new();
- let mut accepted_tokens: Vec<u64> = Vec::new();
+ let sampling_row = (tokens_length
+ - prefill_step_size * number_of_prefill_steps.saturating_sub(1))
+ .saturating_sub(1);
let mut current_token_index: isize = -1;
+ let mut accepted_token_indices: Vec<usize> = vec![sampling_row];
+ let mut accepted_tokens: Vec<u64> = Vec::new();
loop {
let current_index_in_window =
((current_token_index + tokens_length as isize)
@@ -230,7 +229,7 @@ impl Generator {
}
}
- self.update_kv_cache(&mut final_state, &accepted_token_indices);
+ self.update_kv_cache(&accepted_token_indices, false);
self.tokens.extend(accepted_tokens.clone());
@@ -310,7 +309,7 @@ impl Generator {
}
}
- self.update_kv_cache(&mut state, &accepted_token_indices);
+ self.update_kv_cache(&accepted_token_indices, false);
self.tokens.extend(accepted_tokens.clone());
self.sync_prefix();
@@ -332,7 +331,6 @@ impl Generator {
self.tokens.clear();
self.registered_prefix_len = 0;
self.encoded_tasks.clear();
- self.pending_kv_update = None;
self.gpu_capture.reset();
}
@@ -383,7 +381,7 @@ impl Generator {
}
if let Some(_) = self.encoded_tasks.remove(&encoded_task_key) {
- //Nothing
+ // Nothing
} else {
self.context.reset_command_buffer();
@@ -398,13 +396,6 @@ impl Generator {
let root_command_buffer =
self.context.command_buffer.root_command_buffer().to_owned();
- if let Some(signal_value) = self.pending_kv_update.take() {
- root_command_buffer.encode_wait_for_event(
- &self.context.kv_update_event,
- signal_value,
- );
- }
-
if !warmup {
self.context.gpu_sampler.encode(
&mut state,
@@ -465,30 +456,29 @@ impl Generator {
fn update_kv_cache(
&mut self,
- _state: &mut ForwardPassState,
accepted_token_indices: &[usize],
+ wait_until_completed: bool,
) {
let command_buffer = CommandBuffer::from_command_queue(
&self.context.mtl_context.command_queue,
);
- let root_command_buffer = command_buffer.command_buffer().to_owned();
+ let root_command_buffer =
+ command_buffer.root_command_buffer().to_owned();
{
let mut kv_cache = self.context.kv_cache.borrow_mut();
kv_cache.update_after_acceptance(
accepted_token_indices,
- &command_buffer,
+ &root_command_buffer,
&self.context.kv_cache_update,
);
}
- let signal_value = self.context.kv_update_signal;
- root_command_buffer
- .encode_signal_event(&self.context.kv_update_event, signal_value);
- self.context.kv_update_signal += 1;
- self.pending_kv_update = Some(signal_value);
-
command_buffer.commit();
+
+ if wait_until_completed {
+ command_buffer.root_command_buffer().wait_until_completed();
+ }
}
fn allow_pre_encode(&self) -> bool {
diff --git a/crates/uzu/src/session/parameter/prefill_step_size.rs b/crates/uzu/src/session/parameter/prefill_step_size.rs
index 9f75489da..62d2e5d38 100644
--- a/crates/uzu/src/session/parameter/prefill_step_size.rs
+++ b/crates/uzu/src/session/parameter/prefill_step_size.rs
@@ -43,7 +43,7 @@ impl ConfigResolvableValue<LanguageModelConfig, usize> for PrefillStepSize {
if cfg!(target_os = "ios") {
64
} else {
- 64
+ 256
}
},
PrefillStepSize::Maximal => maximal_value,
diff --git a/crates/uzu/src/session/session.rs b/crates/uzu/src/session/session.rs
index 08596b5e4..c0d689726 100644
--- a/crates/uzu/src/session/session.rs
+++ b/crates/uzu/src/session/session.rs
@@ -341,7 +341,7 @@ impl Session {
for (ctx_layer, gen_layer) in
ctx.kv_cache.data.iter().zip(generator_cache.data.iter_mut())
{
- let copy_rows = ctx_layer.effective_prefix_length();
+ let copy_rows = ctx_layer.prefix_segment_length();
if copy_rows > 0 {
gen_layer.keys.borrow_mut().copy_slice(
&ctx_layer.keys.borrow(),
diff --git a/crates/uzu/tests/kv_cache_state_test.rs b/crates/uzu/tests/kv_cache_state_test.rs
new file mode 100644
index 000000000..7c0dcbc1a
--- /dev/null
+++ b/crates/uzu/tests/kv_cache_state_test.rs
@@ -0,0 +1,431 @@
+#![cfg(any(target_os = "macos", target_os = "ios"))]
+
+use metal::Device;
+use mpsgraph::CommandBuffer;
+use uzu::{
+ Array, DataType, DeviceContext,
+ backends::metal::{
+ KVCacheUpdate, KernelDataType, MTLContext,
+ forward_pass::{
+ KVCacheLayer, KVCacheLayerState, kv_cache::INVALID_POSITION,
+ },
+ },
+};
+
+#[derive(Debug)]
+struct Scenario {
+ name: &'static str,
+ state: KVCacheLayerState,
+ prefix_capacity: usize,
+ suffix_capacity: usize,
+ initial_prefix_positions: Vec<usize>,
+ accepted_suffix_indices: Vec<usize>,
+ accepted_token_positions: Vec<usize>,
+ suffix_token_positions: Vec<usize>,
+ expected_prefix_positions: Vec<usize>,
+ expected_mask: Vec<Vec<bool>>,
+ expected_ring_offset: Option<usize>,
+ expected_ring_length: Option<usize>,
+ expected_prefix_len: Option<usize>,
+ expected_prefix_segment_length: usize,
+}
+
+fn create_test_context() -> Option<MTLContext> {
+ let device = Device::system_default()?;
+ let command_queue = device.new_command_queue();
+ match MTLContext::new(device, command_queue) {
+ Ok(ctx) => Some(ctx),
+ Err(e) => {
+ eprintln!(
+ "Skipping KV cache tests: failed to create Metal context: {:?}",
+ e
+ );
+ None
+ },
+ }
+}
+
+fn make_test_layer(
+ context: &MTLContext,
+ state: KVCacheLayerState,
+ prefix_capacity: usize,
+ suffix_capacity: usize,
+) -> KVCacheLayer {
+ let total_len = match &state {
+ KVCacheLayerState::Full {
+ ..
+ } => prefix_capacity + suffix_capacity,
+ KVCacheLayerState::Windowed {
+ window_length,
+ ..
+ } => window_length + suffix_capacity,
+ };
+ let shape = [1, total_len.max(1), 1];
+
+ let keys = std::cell::RefCell::new(context.array(&shape, DataType::F32));
+ let values = std::cell::RefCell::new(context.array(&shape, DataType::F32));
+
+ let prefix_token_positions = match &state {
+ KVCacheLayerState::Full {
+ ..
+ } => Vec::with_capacity(prefix_capacity),
+ KVCacheLayerState::Windowed {
+ window_length,
+ ..
+ } => vec![INVALID_POSITION; *window_length],
+ };
+
+ KVCacheLayer {
+ state,
+ keys,
+ values,
+ prefix_token_positions,
+ max_suffix_length: suffix_capacity,
+ }
+}
+
+fn fill_arrays(layer: &mut KVCacheLayer) -> (Vec<f32>, Vec<f32>) {
+ let initial_keys = {
+ let mut keys_ref = layer.keys.borrow_mut();
+ let slice = keys_ref.as_slice_mut::<f32>().unwrap();
+ for (idx, value) in slice.iter_mut().enumerate() {
+ *value = 1_000.0 + idx as f32;
+ }
+ slice.to_vec()
+ };
+
+ let initial_values = {
+ let mut values_ref = layer.values.borrow_mut();
+ let slice = values_ref.as_slice_mut::<f32>().unwrap();
+ for (idx, value) in slice.iter_mut().enumerate() {
+ *value = 2_000.0 + idx as f32;
+ }
+ slice.to_vec()
+ };
+
+ (initial_keys, initial_values)
+}
+
+fn expected_after_update(
+ state: &KVCacheLayerState,
+ accepted_indices: &[usize],
+ initial: &[f32],
+) -> Vec<f32> {
+ let mut expected = initial.to_vec();
+ let effective_indices: Vec<usize> = if accepted_indices.is_empty()
+ && matches!(state, KVCacheLayerState::Windowed { .. })
+ {
+ vec![0]
+ } else {
+ accepted_indices.to_vec()
+ };
+ match state {
+ KVCacheLayerState::Full {
+ prefix_len,
+ } => {
+ let prefix_start = *prefix_len;
+ for (offset, suffix_idx) in effective_indices.iter().enumerate() {
+ let src = prefix_start + *suffix_idx;
+ let dst = prefix_start + offset;
+ if src != dst {
+ expected.swap(dst, src);
+ }
+ }
+ },
+ KVCacheLayerState::Windowed {
+ ring_offset,
+ ring_length,
+ window_length,
+ } => {
+ for (offset, suffix_idx) in effective_indices.iter().enumerate() {
+ let src = window_length + *suffix_idx;
+ let dst = (ring_length + ring_offset + offset) % *window_length;
+ expected.swap(dst, src);
+ }
+ },
+ }
+ expected
+}
+
+fn collect_mask(
+ layer: &KVCacheLayer,
+ suffix_positions: &[usize],
+) -> Vec<Vec<bool>> {
+ let suffix_length = suffix_positions.len();
+ let prefix_segment_length = layer.prefix_segment_length();
+
+ (0..suffix_length)
+ .map(|row| {
+ (0..prefix_segment_length + suffix_length)
+ .map(|col| {
+ layer.bias_should_be_neg_inf(row, col, suffix_positions)
+ })
+ .collect()
+ })
+ .collect()
+}
+
+fn run_scenario(
+ context: &MTLContext,
+ scenario: &Scenario,
+) {
+ let mut layer = make_test_layer(
+ context,
+ scenario.state.clone(),
+ scenario.prefix_capacity,
+ scenario.suffix_capacity,
+ );
+
+ layer.prefix_token_positions = scenario.initial_prefix_positions.clone();
+
+ let state_before_update = layer.state.clone();
+
+ let (initial_keys, initial_values) = fill_arrays(&mut layer);
+
+ let expected_keys = expected_after_update(
+ &state_before_update,
+ &scenario.accepted_suffix_indices,
+ &initial_keys,
+ );
+ let expected_values = expected_after_update(
+ &state_before_update,
+ &scenario.accepted_suffix_indices,
+ &initial_values,
+ );
+
+ let total_sequence_length = match &layer.state {
+ KVCacheLayerState::Full {
+ ..
+ } => scenario.prefix_capacity + scenario.suffix_capacity,
+ KVCacheLayerState::Windowed {
+ window_length,
+ ..
+ } => window_length + scenario.suffix_capacity,
+ };
+
+ let mask = collect_mask(&layer, &scenario.suffix_token_positions);
+ assert_eq!(
+ mask, scenario.expected_mask,
+ "{}: bias mask mismatch",
+ scenario.name
+ );
+
+ let kv_cache_update = match KVCacheUpdate::new(
+ context,
+ KernelDataType::Float32,
+ total_sequence_length,
+ ) {
+ Ok(update) => update,
+ Err(e) => {
+ panic!(
+ "Failed to create KV cache update for scenario {}: {:?}",
+ scenario.name, e
+ );
+ },
+ };
+
+ let command_buffer =
+ CommandBuffer::from_command_queue(&context.command_queue);
+
+ let root_command_buffer = command_buffer.root_command_buffer().to_owned();
+ layer.update_after_acceptance(
+ &scenario.accepted_suffix_indices,
+ &root_command_buffer,
+ &kv_cache_update,
+ );
+
+ command_buffer.commit();
+ root_command_buffer.wait_until_completed();
+
+ layer.register_accepted_tokens(&scenario.accepted_token_positions);
+
+ let actual_keys = {
+ let keys_ref = layer.keys.borrow();
+ keys_ref.as_slice::<f32>().unwrap().to_vec()
+ };
+ assert_eq!(
+ actual_keys, expected_keys,
+ "{}: key buffer mismatch",
+ scenario.name
+ );
+
+ let actual_values = {
+ let values_ref = layer.values.borrow();
+ values_ref.as_slice::<f32>().unwrap().to_vec()
+ };
+ assert_eq!(
+ actual_values, expected_values,
+ "{}: value buffer mismatch",
+ scenario.name
+ );
+
+ assert_eq!(
+ layer.prefix_token_positions, scenario.expected_prefix_positions,
+ "{}: prefix positions mismatch",
+ scenario.name
+ );
+
+ match &layer.state {
+ KVCacheLayerState::Full {
+ prefix_len,
+ } => {
+ if let Some(expected_len) = scenario.expected_prefix_len {
+ assert_eq!(
+ *prefix_len, expected_len,
+ "{}: prefix length mismatch",
+ scenario.name
+ );
+ }
+ },
+ KVCacheLayerState::Windowed {
+ ring_offset,
+ ring_length,
+ ..
+ } => {
+ if let Some(expected_offset) = scenario.expected_ring_offset {
+ assert_eq!(
+ *ring_offset, expected_offset,
+ "{}: ring offset mismatch",
+ scenario.name
+ );
+ }
+ if let Some(expected_length) = scenario.expected_ring_length {
+ assert_eq!(
+ *ring_length, expected_length,
+ "{}: ring length mismatch",
+ scenario.name
+ );
+ }
+ },
+ }
+
+ assert_eq!(
+ layer.prefix_segment_length(),
+ scenario.expected_prefix_segment_length,
+ "{}: prefix segment length mismatch",
+ scenario.name
+ );
+}
+
+#[test]
+fn kv_cache_state_and_mask_scenarios() {
+ let Some(context) = create_test_context() else {
+ return;
+ };
+
+ let scenarios = vec![
+ Scenario {
+ name: "windowed_after_crossing_window",
+ state: KVCacheLayerState::Windowed {
+ ring_offset: 1,
+ ring_length: 6,
+ window_length: 6,
+ },
+ prefix_capacity: 6,
+ suffix_capacity: 3,
+ initial_prefix_positions: vec![6, 1, 2, 3, 4, 5],
+ accepted_suffix_indices: vec![0, 1, 2],
+ accepted_token_positions: vec![7, 8, 9],
+ suffix_token_positions: vec![7, 8, 9],
+ expected_prefix_positions: vec![6, 7, 8, 9, 4, 5],
+ expected_mask: vec![
+ vec![
+ false, true, false, false, false, false, false, true, true,
+ ],
+ vec![
+ false, true, true, false, false, false, false, false, true,
+ ],
+ vec![
+ false, true, true, true, false, false, false, false, false,
+ ],
+ ],
+ expected_ring_offset: Some(4),
+ expected_ring_length: Some(6),
+ expected_prefix_len: None,
+ expected_prefix_segment_length: 6,
+ },
+ Scenario {
+ name: "windowed_during_crossing_window",
+ state: KVCacheLayerState::Windowed {
+ ring_offset: 0,
+ ring_length: 4,
+ window_length: 6,
+ },
+ prefix_capacity: 6,
+ suffix_capacity: 3,
+ initial_prefix_positions: vec![
+ 0,
+ 1,
+ 2,
+ 3,
+ INVALID_POSITION,
+ INVALID_POSITION,
+ ],
+ accepted_suffix_indices: vec![0, 1, 2],
+ accepted_token_positions: vec![4, 5, 6],
+ suffix_token_positions: vec![4, 5, 6],
+ expected_prefix_positions: vec![6, 1, 2, 3, 4, 5],
+ expected_mask: vec![
+ vec![false, false, false, false, true, true, false, true, true],
+ vec![
+ false, false, false, false, true, true, false, false, true,
+ ],
+ vec![
+ true, false, false, false, true, true, false, false, false,
+ ],
+ ],
+ expected_ring_offset: Some(1),
+ expected_ring_length: Some(6),
+ expected_prefix_len: None,
+ expected_prefix_segment_length: 6,
+ },
+ Scenario {
+ name: "windowed_single_accept_second_wrap",
+ state: KVCacheLayerState::Windowed {
+ ring_offset: 5,
+ ring_length: 6,
+ window_length: 6,
+ },
+ prefix_capacity: 6,
+ suffix_capacity: 1,
+ initial_prefix_positions: vec![11, 12, 13, 14, 15, 10],
+ accepted_suffix_indices: vec![0],
+ accepted_token_positions: vec![16],
+ suffix_token_positions: vec![16],
+ expected_prefix_positions: vec![11, 12, 13, 14, 15, 16],
+ expected_mask: vec![vec![
+ false, false, false, false, false, true, false,
+ ]],
+ expected_ring_offset: Some(0),
+ expected_ring_length: Some(6),
+ expected_prefix_len: None,
+ expected_prefix_segment_length: 6,
+ },
+ Scenario {
+ name: "full_mode_pure_causal",
+ state: KVCacheLayerState::Full {
+ prefix_len: 4,
+ },
+ prefix_capacity: 6,
+ suffix_capacity: 3,
+ initial_prefix_positions: vec![0, 1, 2, 3],
+ accepted_suffix_indices: vec![0, 1, 2],
+ accepted_token_positions: vec![4, 5, 6],
+ suffix_token_positions: vec![4, 5, 6],
+ expected_prefix_positions: vec![0, 1, 2, 3, 4, 5, 6],
+ expected_mask: vec![
+ vec![false, false, false, false, false, true, true],
+ vec![false, false, false, false, false, false, true],
+ vec![false, false, false, false, false, false, false],
+ ],
+ expected_ring_offset: None,
+ expected_ring_length: None,
+ expected_prefix_len: Some(7),
+ expected_prefix_segment_length: 7,
+ },
+ ];
+
+ for scenario in &scenarios {
+ run_scenario(&context, scenario);
+ }
+}