A high-performance inference engine for AI models
using computed suffix start for prefill update in full layer (#39)
LuckyIYI520ca657c768
diff --git a/crates/uzu/src/backends/metal/forward_pass/kv_cache.rs b/crates/uzu/src/backends/metal/forward_pass/kv_cache.rsindex 790662b66..959e6d206 100644--- a/crates/uzu/src/backends/metal/forward_pass/kv_cache.rs+++ b/crates/uzu/src/backends/metal/forward_pass/kv_cache.rs@@ -159,6 +159,7 @@ impl KVCacheLayer {pub fn update_after_acceptance(&mut self,accepted_suffix_indices: &[usize],+ suffix_start: Option<usize>,command_buffer: &MTLCommandBuffer,kv_cache_update: &KVCacheUpdate,) {@@ -170,13 +171,13 @@ impl KVCacheLayer {return;}- // Absolute positions of the *source* rows (still in suffix part).+ // Absolute positions of the *source* rows.let source_indices: Vec<usize> = accepted_suffix_indices.iter()- .map(|i| i + *prefix_len)+ .map(|i| i + suffix_start.unwrap_or(*prefix_len)).collect();- // Absolute positions of the *destination* rows in the prefix.+ // Absolute positions of the *destination* rows.let destination_indices: Vec<usize> = (*prefix_len..*prefix_len + accepted_suffix_indices.len()).collect();@@ -423,12 +424,14 @@ impl KVCache {pub fn update_after_acceptance(&mut self,accepted_suffix_indices: &[usize],+ suffix_start: Option<usize>,command_buffer: &MTLCommandBuffer,kv_cache_update: &KVCacheUpdate,) {for layer in self.data.iter_mut() {layer.update_after_acceptance(accepted_suffix_indices,+ suffix_start,command_buffer,kv_cache_update,);diff --git a/crates/uzu/src/generator/generator.rs b/crates/uzu/src/generator/generator.rsindex 468f3cc72..a5f71fe0d 100644--- a/crates/uzu/src/generator/generator.rs+++ b/crates/uzu/src/generator/generator.rs@@ -181,7 +181,11 @@ impl Generator {let accept_indices_for_step: Vec<usize> =(0..positions_for_step.len()).collect();if !accept_indices_for_step.is_empty() {- self.update_kv_cache(&accept_indices_for_step, true);+ self.update_kv_cache(+ &accept_indices_for_step,+ None,+ true,+ );}self.context@@ -202,9 +206,10 @@ impl Generator {let mut final_state = last_state.ok_or(Error::PrefillFailed)?;let sampled_tokens = self.sample(&mut final_state)?;- let sampling_row = (tokens_length- - prefill_step_size * number_of_prefill_steps.saturating_sub(1))- .saturating_sub(1);+ let last_suffix_start =+ prefill_step_size * (number_of_prefill_steps - 1);++ let sampling_row = (tokens_length - last_suffix_start) - 1;let mut current_token_index: isize = -1;let mut accepted_token_indices: Vec<usize> = vec![sampling_row];let mut accepted_tokens: Vec<u64> = Vec::new();@@ -229,7 +234,11 @@ impl Generator {}}- self.update_kv_cache(&accepted_token_indices, false);+ self.update_kv_cache(+ &accepted_token_indices,+ Some(last_suffix_start),+ false,+ );self.tokens.extend(accepted_tokens.clone());@@ -309,7 +318,7 @@ impl Generator {}}- self.update_kv_cache(&accepted_token_indices, false);+ self.update_kv_cache(&accepted_token_indices, None, false);self.tokens.extend(accepted_tokens.clone());self.sync_prefix();@@ -457,6 +466,7 @@ impl Generator {fn update_kv_cache(&mut self,accepted_token_indices: &[usize],+ suffix_start: Option<usize>,wait_until_completed: bool,) {let command_buffer = CommandBuffer::from_command_queue(@@ -469,6 +479,7 @@ impl Generator {let mut kv_cache = self.context.kv_cache.borrow_mut();kv_cache.update_after_acceptance(accepted_token_indices,+ suffix_start,&root_command_buffer,&self.context.kv_cache_update,);diff --git a/crates/uzu/tests/kv_cache_state_test.rs b/crates/uzu/tests/kv_cache_state_test.rsindex 7c0dcbc1a..ae95ec293 100644--- a/crates/uzu/tests/kv_cache_state_test.rs+++ b/crates/uzu/tests/kv_cache_state_test.rs@@ -22,6 +22,7 @@ struct Scenario {accepted_suffix_indices: Vec<usize>,accepted_token_positions: Vec<usize>,suffix_token_positions: Vec<usize>,+ suffix_start: Option<usize>,expected_prefix_positions: Vec<usize>,expected_mask: Vec<Vec<bool>>,expected_ring_offset: Option<usize>,@@ -230,6 +231,7 @@ fn run_scenario(let root_command_buffer = command_buffer.root_command_buffer().to_owned();layer.update_after_acceptance(&scenario.accepted_suffix_indices,+ scenario.suffix_start,&root_command_buffer,&kv_cache_update,);@@ -327,6 +329,7 @@ fn kv_cache_state_and_mask_scenarios() {accepted_suffix_indices: vec![0, 1, 2],accepted_token_positions: vec![7, 8, 9],suffix_token_positions: vec![7, 8, 9],+ suffix_start: None,expected_prefix_positions: vec![6, 7, 8, 9, 4, 5],expected_mask: vec![vec![@@ -364,6 +367,7 @@ fn kv_cache_state_and_mask_scenarios() {accepted_suffix_indices: vec![0, 1, 2],accepted_token_positions: vec![4, 5, 6],suffix_token_positions: vec![4, 5, 6],+ suffix_start: None,expected_prefix_positions: vec![6, 1, 2, 3, 4, 5],expected_mask: vec![vec![false, false, false, false, true, true, false, true, true],@@ -392,6 +396,7 @@ fn kv_cache_state_and_mask_scenarios() {accepted_suffix_indices: vec![0],accepted_token_positions: vec![16],suffix_token_positions: vec![16],+ suffix_start: None,expected_prefix_positions: vec![11, 12, 13, 14, 15, 16],expected_mask: vec![vec![false, false, false, false, false, true, false,@@ -412,6 +417,7 @@ fn kv_cache_state_and_mask_scenarios() {accepted_suffix_indices: vec![0, 1, 2],accepted_token_positions: vec![4, 5, 6],suffix_token_positions: vec![4, 5, 6],+ suffix_start: None,expected_prefix_positions: vec![0, 1, 2, 3, 4, 5, 6],expected_mask: vec![vec![false, false, false, false, false, true, true],