Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

using computed suffix start for prefill update in full layer (#39)

LuckyIYI520ca657c768
diff --git a/crates/uzu/src/backends/metal/forward_pass/kv_cache.rs b/crates/uzu/src/backends/metal/forward_pass/kv_cache.rs
index 790662b66..959e6d206 100644
--- a/crates/uzu/src/backends/metal/forward_pass/kv_cache.rs
+++ b/crates/uzu/src/backends/metal/forward_pass/kv_cache.rs
@@ -159,6 +159,7 @@ impl KVCacheLayer {
pub fn update_after_acceptance(
&mut self,
accepted_suffix_indices: &[usize],
+ suffix_start: Option<usize>,
command_buffer: &MTLCommandBuffer,
kv_cache_update: &KVCacheUpdate,
) {
@@ -170,13 +171,13 @@ impl KVCacheLayer {
return;
}
- // Absolute positions of the *source* rows (still in suffix part).
+ // Absolute positions of the *source* rows.
let source_indices: Vec<usize> = accepted_suffix_indices
.iter()
- .map(|i| i + *prefix_len)
+ .map(|i| i + suffix_start.unwrap_or(*prefix_len))
.collect();
- // Absolute positions of the *destination* rows in the prefix.
+ // Absolute positions of the *destination* rows.
let destination_indices: Vec<usize> = (*prefix_len
..*prefix_len + accepted_suffix_indices.len())
.collect();
@@ -423,12 +424,14 @@ impl KVCache {
pub fn update_after_acceptance(
&mut self,
accepted_suffix_indices: &[usize],
+ suffix_start: Option<usize>,
command_buffer: &MTLCommandBuffer,
kv_cache_update: &KVCacheUpdate,
) {
for layer in self.data.iter_mut() {
layer.update_after_acceptance(
accepted_suffix_indices,
+ suffix_start,
command_buffer,
kv_cache_update,
);
diff --git a/crates/uzu/src/generator/generator.rs b/crates/uzu/src/generator/generator.rs
index 468f3cc72..a5f71fe0d 100644
--- a/crates/uzu/src/generator/generator.rs
+++ b/crates/uzu/src/generator/generator.rs
@@ -181,7 +181,11 @@ impl Generator {
let accept_indices_for_step: Vec<usize> =
(0..positions_for_step.len()).collect();
if !accept_indices_for_step.is_empty() {
- self.update_kv_cache(&accept_indices_for_step, true);
+ self.update_kv_cache(
+ &accept_indices_for_step,
+ None,
+ true,
+ );
}
self.context
@@ -202,9 +206,10 @@ impl Generator {
let mut final_state = last_state.ok_or(Error::PrefillFailed)?;
let sampled_tokens = self.sample(&mut final_state)?;
- let sampling_row = (tokens_length
- - prefill_step_size * number_of_prefill_steps.saturating_sub(1))
- .saturating_sub(1);
+ let last_suffix_start =
+ prefill_step_size * (number_of_prefill_steps - 1);
+
+ let sampling_row = (tokens_length - last_suffix_start) - 1;
let mut current_token_index: isize = -1;
let mut accepted_token_indices: Vec<usize> = vec![sampling_row];
let mut accepted_tokens: Vec<u64> = Vec::new();
@@ -229,7 +234,11 @@ impl Generator {
}
}
- self.update_kv_cache(&accepted_token_indices, false);
+ self.update_kv_cache(
+ &accepted_token_indices,
+ Some(last_suffix_start),
+ false,
+ );
self.tokens.extend(accepted_tokens.clone());
@@ -309,7 +318,7 @@ impl Generator {
}
}
- self.update_kv_cache(&accepted_token_indices, false);
+ self.update_kv_cache(&accepted_token_indices, None, false);
self.tokens.extend(accepted_tokens.clone());
self.sync_prefix();
@@ -457,6 +466,7 @@ impl Generator {
fn update_kv_cache(
&mut self,
accepted_token_indices: &[usize],
+ suffix_start: Option<usize>,
wait_until_completed: bool,
) {
let command_buffer = CommandBuffer::from_command_queue(
@@ -469,6 +479,7 @@ impl Generator {
let mut kv_cache = self.context.kv_cache.borrow_mut();
kv_cache.update_after_acceptance(
accepted_token_indices,
+ suffix_start,
&root_command_buffer,
&self.context.kv_cache_update,
);
diff --git a/crates/uzu/tests/kv_cache_state_test.rs b/crates/uzu/tests/kv_cache_state_test.rs
index 7c0dcbc1a..ae95ec293 100644
--- a/crates/uzu/tests/kv_cache_state_test.rs
+++ b/crates/uzu/tests/kv_cache_state_test.rs
@@ -22,6 +22,7 @@ struct Scenario {
accepted_suffix_indices: Vec<usize>,
accepted_token_positions: Vec<usize>,
suffix_token_positions: Vec<usize>,
+ suffix_start: Option<usize>,
expected_prefix_positions: Vec<usize>,
expected_mask: Vec<Vec<bool>>,
expected_ring_offset: Option<usize>,
@@ -230,6 +231,7 @@ fn run_scenario(
let root_command_buffer = command_buffer.root_command_buffer().to_owned();
layer.update_after_acceptance(
&scenario.accepted_suffix_indices,
+ scenario.suffix_start,
&root_command_buffer,
&kv_cache_update,
);
@@ -327,6 +329,7 @@ fn kv_cache_state_and_mask_scenarios() {
accepted_suffix_indices: vec![0, 1, 2],
accepted_token_positions: vec![7, 8, 9],
suffix_token_positions: vec![7, 8, 9],
+ suffix_start: None,
expected_prefix_positions: vec![6, 7, 8, 9, 4, 5],
expected_mask: vec![
vec![
@@ -364,6 +367,7 @@ fn kv_cache_state_and_mask_scenarios() {
accepted_suffix_indices: vec![0, 1, 2],
accepted_token_positions: vec![4, 5, 6],
suffix_token_positions: vec![4, 5, 6],
+ suffix_start: None,
expected_prefix_positions: vec![6, 1, 2, 3, 4, 5],
expected_mask: vec![
vec![false, false, false, false, true, true, false, true, true],
@@ -392,6 +396,7 @@ fn kv_cache_state_and_mask_scenarios() {
accepted_suffix_indices: vec![0],
accepted_token_positions: vec![16],
suffix_token_positions: vec![16],
+ suffix_start: None,
expected_prefix_positions: vec![11, 12, 13, 14, 15, 16],
expected_mask: vec![vec![
false, false, false, false, false, true, false,
@@ -412,6 +417,7 @@ fn kv_cache_state_and_mask_scenarios() {
accepted_suffix_indices: vec![0, 1, 2],
accepted_token_positions: vec![4, 5, 6],
suffix_token_positions: vec![4, 5, 6],
+ suffix_start: None,
expected_prefix_positions: vec![0, 1, 2, 3, 4, 5, 6],
expected_mask: vec![
vec![false, false, false, false, false, true, true],