A high-performance inference engine for AI models
added kv_cache update to prefill and implemented proper syncronization (#32)
LuckyIYIbc686d38211d
diff --git a/crates/uzu/src/generator/context.rs b/crates/uzu/src/generator/context.rsindex fae452e5e..bea5a510d 100644--- a/crates/uzu/src/generator/context.rs+++ b/crates/uzu/src/generator/context.rs@@ -1,5 +1,6 @@use std::{cell::RefCell, fs::File, io::BufReader, path::Path, rc::Rc};+use metal::SharedEvent;use mpsgraph::CommandBuffer as MPSCommandBuffer;use objc2::rc::Retained;@@ -20,6 +21,8 @@ use crate::{pub struct GeneratorContext {pub mtl_context: Rc<MTLContext>,pub command_buffer: Retained<MPSCommandBuffer>,+ pub kv_update_event: SharedEvent,+ pub kv_update_signal: u64,pub kv_cache: Rc<RefCell<KVCache>>,pub shared_buffers: Rc<RefCell<SharedBuffers>>,@@ -129,9 +132,14 @@ impl GeneratorContext {).map_err(|_| GeneratorError::UnableToCreateMetalContext)?;+ let kv_update_event = mtl_context.device.new_shared_event();+ let kv_update_signal = 1;+let context = Self {mtl_context,command_buffer,+ kv_update_event,+ kv_update_signal,kv_cache,shared_buffers,scratch_buffers,diff --git a/crates/uzu/src/generator/generator.rs b/crates/uzu/src/generator/generator.rsindex fa050941a..dc5208574 100644--- a/crates/uzu/src/generator/generator.rs+++ b/crates/uzu/src/generator/generator.rs@@ -30,6 +30,7 @@ pub struct Generator {pub context: GeneratorContext,encoded_tasks: HashMap<String, GeneratorEncodedTask>,registered_prefix_len: usize,+ pending_kv_update: Option<u64>,}impl Generator {@@ -48,6 +49,7 @@ impl Generator {context,encoded_tasks: HashMap::new(),registered_prefix_len: 0,+ pending_kv_update: None,};//Warmup@@ -126,7 +128,7 @@ impl Generator {expected_amount_of_new_tokens: self.config.prefill_step_size,};- let (state, run_time) = self.run_model(+ let (mut state, run_time) = self.run_model(task,false,self.allow_pre_encode(),@@ -155,6 +157,14 @@ impl Generator {.kv_cache.borrow_mut().register_accepted_tokens(&positions_for_step);+ let accept_indices_for_step: Vec<usize> =+ (0..positions_for_step.len()).collect();+ if !accept_indices_for_step.is_empty() {+ self.update_kv_cache(+ &mut state,+ &accept_indices_for_step,+ );+ }if let Some(&last_idx) = positions_for_step.last() {self.registered_prefix_len = last_idx + 1;}@@ -299,6 +309,7 @@ impl Generator {self.tokens.clear();self.registered_prefix_len = 0;self.encoded_tasks.clear();+ self.pending_kv_update = None;}pub fn prefix_len(&self) -> usize {@@ -349,6 +360,13 @@ impl Generator {let root_command_buffer =self.context.command_buffer.root_command_buffer().to_owned();+ if let Some(signal_value) = self.pending_kv_update.take() {+ root_command_buffer.encode_wait_for_event(+ &self.context.kv_update_event,+ signal_value,+ );+ }+if !warmup {self.context.gpu_sampler.encode(&mut state,@@ -409,12 +427,22 @@ impl Generator {let command_buffer = CommandBuffer::from_command_queue(&self.context.mtl_context.command_queue,);+ let root_command_buffer = command_buffer.command_buffer().to_owned();++ {+ let mut kv_cache = self.context.kv_cache.borrow_mut();+ kv_cache.update_after_acceptance(+ accepted_token_indices,+ &command_buffer,+ &self.context.kv_cache_update,+ );+ }- self.context.kv_cache.borrow_mut().update_after_acceptance(- accepted_token_indices,- &command_buffer,- &self.context.kv_cache_update,- );+ let signal_value = self.context.kv_update_signal;+ root_command_buffer+ .encode_signal_event(&self.context.kv_update_event, signal_value);+ self.context.kv_update_signal += 1;+ self.pending_kv_update = Some(signal_value);command_buffer.commit();}