A high-performance inference engine for AI models
owned commandbuffer (#230)
uuuvnf06b0a6fd4b0
diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rsindex fad69fc8f..af2fc72a4 100644--- a/crates/uzu/src/language_model/language_model_generator.rs+++ b/crates/uzu/src/language_model/language_model_generator.rs@@ -2,7 +2,7 @@ use std::{any::Any,collections::HashMap,iter::repeat_n,- ops::{Deref, DerefMut, Range},+ ops::{Deref, Range},path::Path,sync::Arc,time::Instant,@@ -492,23 +492,20 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {// Wait on previous pass if this is a continuationif is_continuation {- self.context- .command_buffer- .borrow_mut()- .encode_wait_for_event(&self.context.async_buffers.event, current_counter);+ self.context.command_buffer.encode_wait_for_event(&self.context.async_buffers.event, current_counter);}self.context.executables.encode(&mut state,&EncodingParameters::new(false, false, false),- self.context.command_buffer.borrow_mut().deref_mut(),+ &mut self.context.command_buffer,);// Encode samplingself.context.gpu_sampler.encode(&mut state,&EncodingParameters::new(false, false, false),- self.context.command_buffer.borrow_mut().deref_mut(),+ &mut self.context.command_buffer,);// Copy sampled token: sampling_output → token_ids (for next pass)@@ -521,7 +518,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {let token_ids_buf_rc = token_ids_binding.buffer();let token_ids_buf_borrow = token_ids_buf_rc.borrow();- self.context.command_buffer.borrow_mut().with_compute_encoder(|encoder| {+ self.context.command_buffer.with_compute_encoder(|encoder| {self.context.token_copy_sampled.encode(sampling_output_buf_borrow.deref(),token_ids_buf_borrow.deref(),@@ -539,12 +536,12 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {self.context.cache_layers.borrow_mut().update_after_acceptance(&[0],None,- self.context.command_buffer.borrow_mut().deref_mut(),+ &mut self.context.command_buffer,&self.context.kv_cache_update,);self.context.cache_layers.borrow_mut().register_accepted_tokens(&[token_position]);- self.context.command_buffer.borrow_mut().with_compute_encoder(|encoder| {+ self.context.command_buffer.with_compute_encoder(|encoder| {if let Some(mask_update) = &self.context.mask_update {let updates: Vec<AttentionBiasUpdate> =self.context.cache_layers.borrow().attention_bias_updates_after_acceptance(1);@@ -562,7 +559,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {// Signal event for next passlet next_counter = current_counter + 1;- self.context.command_buffer.borrow_mut().encode_signal_event(&self.context.async_buffers.event, next_counter);+ self.context.command_buffer.encode_signal_event(&self.context.async_buffers.event, next_counter);self.context.async_buffers.counter.set(next_counter);// Add completion handler@@ -579,11 +576,11 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {}};- self.context.command_buffer.borrow_mut().add_completion_handler(handler);- self.context.command_buffer.borrow().submit();+ self.context.command_buffer.add_completion_handler(handler);+ self.context.command_buffer.submit();if should_capture {- self.context.command_buffer.borrow().wait_until_completed();+ self.context.command_buffer.wait_until_completed();self.gpu_capture.stop_capture(&self.context.context, "decode").map_err(|_| Error::CaptureFailed)?;}@@ -780,43 +777,44 @@ impl<B: Backend> LanguageModelGenerator<B> {self.context.reset_command_buffer();_ = task.build_encoded_task(- &self.context,+ &mut self.context,&mut state,&EncodingParameters::new(warmup, true, false),encoded_task_key.clone(),);}- let command_buffer = self.context.command_buffer.clone();-if !warmup {if !task.is_prefilling {self.context.gpu_sampler.encode(&mut state,&EncodingParameters::new(warmup, true, false),- self.context.command_buffer.borrow_mut().deref_mut(),+ &mut self.context.command_buffer,);}}- command_buffer.borrow().submit();+ self.context.command_buffer.submit();if allow_pre_encode {- self.context.reset_command_buffer();+ let in_flight_command_buffer = self.context.reset_command_buffer();let next_task_key: String = task.encoded_task_key(self.tokens.len() + 1);let next_encoded_task = task.build_encoded_task(- &self.context,+ &mut self.context,&mut state,&EncodingParameters::new(warmup, false, false).with_projection(1),next_task_key.clone(),);self.encoded_tasks.insert(next_task_key.clone(), next_encoded_task);++ in_flight_command_buffer.wait_until_completed();+ } else {+ self.context.command_buffer.wait_until_completed();}- command_buffer.borrow().wait_until_completed();let run_time = run_start.elapsed().as_secs_f64();if should_capture {self.gpu_capturediff --git a/crates/uzu/src/language_model/language_model_generator_context.rs b/crates/uzu/src/language_model/language_model_generator_context.rsindex 7f7166b05..b5d576b0d 100644--- a/crates/uzu/src/language_model/language_model_generator_context.rs+++ b/crates/uzu/src/language_model/language_model_generator_context.rs@@ -122,7 +122,7 @@ impl<B: Backend> AsyncBuffers<B> {pub struct LanguageModelGeneratorContext<B: Backend> {pub context: Rc<B::Context>,- pub command_buffer: Rc<RefCell<B::CommandBuffer>>,+ pub command_buffer: B::CommandBuffer,pub cache_layers: Rc<RefCell<CacheLayers<B>>>,pub shared_buffers: Rc<RefCell<SharedBuffers<B>>>,@@ -151,8 +151,7 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {decoding_config: &DecodingConfig,) -> Result<Self, Error> {let context = B::Context::new().map_err(|_| Error::UnableToCreateBackendContext)?;- let command_buffer =- Rc::new(RefCell::new(context.create_command_buffer().expect("Failed to create command buffer")));+ let command_buffer = context.create_command_buffer().expect("Failed to create command buffer");let config_path = model_path.join("config.json");if !config_path.exists() {@@ -248,8 +247,10 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {Ok(context)}- pub fn reset_command_buffer(&mut self) {- self.command_buffer =- Rc::new(RefCell::new(self.context.create_command_buffer().expect("Failed to create command buffer")));+ pub fn reset_command_buffer(&mut self) -> B::CommandBuffer {+ std::mem::replace(+ &mut self.command_buffer,+ self.context.create_command_buffer().expect("Failed to create command buffer"),+ )}}diff --git a/crates/uzu/src/language_model/tasks.rs b/crates/uzu/src/language_model/tasks.rsindex 293afef74..f8395b963 100644--- a/crates/uzu/src/language_model/tasks.rs+++ b/crates/uzu/src/language_model/tasks.rs@@ -1,4 +1,4 @@-use std::{mem::size_of, ops::DerefMut};+use std::mem::size_of;use super::language_model_generator_context::LanguageModelGeneratorContext;use crate::{@@ -109,12 +109,12 @@ impl<'a> LanguageModelGeneratorRunTask<'a> {pub fn build_encoded_task<B: Backend>(&self,- context: &LanguageModelGeneratorContext<B>,+ context: &mut LanguageModelGeneratorContext<B>,state: &mut ForwardPassState<B>,parameters: &EncodingParameters<B>,key: String,) -> LanguageModelGeneratorEncodedTask<B> {- context.executables.encode(state, parameters, context.command_buffer.borrow_mut().deref_mut());+ context.executables.encode(state, parameters, &mut context.command_buffer);LanguageModelGeneratorEncodedTask {key,