A high-performance inference engine for AI models
CommandBuffer refactoring (#223)
Aleksandr Golokozf0b5a8d64aed
diff --git a/crates/uzu/src/backends/common/command_buffer.rs b/crates/uzu/src/backends/common/command_buffer.rsindex ad3959698..363d77d86 100644--- a/crates/uzu/src/backends/common/command_buffer.rs+++ b/crates/uzu/src/backends/common/command_buffer.rs@@ -1,6 +1,6 @@use super::Backend;-pub trait CommandBuffer: Clone {+pub trait CommandBuffer {type Backend: Backend;fn with_compute_encoder<T>(@@ -14,23 +14,24 @@ pub trait CommandBuffer: Clone {) -> T;fn encode_wait_for_event(- &self,+ &mut self,event: &<Self::Backend as Backend>::Event,value: u64,);fn encode_signal_event(- &self,+ &mut self,event: &<Self::Backend as Backend>::Event,value: u64,);- fn add_completed_handler(- &self,+ fn add_completion_handler(+ &mut self,handler: impl Fn() + 'static,);fn submit(&self);+fn wait_until_completed(&self);fn gpu_execution_time_ms(&self) -> Option<f64>;diff --git a/crates/uzu/src/backends/metal/command_buffer.rs b/crates/uzu/src/backends/metal/command_buffer.rsindex bf564978c..ab89ccd33 100644--- a/crates/uzu/src/backends/metal/command_buffer.rs+++ b/crates/uzu/src/backends/metal/command_buffer.rs@@ -36,7 +36,7 @@ impl CommandBuffer for Retained<ProtocolObject<dyn MTLCommandBuffer>> {}fn encode_wait_for_event(- &self,+ &mut self,event: &Retained<ProtocolObject<dyn MTLEvent>>,value: u64,) {@@ -44,15 +44,15 @@ impl CommandBuffer for Retained<ProtocolObject<dyn MTLCommandBuffer>> {}fn encode_signal_event(- &self,+ &mut self,event: &Retained<ProtocolObject<dyn MTLEvent>>,value: u64,) {self.encode_signal_event_value(event, value);}- fn add_completed_handler(- &self,+ fn add_completion_handler(+ &mut self,handler: impl Fn() + 'static,) {self.deref().add_completed_handler(&MTLCommandBufferHandler::new(move |_| handler()));diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rsindex ec12566c8..31450f93d 100644--- a/crates/uzu/src/language_model/language_model_generator.rs+++ b/crates/uzu/src/language_model/language_model_generator.rs@@ -1,4 +1,4 @@-use std::{collections::HashMap, iter::repeat_n, path::Path, sync::Arc, time::Instant};+use std::{collections::HashMap, iter::repeat_n, ops::Deref, path::Path, sync::Arc, time::Instant};use itertools::{Either, Itertools, izip};@@ -456,24 +456,26 @@ where}self.context.reset_command_buffer();- let root_command_buffer = &self.context.command_buffer;// Wait on previous pass if this is a continuationif is_continuation {- root_command_buffer.encode_wait_for_event(&self.context.async_buffers.event, current_counter);+ self.context+ .command_buffer+ .borrow_mut()+ .encode_wait_for_event(&self.context.async_buffers.event, current_counter);}self.context.executables.encode(&mut state,&EncodingParameters::new(false, false, false),- &root_command_buffer,+ self.context.command_buffer.borrow().deref(),);// Encode samplingself.context.gpu_sampler.encode(&mut state,&EncodingParameters::new(false, false, false),- &root_command_buffer,+ self.context.command_buffer.borrow().deref(),);// Copy sampled token: sampling_output → token_ids (for next pass)@@ -484,7 +486,7 @@ wherelet token_ids_binding = self.context.scratch_buffers.token_ids.borrow();let token_ids_buffer = token_ids_binding.buffer();- root_command_buffer.with_compute_encoder(|encoder| {+ self.context.command_buffer.borrow().with_compute_encoder(|encoder| {self.context.token_copy_sampled.encode(sampling_output_buffer, token_ids_buffer, encoder);let results_offset = slot * std::mem::size_of::<u32>();self.context.token_copy_results.encode(@@ -498,12 +500,12 @@ whereself.context.cache_layers.borrow_mut().update_after_acceptance(&[0],None,- &root_command_buffer,+ self.context.command_buffer.borrow().deref(),&self.context.kv_cache_update,);self.context.cache_layers.borrow_mut().register_accepted_tokens(&[token_position]);- root_command_buffer.with_compute_encoder(|encoder| {+ self.context.command_buffer.borrow().with_compute_encoder(|encoder| {if let Some(mask_update) = &self.context.mask_update {let updates: Vec<AttentionBiasUpdate> =self.context.cache_layers.borrow().attention_bias_updates_after_acceptance(1);@@ -524,7 +526,7 @@ where// Signal event for next passlet next_counter = current_counter + 1;- root_command_buffer.encode_signal_event(&self.context.async_buffers.event, next_counter);+ self.context.command_buffer.borrow_mut().encode_signal_event(&self.context.async_buffers.event, next_counter);self.context.async_buffers.counter.set(next_counter);// Add completion handler@@ -541,11 +543,11 @@ where}};- root_command_buffer.add_completed_handler(handler);- root_command_buffer.submit();+ self.context.command_buffer.borrow_mut().add_completion_handler(handler);+ self.context.command_buffer.borrow().submit();if should_capture {- root_command_buffer.wait_until_completed();+ self.context.command_buffer.borrow().wait_until_completed();self.gpu_capture.stop_capture(&self.context.context, "decode").map_err(|_| Error::CaptureFailed)?;}@@ -643,19 +645,19 @@ where);}- let root_command_buffer = self.context.command_buffer.clone();+ let command_buffer = self.context.command_buffer.clone();if !warmup {if !task.is_prefilling {self.context.gpu_sampler.encode(&mut state,&EncodingParameters::new(warmup, true, false),- &self.context.command_buffer,+ &self.context.command_buffer.as_ref().borrow(),);}}- root_command_buffer.submit();+ command_buffer.borrow().submit();if allow_pre_encode {self.context.reset_command_buffer();@@ -672,7 +674,7 @@ whereself.encoded_tasks.insert(next_task_key.clone(), next_encoded_task);}- root_command_buffer.wait_until_completed();+ command_buffer.borrow().wait_until_completed();let run_time = run_start.elapsed().as_secs_f64();if should_capture {self.gpu_capture@@ -712,14 +714,13 @@ wherewait_until_completed: bool,) {let command_buffer = self.context.context.create_command_buffer().expect("Failed to create command buffer");- let root_command_buffer = &command_buffer;{let mut cache_layers = self.context.cache_layers.borrow_mut();cache_layers.update_after_acceptance(accepted_token_indices,suffix_start,- &root_command_buffer,+ &command_buffer,&self.context.kv_cache_update,);}diff --git a/crates/uzu/src/language_model/language_model_generator_context.rs b/crates/uzu/src/language_model/language_model_generator_context.rsindex cf20f3a9a..1e97a1207 100644--- a/crates/uzu/src/language_model/language_model_generator_context.rs+++ b/crates/uzu/src/language_model/language_model_generator_context.rs@@ -125,7 +125,7 @@ impl<B: Backend> AsyncBuffers<B> {pub struct LanguageModelGeneratorContext<B: Backend> {pub context: Rc<B::Context>,- pub command_buffer: B::CommandBuffer,+ pub command_buffer: Rc<RefCell<B::CommandBuffer>>,pub cache_layers: Rc<RefCell<CacheLayers<B>>>,pub shared_buffers: Rc<RefCell<SharedBuffers<B>>>,@@ -157,8 +157,8 @@ wheredecoding_config: &DecodingConfig,) -> Result<Self, Error> {let context = B::Context::new().map_err(|_| Error::UnableToCreateBackendContext)?;-- let command_buffer = context.create_command_buffer().expect("Failed to create command buffer");+ let command_buffer =+ Rc::new(RefCell::new(context.create_command_buffer().expect("Failed to create command buffer")));let config_path = model_path.join("config.json");if !config_path.exists() {@@ -255,6 +255,7 @@ where}pub fn reset_command_buffer(&mut self) {- self.command_buffer = self.context.create_command_buffer().expect("Failed to create command buffer");+ self.command_buffer =+ Rc::new(RefCell::new(self.context.create_command_buffer().expect("Failed to create command buffer")));}}diff --git a/crates/uzu/src/language_model/tasks.rs b/crates/uzu/src/language_model/tasks.rsindex 681bb6db0..e34b9ac29 100644--- a/crates/uzu/src/language_model/tasks.rs+++ b/crates/uzu/src/language_model/tasks.rs@@ -1,4 +1,4 @@-use std::mem::size_of;+use std::{mem::size_of, ops::Deref};use super::LanguageModelGeneratorContext;use crate::{@@ -114,7 +114,7 @@ impl<'a> LanguageModelGeneratorRunTask<'a> {parameters: &EncodingParameters<B>,key: String,) -> LanguageModelGeneratorEncodedTask<B> {- context.executables.encode(state, parameters, &context.command_buffer);+ context.executables.encode(state, parameters, context.command_buffer.borrow().deref());LanguageModelGeneratorEncodedTask {key,