Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

CommandBuffer refactoring (#223)

Aleksandr Golokozf0b5a8d64aed
diff --git a/crates/uzu/src/backends/common/command_buffer.rs b/crates/uzu/src/backends/common/command_buffer.rs
index ad3959698..363d77d86 100644
--- a/crates/uzu/src/backends/common/command_buffer.rs
+++ b/crates/uzu/src/backends/common/command_buffer.rs
@@ -1,6 +1,6 @@
use super::Backend;
-pub trait CommandBuffer: Clone {
+pub trait CommandBuffer {
type Backend: Backend;
fn with_compute_encoder<T>(
@@ -14,23 +14,24 @@ pub trait CommandBuffer: Clone {
) -> T;
fn encode_wait_for_event(
- &self,
+ &mut self,
event: &<Self::Backend as Backend>::Event,
value: u64,
);
fn encode_signal_event(
- &self,
+ &mut self,
event: &<Self::Backend as Backend>::Event,
value: u64,
);
- fn add_completed_handler(
- &self,
+ fn add_completion_handler(
+ &mut self,
handler: impl Fn() + 'static,
);
fn submit(&self);
+
fn wait_until_completed(&self);
fn gpu_execution_time_ms(&self) -> Option<f64>;
diff --git a/crates/uzu/src/backends/metal/command_buffer.rs b/crates/uzu/src/backends/metal/command_buffer.rs
index bf564978c..ab89ccd33 100644
--- a/crates/uzu/src/backends/metal/command_buffer.rs
+++ b/crates/uzu/src/backends/metal/command_buffer.rs
@@ -36,7 +36,7 @@ impl CommandBuffer for Retained<ProtocolObject<dyn MTLCommandBuffer>> {
}
fn encode_wait_for_event(
- &self,
+ &mut self,
event: &Retained<ProtocolObject<dyn MTLEvent>>,
value: u64,
) {
@@ -44,15 +44,15 @@ impl CommandBuffer for Retained<ProtocolObject<dyn MTLCommandBuffer>> {
}
fn encode_signal_event(
- &self,
+ &mut self,
event: &Retained<ProtocolObject<dyn MTLEvent>>,
value: u64,
) {
self.encode_signal_event_value(event, value);
}
- fn add_completed_handler(
- &self,
+ fn add_completion_handler(
+ &mut self,
handler: impl Fn() + 'static,
) {
self.deref().add_completed_handler(&MTLCommandBufferHandler::new(move |_| handler()));
diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rs
index ec12566c8..31450f93d 100644
--- a/crates/uzu/src/language_model/language_model_generator.rs
+++ b/crates/uzu/src/language_model/language_model_generator.rs
@@ -1,4 +1,4 @@
-use std::{collections::HashMap, iter::repeat_n, path::Path, sync::Arc, time::Instant};
+use std::{collections::HashMap, iter::repeat_n, ops::Deref, path::Path, sync::Arc, time::Instant};
use itertools::{Either, Itertools, izip};
@@ -456,24 +456,26 @@ where
}
self.context.reset_command_buffer();
- let root_command_buffer = &self.context.command_buffer;
// Wait on previous pass if this is a continuation
if is_continuation {
- root_command_buffer.encode_wait_for_event(&self.context.async_buffers.event, current_counter);
+ self.context
+ .command_buffer
+ .borrow_mut()
+ .encode_wait_for_event(&self.context.async_buffers.event, current_counter);
}
self.context.executables.encode(
&mut state,
&EncodingParameters::new(false, false, false),
- &root_command_buffer,
+ self.context.command_buffer.borrow().deref(),
);
// Encode sampling
self.context.gpu_sampler.encode(
&mut state,
&EncodingParameters::new(false, false, false),
- &root_command_buffer,
+ self.context.command_buffer.borrow().deref(),
);
// Copy sampled token: sampling_output → token_ids (for next pass)
@@ -484,7 +486,7 @@ where
let token_ids_binding = self.context.scratch_buffers.token_ids.borrow();
let token_ids_buffer = token_ids_binding.buffer();
- root_command_buffer.with_compute_encoder(|encoder| {
+ self.context.command_buffer.borrow().with_compute_encoder(|encoder| {
self.context.token_copy_sampled.encode(sampling_output_buffer, token_ids_buffer, encoder);
let results_offset = slot * std::mem::size_of::<u32>();
self.context.token_copy_results.encode(
@@ -498,12 +500,12 @@ where
self.context.cache_layers.borrow_mut().update_after_acceptance(
&[0],
None,
- &root_command_buffer,
+ self.context.command_buffer.borrow().deref(),
&self.context.kv_cache_update,
);
self.context.cache_layers.borrow_mut().register_accepted_tokens(&[token_position]);
- root_command_buffer.with_compute_encoder(|encoder| {
+ self.context.command_buffer.borrow().with_compute_encoder(|encoder| {
if let Some(mask_update) = &self.context.mask_update {
let updates: Vec<AttentionBiasUpdate> =
self.context.cache_layers.borrow().attention_bias_updates_after_acceptance(1);
@@ -524,7 +526,7 @@ where
// Signal event for next pass
let next_counter = current_counter + 1;
- root_command_buffer.encode_signal_event(&self.context.async_buffers.event, next_counter);
+ self.context.command_buffer.borrow_mut().encode_signal_event(&self.context.async_buffers.event, next_counter);
self.context.async_buffers.counter.set(next_counter);
// Add completion handler
@@ -541,11 +543,11 @@ where
}
};
- root_command_buffer.add_completed_handler(handler);
- root_command_buffer.submit();
+ self.context.command_buffer.borrow_mut().add_completion_handler(handler);
+ self.context.command_buffer.borrow().submit();
if should_capture {
- root_command_buffer.wait_until_completed();
+ self.context.command_buffer.borrow().wait_until_completed();
self.gpu_capture.stop_capture(&self.context.context, "decode").map_err(|_| Error::CaptureFailed)?;
}
@@ -643,19 +645,19 @@ where
);
}
- let root_command_buffer = self.context.command_buffer.clone();
+ let command_buffer = self.context.command_buffer.clone();
if !warmup {
if !task.is_prefilling {
self.context.gpu_sampler.encode(
&mut state,
&EncodingParameters::new(warmup, true, false),
- &self.context.command_buffer,
+ &self.context.command_buffer.as_ref().borrow(),
);
}
}
- root_command_buffer.submit();
+ command_buffer.borrow().submit();
if allow_pre_encode {
self.context.reset_command_buffer();
@@ -672,7 +674,7 @@ where
self.encoded_tasks.insert(next_task_key.clone(), next_encoded_task);
}
- root_command_buffer.wait_until_completed();
+ command_buffer.borrow().wait_until_completed();
let run_time = run_start.elapsed().as_secs_f64();
if should_capture {
self.gpu_capture
@@ -712,14 +714,13 @@ where
wait_until_completed: bool,
) {
let command_buffer = self.context.context.create_command_buffer().expect("Failed to create command buffer");
- let root_command_buffer = &command_buffer;
{
let mut cache_layers = self.context.cache_layers.borrow_mut();
cache_layers.update_after_acceptance(
accepted_token_indices,
suffix_start,
- &root_command_buffer,
+ &command_buffer,
&self.context.kv_cache_update,
);
}
diff --git a/crates/uzu/src/language_model/language_model_generator_context.rs b/crates/uzu/src/language_model/language_model_generator_context.rs
index cf20f3a9a..1e97a1207 100644
--- a/crates/uzu/src/language_model/language_model_generator_context.rs
+++ b/crates/uzu/src/language_model/language_model_generator_context.rs
@@ -125,7 +125,7 @@ impl<B: Backend> AsyncBuffers<B> {
pub struct LanguageModelGeneratorContext<B: Backend> {
pub context: Rc<B::Context>,
- pub command_buffer: B::CommandBuffer,
+ pub command_buffer: Rc<RefCell<B::CommandBuffer>>,
pub cache_layers: Rc<RefCell<CacheLayers<B>>>,
pub shared_buffers: Rc<RefCell<SharedBuffers<B>>>,
@@ -157,8 +157,8 @@ where
decoding_config: &DecodingConfig,
) -> Result<Self, Error> {
let context = B::Context::new().map_err(|_| Error::UnableToCreateBackendContext)?;
-
- let command_buffer = context.create_command_buffer().expect("Failed to create command buffer");
+ let command_buffer =
+ Rc::new(RefCell::new(context.create_command_buffer().expect("Failed to create command buffer")));
let config_path = model_path.join("config.json");
if !config_path.exists() {
@@ -255,6 +255,7 @@ where
}
pub fn reset_command_buffer(&mut self) {
- self.command_buffer = self.context.create_command_buffer().expect("Failed to create command buffer");
+ self.command_buffer =
+ Rc::new(RefCell::new(self.context.create_command_buffer().expect("Failed to create command buffer")));
}
}
diff --git a/crates/uzu/src/language_model/tasks.rs b/crates/uzu/src/language_model/tasks.rs
index 681bb6db0..e34b9ac29 100644
--- a/crates/uzu/src/language_model/tasks.rs
+++ b/crates/uzu/src/language_model/tasks.rs
@@ -1,4 +1,4 @@
-use std::mem::size_of;
+use std::{mem::size_of, ops::Deref};
use super::LanguageModelGeneratorContext;
use crate::{
@@ -114,7 +114,7 @@ impl<'a> LanguageModelGeneratorRunTask<'a> {
parameters: &EncodingParameters<B>,
key: String,
) -> LanguageModelGeneratorEncodedTask<B> {
- context.executables.encode(state, parameters, &context.command_buffer);
+ context.executables.encode(state, parameters, context.command_buffer.borrow().deref());
LanguageModelGeneratorEncodedTask {
key,