Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

owned commandbuffer (#230)

uuuvnf06b0a6fd4b0
diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rs
index fad69fc8f..af2fc72a4 100644
--- a/crates/uzu/src/language_model/language_model_generator.rs
+++ b/crates/uzu/src/language_model/language_model_generator.rs
@@ -2,7 +2,7 @@ use std::{
any::Any,
collections::HashMap,
iter::repeat_n,
- ops::{Deref, DerefMut, Range},
+ ops::{Deref, Range},
path::Path,
sync::Arc,
time::Instant,
@@ -492,23 +492,20 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
// Wait on previous pass if this is a continuation
if is_continuation {
- self.context
- .command_buffer
- .borrow_mut()
- .encode_wait_for_event(&self.context.async_buffers.event, current_counter);
+ self.context.command_buffer.encode_wait_for_event(&self.context.async_buffers.event, current_counter);
}
self.context.executables.encode(
&mut state,
&EncodingParameters::new(false, false, false),
- self.context.command_buffer.borrow_mut().deref_mut(),
+ &mut self.context.command_buffer,
);
// Encode sampling
self.context.gpu_sampler.encode(
&mut state,
&EncodingParameters::new(false, false, false),
- self.context.command_buffer.borrow_mut().deref_mut(),
+ &mut self.context.command_buffer,
);
// Copy sampled token: sampling_output → token_ids (for next pass)
@@ -521,7 +518,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
let token_ids_buf_rc = token_ids_binding.buffer();
let token_ids_buf_borrow = token_ids_buf_rc.borrow();
- self.context.command_buffer.borrow_mut().with_compute_encoder(|encoder| {
+ self.context.command_buffer.with_compute_encoder(|encoder| {
self.context.token_copy_sampled.encode(
sampling_output_buf_borrow.deref(),
token_ids_buf_borrow.deref(),
@@ -539,12 +536,12 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
self.context.cache_layers.borrow_mut().update_after_acceptance(
&[0],
None,
- self.context.command_buffer.borrow_mut().deref_mut(),
+ &mut self.context.command_buffer,
&self.context.kv_cache_update,
);
self.context.cache_layers.borrow_mut().register_accepted_tokens(&[token_position]);
- self.context.command_buffer.borrow_mut().with_compute_encoder(|encoder| {
+ self.context.command_buffer.with_compute_encoder(|encoder| {
if let Some(mask_update) = &self.context.mask_update {
let updates: Vec<AttentionBiasUpdate> =
self.context.cache_layers.borrow().attention_bias_updates_after_acceptance(1);
@@ -562,7 +559,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
// Signal event for next pass
let next_counter = current_counter + 1;
- self.context.command_buffer.borrow_mut().encode_signal_event(&self.context.async_buffers.event, next_counter);
+ self.context.command_buffer.encode_signal_event(&self.context.async_buffers.event, next_counter);
self.context.async_buffers.counter.set(next_counter);
// Add completion handler
@@ -579,11 +576,11 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
}
};
- self.context.command_buffer.borrow_mut().add_completion_handler(handler);
- self.context.command_buffer.borrow().submit();
+ self.context.command_buffer.add_completion_handler(handler);
+ self.context.command_buffer.submit();
if should_capture {
- self.context.command_buffer.borrow().wait_until_completed();
+ self.context.command_buffer.wait_until_completed();
self.gpu_capture.stop_capture(&self.context.context, "decode").map_err(|_| Error::CaptureFailed)?;
}
@@ -780,43 +777,44 @@ impl<B: Backend> LanguageModelGenerator<B> {
self.context.reset_command_buffer();
_ = task.build_encoded_task(
- &self.context,
+ &mut self.context,
&mut state,
&EncodingParameters::new(warmup, true, false),
encoded_task_key.clone(),
);
}
- let command_buffer = self.context.command_buffer.clone();
-
if !warmup {
if !task.is_prefilling {
self.context.gpu_sampler.encode(
&mut state,
&EncodingParameters::new(warmup, true, false),
- self.context.command_buffer.borrow_mut().deref_mut(),
+ &mut self.context.command_buffer,
);
}
}
- command_buffer.borrow().submit();
+ self.context.command_buffer.submit();
if allow_pre_encode {
- self.context.reset_command_buffer();
+ let in_flight_command_buffer = self.context.reset_command_buffer();
let next_task_key: String = task.encoded_task_key(self.tokens.len() + 1);
let next_encoded_task = task.build_encoded_task(
- &self.context,
+ &mut self.context,
&mut state,
&EncodingParameters::new(warmup, false, false).with_projection(1),
next_task_key.clone(),
);
self.encoded_tasks.insert(next_task_key.clone(), next_encoded_task);
+
+ in_flight_command_buffer.wait_until_completed();
+ } else {
+ self.context.command_buffer.wait_until_completed();
}
- command_buffer.borrow().wait_until_completed();
let run_time = run_start.elapsed().as_secs_f64();
if should_capture {
self.gpu_capture
diff --git a/crates/uzu/src/language_model/language_model_generator_context.rs b/crates/uzu/src/language_model/language_model_generator_context.rs
index 7f7166b05..b5d576b0d 100644
--- a/crates/uzu/src/language_model/language_model_generator_context.rs
+++ b/crates/uzu/src/language_model/language_model_generator_context.rs
@@ -122,7 +122,7 @@ impl<B: Backend> AsyncBuffers<B> {
pub struct LanguageModelGeneratorContext<B: Backend> {
pub context: Rc<B::Context>,
- pub command_buffer: Rc<RefCell<B::CommandBuffer>>,
+ pub command_buffer: B::CommandBuffer,
pub cache_layers: Rc<RefCell<CacheLayers<B>>>,
pub shared_buffers: Rc<RefCell<SharedBuffers<B>>>,
@@ -151,8 +151,7 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {
decoding_config: &DecodingConfig,
) -> Result<Self, Error> {
let context = B::Context::new().map_err(|_| Error::UnableToCreateBackendContext)?;
- let command_buffer =
- Rc::new(RefCell::new(context.create_command_buffer().expect("Failed to create command buffer")));
+ let command_buffer = context.create_command_buffer().expect("Failed to create command buffer");
let config_path = model_path.join("config.json");
if !config_path.exists() {
@@ -248,8 +247,10 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {
Ok(context)
}
- pub fn reset_command_buffer(&mut self) {
- self.command_buffer =
- Rc::new(RefCell::new(self.context.create_command_buffer().expect("Failed to create command buffer")));
+ pub fn reset_command_buffer(&mut self) -> B::CommandBuffer {
+ std::mem::replace(
+ &mut self.command_buffer,
+ self.context.create_command_buffer().expect("Failed to create command buffer"),
+ )
}
}
diff --git a/crates/uzu/src/language_model/tasks.rs b/crates/uzu/src/language_model/tasks.rs
index 293afef74..f8395b963 100644
--- a/crates/uzu/src/language_model/tasks.rs
+++ b/crates/uzu/src/language_model/tasks.rs
@@ -1,4 +1,4 @@
-use std::{mem::size_of, ops::DerefMut};
+use std::mem::size_of;
use super::language_model_generator_context::LanguageModelGeneratorContext;
use crate::{
@@ -109,12 +109,12 @@ impl<'a> LanguageModelGeneratorRunTask<'a> {
pub fn build_encoded_task<B: Backend>(
&self,
- context: &LanguageModelGeneratorContext<B>,
+ context: &mut LanguageModelGeneratorContext<B>,
state: &mut ForwardPassState<B>,
parameters: &EncodingParameters<B>,
key: String,
) -> LanguageModelGeneratorEncodedTask<B> {
- context.executables.encode(state, parameters, context.command_buffer.borrow_mut().deref_mut());
+ context.executables.encode(state, parameters, &mut context.command_buffer);
LanguageModelGeneratorEncodedTask {
key,