Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

small (LanguageModel/Classifier)Context cleanup (#239)

uuuvn3817d86c6d8b
diff --git a/crates/uzu/src/classifier/classifier.rs b/crates/uzu/src/classifier/classifier.rs
index 6c5bf3e02..fc2295746 100644
--- a/crates/uzu/src/classifier/classifier.rs
+++ b/crates/uzu/src/classifier/classifier.rs
@@ -9,7 +9,9 @@ use super::ActivationTrace;
use super::{ClassificationOutput, ClassificationStats, ClassifierContext};
use crate::{
DataType,
- backends::common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending},
+ backends::common::{
+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,
+ },
encodable_block::{EncodableBlock, EncodingParameters},
forward_pass::state::{ArrayId, ForwardPassState},
session::types::Error,
@@ -113,55 +115,53 @@ impl<B: Backend> Classifier<B> {
let encoding_params = EncodingParameters::new();
+ let mut command_buffer = self
+ .context
+ .context
+ .create_command_buffer()
+ .map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?
+ .start_encoding();
+
self.context
.embed
- .encode(&mut state, &encoding_params, &mut self.context.command_buffer)
+ .encode(&mut state, &encoding_params, &mut command_buffer)
.map_err(|e| Error::EncodeFailed(Box::new(e)))?;
self.context
.embedding_norm
- .encode(&mut state, &encoding_params, &mut self.context.command_buffer)
+ .encode(&mut state, &encoding_params, &mut command_buffer)
.map_err(|e| Error::EncodeFailed(Box::new(e)))?;
#[cfg(feature = "tracing")]
{
let traces = state.traces().clone();
- state.encode_copy_array(
- &mut self.context.command_buffer,
- ArrayId::Main,
- traces.borrow().embedding_norm().clone(),
- );
+ state.encode_copy_array(&mut command_buffer, ArrayId::Main, traces.borrow().embedding_norm().clone());
}
for layer in self.context.layers.iter() {
layer
- .encode(&mut state, &encoding_params, &mut self.context.command_buffer)
+ .encode(&mut state, &encoding_params, &mut command_buffer)
.map_err(|e| Error::EncodeFailed(Box::new(e)))?;
}
self.context
.output_norm
- .encode(&mut state, &encoding_params, &mut self.context.command_buffer)
+ .encode(&mut state, &encoding_params, &mut command_buffer)
.map_err(|e| Error::EncodeFailed(Box::new(e)))?;
#[cfg(feature = "tracing")]
{
let traces = state.traces().clone();
- state.encode_copy_array(
- &mut self.context.command_buffer,
- ArrayId::Main,
- traces.borrow().output_norm.clone(),
- );
+ state.encode_copy_array(&mut command_buffer, ArrayId::Main, traces.borrow().output_norm.clone());
}
self.context
.pooling
- .encode(&mut state, &encoding_params, &mut self.context.command_buffer)
+ .encode(&mut state, &encoding_params, &mut command_buffer)
.map_err(|e| Error::EncodeFailed(Box::new(e)))?;
self.context
.prediction_head
- .encode(&mut state, &encoding_params, &mut self.context.command_buffer)
+ .encode(&mut state, &encoding_params, &mut command_buffer)
.map_err(|e| Error::EncodeFailed(Box::new(e)))?;
- self.context
- .replace_command_buffer()
+ command_buffer
.end_encoding()
.submit()
.wait_until_completed()
@@ -195,34 +195,40 @@ impl<B: Backend> Classifier<B> {
let encoding_params = EncodingParameters::new();
+ let mut command_buffer = self
+ .context
+ .context
+ .create_command_buffer()
+ .map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?
+ .start_encoding();
+
self.context
.embed
- .encode(&mut state, &encoding_params, &mut self.context.command_buffer)
+ .encode(&mut state, &encoding_params, &mut command_buffer)
.map_err(|e| Error::EncodeFailed(Box::new(e)))?;
self.context
.embedding_norm
- .encode(&mut state, &encoding_params, &mut self.context.command_buffer)
+ .encode(&mut state, &encoding_params, &mut command_buffer)
.map_err(|e| Error::EncodeFailed(Box::new(e)))?;
for layer in self.context.layers.iter() {
layer
- .encode(&mut state, &encoding_params, &mut self.context.command_buffer)
+ .encode(&mut state, &encoding_params, &mut command_buffer)
.map_err(|e| Error::EncodeFailed(Box::new(e)))?;
}
self.context
.output_norm
- .encode(&mut state, &encoding_params, &mut self.context.command_buffer)
+ .encode(&mut state, &encoding_params, &mut command_buffer)
.map_err(|e| Error::EncodeFailed(Box::new(e)))?;
self.context
.pooling
- .encode(&mut state, &encoding_params, &mut self.context.command_buffer)
+ .encode(&mut state, &encoding_params, &mut command_buffer)
.map_err(|e| Error::EncodeFailed(Box::new(e)))?;
self.context
.prediction_head
- .encode(&mut state, &encoding_params, &mut self.context.command_buffer)
+ .encode(&mut state, &encoding_params, &mut command_buffer)
.map_err(|e| Error::EncodeFailed(Box::new(e)))?;
- self.context
- .replace_command_buffer()
+ command_buffer
.end_encoding()
.submit()
.wait_until_completed()
diff --git a/crates/uzu/src/classifier/classifier_context.rs b/crates/uzu/src/classifier/classifier_context.rs
index 4e73fa3c7..3edbfd5d1 100644
--- a/crates/uzu/src/classifier/classifier_context.rs
+++ b/crates/uzu/src/classifier/classifier_context.rs
@@ -2,7 +2,7 @@ use std::{cell::RefCell, fs::File, io::BufReader, path::Path, rc::Rc};
use crate::{
DataType,
- backends::common::{Backend, CommandBuffer, CommandBufferInitial, Context, Kernels, kernel::SigmoidKernel},
+ backends::common::{Backend, Context, Kernels, kernel::SigmoidKernel},
classifier::ClassifierError,
config::{ClassifierModelConfig, ModelMetadata},
encodable_block::{
@@ -20,7 +20,6 @@ use crate::{
pub struct ClassifierContext<B: Backend> {
pub context: Rc<B::Context>,
- pub command_buffer: <B::CommandBuffer as CommandBuffer>::Encoding,
pub shared_buffers: Rc<RefCell<SharedBuffers<B>>>,
pub scratch_buffers: ScratchBuffers<B>,
@@ -43,10 +42,7 @@ pub struct ClassifierContext<B: Backend> {
impl<B: Backend> ClassifierContext<B> {
pub fn new(model_path: &Path) -> Result<Self, Error> {
- let context = B::Context::new().map_err(|_| Error::UnableToCreateBackendContext)?;
-
- let command_buffer =
- context.create_command_buffer().map_err(|_| Error::UnableToCreateBackendContext)?.start_encoding();
+ let context = B::Context::new().map_err(|e| Error::UnableToCreateContext(e.into()))?;
let config_path = model_path.join("config.json");
if !config_path.exists() {
@@ -254,7 +250,7 @@ impl<B: Backend> ClassifierContext<B> {
let sigmoid_kernel = <B::Kernels as Kernels>::SigmoidKernel::new(&context, data_type.into()).map_err(|e| {
eprintln!("Failed to create sigmoid kernel: {:?}", e);
- Error::UnableToCreateBackendContext
+ Error::UnableToCreateContext(e.into())
})?;
let pooling = Box::new(
@@ -266,7 +262,7 @@ impl<B: Backend> ClassifierContext<B> {
)
.map_err(|e| {
eprintln!("Failed to create pooling: {:?}", e);
- Error::UnableToCreateBackendContext
+ Error::UnableToCreateContext(e.into())
})?,
);
@@ -282,7 +278,6 @@ impl<B: Backend> ClassifierContext<B> {
Ok(Self {
context,
- command_buffer,
shared_buffers,
scratch_buffers,
model_config: classifier_model_config.clone(),
@@ -310,11 +305,4 @@ impl<B: Backend> ClassifierContext<B> {
);
Ok(Rc::new(rotation))
}
-
- pub fn replace_command_buffer(&mut self) -> <B::CommandBuffer as CommandBuffer>::Encoding {
- std::mem::replace(
- &mut self.command_buffer,
- self.context.create_command_buffer().expect("Failed to create command buffer").start_encoding(),
- )
- }
}
diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rs
index 8a3685efd..87e31a7e7 100644
--- a/crates/uzu/src/language_model/language_model_generator.rs
+++ b/crates/uzu/src/language_model/language_model_generator.rs
@@ -1,6 +1,5 @@
use std::{
any::Any,
- collections::HashMap,
iter::repeat_n,
ops::{Deref, DerefMut, Range},
path::Path,
@@ -15,7 +14,6 @@ use super::{
language_model_generator_context::LanguageModelGeneratorContext,
result::{GenerateResult, PrefillResult},
rng::PRng,
- tasks::{LanguageModelGeneratorEncodedTask, LanguageModelGeneratorRunTask},
};
use crate::{
backends::common::{
@@ -38,12 +36,38 @@ use crate::{
trie::{TrieCreationConfig, TrieNode},
};
+#[derive(Debug, Clone)]
+struct Task<'a> {
+ token_ids: &'a [u64],
+ token_positions: &'a [usize],
+ token_bitmask: Option<&'a [u32]>,
+ token_seeds: &'a [u64],
+ expected_number_of_new_tokens: usize,
+ active_suffix_length: usize,
+ sampling_start: usize,
+ sampling_length: usize,
+ is_prefilling: bool,
+}
+
+#[derive(Debug, Clone, PartialEq)]
+struct TaskEncodingKey {
+ context_len: usize,
+ batch_size: usize,
+ expected_number_of_new_tokens: usize,
+ active_suffix_len: usize,
+ sampling_method: SamplingMethod,
+ sampling_start: usize,
+ sampling_len: usize,
+ has_bitmask: bool,
+ is_prefilling: bool,
+}
+
pub struct LanguageModelGenerator<B: Backend> {
pub decoding_config: DecodingConfig,
pub tokens: Vec<u64>,
pub context: LanguageModelGeneratorContext<B>,
- encoded_tasks: HashMap<String, LanguageModelGeneratorEncodedTask<B>>,
+ pre_encoded_task: Option<(TaskEncodingKey, <B::CommandBuffer as CommandBuffer>::Executable)>,
registered_prefix_len: usize,
gpu_capture: GpuCaptureManager<B>,
}
@@ -217,7 +241,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
let _ = last_state.take();
});
- let task = LanguageModelGeneratorRunTask {
+ let task = Task {
token_ids: &step_token_ids,
token_positions: &step_token_positions,
token_bitmask: step_token_bitmask.as_deref(),
@@ -279,7 +303,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
forwardpass_durations: run_times,
});
}
- let sampled_tokens = self.sample(&mut final_state)?;
+ let sampled_tokens = self.read_sampling_output(&mut final_state)?;
let last_suffix_start = prefill_step_size * (prefill_steps - 1);
let suffix_root_index = (tokens_length - last_suffix_start) - 1;
@@ -353,7 +377,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
let token_seeds =
flat_trie.token_seeds().chain(repeat_n(0, suffix_length - active_suffix_length)).collect::<Box<[u64]>>();
- let task = LanguageModelGeneratorRunTask {
+ let task = Task {
token_ids: &token_ids,
token_positions: &token_positions,
token_bitmask: token_bitmask.as_deref(),
@@ -368,7 +392,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
let (mut state, run_time) =
self.run_model(task, false, self.allow_pre_encode(), sampling_method, self.should_fill_attention_bias())?;
- let sampled_tokens = self.sample(&mut state)?;
+ let sampled_tokens = self.read_sampling_output(&mut state)?;
let (accepted_tokens, accepted_token_indices) =
flat_trie.accept(&sampled_tokens, compiled_grammar.as_deref_mut());
@@ -439,7 +463,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
*ptr.add(pass_idx) as usize
};
- let task = LanguageModelGeneratorRunTask {
+ let task = Task {
token_ids: &[last_token],
token_positions: &[token_position],
token_bitmask: None,
@@ -488,20 +512,27 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
*sm = Some(sampling_method);
}
+ let mut command_buffer = self
+ .context
+ .context
+ .create_command_buffer()
+ .map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?
+ .start_encoding();
+
// Wait on previous pass if this is a continuation
if is_continuation {
- self.context.command_buffer.encode_wait_for_event(&self.context.async_buffers.event, current_counter);
+ command_buffer.encode_wait_for_event(&self.context.async_buffers.event, current_counter);
}
self.context
.executables
- .encode(&mut state, &EncodingParameters::new(), &mut self.context.command_buffer)
+ .encode(&mut state, &EncodingParameters::new(), &mut command_buffer)
.map_err(|e| Error::EncodeFailed(Box::new(e)))?;
// Encode sampling
self.context
.gpu_sampler
- .encode(&mut state, &EncodingParameters::new(), &mut self.context.command_buffer)
+ .encode(&mut state, &EncodingParameters::new(), &mut command_buffer)
.map_err(|e| Error::EncodeFailed(Box::new(e)))?;
// Copy sampled token: sampling_output → token_ids (for next pass)
@@ -517,20 +548,20 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
self.context.token_copy_sampled.encode(
sampling_output_buf_borrow.deref(),
token_ids_buf_borrow.deref_mut(),
- &mut self.context.command_buffer,
+ &mut command_buffer,
);
let results_offset = slot * std::mem::size_of::<u32>();
self.context.token_copy_results.encode(
sampling_output_buf_borrow.deref(),
(results_buffer.borrow_mut().deref_mut(), results_offset),
- &mut self.context.command_buffer,
+ &mut command_buffer,
);
// Scatter + register for all transformer layers
self.context.cache_layers.borrow_mut().update_after_acceptance(
&[0],
None,
- &mut self.context.command_buffer,
+ &mut command_buffer,
&self.context.kv_cache_update,
);
self.context.cache_layers.borrow_mut().register_accepted_tokens(&[token_position]);
@@ -547,7 +578,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
mask_buf_borrow.deref_mut(),
update.unmask_col,
update.mask_col,
- &mut self.context.command_buffer,
+ &mut command_buffer,
);
}
}
@@ -556,7 +587,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
// Signal event for next pass
let next_counter = current_counter + 1;
- self.context.command_buffer.encode_signal_event(&self.context.async_buffers.event, next_counter);
+ command_buffer.encode_signal_event(&self.context.async_buffers.event, next_counter);
self.context.async_buffers.counter.set(next_counter);
// Add completion handler
@@ -568,9 +599,10 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
on_complete(token);
};
- self.context.command_buffer.add_completion_handler(handler);
+ command_buffer.add_completion_handler(handler);
drop(token_ids_binding);
- let pending = self.context.replace_command_buffer().end_encoding().submit();
+
+ let pending = command_buffer.end_encoding().submit();
if should_capture {
pending.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;
@@ -582,7 +614,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
fn clear_cache(&mut self) {
objc2::rc::autoreleasepool(|_pool| {
- self.encoded_tasks.clear();
+ self.pre_encoded_task = None;
});
}
@@ -590,7 +622,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
self.context.cache_layers.borrow_mut().clear();
self.tokens.clear();
self.registered_prefix_len = 0;
- self.encoded_tasks.clear();
+ self.pre_encoded_task = None;
self.gpu_capture.reset();
let seed = self.decoding_config.sampling_seed.resolve();
@@ -700,7 +732,7 @@ impl<B: Backend> LanguageModelGenerator<B> {
decoding_config,
tokens: Vec::new(),
context,
- encoded_tasks: HashMap::new(),
+ pre_encoded_task: None,
registered_prefix_len: 0,
gpu_capture,
};
@@ -720,7 +752,7 @@ impl<B: Backend> LanguageModelGenerator<B> {
let token_positions: Vec<usize> = (0..suffix_length).collect();
let token_seeds: Vec<u64> = vec![0; suffix_length];
- let task = LanguageModelGeneratorRunTask {
+ let task = Task {
token_ids: &token_ids,
token_positions: &token_positions,
token_bitmask: None,
@@ -738,7 +770,7 @@ impl<B: Backend> LanguageModelGenerator<B> {
fn run_model(
&mut self,
- task: LanguageModelGeneratorRunTask,
+ task: Task,
warmup: bool,
allow_pre_encode: bool,
sampling_method: SamplingMethod,
@@ -747,71 +779,79 @@ impl<B: Backend> LanguageModelGenerator<B> {
objc2::rc::autoreleasepool(|_pool| {
let run_start = Instant::now();
- let mut state = task.create_state(&mut self.context, None, should_fill_attention_bias);
+ let mut state = ForwardPassState::new_llm(
+ self.context.context.clone(),
+ &self.context.decoder_config,
+ &self.context.model_shape,
+ &self.context.scratch_buffers,
+ self.context.cache_layers.clone(),
+ self.context.shared_buffers.clone(),
+ task.token_ids,
+ task.token_positions,
+ task.token_bitmask,
+ task.token_seeds,
+ task.active_suffix_length,
+ task.sampling_start,
+ task.sampling_length,
+ task.is_prefilling,
+ None,
+ false,
+ should_fill_attention_bias,
+ None,
+ None,
+ );
+
if let Some(method) = state.sampling_method_mut() {
*method = Some(sampling_method);
}
+ let encoding_key = TaskEncodingKey {
+ context_len: self.tokens.len(),
+ batch_size: task.token_ids.len(),
+ expected_number_of_new_tokens: task.expected_number_of_new_tokens,
+ active_suffix_len: task.active_suffix_length,
+ sampling_method,
+ sampling_start: task.sampling_start,
+ sampling_len: task.sampling_length,
+ has_bitmask: task.token_bitmask.is_some(),
+ is_prefilling: task.is_prefilling,
+ };
+
let is_first_decode = !warmup && task.token_ids.len() == 1;
let should_capture = self.gpu_capture.should_capture_decode(is_first_decode);
if should_capture {
- let _ = self.gpu_capture.start_capture(&self.context.context, "decode");
+ self.gpu_capture.start_capture(&self.context.context, "decode").map_err(|_| Error::CaptureFailed)?;
+ self.pre_encoded_task = None;
}
- let encoded_task_key = task.encoded_task_key(self.tokens.len());
-
- if should_capture {
- self.encoded_tasks.remove(&encoded_task_key);
- }
+ let sample = !warmup && !task.is_prefilling;
- if let Some(_) = self.encoded_tasks.remove(&encoded_task_key) {
- // Nothing
+ let executable = if let Some((pre_encoded_key, pre_encoded_executable)) = self.pre_encoded_task.take()
+ && pre_encoded_key == encoding_key
+ {
+ pre_encoded_executable
} else {
- self.context.replace_command_buffer();
+ self.encode_forward_pass(&mut state, &EncodingParameters::new(), sample)?
+ };
- task.build_encoded_task(
- &mut self.context,
- &mut state,
- &EncodingParameters::new(),
- encoded_task_key.clone(),
- )
- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
- }
+ let pending = executable.submit();
- if !warmup {
- if !task.is_prefilling {
- self.context
- .gpu_sampler
- .encode(&mut state, &EncodingParameters::new(), &mut self.context.command_buffer)
- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
- }
- }
+ if allow_pre_encode {
+ let mut next_encoding_key = encoding_key;
- let command_buffer = std::mem::replace(
- &mut self.context.command_buffer,
- self.context.context.create_command_buffer().expect("Failed to create command buffer").start_encoding(),
- );
- let pending = command_buffer.end_encoding().submit();
+ next_encoding_key.context_len += 1;
- if allow_pre_encode {
- let next_task_key: String = task.encoded_task_key(self.tokens.len() + 1);
-
- let next_encoded_task = task
- .build_encoded_task(
- &mut self.context,
- &mut state,
- &EncodingParameters::new().with_projection(1),
- next_task_key.clone(),
- )
- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
-
- self.encoded_tasks.insert(next_task_key.clone(), next_encoded_task);
+ let next_executable =
+ self.encode_forward_pass(&mut state, &EncodingParameters::new().with_projection(1), sample)?;
+
+ self.pre_encoded_task = Some((next_encoding_key, next_executable));
}
pending.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;
let run_time = run_start.elapsed().as_secs_f64();
+
if should_capture {
self.gpu_capture.stop_capture(&self.context.context, "decode").map_err(|_| Error::CaptureFailed)?;
}
@@ -820,7 +860,37 @@ impl<B: Backend> LanguageModelGenerator<B> {
})
}
- fn sample(
+ fn encode_forward_pass(
+ &self,
+ state: &mut ForwardPassState<B>,
+ parameters: &EncodingParameters,
+ sample: bool,
+ ) -> Result<<B::CommandBuffer as CommandBuffer>::Executable, Error> {
+ let mut command_buffer = self
+ .context
+ .context
+ .create_command_buffer()
+ .map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?
+ .start_encoding();
+
+ self.context
+ .executables
+ .encode(state, parameters, &mut command_buffer)
+ .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
+
+ if sample {
+ self.context
+ .gpu_sampler
+ .encode(state, parameters, &mut command_buffer)
+ .map_err(|e| Error::EncodeFailed(Box::new(e)))?;
+ }
+
+ let executable = command_buffer.end_encoding();
+
+ Ok(executable)
+ }
+
+ fn read_sampling_output(
&mut self,
state: &mut ForwardPassState<B>,
) -> Result<Vec<u64>, Error> {
diff --git a/crates/uzu/src/language_model/language_model_generator_context.rs b/crates/uzu/src/language_model/language_model_generator_context.rs
index f473fea09..296491dd2 100644
--- a/crates/uzu/src/language_model/language_model_generator_context.rs
+++ b/crates/uzu/src/language_model/language_model_generator_context.rs
@@ -9,7 +9,7 @@ use std::{
use crate::{
DataType,
backends::common::{
- Backend, Buffer, CommandBuffer, CommandBufferInitial, Context, Kernels,
+ Backend, Buffer, Context, Kernels,
kernel::{MaskUpdateKernel, TokenCopySampledKernel, TokenCopyToResultsKernel, kv_cache_update::KVCacheUpdate},
},
config::{DecoderConfig, LanguageModelConfig, ModelMetadata},
@@ -122,7 +122,6 @@ impl<B: Backend> AsyncBuffers<B> {
pub struct LanguageModelGeneratorContext<B: Backend> {
pub context: Rc<B::Context>,
- pub command_buffer: <B::CommandBuffer as CommandBuffer>::Encoding,
pub cache_layers: Rc<RefCell<CacheLayers<B>>>,
pub shared_buffers: Rc<RefCell<SharedBuffers<B>>>,
@@ -150,8 +149,7 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {
model_path: &Path,
decoding_config: &DecodingConfig,
) -> Result<Self, Error> {
- let context = B::Context::new().map_err(|_| Error::UnableToCreateBackendContext)?;
- let command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();
+ let context = B::Context::new().map_err(|e| Error::UnableToCreateContext(e.into()))?;
let config_path = model_path.join("config.json");
if !config_path.exists() {
@@ -198,23 +196,23 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {
let intermediate_data_type: DataType = decoder_config.output_norm_config.scale_precision.into();
let kv_cache_update = Box::new(
KVCacheUpdate::new(context.as_ref(), intermediate_data_type, max_prefix_length)
- .map_err(|_| Error::UnableToCreateBackendContext)?,
+ .map_err(|e| Error::UnableToCreateContext(e.into()))?,
);
let gpu_sampler =
Sampling::<B>::new(&context, intermediate_data_type, max_suffix_length, decoder_config.vocab_size)
- .map_err(|_| Error::UnableToCreateBackendContext)?;
+ .map_err(|e| Error::UnableToCreateContext(e.into()))?;
let token_copy_sampled = <B::Kernels as Kernels>::TokenCopySampledKernel::new(&context)
- .map_err(|_| Error::UnableToCreateBackendContext)?;
+ .map_err(|e| Error::UnableToCreateContext(e.into()))?;
let token_copy_results = <B::Kernels as Kernels>::TokenCopyToResultsKernel::new(&context)
- .map_err(|_| Error::UnableToCreateBackendContext)?;
+ .map_err(|e| Error::UnableToCreateContext(e.into()))?;
// Create mask update kernel if model has attention layers
let mask_update = if decoder_config.has_attention_layers() {
Some(
<B::Kernels as Kernels>::MaskUpdateKernel::new(&context, intermediate_data_type)
- .map_err(|_| Error::UnableToCreateBackendContext)?,
+ .map_err(|e| Error::UnableToCreateContext(e.into()))?,
)
} else {
None
@@ -227,7 +225,6 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {
let context = Self {
context,
- command_buffer,
cache_layers,
shared_buffers,
scratch_buffers,
@@ -246,11 +243,4 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {
Ok(context)
}
-
- pub fn replace_command_buffer(&mut self) -> <B::CommandBuffer as CommandBuffer>::Encoding {
- std::mem::replace(
- &mut self.command_buffer,
- self.context.create_command_buffer().expect("Failed to create command buffer").start_encoding(),
- )
- }
}
diff --git a/crates/uzu/src/language_model/mod.rs b/crates/uzu/src/language_model/mod.rs
index 8c30f89ef..246f4722c 100644
--- a/crates/uzu/src/language_model/mod.rs
+++ b/crates/uzu/src/language_model/mod.rs
@@ -6,7 +6,6 @@ pub mod language_model_generator_context;
pub mod result;
pub mod rng;
pub mod sampler;
-pub mod tasks;
// Re-export main types
pub use language_model_generator::{LanguageModelGenerator, LanguageModelGeneratorTrait};
diff --git a/crates/uzu/src/language_model/tasks.rs b/crates/uzu/src/language_model/tasks.rs
deleted file mode 100644
index 3df98449f..000000000
--- a/crates/uzu/src/language_model/tasks.rs
+++ /dev/null
@@ -1,127 +0,0 @@
-use std::mem::size_of;
-
-use super::language_model_generator_context::LanguageModelGeneratorContext;
-use crate::{
- backends::common::{Backend, Buffer, Context},
- encodable_block::{EncodableBlock, EncodingParameters},
- forward_pass::state::ForwardPassState,
-};
-
-pub struct LanguageModelGeneratorEncodedTask<B: Backend> {
- pub key: String,
- predicate_buffer: B::Buffer,
-}
-
-impl<B: Backend> LanguageModelGeneratorEncodedTask<B> {
- pub fn predicate_buffer(&self) -> &B::Buffer {
- &self.predicate_buffer
- }
-
- pub fn disable_execution(&mut self) {
- let ptr = self.predicate_buffer.cpu_ptr().as_ptr() as *mut u32;
-
- unsafe {
- *ptr = 1;
- }
- }
-}
-
-#[derive(Debug, Clone)]
-pub struct LanguageModelGeneratorRunTask<'a> {
- pub token_ids: &'a [u64],
- pub token_positions: &'a [usize],
- pub token_bitmask: Option<&'a [u32]>,
- pub token_seeds: &'a [u64],
- pub expected_number_of_new_tokens: usize,
- pub active_suffix_length: usize,
- /// Start index (within this task's suffix batch) from which we need logits
- /// and sampling results.
- ///
- /// - For normal decode/generate: 0
- /// - For prefill+sample: index of the "suffix root" token inside the last prefill step
- pub sampling_start: usize,
- /// Number of batch items to produce logits + sampling results for.
- ///
- /// Typically `active_suffix_length - sampling_start`, but stored explicitly
- /// so it can be validated and used consistently by downstream encoders.
- pub sampling_length: usize,
- pub is_prefilling: bool,
-}
-
-impl<'a> LanguageModelGeneratorRunTask<'a> {
- pub fn speculate_next_task(&self) -> Self {
- LanguageModelGeneratorRunTask {
- token_ids: self.token_ids,
- token_positions: self.token_positions,
- token_bitmask: self.token_bitmask,
- token_seeds: self.token_seeds,
- expected_number_of_new_tokens: self.expected_number_of_new_tokens,
- active_suffix_length: self.active_suffix_length,
- sampling_start: self.sampling_start,
- sampling_length: self.sampling_length,
- is_prefilling: self.is_prefilling,
- }
- }
-
- pub fn encoded_task_key(
- &self,
- tokens_count: usize,
- ) -> String {
- format!(
- "tokens:{}_suffix:{}_active:{}_prefill:{}_sample_start:{}_sample_len:{}",
- tokens_count,
- self.expected_number_of_new_tokens,
- self.active_suffix_length,
- self.is_prefilling,
- self.sampling_start,
- self.sampling_length
- )
- }
-
- pub fn create_state<B: Backend>(
- &self,
- context: &mut LanguageModelGeneratorContext<B>,
- external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,
- should_fill_attention_bias: bool,
- ) -> ForwardPassState<B> {
- ForwardPassState::new_llm(
- context.context.clone(),
- &context.decoder_config,
- &context.model_shape,
- &context.scratch_buffers,
- context.cache_layers.clone(),
- context.shared_buffers.clone(),
- self.token_ids,
- self.token_positions,
- self.token_bitmask,
- self.token_seeds,
- self.active_suffix_length,
- self.sampling_start,
- self.sampling_length,
- self.is_prefilling,
- external_bias_fn,
- false,
- should_fill_attention_bias,
- None,
- None,
- )
- }
-
- pub fn build_encoded_task<B: Backend>(
- &self,
- context: &mut LanguageModelGeneratorContext<B>,
- state: &mut ForwardPassState<B>,
- parameters: &EncodingParameters,
- key: String,
- ) -> Result<LanguageModelGeneratorEncodedTask<B>, B::Error> {
- context.executables.encode(state, parameters, &mut context.command_buffer)?;
-
- Ok(LanguageModelGeneratorEncodedTask {
- key,
- predicate_buffer: context
- .context
- .create_buffer(size_of::<u32>())
- .expect("Failed to create predicate buffer"),
- })
- }
-}
diff --git a/crates/uzu/src/session/chat_session.rs b/crates/uzu/src/session/chat_session.rs
index 5150a184b..586c336e8 100644
--- a/crates/uzu/src/session/chat_session.rs
+++ b/crates/uzu/src/session/chat_session.rs
@@ -65,7 +65,7 @@ impl ChatSession {
model_path: PathBuf,
decoding_config: DecodingConfig,
) -> Result<Self, Error> {
- select_backend!(Self::new_with_backend::<B>(model_path, decoding_config), Error::UnableToCreateBackendContext)
+ select_backend!(Self::new_with_backend::<B>(model_path, decoding_config), Error::UnableToOpenAnyBackend)
}
pub fn new_with_backend<B: Backend>(
diff --git a/crates/uzu/src/session/classification_session.rs b/crates/uzu/src/session/classification_session.rs
index 57fff930b..cc3de24aa 100644
--- a/crates/uzu/src/session/classification_session.rs
+++ b/crates/uzu/src/session/classification_session.rs
@@ -24,7 +24,7 @@ pub struct ClassificationSession {
impl ClassificationSession {
pub fn new(model_path: PathBuf) -> Result<Self, Error> {
- select_backend!(Self::new_with_backend::<B>(model_path), Error::UnableToCreateBackendContext)
+ select_backend!(Self::new_with_backend::<B>(model_path), Error::UnableToOpenAnyBackend)
}
pub fn new_with_backend<B: Backend>(model_path: PathBuf) -> Result<Self, Error> {
diff --git a/crates/uzu/src/session/types/error.rs b/crates/uzu/src/session/types/error.rs
index af0d03cf8..3e634ed4c 100644
--- a/crates/uzu/src/session/types/error.rs
+++ b/crates/uzu/src/session/types/error.rs
@@ -4,8 +4,12 @@ use crate::classifier::ClassifierError;
pub enum Error {
#[error("Model folder not found")]
ModelFolderNotFound,
- #[error("Unable to create Backend context")]
- UnableToCreateBackendContext,
+ #[error("Unable to open any backend")]
+ UnableToOpenAnyBackend,
+ #[error("Unable to create Context: {0}")]
+ UnableToCreateContext(Box<dyn std::error::Error>),
+ #[error("Unable to create CommandBuffer: {0}")]
+ UnableToCreateCommandBuffer(Box<dyn std::error::Error>),
#[error("Unable to load model configuration")]
UnableToLoadConfig,
#[error("Unable to load model weights")]