A high-performance inference engine for AI models
small (LanguageModel/Classifier)Context cleanup (#239)
uuuvn3817d86c6d8b
diff --git a/crates/uzu/src/classifier/classifier.rs b/crates/uzu/src/classifier/classifier.rsindex 6c5bf3e02..fc2295746 100644--- a/crates/uzu/src/classifier/classifier.rs+++ b/crates/uzu/src/classifier/classifier.rs@@ -9,7 +9,9 @@ use super::ActivationTrace;use super::{ClassificationOutput, ClassificationStats, ClassifierContext};use crate::{DataType,- backends::common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending},+ backends::common::{+ Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending, Context,+ },encodable_block::{EncodableBlock, EncodingParameters},forward_pass::state::{ArrayId, ForwardPassState},session::types::Error,@@ -113,55 +115,53 @@ impl<B: Backend> Classifier<B> {let encoding_params = EncodingParameters::new();+ let mut command_buffer = self+ .context+ .context+ .create_command_buffer()+ .map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?+ .start_encoding();+self.context.embed- .encode(&mut state, &encoding_params, &mut self.context.command_buffer)+ .encode(&mut state, &encoding_params, &mut command_buffer).map_err(|e| Error::EncodeFailed(Box::new(e)))?;self.context.embedding_norm- .encode(&mut state, &encoding_params, &mut self.context.command_buffer)+ .encode(&mut state, &encoding_params, &mut command_buffer).map_err(|e| Error::EncodeFailed(Box::new(e)))?;#[cfg(feature = "tracing")]{let traces = state.traces().clone();- state.encode_copy_array(- &mut self.context.command_buffer,- ArrayId::Main,- traces.borrow().embedding_norm().clone(),- );+ state.encode_copy_array(&mut command_buffer, ArrayId::Main, traces.borrow().embedding_norm().clone());}for layer in self.context.layers.iter() {layer- .encode(&mut state, &encoding_params, &mut self.context.command_buffer)+ .encode(&mut state, &encoding_params, &mut command_buffer).map_err(|e| Error::EncodeFailed(Box::new(e)))?;}self.context.output_norm- .encode(&mut state, &encoding_params, &mut self.context.command_buffer)+ .encode(&mut state, &encoding_params, &mut command_buffer).map_err(|e| Error::EncodeFailed(Box::new(e)))?;#[cfg(feature = "tracing")]{let traces = state.traces().clone();- state.encode_copy_array(- &mut self.context.command_buffer,- ArrayId::Main,- traces.borrow().output_norm.clone(),- );+ state.encode_copy_array(&mut command_buffer, ArrayId::Main, traces.borrow().output_norm.clone());}self.context.pooling- .encode(&mut state, &encoding_params, &mut self.context.command_buffer)+ .encode(&mut state, &encoding_params, &mut command_buffer).map_err(|e| Error::EncodeFailed(Box::new(e)))?;self.context.prediction_head- .encode(&mut state, &encoding_params, &mut self.context.command_buffer)+ .encode(&mut state, &encoding_params, &mut command_buffer).map_err(|e| Error::EncodeFailed(Box::new(e)))?;- self.context- .replace_command_buffer()+ command_buffer.end_encoding().submit().wait_until_completed()@@ -195,34 +195,40 @@ impl<B: Backend> Classifier<B> {let encoding_params = EncodingParameters::new();+ let mut command_buffer = self+ .context+ .context+ .create_command_buffer()+ .map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?+ .start_encoding();+self.context.embed- .encode(&mut state, &encoding_params, &mut self.context.command_buffer)+ .encode(&mut state, &encoding_params, &mut command_buffer).map_err(|e| Error::EncodeFailed(Box::new(e)))?;self.context.embedding_norm- .encode(&mut state, &encoding_params, &mut self.context.command_buffer)+ .encode(&mut state, &encoding_params, &mut command_buffer).map_err(|e| Error::EncodeFailed(Box::new(e)))?;for layer in self.context.layers.iter() {layer- .encode(&mut state, &encoding_params, &mut self.context.command_buffer)+ .encode(&mut state, &encoding_params, &mut command_buffer).map_err(|e| Error::EncodeFailed(Box::new(e)))?;}self.context.output_norm- .encode(&mut state, &encoding_params, &mut self.context.command_buffer)+ .encode(&mut state, &encoding_params, &mut command_buffer).map_err(|e| Error::EncodeFailed(Box::new(e)))?;self.context.pooling- .encode(&mut state, &encoding_params, &mut self.context.command_buffer)+ .encode(&mut state, &encoding_params, &mut command_buffer).map_err(|e| Error::EncodeFailed(Box::new(e)))?;self.context.prediction_head- .encode(&mut state, &encoding_params, &mut self.context.command_buffer)+ .encode(&mut state, &encoding_params, &mut command_buffer).map_err(|e| Error::EncodeFailed(Box::new(e)))?;- self.context- .replace_command_buffer()+ command_buffer.end_encoding().submit().wait_until_completed()diff --git a/crates/uzu/src/classifier/classifier_context.rs b/crates/uzu/src/classifier/classifier_context.rsindex 4e73fa3c7..3edbfd5d1 100644--- a/crates/uzu/src/classifier/classifier_context.rs+++ b/crates/uzu/src/classifier/classifier_context.rs@@ -2,7 +2,7 @@ use std::{cell::RefCell, fs::File, io::BufReader, path::Path, rc::Rc};use crate::{DataType,- backends::common::{Backend, CommandBuffer, CommandBufferInitial, Context, Kernels, kernel::SigmoidKernel},+ backends::common::{Backend, Context, Kernels, kernel::SigmoidKernel},classifier::ClassifierError,config::{ClassifierModelConfig, ModelMetadata},encodable_block::{@@ -20,7 +20,6 @@ use crate::{pub struct ClassifierContext<B: Backend> {pub context: Rc<B::Context>,- pub command_buffer: <B::CommandBuffer as CommandBuffer>::Encoding,pub shared_buffers: Rc<RefCell<SharedBuffers<B>>>,pub scratch_buffers: ScratchBuffers<B>,@@ -43,10 +42,7 @@ pub struct ClassifierContext<B: Backend> {impl<B: Backend> ClassifierContext<B> {pub fn new(model_path: &Path) -> Result<Self, Error> {- let context = B::Context::new().map_err(|_| Error::UnableToCreateBackendContext)?;-- let command_buffer =- context.create_command_buffer().map_err(|_| Error::UnableToCreateBackendContext)?.start_encoding();+ let context = B::Context::new().map_err(|e| Error::UnableToCreateContext(e.into()))?;let config_path = model_path.join("config.json");if !config_path.exists() {@@ -254,7 +250,7 @@ impl<B: Backend> ClassifierContext<B> {let sigmoid_kernel = <B::Kernels as Kernels>::SigmoidKernel::new(&context, data_type.into()).map_err(|e| {eprintln!("Failed to create sigmoid kernel: {:?}", e);- Error::UnableToCreateBackendContext+ Error::UnableToCreateContext(e.into())})?;let pooling = Box::new(@@ -266,7 +262,7 @@ impl<B: Backend> ClassifierContext<B> {).map_err(|e| {eprintln!("Failed to create pooling: {:?}", e);- Error::UnableToCreateBackendContext+ Error::UnableToCreateContext(e.into())})?,);@@ -282,7 +278,6 @@ impl<B: Backend> ClassifierContext<B> {Ok(Self {context,- command_buffer,shared_buffers,scratch_buffers,model_config: classifier_model_config.clone(),@@ -310,11 +305,4 @@ impl<B: Backend> ClassifierContext<B> {);Ok(Rc::new(rotation))}-- pub fn replace_command_buffer(&mut self) -> <B::CommandBuffer as CommandBuffer>::Encoding {- std::mem::replace(- &mut self.command_buffer,- self.context.create_command_buffer().expect("Failed to create command buffer").start_encoding(),- )- }}diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rsindex 8a3685efd..87e31a7e7 100644--- a/crates/uzu/src/language_model/language_model_generator.rs+++ b/crates/uzu/src/language_model/language_model_generator.rs@@ -1,6 +1,5 @@use std::{any::Any,- collections::HashMap,iter::repeat_n,ops::{Deref, DerefMut, Range},path::Path,@@ -15,7 +14,6 @@ use super::{language_model_generator_context::LanguageModelGeneratorContext,result::{GenerateResult, PrefillResult},rng::PRng,- tasks::{LanguageModelGeneratorEncodedTask, LanguageModelGeneratorRunTask},};use crate::{backends::common::{@@ -38,12 +36,38 @@ use crate::{trie::{TrieCreationConfig, TrieNode},};+#[derive(Debug, Clone)]+struct Task<'a> {+ token_ids: &'a [u64],+ token_positions: &'a [usize],+ token_bitmask: Option<&'a [u32]>,+ token_seeds: &'a [u64],+ expected_number_of_new_tokens: usize,+ active_suffix_length: usize,+ sampling_start: usize,+ sampling_length: usize,+ is_prefilling: bool,+}++#[derive(Debug, Clone, PartialEq)]+struct TaskEncodingKey {+ context_len: usize,+ batch_size: usize,+ expected_number_of_new_tokens: usize,+ active_suffix_len: usize,+ sampling_method: SamplingMethod,+ sampling_start: usize,+ sampling_len: usize,+ has_bitmask: bool,+ is_prefilling: bool,+}+pub struct LanguageModelGenerator<B: Backend> {pub decoding_config: DecodingConfig,pub tokens: Vec<u64>,pub context: LanguageModelGeneratorContext<B>,- encoded_tasks: HashMap<String, LanguageModelGeneratorEncodedTask<B>>,+ pre_encoded_task: Option<(TaskEncodingKey, <B::CommandBuffer as CommandBuffer>::Executable)>,registered_prefix_len: usize,gpu_capture: GpuCaptureManager<B>,}@@ -217,7 +241,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {let _ = last_state.take();});- let task = LanguageModelGeneratorRunTask {+ let task = Task {token_ids: &step_token_ids,token_positions: &step_token_positions,token_bitmask: step_token_bitmask.as_deref(),@@ -279,7 +303,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {forwardpass_durations: run_times,});}- let sampled_tokens = self.sample(&mut final_state)?;+ let sampled_tokens = self.read_sampling_output(&mut final_state)?;let last_suffix_start = prefill_step_size * (prefill_steps - 1);let suffix_root_index = (tokens_length - last_suffix_start) - 1;@@ -353,7 +377,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {let token_seeds =flat_trie.token_seeds().chain(repeat_n(0, suffix_length - active_suffix_length)).collect::<Box<[u64]>>();- let task = LanguageModelGeneratorRunTask {+ let task = Task {token_ids: &token_ids,token_positions: &token_positions,token_bitmask: token_bitmask.as_deref(),@@ -368,7 +392,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {let (mut state, run_time) =self.run_model(task, false, self.allow_pre_encode(), sampling_method, self.should_fill_attention_bias())?;- let sampled_tokens = self.sample(&mut state)?;+ let sampled_tokens = self.read_sampling_output(&mut state)?;let (accepted_tokens, accepted_token_indices) =flat_trie.accept(&sampled_tokens, compiled_grammar.as_deref_mut());@@ -439,7 +463,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {*ptr.add(pass_idx) as usize};- let task = LanguageModelGeneratorRunTask {+ let task = Task {token_ids: &[last_token],token_positions: &[token_position],token_bitmask: None,@@ -488,20 +512,27 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {*sm = Some(sampling_method);}+ let mut command_buffer = self+ .context+ .context+ .create_command_buffer()+ .map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?+ .start_encoding();+// Wait on previous pass if this is a continuationif is_continuation {- self.context.command_buffer.encode_wait_for_event(&self.context.async_buffers.event, current_counter);+ command_buffer.encode_wait_for_event(&self.context.async_buffers.event, current_counter);}self.context.executables- .encode(&mut state, &EncodingParameters::new(), &mut self.context.command_buffer)+ .encode(&mut state, &EncodingParameters::new(), &mut command_buffer).map_err(|e| Error::EncodeFailed(Box::new(e)))?;// Encode samplingself.context.gpu_sampler- .encode(&mut state, &EncodingParameters::new(), &mut self.context.command_buffer)+ .encode(&mut state, &EncodingParameters::new(), &mut command_buffer).map_err(|e| Error::EncodeFailed(Box::new(e)))?;// Copy sampled token: sampling_output → token_ids (for next pass)@@ -517,20 +548,20 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {self.context.token_copy_sampled.encode(sampling_output_buf_borrow.deref(),token_ids_buf_borrow.deref_mut(),- &mut self.context.command_buffer,+ &mut command_buffer,);let results_offset = slot * std::mem::size_of::<u32>();self.context.token_copy_results.encode(sampling_output_buf_borrow.deref(),(results_buffer.borrow_mut().deref_mut(), results_offset),- &mut self.context.command_buffer,+ &mut command_buffer,);// Scatter + register for all transformer layersself.context.cache_layers.borrow_mut().update_after_acceptance(&[0],None,- &mut self.context.command_buffer,+ &mut command_buffer,&self.context.kv_cache_update,);self.context.cache_layers.borrow_mut().register_accepted_tokens(&[token_position]);@@ -547,7 +578,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {mask_buf_borrow.deref_mut(),update.unmask_col,update.mask_col,- &mut self.context.command_buffer,+ &mut command_buffer,);}}@@ -556,7 +587,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {// Signal event for next passlet next_counter = current_counter + 1;- self.context.command_buffer.encode_signal_event(&self.context.async_buffers.event, next_counter);+ command_buffer.encode_signal_event(&self.context.async_buffers.event, next_counter);self.context.async_buffers.counter.set(next_counter);// Add completion handler@@ -568,9 +599,10 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {on_complete(token);};- self.context.command_buffer.add_completion_handler(handler);+ command_buffer.add_completion_handler(handler);drop(token_ids_binding);- let pending = self.context.replace_command_buffer().end_encoding().submit();++ let pending = command_buffer.end_encoding().submit();if should_capture {pending.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;@@ -582,7 +614,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {fn clear_cache(&mut self) {objc2::rc::autoreleasepool(|_pool| {- self.encoded_tasks.clear();+ self.pre_encoded_task = None;});}@@ -590,7 +622,7 @@ impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {self.context.cache_layers.borrow_mut().clear();self.tokens.clear();self.registered_prefix_len = 0;- self.encoded_tasks.clear();+ self.pre_encoded_task = None;self.gpu_capture.reset();let seed = self.decoding_config.sampling_seed.resolve();@@ -700,7 +732,7 @@ impl<B: Backend> LanguageModelGenerator<B> {decoding_config,tokens: Vec::new(),context,- encoded_tasks: HashMap::new(),+ pre_encoded_task: None,registered_prefix_len: 0,gpu_capture,};@@ -720,7 +752,7 @@ impl<B: Backend> LanguageModelGenerator<B> {let token_positions: Vec<usize> = (0..suffix_length).collect();let token_seeds: Vec<u64> = vec![0; suffix_length];- let task = LanguageModelGeneratorRunTask {+ let task = Task {token_ids: &token_ids,token_positions: &token_positions,token_bitmask: None,@@ -738,7 +770,7 @@ impl<B: Backend> LanguageModelGenerator<B> {fn run_model(&mut self,- task: LanguageModelGeneratorRunTask,+ task: Task,warmup: bool,allow_pre_encode: bool,sampling_method: SamplingMethod,@@ -747,71 +779,79 @@ impl<B: Backend> LanguageModelGenerator<B> {objc2::rc::autoreleasepool(|_pool| {let run_start = Instant::now();- let mut state = task.create_state(&mut self.context, None, should_fill_attention_bias);+ let mut state = ForwardPassState::new_llm(+ self.context.context.clone(),+ &self.context.decoder_config,+ &self.context.model_shape,+ &self.context.scratch_buffers,+ self.context.cache_layers.clone(),+ self.context.shared_buffers.clone(),+ task.token_ids,+ task.token_positions,+ task.token_bitmask,+ task.token_seeds,+ task.active_suffix_length,+ task.sampling_start,+ task.sampling_length,+ task.is_prefilling,+ None,+ false,+ should_fill_attention_bias,+ None,+ None,+ );+if let Some(method) = state.sampling_method_mut() {*method = Some(sampling_method);}+ let encoding_key = TaskEncodingKey {+ context_len: self.tokens.len(),+ batch_size: task.token_ids.len(),+ expected_number_of_new_tokens: task.expected_number_of_new_tokens,+ active_suffix_len: task.active_suffix_length,+ sampling_method,+ sampling_start: task.sampling_start,+ sampling_len: task.sampling_length,+ has_bitmask: task.token_bitmask.is_some(),+ is_prefilling: task.is_prefilling,+ };+let is_first_decode = !warmup && task.token_ids.len() == 1;let should_capture = self.gpu_capture.should_capture_decode(is_first_decode);if should_capture {- let _ = self.gpu_capture.start_capture(&self.context.context, "decode");+ self.gpu_capture.start_capture(&self.context.context, "decode").map_err(|_| Error::CaptureFailed)?;+ self.pre_encoded_task = None;}- let encoded_task_key = task.encoded_task_key(self.tokens.len());-- if should_capture {- self.encoded_tasks.remove(&encoded_task_key);- }+ let sample = !warmup && !task.is_prefilling;- if let Some(_) = self.encoded_tasks.remove(&encoded_task_key) {- // Nothing+ let executable = if let Some((pre_encoded_key, pre_encoded_executable)) = self.pre_encoded_task.take()+ && pre_encoded_key == encoding_key+ {+ pre_encoded_executable} else {- self.context.replace_command_buffer();+ self.encode_forward_pass(&mut state, &EncodingParameters::new(), sample)?+ };- task.build_encoded_task(- &mut self.context,- &mut state,- &EncodingParameters::new(),- encoded_task_key.clone(),- )- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;- }+ let pending = executable.submit();- if !warmup {- if !task.is_prefilling {- self.context- .gpu_sampler- .encode(&mut state, &EncodingParameters::new(), &mut self.context.command_buffer)- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;- }- }+ if allow_pre_encode {+ let mut next_encoding_key = encoding_key;- let command_buffer = std::mem::replace(- &mut self.context.command_buffer,- self.context.context.create_command_buffer().expect("Failed to create command buffer").start_encoding(),- );- let pending = command_buffer.end_encoding().submit();+ next_encoding_key.context_len += 1;- if allow_pre_encode {- let next_task_key: String = task.encoded_task_key(self.tokens.len() + 1);-- let next_encoded_task = task- .build_encoded_task(- &mut self.context,- &mut state,- &EncodingParameters::new().with_projection(1),- next_task_key.clone(),- )- .map_err(|e| Error::EncodeFailed(Box::new(e)))?;-- self.encoded_tasks.insert(next_task_key.clone(), next_encoded_task);+ let next_executable =+ self.encode_forward_pass(&mut state, &EncodingParameters::new().with_projection(1), sample)?;++ self.pre_encoded_task = Some((next_encoding_key, next_executable));}pending.wait_until_completed().map_err(|e| Error::CommandBufferFailed(Box::new(e)))?;let run_time = run_start.elapsed().as_secs_f64();+if should_capture {self.gpu_capture.stop_capture(&self.context.context, "decode").map_err(|_| Error::CaptureFailed)?;}@@ -820,7 +860,37 @@ impl<B: Backend> LanguageModelGenerator<B> {})}- fn sample(+ fn encode_forward_pass(+ &self,+ state: &mut ForwardPassState<B>,+ parameters: &EncodingParameters,+ sample: bool,+ ) -> Result<<B::CommandBuffer as CommandBuffer>::Executable, Error> {+ let mut command_buffer = self+ .context+ .context+ .create_command_buffer()+ .map_err(|e| Error::UnableToCreateCommandBuffer(e.into()))?+ .start_encoding();++ self.context+ .executables+ .encode(state, parameters, &mut command_buffer)+ .map_err(|e| Error::EncodeFailed(Box::new(e)))?;++ if sample {+ self.context+ .gpu_sampler+ .encode(state, parameters, &mut command_buffer)+ .map_err(|e| Error::EncodeFailed(Box::new(e)))?;+ }++ let executable = command_buffer.end_encoding();++ Ok(executable)+ }++ fn read_sampling_output(&mut self,state: &mut ForwardPassState<B>,) -> Result<Vec<u64>, Error> {diff --git a/crates/uzu/src/language_model/language_model_generator_context.rs b/crates/uzu/src/language_model/language_model_generator_context.rsindex f473fea09..296491dd2 100644--- a/crates/uzu/src/language_model/language_model_generator_context.rs+++ b/crates/uzu/src/language_model/language_model_generator_context.rs@@ -9,7 +9,7 @@ use std::{use crate::{DataType,backends::common::{- Backend, Buffer, CommandBuffer, CommandBufferInitial, Context, Kernels,+ Backend, Buffer, Context, Kernels,kernel::{MaskUpdateKernel, TokenCopySampledKernel, TokenCopyToResultsKernel, kv_cache_update::KVCacheUpdate},},config::{DecoderConfig, LanguageModelConfig, ModelMetadata},@@ -122,7 +122,6 @@ impl<B: Backend> AsyncBuffers<B> {pub struct LanguageModelGeneratorContext<B: Backend> {pub context: Rc<B::Context>,- pub command_buffer: <B::CommandBuffer as CommandBuffer>::Encoding,pub cache_layers: Rc<RefCell<CacheLayers<B>>>,pub shared_buffers: Rc<RefCell<SharedBuffers<B>>>,@@ -150,8 +149,7 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {model_path: &Path,decoding_config: &DecodingConfig,) -> Result<Self, Error> {- let context = B::Context::new().map_err(|_| Error::UnableToCreateBackendContext)?;- let command_buffer = context.create_command_buffer().expect("Failed to create command buffer").start_encoding();+ let context = B::Context::new().map_err(|e| Error::UnableToCreateContext(e.into()))?;let config_path = model_path.join("config.json");if !config_path.exists() {@@ -198,23 +196,23 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {let intermediate_data_type: DataType = decoder_config.output_norm_config.scale_precision.into();let kv_cache_update = Box::new(KVCacheUpdate::new(context.as_ref(), intermediate_data_type, max_prefix_length)- .map_err(|_| Error::UnableToCreateBackendContext)?,+ .map_err(|e| Error::UnableToCreateContext(e.into()))?,);let gpu_sampler =Sampling::<B>::new(&context, intermediate_data_type, max_suffix_length, decoder_config.vocab_size)- .map_err(|_| Error::UnableToCreateBackendContext)?;+ .map_err(|e| Error::UnableToCreateContext(e.into()))?;let token_copy_sampled = <B::Kernels as Kernels>::TokenCopySampledKernel::new(&context)- .map_err(|_| Error::UnableToCreateBackendContext)?;+ .map_err(|e| Error::UnableToCreateContext(e.into()))?;let token_copy_results = <B::Kernels as Kernels>::TokenCopyToResultsKernel::new(&context)- .map_err(|_| Error::UnableToCreateBackendContext)?;+ .map_err(|e| Error::UnableToCreateContext(e.into()))?;// Create mask update kernel if model has attention layerslet mask_update = if decoder_config.has_attention_layers() {Some(<B::Kernels as Kernels>::MaskUpdateKernel::new(&context, intermediate_data_type)- .map_err(|_| Error::UnableToCreateBackendContext)?,+ .map_err(|e| Error::UnableToCreateContext(e.into()))?,)} else {None@@ -227,7 +225,6 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {let context = Self {context,- command_buffer,cache_layers,shared_buffers,scratch_buffers,@@ -246,11 +243,4 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {Ok(context)}-- pub fn replace_command_buffer(&mut self) -> <B::CommandBuffer as CommandBuffer>::Encoding {- std::mem::replace(- &mut self.command_buffer,- self.context.create_command_buffer().expect("Failed to create command buffer").start_encoding(),- )- }}diff --git a/crates/uzu/src/language_model/mod.rs b/crates/uzu/src/language_model/mod.rsindex 8c30f89ef..246f4722c 100644--- a/crates/uzu/src/language_model/mod.rs+++ b/crates/uzu/src/language_model/mod.rs@@ -6,7 +6,6 @@ pub mod language_model_generator_context;pub mod result;pub mod rng;pub mod sampler;-pub mod tasks;// Re-export main typespub use language_model_generator::{LanguageModelGenerator, LanguageModelGeneratorTrait};diff --git a/crates/uzu/src/language_model/tasks.rs b/crates/uzu/src/language_model/tasks.rsdeleted file mode 100644index 3df98449f..000000000--- a/crates/uzu/src/language_model/tasks.rs+++ /dev/null@@ -1,127 +0,0 @@-use std::mem::size_of;--use super::language_model_generator_context::LanguageModelGeneratorContext;-use crate::{- backends::common::{Backend, Buffer, Context},- encodable_block::{EncodableBlock, EncodingParameters},- forward_pass::state::ForwardPassState,-};--pub struct LanguageModelGeneratorEncodedTask<B: Backend> {- pub key: String,- predicate_buffer: B::Buffer,-}--impl<B: Backend> LanguageModelGeneratorEncodedTask<B> {- pub fn predicate_buffer(&self) -> &B::Buffer {- &self.predicate_buffer- }-- pub fn disable_execution(&mut self) {- let ptr = self.predicate_buffer.cpu_ptr().as_ptr() as *mut u32;-- unsafe {- *ptr = 1;- }- }-}--#[derive(Debug, Clone)]-pub struct LanguageModelGeneratorRunTask<'a> {- pub token_ids: &'a [u64],- pub token_positions: &'a [usize],- pub token_bitmask: Option<&'a [u32]>,- pub token_seeds: &'a [u64],- pub expected_number_of_new_tokens: usize,- pub active_suffix_length: usize,- /// Start index (within this task's suffix batch) from which we need logits- /// and sampling results.- ///- /// - For normal decode/generate: 0- /// - For prefill+sample: index of the "suffix root" token inside the last prefill step- pub sampling_start: usize,- /// Number of batch items to produce logits + sampling results for.- ///- /// Typically `active_suffix_length - sampling_start`, but stored explicitly- /// so it can be validated and used consistently by downstream encoders.- pub sampling_length: usize,- pub is_prefilling: bool,-}--impl<'a> LanguageModelGeneratorRunTask<'a> {- pub fn speculate_next_task(&self) -> Self {- LanguageModelGeneratorRunTask {- token_ids: self.token_ids,- token_positions: self.token_positions,- token_bitmask: self.token_bitmask,- token_seeds: self.token_seeds,- expected_number_of_new_tokens: self.expected_number_of_new_tokens,- active_suffix_length: self.active_suffix_length,- sampling_start: self.sampling_start,- sampling_length: self.sampling_length,- is_prefilling: self.is_prefilling,- }- }-- pub fn encoded_task_key(- &self,- tokens_count: usize,- ) -> String {- format!(- "tokens:{}_suffix:{}_active:{}_prefill:{}_sample_start:{}_sample_len:{}",- tokens_count,- self.expected_number_of_new_tokens,- self.active_suffix_length,- self.is_prefilling,- self.sampling_start,- self.sampling_length- )- }-- pub fn create_state<B: Backend>(- &self,- context: &mut LanguageModelGeneratorContext<B>,- external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,- should_fill_attention_bias: bool,- ) -> ForwardPassState<B> {- ForwardPassState::new_llm(- context.context.clone(),- &context.decoder_config,- &context.model_shape,- &context.scratch_buffers,- context.cache_layers.clone(),- context.shared_buffers.clone(),- self.token_ids,- self.token_positions,- self.token_bitmask,- self.token_seeds,- self.active_suffix_length,- self.sampling_start,- self.sampling_length,- self.is_prefilling,- external_bias_fn,- false,- should_fill_attention_bias,- None,- None,- )- }-- pub fn build_encoded_task<B: Backend>(- &self,- context: &mut LanguageModelGeneratorContext<B>,- state: &mut ForwardPassState<B>,- parameters: &EncodingParameters,- key: String,- ) -> Result<LanguageModelGeneratorEncodedTask<B>, B::Error> {- context.executables.encode(state, parameters, &mut context.command_buffer)?;-- Ok(LanguageModelGeneratorEncodedTask {- key,- predicate_buffer: context- .context- .create_buffer(size_of::<u32>())- .expect("Failed to create predicate buffer"),- })- }-}diff --git a/crates/uzu/src/session/chat_session.rs b/crates/uzu/src/session/chat_session.rsindex 5150a184b..586c336e8 100644--- a/crates/uzu/src/session/chat_session.rs+++ b/crates/uzu/src/session/chat_session.rs@@ -65,7 +65,7 @@ impl ChatSession {model_path: PathBuf,decoding_config: DecodingConfig,) -> Result<Self, Error> {- select_backend!(Self::new_with_backend::<B>(model_path, decoding_config), Error::UnableToCreateBackendContext)+ select_backend!(Self::new_with_backend::<B>(model_path, decoding_config), Error::UnableToOpenAnyBackend)}pub fn new_with_backend<B: Backend>(diff --git a/crates/uzu/src/session/classification_session.rs b/crates/uzu/src/session/classification_session.rsindex 57fff930b..cc3de24aa 100644--- a/crates/uzu/src/session/classification_session.rs+++ b/crates/uzu/src/session/classification_session.rs@@ -24,7 +24,7 @@ pub struct ClassificationSession {impl ClassificationSession {pub fn new(model_path: PathBuf) -> Result<Self, Error> {- select_backend!(Self::new_with_backend::<B>(model_path), Error::UnableToCreateBackendContext)+ select_backend!(Self::new_with_backend::<B>(model_path), Error::UnableToOpenAnyBackend)}pub fn new_with_backend<B: Backend>(model_path: PathBuf) -> Result<Self, Error> {diff --git a/crates/uzu/src/session/types/error.rs b/crates/uzu/src/session/types/error.rsindex af0d03cf8..3e634ed4c 100644--- a/crates/uzu/src/session/types/error.rs+++ b/crates/uzu/src/session/types/error.rs@@ -4,8 +4,12 @@ use crate::classifier::ClassifierError;pub enum Error {#[error("Model folder not found")]ModelFolderNotFound,- #[error("Unable to create Backend context")]- UnableToCreateBackendContext,+ #[error("Unable to open any backend")]+ UnableToOpenAnyBackend,+ #[error("Unable to create Context: {0}")]+ UnableToCreateContext(Box<dyn std::error::Error>),+ #[error("Unable to create CommandBuffer: {0}")]+ UnableToCreateCommandBuffer(Box<dyn std::error::Error>),#[error("Unable to load model configuration")]UnableToLoadConfig,#[error("Unable to load model weights")]