A high-performance inference engine for AI models
Dynamic dispatch backends (#222)
uuuvn874dc65b2047
diff --git a/crates/uzu/src/backends/common/backend.rs b/crates/uzu/src/backends/common/backend.rsindex 62f878678..9d811eb5c 100644--- a/crates/uzu/src/backends/common/backend.rs+++ b/crates/uzu/src/backends/common/backend.rs@@ -1,15 +1,15 @@use std::{error::Error, fmt::Debug};-use super::{CommandBuffer, Context, CopyEncoder, Event, Kernels, NativeBuffer};+use super::{CommandBuffer, Context, CopyEncoder, Event, Kernels, NativeBuffer, kernel::matmul::MatmulKernels};-pub trait Backend: Debug + Clone {+pub trait Backend: Debug + Clone + 'static {type Context: Context<Backend = Self>;type NativeBuffer: NativeBuffer<Backend = Self>;type CommandBuffer: CommandBuffer<Backend = Self>;type ComputeEncoder;type CopyEncoder: CopyEncoder<Backend = Self>;type Event: Event<Backend = Self>;- type Kernels: Kernels<Backend = Self>;+ type Kernels: Kernels<Backend = Self> + MatmulKernels;type Error: Error + Debug;const MAX_INLINE_BYTES: usize;diff --git a/crates/uzu/src/backends/mod.rs b/crates/uzu/src/backends/mod.rsindex d69e0ae69..5560db88f 100644--- a/crates/uzu/src/backends/mod.rs+++ b/crates/uzu/src/backends/mod.rs@@ -3,3 +3,17 @@ pub mod common;pub mod cpu;#[cfg(feature = "metal")]pub mod metal;++macro_rules! select_backend {+ ($expr:expr, $unk:expr) => {{+ match std::env::var("UZU_BACKEND").map(|s| s.to_lowercase()).as_deref().unwrap_or("metal") {+ #[cfg(feature = "metal")]+ "metal" => {+ type B = crate::backends::metal::Metal;+ $expr+ },+ _ => Err($unk),+ }+ }};+}+pub(crate) use select_backend;diff --git a/crates/uzu/src/classifier/classifier.rs b/crates/uzu/src/classifier/classifier.rsindex 479d205f7..151d77fa4 100644--- a/crates/uzu/src/classifier/classifier.rs+++ b/crates/uzu/src/classifier/classifier.rs@@ -9,31 +9,26 @@ use super::ActivationTrace;use super::{ClassificationOutput, ClassificationStats, ClassifierContext};use crate::{DataType,- backends::common::{Backend, CommandBuffer, kernel::matmul::MatmulKernels},+ backends::common::{Backend, CommandBuffer},encodable_block::{EncodableBlock, EncodingParameters},forward_pass::state::{ArrayId, ForwardPassState},session::types::Error,};-pub struct Classifier<B: Backend + 'static>-where- B::Kernels: MatmulKernels,-{+pub struct Classifier<B: Backend> {pub context: ClassifierContext<B>,}-impl<B: Backend + 'static> Classifier<B>-where- B::Kernels: MatmulKernels,-{- pub fn new(model_path: &Path) -> Result<Self, Error> {- let context = ClassifierContext::new(model_path)?;- Ok(Self {- context,- })- }+pub trait ClassifierTrait {+ fn classify_tokens(+ &mut self,+ token_ids: Vec<u64>,+ token_positions: Vec<usize>,+ ) -> Result<ClassificationOutput, Error>;+}- pub fn classify_tokens(+impl<B: Backend> ClassifierTrait for Classifier<B> {+ fn classify_tokens(&mut self,token_ids: Vec<u64>,token_positions: Vec<usize>,@@ -78,6 +73,15 @@ where})})}+}++impl<B: Backend> Classifier<B> {+ pub fn new(model_path: &Path) -> Result<Self, Error> {+ let context = ClassifierContext::new(model_path)?;+ Ok(Self {+ context,+ })+ }#[cfg(feature = "tracing")]pub fn forward_pass_with_traces(diff --git a/crates/uzu/src/classifier/classifier_context.rs b/crates/uzu/src/classifier/classifier_context.rsindex 6a95d5c34..ef05c7ef3 100644--- a/crates/uzu/src/classifier/classifier_context.rs+++ b/crates/uzu/src/classifier/classifier_context.rs@@ -2,10 +2,7 @@ use std::{cell::RefCell, fs::File, io::BufReader, path::Path, rc::Rc};use crate::{DataType,- backends::common::{- Backend, Context, Kernels,- kernel::{SigmoidKernel, matmul::MatmulKernels},- },+ backends::common::{Backend, Context, Kernels, kernel::SigmoidKernel},classifier::ClassifierError,config::{ClassifierModelConfig, ModelMetadata},encodable_block::{@@ -44,10 +41,7 @@ pub struct ClassifierContext<B: Backend> {pub sigmoid_kernel: <B::Kernels as Kernels>::SigmoidKernel,}-impl<B: Backend + 'static> ClassifierContext<B>-where- B::Kernels: MatmulKernels,-{+impl<B: Backend> ClassifierContext<B> {pub fn new(model_path: &Path) -> Result<Self, Error> {let context = B::Context::new().map_err(|_| Error::UnableToCreateBackendContext)?;diff --git a/crates/uzu/src/classifier/mod.rs b/crates/uzu/src/classifier/mod.rsindex a8dfc18e8..5857dc8b4 100644--- a/crates/uzu/src/classifier/mod.rs+++ b/crates/uzu/src/classifier/mod.rs@@ -3,7 +3,7 @@ mod classifier_context;mod error;mod result;-pub use classifier::Classifier;+pub use classifier::{Classifier, ClassifierTrait};pub use classifier_context::ClassifierContext;pub use error::ClassifierError;pub use result::{ClassificationOutput, ClassificationStats};diff --git a/crates/uzu/src/encodable_block/classifier_layer.rs b/crates/uzu/src/encodable_block/classifier_layer.rsindex a452cd3ed..e2389ee61 100644--- a/crates/uzu/src/encodable_block/classifier_layer.rs+++ b/crates/uzu/src/encodable_block/classifier_layer.rs@@ -6,7 +6,7 @@ use objc2::rc::autoreleasepool;use crate::backends::common::CommandBuffer;use crate::{DataType,- backends::common::{Backend, kernel::matmul::MatmulKernels},+ backends::common::Backend,config::TransformerLayerConfig,encodable_block::{Attention, EncodableBlock, EncodingParameters, Normalization, QKNorm, TensorAddSwap, TensorCopy,@@ -35,10 +35,7 @@ pub struct ClassifierLayer<B: Backend> {mlp_residual_add: Box<dyn EncodableBlock<B>>,}-impl<B: Backend + 'static> ClassifierLayer<B>-where- B::Kernels: MatmulKernels,-{+impl<B: Backend> ClassifierLayer<B> {pub fn new(context: Rc<B::Context>,layer_config: &TransformerLayerConfig,diff --git a/crates/uzu/src/encodable_block/decoder.rs b/crates/uzu/src/encodable_block/decoder.rsindex 39c8e0f3d..ac62d59fc 100644--- a/crates/uzu/src/encodable_block/decoder.rs+++ b/crates/uzu/src/encodable_block/decoder.rs@@ -4,7 +4,7 @@ use std::rc::Rc;use crate::{DataType, DecoderConfig,- backends::common::{Backend, kernel::matmul::MatmulKernels},+ backends::common::Backend,config::{DecoderLayerType, MixerConfig},encodable_block::{EncodableBlock, EncodingParameters, LayerExecutables, RMSNorm, Rope,@@ -27,10 +27,7 @@ pub struct Decoder<B: Backend> {pub local_rope: Option<Rc<Box<dyn EncodableBlock<B>>>>,}-impl<B: Backend + 'static> Decoder<B>-where- B::Kernels: MatmulKernels,-{+impl<B: Backend> Decoder<B> {pub fn new(context: Rc<B::Context>,decoder_config: Rc<DecoderConfig>,diff --git a/crates/uzu/src/encodable_block/full_precision_embedding_readout.rs b/crates/uzu/src/encodable_block/full_precision_embedding_readout.rsindex 7262b584a..5342b58d3 100644--- a/crates/uzu/src/encodable_block/full_precision_embedding_readout.rs+++ b/crates/uzu/src/encodable_block/full_precision_embedding_readout.rs@@ -36,20 +36,14 @@ pub enum FullPrecisionEmbeddingReadoutError<B: Backend> {},}-pub struct FullPrecisionEmbeddingReadout<B: Backend>-where- B::Kernels: MatmulKernels,-{+pub struct FullPrecisionEmbeddingReadout<B: Backend> {kernel: RefCell<<B::Kernels as MatmulKernels>::FullPrecisionMatmulKernel>,weights_buffer: Rc<RefCell<B::NativeBuffer>>,vocab_size: usize,model_dim: usize,}-impl<B: Backend> FullPrecisionEmbeddingReadout<B>-where- B::Kernels: MatmulKernels,-{+impl<B: Backend> FullPrecisionEmbeddingReadout<B> {pub fn new(context: &B::Context,data_type: DataType,@@ -96,10 +90,7 @@ where}}-impl<B: Backend> EncodableBlock<B> for FullPrecisionEmbeddingReadout<B>-where- B::Kernels: MatmulKernels,-{+impl<B: Backend> EncodableBlock<B> for FullPrecisionEmbeddingReadout<B> {fn supports_shared_encoder(&self) -> bool {true}diff --git a/crates/uzu/src/encodable_block/full_precision_linear.rs b/crates/uzu/src/encodable_block/full_precision_linear.rsindex fd9c37509..6b748773a 100644--- a/crates/uzu/src/encodable_block/full_precision_linear.rs+++ b/crates/uzu/src/encodable_block/full_precision_linear.rs@@ -44,10 +44,7 @@ pub enum FullPrecisionLinearError<B: Backend> {},}-pub struct FullPrecisionLinear<B: Backend>-where- B::Kernels: MatmulKernels,-{+pub struct FullPrecisionLinear<B: Backend> {kernel: RefCell<<B::Kernels as MatmulKernels>::FullPrecisionMatmulKernel>,bias_buffer: Option<Rc<RefCell<B::NativeBuffer>>>,weights_buffer: Rc<RefCell<B::NativeBuffer>>,@@ -57,10 +54,7 @@ whereoutput_array_id: ArrayId,}-impl<B: Backend> FullPrecisionLinear<B>-where- B::Kernels: MatmulKernels,-{+impl<B: Backend> FullPrecisionLinear<B> {pub fn new(context: &B::Context,precision: DataType,@@ -128,10 +122,7 @@ where}}-impl<B: Backend> EncodableBlock<B> for FullPrecisionLinear<B>-where- B::Kernels: MatmulKernels,-{+impl<B: Backend> EncodableBlock<B> for FullPrecisionLinear<B> {fn supports_shared_encoder(&self) -> bool {true}diff --git a/crates/uzu/src/encodable_block/layer/executables.rs b/crates/uzu/src/encodable_block/layer/executables.rsindex acb6db004..f49b3dc76 100644--- a/crates/uzu/src/encodable_block/layer/executables.rs+++ b/crates/uzu/src/encodable_block/layer/executables.rs@@ -9,7 +9,7 @@ use super::MixerExecutables;use crate::backends::common::CommandBuffer;use crate::{DataType, DecoderLayerConfig,- backends::common::{Backend, kernel::matmul::MatmulKernels},+ backends::common::Backend,config::{DecoderLayerType, MixerConfig},encodable_block::{Attention, EncodableBlock, EncodingParameters, MambaMixer, QKNorm, RMSNorm, ShortConvMixer, TensorAddSwap,@@ -33,10 +33,7 @@ pub struct LayerExecutables<B: Backend> {pub post_mlp_norm: Option<Box<dyn EncodableBlock<B>>>,}-impl<B: Backend + 'static> LayerExecutables<B>-where- B::Kernels: MatmulKernels,-{+impl<B: Backend> LayerExecutables<B> {#[allow(clippy::too_many_arguments)]pub fn new(context: Rc<B::Context>,diff --git a/crates/uzu/src/encodable_block/mamba_mixer.rs b/crates/uzu/src/encodable_block/mamba_mixer.rsindex c248bebce..f9e0c1fc2 100644--- a/crates/uzu/src/encodable_block/mamba_mixer.rs+++ b/crates/uzu/src/encodable_block/mamba_mixer.rs@@ -9,7 +9,6 @@ use crate::{Backend, CommandBuffer, Kernels,kernel::{Conv1dDecodeKernel, Conv1dPackKernel, Conv1dScanKernel, SSDUpdateKernel, SplitInProjKernel,- matmul::MatmulKernels,ssd_prefill::{SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode},},},@@ -37,10 +36,7 @@ pub(crate) struct MambaMixer<B: Backend> {prefill_mode: SSDPrefillMode,}-impl<B: Backend + 'static> MambaMixer<B>-where- B::Kernels: MatmulKernels,-{+impl<B: Backend> MambaMixer<B> {#[allow(clippy::too_many_arguments)]pub(crate) fn new(context: &B::Context,diff --git a/crates/uzu/src/encodable_block/short_conv_mixer.rs b/crates/uzu/src/encodable_block/short_conv_mixer.rsindex 6b79f4437..dccc21457 100644--- a/crates/uzu/src/encodable_block/short_conv_mixer.rs+++ b/crates/uzu/src/encodable_block/short_conv_mixer.rs@@ -5,10 +5,7 @@ use crate::{array::Array,backends::common::{Backend, CommandBuffer, Context, Kernels,- kernel::{- ShortConvDecodeKernel, ShortConvPackKernel, ShortConvPrefillKernel, ShortConvTrieKernel,- matmul::MatmulKernels,- },+ kernel::{ShortConvDecodeKernel, ShortConvPackKernel, ShortConvPrefillKernel, ShortConvTrieKernel},},config::{DecoderLayerType, ShortConvConfig},encodable_block::{EncodableBlock, EncodingParameters, transformer_layer::linear_block},@@ -30,10 +27,7 @@ pub struct ShortConvMixer<B: Backend> {conv_bias: Option<Array<B>>,}-impl<B: Backend + 'static> ShortConvMixer<B>-where- B::Kernels: MatmulKernels,-{+impl<B: Backend> ShortConvMixer<B> {pub fn new(context: &B::Context,layer_type: DecoderLayerType,diff --git a/crates/uzu/src/encodable_block/transformer_layer.rs b/crates/uzu/src/encodable_block/transformer_layer.rsindex 705abd0e1..736fe8fb3 100644--- a/crates/uzu/src/encodable_block/transformer_layer.rs+++ b/crates/uzu/src/encodable_block/transformer_layer.rs@@ -6,10 +6,7 @@ use super::{};use crate::{DataType,- backends::common::{- Backend,- kernel::{matmul::MatmulKernels, mlp_gate_act_mul::MlpGateActMulEncodable},- },+ backends::common::{Backend, kernel::mlp_gate_act_mul::MlpGateActMulEncodable},config::{DecoderConfig, EmbeddingConfig, LinearConfig, MLPConfig},encodable_block::{EncodableBlock, FullPrecisionLinearError, MlpBlock, MoeBlock, QuantizedLinearError, moe_block::MoeBlockError,@@ -34,7 +31,7 @@ pub enum LayerError<B: Backend> {QLoRaNotSupported,}-pub fn linear_block<const N: usize, B: Backend + 'static>(+pub fn linear_block<const N: usize, B: Backend>(config: &LinearConfig,_has_biases: bool,input_dimension: usize,@@ -43,10 +40,7 @@ pub fn linear_block<const N: usize, B: Backend + 'static>(parameter_tree: &ParameterTree<B::Context>,input_array_id: ArrayId,output_array_id: ArrayId,-) -> Result<Box<dyn EncodableBlock<B>>, LayerError<B>>-where- B::Kernels: MatmulKernels,-{+) -> Result<Box<dyn EncodableBlock<B>>, LayerError<B>> {let output_dimension_sum: usize = output_dimensions.iter().sum();match config {LinearConfig::Quantized(quantization_config) | LinearConfig::MLXQuantized(quantization_config) => {@@ -84,16 +78,13 @@ where}/// Creates an MLP block using the unfused implementation (separate up, activation, down)-pub fn mlp_block<B: Backend + 'static>(+pub fn mlp_block<B: Backend>(config: &MLPConfig,model_dimension: usize,hidden_dimension: usize,context: &B::Context,parameter_tree: &ParameterTree<B::Context>,-) -> Result<Box<dyn EncodableBlock<B>>, LayerError<B>>-where- B::Kernels: MatmulKernels,-{+) -> Result<Box<dyn EncodableBlock<B>>, LayerError<B>> {if let crate::config::MLPConfig::Dense(dense_config) = config {let data_type: DataType = dense_config.linear_config.activation_precision().into();@@ -139,7 +130,7 @@ whereunreachable!("Unknown MLP config")}-pub fn embed_block<B: Backend + 'static>(+pub fn embed_block<B: Backend>(config: &DecoderConfig,context: &B::Context,parameter_tree: &ParameterTree<B::Context>,@@ -279,14 +270,11 @@ pub fn embed_block<B: Backend + 'static>(}}-pub fn readout_block<B: Backend + 'static>(+pub fn readout_block<B: Backend>(config: &DecoderConfig,context: &B::Context,parameter_tree: &ParameterTree<B::Context>,-) -> Box<dyn EncodableBlock<B>>-where- B::Kernels: MatmulKernels,-{+) -> Box<dyn EncodableBlock<B>> {match &config.embedding_config {EmbeddingConfig::Tied {precision,diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rsindex 5cb0c89ea..dd27362c8 100644--- a/crates/uzu/src/language_model/language_model_generator.rs+++ b/crates/uzu/src/language_model/language_model_generator.rs@@ -1,7 +1,8 @@use std::{+ any::Any,collections::HashMap,iter::repeat_n,- ops::{Deref, DerefMut},+ ops::{Deref, DerefMut, Range},path::Path,sync::Arc,time::Instant,@@ -10,9 +11,9 @@ use std::{use itertools::{Either, Itertools, izip};use super::{- LanguageModelGeneratorContext,gpu_capture::GpuCaptureManager,grammar::CompiledGrammar,+ language_model_generator_context::LanguageModelGeneratorContext,result::{GenerateResult, PrefillResult},rng::PRng,tasks::{LanguageModelGeneratorEncodedTask, LanguageModelGeneratorRunTask},@@ -20,25 +21,24 @@ use super::{use crate::{backends::common::{Backend, CommandBuffer, Context, NativeBuffer,- kernel::{MaskUpdateKernel, TokenCopySampledKernel, TokenCopyToResultsKernel, matmul::MatmulKernels},+ kernel::{MaskUpdateKernel, TokenCopySampledKernel, TokenCopyToResultsKernel},},encodable_block::{EncodableBlock, EncodingParameters},forward_pass::{+ cache_layers::{CacheLayer, CacheLayersSlice},kv_cache_layer::{AttentionBiasUpdate, INVALID_POSITION},state::ForwardPassState,},session::{config::DecodingConfig,+ helpers::Context as LlmContext,parameter::{ConfigResolvableValue, ResolvableValue, SamplingMethod},types::Error,},trie::{TrieCreationConfig, TrieNode},};-pub struct LanguageModelGenerator<B: Backend + 'static>-where- B::Kernels: MatmulKernels,-{+pub struct LanguageModelGenerator<B: Backend> {pub decoding_config: DecodingConfig,pub tokens: Vec<u64>,@@ -48,37 +48,66 @@ wheregpu_capture: GpuCaptureManager<B>,}-impl<B: Backend + 'static> LanguageModelGenerator<B>-where- B::Kernels: MatmulKernels,-{- pub fn new(- model_path: &Path,- decoding_config: DecodingConfig,- ) -> Result<Self, Error> {- let gpu_capture = GpuCaptureManager::new();+pub trait LanguageModelGeneratorTrait {+ fn prefill(+ &mut self,+ tokens: Vec<u64>,+ compiled_grammar: Option<&mut CompiledGrammar>,+ sampling_method: SamplingMethod,+ prefix_offset: usize,+ sample_suffix: bool,+ ) -> Result<PrefillResult, Error>;- let context = LanguageModelGeneratorContext::new(model_path, &decoding_config)?;- let prefill_step_size = decoding_config.prefill_step_size.resolve(&context.model_config);- let generate_suffix_length = decoding_config.generate_suffix_length();+ fn generate(+ &mut self,+ compiled_grammar: Option<&mut CompiledGrammar>,+ sampling_method: SamplingMethod,+ ) -> Result<GenerateResult, Error>;- let mut generator = Self {- decoding_config,- tokens: Vec::new(),- context,- encoded_tasks: HashMap::new(),- registered_prefix_len: 0,- gpu_capture,- };+ fn prepare_async(+ &mut self,+ tokens_to_generate: usize,+ );+ fn async_generate(+ &mut self,+ pass_idx: usize,+ sampling_method: SamplingMethod,+ on_complete: Box<dyn FnOnce(u64) + Send>,+ ) -> Result<(), Error>;- //Warmup- generator.warmup(prefill_step_size);- generator.warmup(generate_suffix_length);+ fn clear_cache(&mut self);+ fn reset_state(&mut self);- Ok(generator)- }+ fn tokens_len(&self) -> usize;+ fn tokens_push(+ &mut self,+ token: u64,+ );+ fn generate_suffix_length(&self) -> usize;+ fn async_batch_size(+ &self,+ model_path: &Path,+ ) -> usize;- pub fn prefill(+ fn get_slice(+ &self,+ range: Range<usize>,+ ) -> Option<Box<dyn Any>>;+ fn apply_slice(+ &mut self,+ slice: &dyn Any,+ range: Range<usize>,+ );++ fn build_llm_context(&self) -> Box<dyn Any>;+ fn reconfigure_from_context(+ &mut self,+ context: &dyn Any,+ );+}++impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {+ fn prefill(&mut self,tokens: Vec<u64>,mut compiled_grammar: Option<&mut CompiledGrammar>,@@ -273,7 +302,7 @@ where})}- pub fn generate(+ fn generate(&mut self,mut compiled_grammar: Option<&mut CompiledGrammar>,sampling_method: SamplingMethod,@@ -357,7 +386,7 @@ where/// Prepares async buffers for generation./// Must be called after prefill, before async_generate loop.- pub fn prepare_async(+ fn prepare_async(&mut self,tokens_to_generate: usize,) {@@ -385,15 +414,12 @@ where/// - `pass_idx`: Index of this pass (0, 1, 2, ...)/// - `sampling_method`: Sampling configuration/// - `on_complete`: Callback receiving sampled token as u64- pub fn async_generate<F>(+ fn async_generate(&mut self,pass_idx: usize,sampling_method: SamplingMethod,- on_complete: F,- ) -> Result<(), Error>- where- F: FnOnce(u64) + Send + 'static,- {+ on_complete: Box<dyn FnOnce(u64) + Send>,+ ) -> Result<(), Error> {assert_eq!(self.decoding_config.generate_suffix_length(), 1, "async_generate only supports suffix_length=1");// Extract values from async_buffers before mutable borrow@@ -564,17 +590,13 @@ whereOk(())}- pub fn has_attention_layers(&self) -> bool {- self.context.decoder_config.has_attention_layers()- }-- pub fn clear_cache(&mut self) {+ fn clear_cache(&mut self) {objc2::rc::autoreleasepool(|_pool| {self.encoded_tasks.clear();});}- pub fn reset_state(&mut self) {+ fn reset_state(&mut self) {self.context.cache_layers.borrow_mut().clear();self.tokens.clear();self.registered_prefix_len = 0;@@ -586,8 +608,118 @@ whereself.context.async_buffers.reset_counter();}- pub fn prefix_len(&self) -> usize {- self.registered_prefix_len+ fn tokens_len(&self) -> usize {+ self.tokens.len()+ }+ fn tokens_push(+ &mut self,+ token: u64,+ ) {+ self.tokens.push(token);+ }+ fn generate_suffix_length(&self) -> usize {+ self.decoding_config.generate_suffix_length()+ }+ fn async_batch_size(+ &self,+ model_path: &Path,+ ) -> usize {+ self.decoding_config.async_batch_size.resolve::<B>(model_path, self.context.context.as_ref())+ }++ fn get_slice(+ &self,+ range: Range<usize>,+ ) -> Option<Box<dyn Any>> {+ self.context.cache_layers.borrow().slice(&self.context.context, range).map(|s| Box::new(s) as Box<dyn Any>)+ }+ fn apply_slice(+ &mut self,+ slice: &dyn Any,+ range: Range<usize>,+ ) {+ let slice = slice.downcast_ref::<CacheLayersSlice<B>>().unwrap();+ self.context.cache_layers.borrow_mut().apply_slice(slice, Some(range));+ }++ fn build_llm_context(&self) -> Box<dyn Any> {+ let cache_layers = self.context.cache_layers.borrow().clone(&self.context.context);+ let context = LlmContext::new(self.tokens.clone(), cache_layers, self.decoding_config.clone());+ Box::new(context)+ }+ fn reconfigure_from_context(+ &mut self,+ context: &dyn Any,+ ) {+ let ctx = context.downcast_ref::<LlmContext<B>>().unwrap();+ let mut llm_state = self.context.cache_layers.borrow_mut();+ for (_layer_idx, (ctx_layer, gen_layer)) in+ ctx.cache_layers.data.iter().zip(llm_state.data.iter_mut()).enumerate()+ {+ match (ctx_layer, gen_layer) {+ (CacheLayer::Transformer(src), CacheLayer::Transformer(dst)) => {+ let copy_rows = src.prefix_segment_length();+ if copy_rows > 0 {+ {+ let mut dst_keys = dst.keys.borrow_mut();+ let src_keys = src.keys.borrow();+ dst_keys.copy_slice(&src_keys, 1, 0..copy_rows, 0);+ }+ {+ let mut dst_values = dst.values.borrow_mut();+ let src_values = src.values.borrow();+ dst_values.copy_slice(&src_values, 1, 0..copy_rows, 0);+ }+ }+ dst.state = src.state.clone();+ dst.prefix_token_positions = src.prefix_token_positions.clone();+ },+ (CacheLayer::StateSpace(src), CacheLayer::StateSpace(dst)) => {+ {+ let mut dst_conv = dst.conv_state.borrow_mut();+ let src_conv = src.conv_state.borrow();+ dst_conv.copy_from_array(&src_conv);+ }+ {+ let mut dst_ssm = dst.ssm_state.borrow_mut();+ let src_ssm = src.ssm_state.borrow();+ dst_ssm.copy_from_array(&src_ssm);+ }+ },+ _ => panic!("Layer type mismatch when reconfiguring language model generator cache"),+ }+ }+ drop(llm_state);++ self.tokens = ctx.tokens.clone();+ }+}++impl<B: Backend> LanguageModelGenerator<B> {+ pub fn new(+ model_path: &Path,+ decoding_config: DecodingConfig,+ ) -> Result<Self, Error> {+ let gpu_capture = GpuCaptureManager::new();++ let context = LanguageModelGeneratorContext::new(model_path, &decoding_config)?;+ let prefill_step_size = decoding_config.prefill_step_size.resolve(&context.model_config);+ let generate_suffix_length = decoding_config.generate_suffix_length();++ let mut generator = Self {+ decoding_config,+ tokens: Vec::new(),+ context,+ encoded_tasks: HashMap::new(),+ registered_prefix_len: 0,+ gpu_capture,+ };++ //Warmup+ generator.warmup(prefill_step_size);+ generator.warmup(generate_suffix_length);++ Ok(generator)}fn warmup(diff --git a/crates/uzu/src/language_model/language_model_generator_context.rs b/crates/uzu/src/language_model/language_model_generator_context.rsindex 02912efbd..7f7166b05 100644--- a/crates/uzu/src/language_model/language_model_generator_context.rs+++ b/crates/uzu/src/language_model/language_model_generator_context.rs@@ -10,10 +10,7 @@ use crate::{DataType,backends::common::{Backend, Context, Kernels, NativeBuffer,- kernel::{- MaskUpdateKernel, TokenCopySampledKernel, TokenCopyToResultsKernel, kv_cache_update::KVCacheUpdate,- matmul::MatmulKernels,- },+ kernel::{MaskUpdateKernel, TokenCopySampledKernel, TokenCopyToResultsKernel, kv_cache_update::KVCacheUpdate},},config::{DecoderConfig, LanguageModelConfig, ModelMetadata},encodable_block::{Decoder, Sampling},@@ -148,10 +145,7 @@ pub struct LanguageModelGeneratorContext<B: Backend> {pub async_buffers: AsyncBuffers<B>,}-impl<B: Backend + 'static> LanguageModelGeneratorContext<B>-where- B::Kernels: MatmulKernels,-{+impl<B: Backend> LanguageModelGeneratorContext<B> {pub fn new(model_path: &Path,decoding_config: &DecodingConfig,diff --git a/crates/uzu/src/language_model/mod.rs b/crates/uzu/src/language_model/mod.rsindex b2f74ea01..8c30f89ef 100644--- a/crates/uzu/src/language_model/mod.rs+++ b/crates/uzu/src/language_model/mod.rs@@ -9,5 +9,4 @@ pub mod sampler;pub mod tasks;// Re-export main types-pub use language_model_generator::LanguageModelGenerator;-pub use language_model_generator_context::LanguageModelGeneratorContext;+pub use language_model_generator::{LanguageModelGenerator, LanguageModelGeneratorTrait};diff --git a/crates/uzu/src/language_model/tasks.rs b/crates/uzu/src/language_model/tasks.rsindex fc419fa14..293afef74 100644--- a/crates/uzu/src/language_model/tasks.rs+++ b/crates/uzu/src/language_model/tasks.rs@@ -1,6 +1,6 @@use std::{mem::size_of, ops::DerefMut};-use super::LanguageModelGeneratorContext;+use super::language_model_generator_context::LanguageModelGeneratorContext;use crate::{backends::common::{Backend, Context, NativeBuffer},encodable_block::{EncodableBlock, EncodingParameters},diff --git a/crates/uzu/src/session/chat_session.rs b/crates/uzu/src/session/chat_session.rsindex 6034560e8..5150a184b 100644--- a/crates/uzu/src/session/chat_session.rs+++ b/crates/uzu/src/session/chat_session.rs@@ -1,4 +1,5 @@use std::{+ any::Any,fs::File,io::BufReader,path::PathBuf,@@ -19,17 +20,16 @@ use tokenizers::Tokenizer;use xgrammar::TokenizerInfo;use crate::{- backends::metal::Metal,+ backends::{common::Backend, select_backend},config::{MixerConfig, ModelMetadata},- forward_pass::cache_layers::CacheLayer,language_model::{- LanguageModelGenerator,+ LanguageModelGenerator, LanguageModelGeneratorTrait,grammar::CompiledGrammar,result::{GenerateResult, PrefillResult},},session::{config::{DecodingConfig, RunConfig},- helpers::{Context, InputProcessor, InputProcessorDefault, OutputParser, is_directory_fits_ram},+ helpers::{InputProcessor, InputProcessorDefault, OutputParser, is_directory_fits_ram},parameter::{ConfigResolvableValue, ContextMode},types::{Error, FinishReason, Input, Output, RunStats, Stats, StepStats, TotalStats},},@@ -55,14 +55,22 @@ pub struct ChatSession {tokenizer_info: TokenizerInfo,input_processor: Box<dyn InputProcessor>,output_parser: OutputParser,- llm: Option<LanguageModelGenerator<Metal>>,- static_context: Option<Context<Metal>>,+ decoding_config: DecodingConfig,+ llm: Option<Box<dyn LanguageModelGeneratorTrait>>,+ static_context: Option<Box<dyn Any>>,}impl ChatSession {pub fn new(model_path: PathBuf,decoding_config: DecodingConfig,+ ) -> Result<Self, Error> {+ select_backend!(Self::new_with_backend::<B>(model_path, decoding_config), Error::UnableToCreateBackendContext)+ }++ pub fn new_with_backend<B: Backend>(+ model_path: PathBuf,+ decoding_config: DecodingConfig,) -> Result<Self, Error> {if !model_path.exists() {return Err(Error::ModelFolderNotFound);@@ -130,7 +138,8 @@ impl ChatSession {let output_parser =OutputParser::new(language_model_config.message_processor_config.output_parser_regex.clone())?;- let llm = LanguageModelGenerator::new(&model_path, decoding_config).map_err(Error::from)?;+ let llm =+ Box::new(LanguageModelGenerator::<B>::new(&model_path, decoding_config.clone()).map_err(Error::from)?);Ok(Self {model_path,@@ -139,6 +148,7 @@ impl ChatSession {tokenizer_info,input_processor: Box::new(input_processor),output_parser,+ decoding_config,llm: Some(llm),static_context: None,})@@ -153,10 +163,7 @@ impl ChatSession {whereF: Fn(Output) -> bool,{- let context_mode =- self.llm.as_ref().ok_or(Error::LanguageModelGeneratorNotLoaded)?.decoding_config.context_mode.clone();-- match &context_mode {+ match &self.decoding_config.context_mode {ContextMode::None => {let language_model_generator = self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;language_model_generator.reset_state();@@ -181,7 +188,7 @@ impl ChatSession {let output = self.run_internal(input, config, progress)?;- match context_mode {+ match &self.decoding_config.context_mode {ContextMode::None| ContextMode::Static {..@@ -199,15 +206,13 @@ impl ChatSession {language_model_generator.reset_state();Ok(())}-}-impl ChatSession {fn extend(&mut self,input: Input,- context: Option<&Context<Metal>>,+ context: Option<&Box<dyn Any>>,config: RunConfig,- ) -> Result<(Output, Context<Metal>), Error> {+ ) -> Result<(Output, Box<dyn Any>), Error> {self.reconfigure_language_model_generator(context)?;let output = self.run_internal(input, config, None::<fn(Output) -> bool>)?;let new_context = self.build_context_from_language_model_generator()?;@@ -240,12 +245,12 @@ impl ChatSession {self.model_metadata.model_config.as_language_model().ok_or(Error::UnableToLoadConfig)?;let language_model_generator = self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;- let context_length = language_model_generator.decoding_config.context_length.resolve(language_model_config);+ let context_length = self.decoding_config.context_length.resolve(language_model_config);if tokens.len() >= context_length {return Err(Error::ContextLengthExceeded);}- let prefix_offset = language_model_generator.tokens.len();+ let prefix_offset = language_model_generator.tokens_len();let prefix_len_before = prefix_offset;let eos_tokens: Vec<u64> =@@ -272,8 +277,7 @@ impl ChatSession {let prefill_duration = prefill_start.elapsed().as_secs_f64();language_model_generator.clear_cache();- let prefill_suffix_length =- language_model_generator.decoding_config.prefill_step_size.resolve(language_model_config);+ let prefill_suffix_length = self.decoding_config.prefill_step_size.resolve(language_model_config);let run_context = RunContext {eos_tokens,@@ -292,13 +296,13 @@ impl ChatSession {let prefill_finish_reason = if grammar_terminated {Some(FinishReason::Stop)} else {- Self::check_finish_reason(&run_context, language_model_generator, &prefill_tokens)+ Self::check_finish_reason(&run_context, language_model_generator.as_ref(), &prefill_tokens)};let prefill_output = Self::build_output(&self.tokenizer,&self.output_parser,&run_context,- language_model_generator,+ language_model_generator.as_ref(),&[],&[],prefill_finish_reason.clone(),@@ -318,19 +322,15 @@ impl ChatSession {}}- let can_use_async =- language_model_generator.decoding_config.generate_suffix_length() == 1 && compiled_grammar.is_none();+ let can_use_async = language_model_generator.generate_suffix_length() == 1 && compiled_grammar.is_none();let generate_output = if can_use_async {- let batch_size = language_model_generator- .decoding_config- .async_batch_size- .resolve::<Metal>(&self.model_path, language_model_generator.context.context.as_ref());+ let batch_size = language_model_generator.async_batch_size(&self.model_path);Self::run_async_batch(&self.tokenizer,&self.output_parser,&run_context,- language_model_generator,+ language_model_generator.as_mut(),sampling_method,&progress,batch_size,@@ -340,7 +340,7 @@ impl ChatSession {&self.tokenizer,&self.output_parser,&run_context,- language_model_generator,+ language_model_generator.as_mut(),compiled_grammar.as_mut(),sampling_method,&progress,@@ -355,7 +355,7 @@ impl ChatSession {tokenizer: &Tokenizer,output_parser: &OutputParser,run_context: &RunContext,- language_model_generator: &mut LanguageModelGenerator<Metal>,+ language_model_generator: &mut dyn LanguageModelGeneratorTrait,compiled_grammar: Option<&mut CompiledGrammar>,sampling_method: super::parameter::SamplingMethod,progress: &Option<F>,@@ -413,7 +413,7 @@ impl ChatSession {tokenizer: &Tokenizer,output_parser: &OutputParser,run_context: &RunContext,- llm: &mut LanguageModelGenerator<Metal>,+ llm: &mut dyn LanguageModelGeneratorTrait,sampling_method: super::parameter::SamplingMethod,progress: &Option<F>,batch_size: usize,@@ -422,7 +422,7 @@ impl ChatSession {F: Fn(Output) -> bool,{let remaining_by_limit = run_context.tokens_limit.saturating_sub(run_context.prefill_result.tokens.len());- let remaining_by_context = run_context.context_length.saturating_sub(llm.tokens.len());+ let remaining_by_context = run_context.context_length.saturating_sub(llm.tokens_len());let tokens_to_generate = remaining_by_limit.min(remaining_by_context);llm.prepare_async(tokens_to_generate);@@ -442,17 +442,21 @@ impl ChatSession {let batch_submitted = batch_end - next_to_submit;let accepted_before_batch = results.len();- let cache_slice = llm.context.cache_layers.borrow().slice(&llm.context.context, 0..batch_submitted);+ let cache_slice = llm.get_slice(0..batch_submitted);for idx in next_to_submit..batch_end {let batch_sender = sender.clone();let last_callback_nanos = last_nanos.clone();let batch_start_time = start_time;- llm.async_generate(idx, sampling_method, move |token| {- let now_nanos = batch_start_time.elapsed().as_nanos() as u64;- let prev_nanos = last_callback_nanos.swap(now_nanos, Ordering::SeqCst);- let _ = batch_sender.send((idx, token, now_nanos.saturating_sub(prev_nanos)));- })?;+ llm.async_generate(+ idx,+ sampling_method,+ Box::new(move |token| {+ let now_nanos = batch_start_time.elapsed().as_nanos() as u64;+ let prev_nanos = last_callback_nanos.swap(now_nanos, Ordering::SeqCst);+ let _ = batch_sender.send((idx, token, now_nanos.saturating_sub(prev_nanos)));+ }),+ )?;in_flight += 1;}next_to_submit = batch_end;@@ -462,7 +466,7 @@ impl ChatSession {in_flight -= 1;let duration = nanos_to_secs(duration_nanos);- llm.tokens.push(token);+ llm.tokens_push(token);results.push(GenerateResult {tokens: vec![token],forwardpass_duration: duration,@@ -472,7 +476,7 @@ impl ChatSession {let should_stop = if run_context.eos_tokens.contains(&token) {finish_reason = FinishReason::Stop;true- } else if llm.tokens.len() >= run_context.context_length {+ } else if llm.tokens_len() >= run_context.context_length {finish_reason = FinishReason::ContextLimitReached;true} else if let Some(progress_fn) = progress {@@ -496,10 +500,7 @@ impl ChatSession {if let Some(slice) = cache_slice {let accepted_in_batch = results.len().saturating_sub(accepted_before_batch);if accepted_in_batch < batch_submitted {- llm.context- .cache_layers- .borrow_mut()- .apply_slice(&slice, Some(accepted_in_batch..batch_submitted));+ llm.apply_slice(slice.as_ref(), accepted_in_batch..batch_submitted);}}return Self::build_output(@@ -520,78 +521,30 @@ impl ChatSession {fn reconfigure_language_model_generator(&mut self,- context: Option<&Context<Metal>>,+ context: Option<&Box<dyn Any>>,) -> Result<(), Error> {let language_model_generator = self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;language_model_generator.reset_state();if let Some(ctx) = context {- let mut llm_state = language_model_generator.context.cache_layers.borrow_mut();- for (_layer_idx, (ctx_layer, gen_layer)) in- ctx.cache_layers.data.iter().zip(llm_state.data.iter_mut()).enumerate()- {- match (ctx_layer, gen_layer) {- (CacheLayer::Transformer(src), CacheLayer::Transformer(dst)) => {- let copy_rows = src.prefix_segment_length();- if copy_rows > 0 {- {- let mut dst_keys = dst.keys.borrow_mut();- let src_keys = src.keys.borrow();- dst_keys.copy_slice(&src_keys, 1, 0..copy_rows, 0);- }- {- let mut dst_values = dst.values.borrow_mut();- let src_values = src.values.borrow();- dst_values.copy_slice(&src_values, 1, 0..copy_rows, 0);- }- }- dst.state = src.state.clone();- dst.prefix_token_positions = src.prefix_token_positions.clone();- },- (CacheLayer::StateSpace(src), CacheLayer::StateSpace(dst)) => {- {- let mut dst_conv = dst.conv_state.borrow_mut();- let src_conv = src.conv_state.borrow();- dst_conv.copy_from_array(&src_conv);- }- {- let mut dst_ssm = dst.ssm_state.borrow_mut();- let src_ssm = src.ssm_state.borrow();- dst_ssm.copy_from_array(&src_ssm);- }- },- _ => panic!("Layer type mismatch when reconfiguring language model generator cache"),- }- }- drop(llm_state);-- language_model_generator.tokens = ctx.tokens.clone();+ language_model_generator.reconfigure_from_context(ctx.as_ref());}Ok(())}- fn build_context_from_language_model_generator(&self) -> Result<Context<Metal>, Error> {+ fn build_context_from_language_model_generator(&self) -> Result<Box<dyn Any>, Error> {let language_model_generator = self.llm.as_ref().ok_or(Error::LanguageModelGeneratorNotLoaded)?;- let cache_layers =- language_model_generator.context.cache_layers.borrow().clone(&language_model_generator.context.context);- let context = Context::new(- language_model_generator.tokens.clone(),- cache_layers,- language_model_generator.decoding_config.clone(),- );- Ok(context)+ Ok(language_model_generator.build_llm_context())}-}-impl ChatSession {fn check_finish_reason(run_context: &RunContext,- language_model_generator: &LanguageModelGenerator<Metal>,+ language_model_generator: &dyn LanguageModelGeneratorTrait,new_tokens: &[u64],) -> Option<FinishReason> {let start_idx = run_context.prefix_len_before + run_context.input_tokens_len;- let total_new_tokens = language_model_generator.tokens.len().saturating_sub(start_idx);+ let total_new_tokens = language_model_generator.tokens_len().saturating_sub(start_idx);let has_eos = new_tokens.iter().any(|t| run_context.eos_tokens.contains(t));- let context_limit = language_model_generator.tokens.len() >= run_context.context_length;+ let context_limit = language_model_generator.tokens_len() >= run_context.context_length;if has_eos {Some(FinishReason::Stop)@@ -621,7 +574,7 @@ impl ChatSession {tokenizer: &Tokenizer,output_parser: &OutputParser,run_context: &RunContext,- language_model_generator: &LanguageModelGenerator<Metal>,+ language_model_generator: &dyn LanguageModelGeneratorTrait,generate_results: &[GenerateResult],generate_durations: &[f64],finish_reason: Option<FinishReason>,@@ -629,7 +582,7 @@ impl ChatSession {let text = Self::decode_generated_tokens(tokenizer, &run_context.prefill_result.tokens, generate_results)?;let parsed = output_parser.parse(text);let start_idx = run_context.prefix_len_before + run_context.input_tokens_len;- let output_tokens = language_model_generator.tokens.len().saturating_sub(start_idx);+ let output_tokens = language_model_generator.tokens_len().saturating_sub(start_idx);Ok(Output {text: parsed,@@ -639,7 +592,7 @@ impl ChatSession {run_context.prefill_suffix_length,generate_results.to_vec(),generate_durations.to_vec(),- language_model_generator.decoding_config.generate_suffix_length(),+ language_model_generator.generate_suffix_length(),run_context.run_start.elapsed().as_secs_f64(),run_context.input_tokens_len,output_tokens,diff --git a/crates/uzu/src/session/classification_session.rs b/crates/uzu/src/session/classification_session.rsindex c1bd9e835..57fff930b 100644--- a/crates/uzu/src/session/classification_session.rs+++ b/crates/uzu/src/session/classification_session.rs@@ -3,8 +3,8 @@ use std::path::PathBuf;use tokenizers::Tokenizer;use crate::{- backends::metal::Metal,- classifier::{ClassificationOutput, Classifier},+ backends::{common::Backend, select_backend},+ classifier::{ClassificationOutput, Classifier, ClassifierTrait},config::ModelMetadata,session::{helpers::{InputProcessor, InputProcessorDefault},@@ -18,12 +18,16 @@ pub struct ClassificationSession {#[allow(dead_code)]model_metadata: ModelMetadata,tokenizer: Tokenizer,- classifier: Classifier<Metal>,+ classifier: Box<dyn ClassifierTrait>,input_processor: Box<dyn InputProcessor>,}impl ClassificationSession {pub fn new(model_path: PathBuf) -> Result<Self, Error> {+ select_backend!(Self::new_with_backend::<B>(model_path), Error::UnableToCreateBackendContext)+ }++ pub fn new_with_backend<B: Backend>(model_path: PathBuf) -> Result<Self, Error> {let config_path = model_path.join("config.json");if !config_path.exists() {return Err(Error::ModelFolderNotFound);@@ -43,7 +47,7 @@ impl ClassificationSession {let input_processor =Box::new(InputProcessorDefault::new(classifier_model_config.message_processor_config.clone()))as Box<dyn InputProcessor>;- let classifier = Classifier::new(&model_path)?;+ let classifier = Box::new(Classifier::<B>::new(&model_path)?);Ok(Self {model_path,diff --git a/crates/uzu/src/tracer/trace_validator.rs b/crates/uzu/src/tracer/trace_validator.rsindex c3e3f5d43..7f24c91d5 100644--- a/crates/uzu/src/tracer/trace_validator.rs+++ b/crates/uzu/src/tracer/trace_validator.rs@@ -18,10 +18,7 @@ use num_traits::NumCast;use crate::{ArrayElement, DataType,array::Array,- backends::common::{- Backend, CommandBuffer, Context,- kernel::{kv_cache_update::KVCacheUpdate, matmul::MatmulKernels},- },+ backends::common::{Backend, CommandBuffer, Context, kernel::kv_cache_update::KVCacheUpdate},classifier::Classifier,config::ModelMetadata,encodable_block::{EncodableBlock, EncodingParameters, Sampling},@@ -32,7 +29,7 @@ use crate::{traces::ActivationTrace,},language_model::{- LanguageModelGeneratorContext,+ language_model_generator_context::LanguageModelGeneratorContext,sampler::{ArgmaxSampler, LogitsSampler},},parameters::{ParameterLoader, ParameterTree, read_safetensors_metadata},@@ -155,10 +152,7 @@ pub enum ArrayTransform {// Model Context (internal)// ============================================================================-enum ModelContext<B: Backend + 'static>-where- B::Kernels: MatmulKernels,-{+enum ModelContext<B: Backend> {LanguageModelGenerator(LanguageModelGeneratorContext<B>),Classifier(Classifier<B>),}@@ -171,18 +165,12 @@ where////// Automatically detects whether the model is an LLM or classifier and/// runs the appropriate validation.-pub struct TraceValidator<B: Backend + 'static>-where- B::Kernels: MatmulKernels,-{+pub struct TraceValidator<B: Backend> {model_path: PathBuf,context: ModelContext<B>,}-impl<B: Backend + 'static> TraceValidator<B>-where- B::Kernels: MatmulKernels,-{+impl<B: Backend> TraceValidator<B> {/// Create a new trace validator for the given model path.////// Automatically detects the model type from config.json.