Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Dynamic dispatch backends (#222)

uuuvn874dc65b2047
diff --git a/crates/uzu/src/backends/common/backend.rs b/crates/uzu/src/backends/common/backend.rs
index 62f878678..9d811eb5c 100644
--- a/crates/uzu/src/backends/common/backend.rs
+++ b/crates/uzu/src/backends/common/backend.rs
@@ -1,15 +1,15 @@
use std::{error::Error, fmt::Debug};
-use super::{CommandBuffer, Context, CopyEncoder, Event, Kernels, NativeBuffer};
+use super::{CommandBuffer, Context, CopyEncoder, Event, Kernels, NativeBuffer, kernel::matmul::MatmulKernels};
-pub trait Backend: Debug + Clone {
+pub trait Backend: Debug + Clone + 'static {
type Context: Context<Backend = Self>;
type NativeBuffer: NativeBuffer<Backend = Self>;
type CommandBuffer: CommandBuffer<Backend = Self>;
type ComputeEncoder;
type CopyEncoder: CopyEncoder<Backend = Self>;
type Event: Event<Backend = Self>;
- type Kernels: Kernels<Backend = Self>;
+ type Kernels: Kernels<Backend = Self> + MatmulKernels;
type Error: Error + Debug;
const MAX_INLINE_BYTES: usize;
diff --git a/crates/uzu/src/backends/mod.rs b/crates/uzu/src/backends/mod.rs
index d69e0ae69..5560db88f 100644
--- a/crates/uzu/src/backends/mod.rs
+++ b/crates/uzu/src/backends/mod.rs
@@ -3,3 +3,17 @@ pub mod common;
pub mod cpu;
#[cfg(feature = "metal")]
pub mod metal;
+
+macro_rules! select_backend {
+ ($expr:expr, $unk:expr) => {{
+ match std::env::var("UZU_BACKEND").map(|s| s.to_lowercase()).as_deref().unwrap_or("metal") {
+ #[cfg(feature = "metal")]
+ "metal" => {
+ type B = crate::backends::metal::Metal;
+ $expr
+ },
+ _ => Err($unk),
+ }
+ }};
+}
+pub(crate) use select_backend;
diff --git a/crates/uzu/src/classifier/classifier.rs b/crates/uzu/src/classifier/classifier.rs
index 479d205f7..151d77fa4 100644
--- a/crates/uzu/src/classifier/classifier.rs
+++ b/crates/uzu/src/classifier/classifier.rs
@@ -9,31 +9,26 @@ use super::ActivationTrace;
use super::{ClassificationOutput, ClassificationStats, ClassifierContext};
use crate::{
DataType,
- backends::common::{Backend, CommandBuffer, kernel::matmul::MatmulKernels},
+ backends::common::{Backend, CommandBuffer},
encodable_block::{EncodableBlock, EncodingParameters},
forward_pass::state::{ArrayId, ForwardPassState},
session::types::Error,
};
-pub struct Classifier<B: Backend + 'static>
-where
- B::Kernels: MatmulKernels,
-{
+pub struct Classifier<B: Backend> {
pub context: ClassifierContext<B>,
}
-impl<B: Backend + 'static> Classifier<B>
-where
- B::Kernels: MatmulKernels,
-{
- pub fn new(model_path: &Path) -> Result<Self, Error> {
- let context = ClassifierContext::new(model_path)?;
- Ok(Self {
- context,
- })
- }
+pub trait ClassifierTrait {
+ fn classify_tokens(
+ &mut self,
+ token_ids: Vec<u64>,
+ token_positions: Vec<usize>,
+ ) -> Result<ClassificationOutput, Error>;
+}
- pub fn classify_tokens(
+impl<B: Backend> ClassifierTrait for Classifier<B> {
+ fn classify_tokens(
&mut self,
token_ids: Vec<u64>,
token_positions: Vec<usize>,
@@ -78,6 +73,15 @@ where
})
})
}
+}
+
+impl<B: Backend> Classifier<B> {
+ pub fn new(model_path: &Path) -> Result<Self, Error> {
+ let context = ClassifierContext::new(model_path)?;
+ Ok(Self {
+ context,
+ })
+ }
#[cfg(feature = "tracing")]
pub fn forward_pass_with_traces(
diff --git a/crates/uzu/src/classifier/classifier_context.rs b/crates/uzu/src/classifier/classifier_context.rs
index 6a95d5c34..ef05c7ef3 100644
--- a/crates/uzu/src/classifier/classifier_context.rs
+++ b/crates/uzu/src/classifier/classifier_context.rs
@@ -2,10 +2,7 @@ use std::{cell::RefCell, fs::File, io::BufReader, path::Path, rc::Rc};
use crate::{
DataType,
- backends::common::{
- Backend, Context, Kernels,
- kernel::{SigmoidKernel, matmul::MatmulKernels},
- },
+ backends::common::{Backend, Context, Kernels, kernel::SigmoidKernel},
classifier::ClassifierError,
config::{ClassifierModelConfig, ModelMetadata},
encodable_block::{
@@ -44,10 +41,7 @@ pub struct ClassifierContext<B: Backend> {
pub sigmoid_kernel: <B::Kernels as Kernels>::SigmoidKernel,
}
-impl<B: Backend + 'static> ClassifierContext<B>
-where
- B::Kernels: MatmulKernels,
-{
+impl<B: Backend> ClassifierContext<B> {
pub fn new(model_path: &Path) -> Result<Self, Error> {
let context = B::Context::new().map_err(|_| Error::UnableToCreateBackendContext)?;
diff --git a/crates/uzu/src/classifier/mod.rs b/crates/uzu/src/classifier/mod.rs
index a8dfc18e8..5857dc8b4 100644
--- a/crates/uzu/src/classifier/mod.rs
+++ b/crates/uzu/src/classifier/mod.rs
@@ -3,7 +3,7 @@ mod classifier_context;
mod error;
mod result;
-pub use classifier::Classifier;
+pub use classifier::{Classifier, ClassifierTrait};
pub use classifier_context::ClassifierContext;
pub use error::ClassifierError;
pub use result::{ClassificationOutput, ClassificationStats};
diff --git a/crates/uzu/src/encodable_block/classifier_layer.rs b/crates/uzu/src/encodable_block/classifier_layer.rs
index a452cd3ed..e2389ee61 100644
--- a/crates/uzu/src/encodable_block/classifier_layer.rs
+++ b/crates/uzu/src/encodable_block/classifier_layer.rs
@@ -6,7 +6,7 @@ use objc2::rc::autoreleasepool;
use crate::backends::common::CommandBuffer;
use crate::{
DataType,
- backends::common::{Backend, kernel::matmul::MatmulKernels},
+ backends::common::Backend,
config::TransformerLayerConfig,
encodable_block::{
Attention, EncodableBlock, EncodingParameters, Normalization, QKNorm, TensorAddSwap, TensorCopy,
@@ -35,10 +35,7 @@ pub struct ClassifierLayer<B: Backend> {
mlp_residual_add: Box<dyn EncodableBlock<B>>,
}
-impl<B: Backend + 'static> ClassifierLayer<B>
-where
- B::Kernels: MatmulKernels,
-{
+impl<B: Backend> ClassifierLayer<B> {
pub fn new(
context: Rc<B::Context>,
layer_config: &TransformerLayerConfig,
diff --git a/crates/uzu/src/encodable_block/decoder.rs b/crates/uzu/src/encodable_block/decoder.rs
index 39c8e0f3d..ac62d59fc 100644
--- a/crates/uzu/src/encodable_block/decoder.rs
+++ b/crates/uzu/src/encodable_block/decoder.rs
@@ -4,7 +4,7 @@ use std::rc::Rc;
use crate::{
DataType, DecoderConfig,
- backends::common::{Backend, kernel::matmul::MatmulKernels},
+ backends::common::Backend,
config::{DecoderLayerType, MixerConfig},
encodable_block::{
EncodableBlock, EncodingParameters, LayerExecutables, RMSNorm, Rope,
@@ -27,10 +27,7 @@ pub struct Decoder<B: Backend> {
pub local_rope: Option<Rc<Box<dyn EncodableBlock<B>>>>,
}
-impl<B: Backend + 'static> Decoder<B>
-where
- B::Kernels: MatmulKernels,
-{
+impl<B: Backend> Decoder<B> {
pub fn new(
context: Rc<B::Context>,
decoder_config: Rc<DecoderConfig>,
diff --git a/crates/uzu/src/encodable_block/full_precision_embedding_readout.rs b/crates/uzu/src/encodable_block/full_precision_embedding_readout.rs
index 7262b584a..5342b58d3 100644
--- a/crates/uzu/src/encodable_block/full_precision_embedding_readout.rs
+++ b/crates/uzu/src/encodable_block/full_precision_embedding_readout.rs
@@ -36,20 +36,14 @@ pub enum FullPrecisionEmbeddingReadoutError<B: Backend> {
},
}
-pub struct FullPrecisionEmbeddingReadout<B: Backend>
-where
- B::Kernels: MatmulKernels,
-{
+pub struct FullPrecisionEmbeddingReadout<B: Backend> {
kernel: RefCell<<B::Kernels as MatmulKernels>::FullPrecisionMatmulKernel>,
weights_buffer: Rc<RefCell<B::NativeBuffer>>,
vocab_size: usize,
model_dim: usize,
}
-impl<B: Backend> FullPrecisionEmbeddingReadout<B>
-where
- B::Kernels: MatmulKernels,
-{
+impl<B: Backend> FullPrecisionEmbeddingReadout<B> {
pub fn new(
context: &B::Context,
data_type: DataType,
@@ -96,10 +90,7 @@ where
}
}
-impl<B: Backend> EncodableBlock<B> for FullPrecisionEmbeddingReadout<B>
-where
- B::Kernels: MatmulKernels,
-{
+impl<B: Backend> EncodableBlock<B> for FullPrecisionEmbeddingReadout<B> {
fn supports_shared_encoder(&self) -> bool {
true
}
diff --git a/crates/uzu/src/encodable_block/full_precision_linear.rs b/crates/uzu/src/encodable_block/full_precision_linear.rs
index fd9c37509..6b748773a 100644
--- a/crates/uzu/src/encodable_block/full_precision_linear.rs
+++ b/crates/uzu/src/encodable_block/full_precision_linear.rs
@@ -44,10 +44,7 @@ pub enum FullPrecisionLinearError<B: Backend> {
},
}
-pub struct FullPrecisionLinear<B: Backend>
-where
- B::Kernels: MatmulKernels,
-{
+pub struct FullPrecisionLinear<B: Backend> {
kernel: RefCell<<B::Kernels as MatmulKernels>::FullPrecisionMatmulKernel>,
bias_buffer: Option<Rc<RefCell<B::NativeBuffer>>>,
weights_buffer: Rc<RefCell<B::NativeBuffer>>,
@@ -57,10 +54,7 @@ where
output_array_id: ArrayId,
}
-impl<B: Backend> FullPrecisionLinear<B>
-where
- B::Kernels: MatmulKernels,
-{
+impl<B: Backend> FullPrecisionLinear<B> {
pub fn new(
context: &B::Context,
precision: DataType,
@@ -128,10 +122,7 @@ where
}
}
-impl<B: Backend> EncodableBlock<B> for FullPrecisionLinear<B>
-where
- B::Kernels: MatmulKernels,
-{
+impl<B: Backend> EncodableBlock<B> for FullPrecisionLinear<B> {
fn supports_shared_encoder(&self) -> bool {
true
}
diff --git a/crates/uzu/src/encodable_block/layer/executables.rs b/crates/uzu/src/encodable_block/layer/executables.rs
index acb6db004..f49b3dc76 100644
--- a/crates/uzu/src/encodable_block/layer/executables.rs
+++ b/crates/uzu/src/encodable_block/layer/executables.rs
@@ -9,7 +9,7 @@ use super::MixerExecutables;
use crate::backends::common::CommandBuffer;
use crate::{
DataType, DecoderLayerConfig,
- backends::common::{Backend, kernel::matmul::MatmulKernels},
+ backends::common::Backend,
config::{DecoderLayerType, MixerConfig},
encodable_block::{
Attention, EncodableBlock, EncodingParameters, MambaMixer, QKNorm, RMSNorm, ShortConvMixer, TensorAddSwap,
@@ -33,10 +33,7 @@ pub struct LayerExecutables<B: Backend> {
pub post_mlp_norm: Option<Box<dyn EncodableBlock<B>>>,
}
-impl<B: Backend + 'static> LayerExecutables<B>
-where
- B::Kernels: MatmulKernels,
-{
+impl<B: Backend> LayerExecutables<B> {
#[allow(clippy::too_many_arguments)]
pub fn new(
context: Rc<B::Context>,
diff --git a/crates/uzu/src/encodable_block/mamba_mixer.rs b/crates/uzu/src/encodable_block/mamba_mixer.rs
index c248bebce..f9e0c1fc2 100644
--- a/crates/uzu/src/encodable_block/mamba_mixer.rs
+++ b/crates/uzu/src/encodable_block/mamba_mixer.rs
@@ -9,7 +9,6 @@ use crate::{
Backend, CommandBuffer, Kernels,
kernel::{
Conv1dDecodeKernel, Conv1dPackKernel, Conv1dScanKernel, SSDUpdateKernel, SplitInProjKernel,
- matmul::MatmulKernels,
ssd_prefill::{SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode},
},
},
@@ -37,10 +36,7 @@ pub(crate) struct MambaMixer<B: Backend> {
prefill_mode: SSDPrefillMode,
}
-impl<B: Backend + 'static> MambaMixer<B>
-where
- B::Kernels: MatmulKernels,
-{
+impl<B: Backend> MambaMixer<B> {
#[allow(clippy::too_many_arguments)]
pub(crate) fn new(
context: &B::Context,
diff --git a/crates/uzu/src/encodable_block/short_conv_mixer.rs b/crates/uzu/src/encodable_block/short_conv_mixer.rs
index 6b79f4437..dccc21457 100644
--- a/crates/uzu/src/encodable_block/short_conv_mixer.rs
+++ b/crates/uzu/src/encodable_block/short_conv_mixer.rs
@@ -5,10 +5,7 @@ use crate::{
array::Array,
backends::common::{
Backend, CommandBuffer, Context, Kernels,
- kernel::{
- ShortConvDecodeKernel, ShortConvPackKernel, ShortConvPrefillKernel, ShortConvTrieKernel,
- matmul::MatmulKernels,
- },
+ kernel::{ShortConvDecodeKernel, ShortConvPackKernel, ShortConvPrefillKernel, ShortConvTrieKernel},
},
config::{DecoderLayerType, ShortConvConfig},
encodable_block::{EncodableBlock, EncodingParameters, transformer_layer::linear_block},
@@ -30,10 +27,7 @@ pub struct ShortConvMixer<B: Backend> {
conv_bias: Option<Array<B>>,
}
-impl<B: Backend + 'static> ShortConvMixer<B>
-where
- B::Kernels: MatmulKernels,
-{
+impl<B: Backend> ShortConvMixer<B> {
pub fn new(
context: &B::Context,
layer_type: DecoderLayerType,
diff --git a/crates/uzu/src/encodable_block/transformer_layer.rs b/crates/uzu/src/encodable_block/transformer_layer.rs
index 705abd0e1..736fe8fb3 100644
--- a/crates/uzu/src/encodable_block/transformer_layer.rs
+++ b/crates/uzu/src/encodable_block/transformer_layer.rs
@@ -6,10 +6,7 @@ use super::{
};
use crate::{
DataType,
- backends::common::{
- Backend,
- kernel::{matmul::MatmulKernels, mlp_gate_act_mul::MlpGateActMulEncodable},
- },
+ backends::common::{Backend, kernel::mlp_gate_act_mul::MlpGateActMulEncodable},
config::{DecoderConfig, EmbeddingConfig, LinearConfig, MLPConfig},
encodable_block::{
EncodableBlock, FullPrecisionLinearError, MlpBlock, MoeBlock, QuantizedLinearError, moe_block::MoeBlockError,
@@ -34,7 +31,7 @@ pub enum LayerError<B: Backend> {
QLoRaNotSupported,
}
-pub fn linear_block<const N: usize, B: Backend + 'static>(
+pub fn linear_block<const N: usize, B: Backend>(
config: &LinearConfig,
_has_biases: bool,
input_dimension: usize,
@@ -43,10 +40,7 @@ pub fn linear_block<const N: usize, B: Backend + 'static>(
parameter_tree: &ParameterTree<B::Context>,
input_array_id: ArrayId,
output_array_id: ArrayId,
-) -> Result<Box<dyn EncodableBlock<B>>, LayerError<B>>
-where
- B::Kernels: MatmulKernels,
-{
+) -> Result<Box<dyn EncodableBlock<B>>, LayerError<B>> {
let output_dimension_sum: usize = output_dimensions.iter().sum();
match config {
LinearConfig::Quantized(quantization_config) | LinearConfig::MLXQuantized(quantization_config) => {
@@ -84,16 +78,13 @@ where
}
/// Creates an MLP block using the unfused implementation (separate up, activation, down)
-pub fn mlp_block<B: Backend + 'static>(
+pub fn mlp_block<B: Backend>(
config: &MLPConfig,
model_dimension: usize,
hidden_dimension: usize,
context: &B::Context,
parameter_tree: &ParameterTree<B::Context>,
-) -> Result<Box<dyn EncodableBlock<B>>, LayerError<B>>
-where
- B::Kernels: MatmulKernels,
-{
+) -> Result<Box<dyn EncodableBlock<B>>, LayerError<B>> {
if let crate::config::MLPConfig::Dense(dense_config) = config {
let data_type: DataType = dense_config.linear_config.activation_precision().into();
@@ -139,7 +130,7 @@ where
unreachable!("Unknown MLP config")
}
-pub fn embed_block<B: Backend + 'static>(
+pub fn embed_block<B: Backend>(
config: &DecoderConfig,
context: &B::Context,
parameter_tree: &ParameterTree<B::Context>,
@@ -279,14 +270,11 @@ pub fn embed_block<B: Backend + 'static>(
}
}
-pub fn readout_block<B: Backend + 'static>(
+pub fn readout_block<B: Backend>(
config: &DecoderConfig,
context: &B::Context,
parameter_tree: &ParameterTree<B::Context>,
-) -> Box<dyn EncodableBlock<B>>
-where
- B::Kernels: MatmulKernels,
-{
+) -> Box<dyn EncodableBlock<B>> {
match &config.embedding_config {
EmbeddingConfig::Tied {
precision,
diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rs
index 5cb0c89ea..dd27362c8 100644
--- a/crates/uzu/src/language_model/language_model_generator.rs
+++ b/crates/uzu/src/language_model/language_model_generator.rs
@@ -1,7 +1,8 @@
use std::{
+ any::Any,
collections::HashMap,
iter::repeat_n,
- ops::{Deref, DerefMut},
+ ops::{Deref, DerefMut, Range},
path::Path,
sync::Arc,
time::Instant,
@@ -10,9 +11,9 @@ use std::{
use itertools::{Either, Itertools, izip};
use super::{
- LanguageModelGeneratorContext,
gpu_capture::GpuCaptureManager,
grammar::CompiledGrammar,
+ language_model_generator_context::LanguageModelGeneratorContext,
result::{GenerateResult, PrefillResult},
rng::PRng,
tasks::{LanguageModelGeneratorEncodedTask, LanguageModelGeneratorRunTask},
@@ -20,25 +21,24 @@ use super::{
use crate::{
backends::common::{
Backend, CommandBuffer, Context, NativeBuffer,
- kernel::{MaskUpdateKernel, TokenCopySampledKernel, TokenCopyToResultsKernel, matmul::MatmulKernels},
+ kernel::{MaskUpdateKernel, TokenCopySampledKernel, TokenCopyToResultsKernel},
},
encodable_block::{EncodableBlock, EncodingParameters},
forward_pass::{
+ cache_layers::{CacheLayer, CacheLayersSlice},
kv_cache_layer::{AttentionBiasUpdate, INVALID_POSITION},
state::ForwardPassState,
},
session::{
config::DecodingConfig,
+ helpers::Context as LlmContext,
parameter::{ConfigResolvableValue, ResolvableValue, SamplingMethod},
types::Error,
},
trie::{TrieCreationConfig, TrieNode},
};
-pub struct LanguageModelGenerator<B: Backend + 'static>
-where
- B::Kernels: MatmulKernels,
-{
+pub struct LanguageModelGenerator<B: Backend> {
pub decoding_config: DecodingConfig,
pub tokens: Vec<u64>,
@@ -48,37 +48,66 @@ where
gpu_capture: GpuCaptureManager<B>,
}
-impl<B: Backend + 'static> LanguageModelGenerator<B>
-where
- B::Kernels: MatmulKernels,
-{
- pub fn new(
- model_path: &Path,
- decoding_config: DecodingConfig,
- ) -> Result<Self, Error> {
- let gpu_capture = GpuCaptureManager::new();
+pub trait LanguageModelGeneratorTrait {
+ fn prefill(
+ &mut self,
+ tokens: Vec<u64>,
+ compiled_grammar: Option<&mut CompiledGrammar>,
+ sampling_method: SamplingMethod,
+ prefix_offset: usize,
+ sample_suffix: bool,
+ ) -> Result<PrefillResult, Error>;
- let context = LanguageModelGeneratorContext::new(model_path, &decoding_config)?;
- let prefill_step_size = decoding_config.prefill_step_size.resolve(&context.model_config);
- let generate_suffix_length = decoding_config.generate_suffix_length();
+ fn generate(
+ &mut self,
+ compiled_grammar: Option<&mut CompiledGrammar>,
+ sampling_method: SamplingMethod,
+ ) -> Result<GenerateResult, Error>;
- let mut generator = Self {
- decoding_config,
- tokens: Vec::new(),
- context,
- encoded_tasks: HashMap::new(),
- registered_prefix_len: 0,
- gpu_capture,
- };
+ fn prepare_async(
+ &mut self,
+ tokens_to_generate: usize,
+ );
+ fn async_generate(
+ &mut self,
+ pass_idx: usize,
+ sampling_method: SamplingMethod,
+ on_complete: Box<dyn FnOnce(u64) + Send>,
+ ) -> Result<(), Error>;
- //Warmup
- generator.warmup(prefill_step_size);
- generator.warmup(generate_suffix_length);
+ fn clear_cache(&mut self);
+ fn reset_state(&mut self);
- Ok(generator)
- }
+ fn tokens_len(&self) -> usize;
+ fn tokens_push(
+ &mut self,
+ token: u64,
+ );
+ fn generate_suffix_length(&self) -> usize;
+ fn async_batch_size(
+ &self,
+ model_path: &Path,
+ ) -> usize;
- pub fn prefill(
+ fn get_slice(
+ &self,
+ range: Range<usize>,
+ ) -> Option<Box<dyn Any>>;
+ fn apply_slice(
+ &mut self,
+ slice: &dyn Any,
+ range: Range<usize>,
+ );
+
+ fn build_llm_context(&self) -> Box<dyn Any>;
+ fn reconfigure_from_context(
+ &mut self,
+ context: &dyn Any,
+ );
+}
+
+impl<B: Backend> LanguageModelGeneratorTrait for LanguageModelGenerator<B> {
+ fn prefill(
&mut self,
tokens: Vec<u64>,
mut compiled_grammar: Option<&mut CompiledGrammar>,
@@ -273,7 +302,7 @@ where
})
}
- pub fn generate(
+ fn generate(
&mut self,
mut compiled_grammar: Option<&mut CompiledGrammar>,
sampling_method: SamplingMethod,
@@ -357,7 +386,7 @@ where
/// Prepares async buffers for generation.
/// Must be called after prefill, before async_generate loop.
- pub fn prepare_async(
+ fn prepare_async(
&mut self,
tokens_to_generate: usize,
) {
@@ -385,15 +414,12 @@ where
/// - `pass_idx`: Index of this pass (0, 1, 2, ...)
/// - `sampling_method`: Sampling configuration
/// - `on_complete`: Callback receiving sampled token as u64
- pub fn async_generate<F>(
+ fn async_generate(
&mut self,
pass_idx: usize,
sampling_method: SamplingMethod,
- on_complete: F,
- ) -> Result<(), Error>
- where
- F: FnOnce(u64) + Send + 'static,
- {
+ on_complete: Box<dyn FnOnce(u64) + Send>,
+ ) -> Result<(), Error> {
assert_eq!(self.decoding_config.generate_suffix_length(), 1, "async_generate only supports suffix_length=1");
// Extract values from async_buffers before mutable borrow
@@ -564,17 +590,13 @@ where
Ok(())
}
- pub fn has_attention_layers(&self) -> bool {
- self.context.decoder_config.has_attention_layers()
- }
-
- pub fn clear_cache(&mut self) {
+ fn clear_cache(&mut self) {
objc2::rc::autoreleasepool(|_pool| {
self.encoded_tasks.clear();
});
}
- pub fn reset_state(&mut self) {
+ fn reset_state(&mut self) {
self.context.cache_layers.borrow_mut().clear();
self.tokens.clear();
self.registered_prefix_len = 0;
@@ -586,8 +608,118 @@ where
self.context.async_buffers.reset_counter();
}
- pub fn prefix_len(&self) -> usize {
- self.registered_prefix_len
+ fn tokens_len(&self) -> usize {
+ self.tokens.len()
+ }
+ fn tokens_push(
+ &mut self,
+ token: u64,
+ ) {
+ self.tokens.push(token);
+ }
+ fn generate_suffix_length(&self) -> usize {
+ self.decoding_config.generate_suffix_length()
+ }
+ fn async_batch_size(
+ &self,
+ model_path: &Path,
+ ) -> usize {
+ self.decoding_config.async_batch_size.resolve::<B>(model_path, self.context.context.as_ref())
+ }
+
+ fn get_slice(
+ &self,
+ range: Range<usize>,
+ ) -> Option<Box<dyn Any>> {
+ self.context.cache_layers.borrow().slice(&self.context.context, range).map(|s| Box::new(s) as Box<dyn Any>)
+ }
+ fn apply_slice(
+ &mut self,
+ slice: &dyn Any,
+ range: Range<usize>,
+ ) {
+ let slice = slice.downcast_ref::<CacheLayersSlice<B>>().unwrap();
+ self.context.cache_layers.borrow_mut().apply_slice(slice, Some(range));
+ }
+
+ fn build_llm_context(&self) -> Box<dyn Any> {
+ let cache_layers = self.context.cache_layers.borrow().clone(&self.context.context);
+ let context = LlmContext::new(self.tokens.clone(), cache_layers, self.decoding_config.clone());
+ Box::new(context)
+ }
+ fn reconfigure_from_context(
+ &mut self,
+ context: &dyn Any,
+ ) {
+ let ctx = context.downcast_ref::<LlmContext<B>>().unwrap();
+ let mut llm_state = self.context.cache_layers.borrow_mut();
+ for (_layer_idx, (ctx_layer, gen_layer)) in
+ ctx.cache_layers.data.iter().zip(llm_state.data.iter_mut()).enumerate()
+ {
+ match (ctx_layer, gen_layer) {
+ (CacheLayer::Transformer(src), CacheLayer::Transformer(dst)) => {
+ let copy_rows = src.prefix_segment_length();
+ if copy_rows > 0 {
+ {
+ let mut dst_keys = dst.keys.borrow_mut();
+ let src_keys = src.keys.borrow();
+ dst_keys.copy_slice(&src_keys, 1, 0..copy_rows, 0);
+ }
+ {
+ let mut dst_values = dst.values.borrow_mut();
+ let src_values = src.values.borrow();
+ dst_values.copy_slice(&src_values, 1, 0..copy_rows, 0);
+ }
+ }
+ dst.state = src.state.clone();
+ dst.prefix_token_positions = src.prefix_token_positions.clone();
+ },
+ (CacheLayer::StateSpace(src), CacheLayer::StateSpace(dst)) => {
+ {
+ let mut dst_conv = dst.conv_state.borrow_mut();
+ let src_conv = src.conv_state.borrow();
+ dst_conv.copy_from_array(&src_conv);
+ }
+ {
+ let mut dst_ssm = dst.ssm_state.borrow_mut();
+ let src_ssm = src.ssm_state.borrow();
+ dst_ssm.copy_from_array(&src_ssm);
+ }
+ },
+ _ => panic!("Layer type mismatch when reconfiguring language model generator cache"),
+ }
+ }
+ drop(llm_state);
+
+ self.tokens = ctx.tokens.clone();
+ }
+}
+
+impl<B: Backend> LanguageModelGenerator<B> {
+ pub fn new(
+ model_path: &Path,
+ decoding_config: DecodingConfig,
+ ) -> Result<Self, Error> {
+ let gpu_capture = GpuCaptureManager::new();
+
+ let context = LanguageModelGeneratorContext::new(model_path, &decoding_config)?;
+ let prefill_step_size = decoding_config.prefill_step_size.resolve(&context.model_config);
+ let generate_suffix_length = decoding_config.generate_suffix_length();
+
+ let mut generator = Self {
+ decoding_config,
+ tokens: Vec::new(),
+ context,
+ encoded_tasks: HashMap::new(),
+ registered_prefix_len: 0,
+ gpu_capture,
+ };
+
+ //Warmup
+ generator.warmup(prefill_step_size);
+ generator.warmup(generate_suffix_length);
+
+ Ok(generator)
}
fn warmup(
diff --git a/crates/uzu/src/language_model/language_model_generator_context.rs b/crates/uzu/src/language_model/language_model_generator_context.rs
index 02912efbd..7f7166b05 100644
--- a/crates/uzu/src/language_model/language_model_generator_context.rs
+++ b/crates/uzu/src/language_model/language_model_generator_context.rs
@@ -10,10 +10,7 @@ use crate::{
DataType,
backends::common::{
Backend, Context, Kernels, NativeBuffer,
- kernel::{
- MaskUpdateKernel, TokenCopySampledKernel, TokenCopyToResultsKernel, kv_cache_update::KVCacheUpdate,
- matmul::MatmulKernels,
- },
+ kernel::{MaskUpdateKernel, TokenCopySampledKernel, TokenCopyToResultsKernel, kv_cache_update::KVCacheUpdate},
},
config::{DecoderConfig, LanguageModelConfig, ModelMetadata},
encodable_block::{Decoder, Sampling},
@@ -148,10 +145,7 @@ pub struct LanguageModelGeneratorContext<B: Backend> {
pub async_buffers: AsyncBuffers<B>,
}
-impl<B: Backend + 'static> LanguageModelGeneratorContext<B>
-where
- B::Kernels: MatmulKernels,
-{
+impl<B: Backend> LanguageModelGeneratorContext<B> {
pub fn new(
model_path: &Path,
decoding_config: &DecodingConfig,
diff --git a/crates/uzu/src/language_model/mod.rs b/crates/uzu/src/language_model/mod.rs
index b2f74ea01..8c30f89ef 100644
--- a/crates/uzu/src/language_model/mod.rs
+++ b/crates/uzu/src/language_model/mod.rs
@@ -9,5 +9,4 @@ pub mod sampler;
pub mod tasks;
// Re-export main types
-pub use language_model_generator::LanguageModelGenerator;
-pub use language_model_generator_context::LanguageModelGeneratorContext;
+pub use language_model_generator::{LanguageModelGenerator, LanguageModelGeneratorTrait};
diff --git a/crates/uzu/src/language_model/tasks.rs b/crates/uzu/src/language_model/tasks.rs
index fc419fa14..293afef74 100644
--- a/crates/uzu/src/language_model/tasks.rs
+++ b/crates/uzu/src/language_model/tasks.rs
@@ -1,6 +1,6 @@
use std::{mem::size_of, ops::DerefMut};
-use super::LanguageModelGeneratorContext;
+use super::language_model_generator_context::LanguageModelGeneratorContext;
use crate::{
backends::common::{Backend, Context, NativeBuffer},
encodable_block::{EncodableBlock, EncodingParameters},
diff --git a/crates/uzu/src/session/chat_session.rs b/crates/uzu/src/session/chat_session.rs
index 6034560e8..5150a184b 100644
--- a/crates/uzu/src/session/chat_session.rs
+++ b/crates/uzu/src/session/chat_session.rs
@@ -1,4 +1,5 @@
use std::{
+ any::Any,
fs::File,
io::BufReader,
path::PathBuf,
@@ -19,17 +20,16 @@ use tokenizers::Tokenizer;
use xgrammar::TokenizerInfo;
use crate::{
- backends::metal::Metal,
+ backends::{common::Backend, select_backend},
config::{MixerConfig, ModelMetadata},
- forward_pass::cache_layers::CacheLayer,
language_model::{
- LanguageModelGenerator,
+ LanguageModelGenerator, LanguageModelGeneratorTrait,
grammar::CompiledGrammar,
result::{GenerateResult, PrefillResult},
},
session::{
config::{DecodingConfig, RunConfig},
- helpers::{Context, InputProcessor, InputProcessorDefault, OutputParser, is_directory_fits_ram},
+ helpers::{InputProcessor, InputProcessorDefault, OutputParser, is_directory_fits_ram},
parameter::{ConfigResolvableValue, ContextMode},
types::{Error, FinishReason, Input, Output, RunStats, Stats, StepStats, TotalStats},
},
@@ -55,14 +55,22 @@ pub struct ChatSession {
tokenizer_info: TokenizerInfo,
input_processor: Box<dyn InputProcessor>,
output_parser: OutputParser,
- llm: Option<LanguageModelGenerator<Metal>>,
- static_context: Option<Context<Metal>>,
+ decoding_config: DecodingConfig,
+ llm: Option<Box<dyn LanguageModelGeneratorTrait>>,
+ static_context: Option<Box<dyn Any>>,
}
impl ChatSession {
pub fn new(
model_path: PathBuf,
decoding_config: DecodingConfig,
+ ) -> Result<Self, Error> {
+ select_backend!(Self::new_with_backend::<B>(model_path, decoding_config), Error::UnableToCreateBackendContext)
+ }
+
+ pub fn new_with_backend<B: Backend>(
+ model_path: PathBuf,
+ decoding_config: DecodingConfig,
) -> Result<Self, Error> {
if !model_path.exists() {
return Err(Error::ModelFolderNotFound);
@@ -130,7 +138,8 @@ impl ChatSession {
let output_parser =
OutputParser::new(language_model_config.message_processor_config.output_parser_regex.clone())?;
- let llm = LanguageModelGenerator::new(&model_path, decoding_config).map_err(Error::from)?;
+ let llm =
+ Box::new(LanguageModelGenerator::<B>::new(&model_path, decoding_config.clone()).map_err(Error::from)?);
Ok(Self {
model_path,
@@ -139,6 +148,7 @@ impl ChatSession {
tokenizer_info,
input_processor: Box::new(input_processor),
output_parser,
+ decoding_config,
llm: Some(llm),
static_context: None,
})
@@ -153,10 +163,7 @@ impl ChatSession {
where
F: Fn(Output) -> bool,
{
- let context_mode =
- self.llm.as_ref().ok_or(Error::LanguageModelGeneratorNotLoaded)?.decoding_config.context_mode.clone();
-
- match &context_mode {
+ match &self.decoding_config.context_mode {
ContextMode::None => {
let language_model_generator = self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;
language_model_generator.reset_state();
@@ -181,7 +188,7 @@ impl ChatSession {
let output = self.run_internal(input, config, progress)?;
- match context_mode {
+ match &self.decoding_config.context_mode {
ContextMode::None
| ContextMode::Static {
..
@@ -199,15 +206,13 @@ impl ChatSession {
language_model_generator.reset_state();
Ok(())
}
-}
-impl ChatSession {
fn extend(
&mut self,
input: Input,
- context: Option<&Context<Metal>>,
+ context: Option<&Box<dyn Any>>,
config: RunConfig,
- ) -> Result<(Output, Context<Metal>), Error> {
+ ) -> Result<(Output, Box<dyn Any>), Error> {
self.reconfigure_language_model_generator(context)?;
let output = self.run_internal(input, config, None::<fn(Output) -> bool>)?;
let new_context = self.build_context_from_language_model_generator()?;
@@ -240,12 +245,12 @@ impl ChatSession {
self.model_metadata.model_config.as_language_model().ok_or(Error::UnableToLoadConfig)?;
let language_model_generator = self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;
- let context_length = language_model_generator.decoding_config.context_length.resolve(language_model_config);
+ let context_length = self.decoding_config.context_length.resolve(language_model_config);
if tokens.len() >= context_length {
return Err(Error::ContextLengthExceeded);
}
- let prefix_offset = language_model_generator.tokens.len();
+ let prefix_offset = language_model_generator.tokens_len();
let prefix_len_before = prefix_offset;
let eos_tokens: Vec<u64> =
@@ -272,8 +277,7 @@ impl ChatSession {
let prefill_duration = prefill_start.elapsed().as_secs_f64();
language_model_generator.clear_cache();
- let prefill_suffix_length =
- language_model_generator.decoding_config.prefill_step_size.resolve(language_model_config);
+ let prefill_suffix_length = self.decoding_config.prefill_step_size.resolve(language_model_config);
let run_context = RunContext {
eos_tokens,
@@ -292,13 +296,13 @@ impl ChatSession {
let prefill_finish_reason = if grammar_terminated {
Some(FinishReason::Stop)
} else {
- Self::check_finish_reason(&run_context, language_model_generator, &prefill_tokens)
+ Self::check_finish_reason(&run_context, language_model_generator.as_ref(), &prefill_tokens)
};
let prefill_output = Self::build_output(
&self.tokenizer,
&self.output_parser,
&run_context,
- language_model_generator,
+ language_model_generator.as_ref(),
&[],
&[],
prefill_finish_reason.clone(),
@@ -318,19 +322,15 @@ impl ChatSession {
}
}
- let can_use_async =
- language_model_generator.decoding_config.generate_suffix_length() == 1 && compiled_grammar.is_none();
+ let can_use_async = language_model_generator.generate_suffix_length() == 1 && compiled_grammar.is_none();
let generate_output = if can_use_async {
- let batch_size = language_model_generator
- .decoding_config
- .async_batch_size
- .resolve::<Metal>(&self.model_path, language_model_generator.context.context.as_ref());
+ let batch_size = language_model_generator.async_batch_size(&self.model_path);
Self::run_async_batch(
&self.tokenizer,
&self.output_parser,
&run_context,
- language_model_generator,
+ language_model_generator.as_mut(),
sampling_method,
&progress,
batch_size,
@@ -340,7 +340,7 @@ impl ChatSession {
&self.tokenizer,
&self.output_parser,
&run_context,
- language_model_generator,
+ language_model_generator.as_mut(),
compiled_grammar.as_mut(),
sampling_method,
&progress,
@@ -355,7 +355,7 @@ impl ChatSession {
tokenizer: &Tokenizer,
output_parser: &OutputParser,
run_context: &RunContext,
- language_model_generator: &mut LanguageModelGenerator<Metal>,
+ language_model_generator: &mut dyn LanguageModelGeneratorTrait,
compiled_grammar: Option<&mut CompiledGrammar>,
sampling_method: super::parameter::SamplingMethod,
progress: &Option<F>,
@@ -413,7 +413,7 @@ impl ChatSession {
tokenizer: &Tokenizer,
output_parser: &OutputParser,
run_context: &RunContext,
- llm: &mut LanguageModelGenerator<Metal>,
+ llm: &mut dyn LanguageModelGeneratorTrait,
sampling_method: super::parameter::SamplingMethod,
progress: &Option<F>,
batch_size: usize,
@@ -422,7 +422,7 @@ impl ChatSession {
F: Fn(Output) -> bool,
{
let remaining_by_limit = run_context.tokens_limit.saturating_sub(run_context.prefill_result.tokens.len());
- let remaining_by_context = run_context.context_length.saturating_sub(llm.tokens.len());
+ let remaining_by_context = run_context.context_length.saturating_sub(llm.tokens_len());
let tokens_to_generate = remaining_by_limit.min(remaining_by_context);
llm.prepare_async(tokens_to_generate);
@@ -442,17 +442,21 @@ impl ChatSession {
let batch_submitted = batch_end - next_to_submit;
let accepted_before_batch = results.len();
- let cache_slice = llm.context.cache_layers.borrow().slice(&llm.context.context, 0..batch_submitted);
+ let cache_slice = llm.get_slice(0..batch_submitted);
for idx in next_to_submit..batch_end {
let batch_sender = sender.clone();
let last_callback_nanos = last_nanos.clone();
let batch_start_time = start_time;
- llm.async_generate(idx, sampling_method, move |token| {
- let now_nanos = batch_start_time.elapsed().as_nanos() as u64;
- let prev_nanos = last_callback_nanos.swap(now_nanos, Ordering::SeqCst);
- let _ = batch_sender.send((idx, token, now_nanos.saturating_sub(prev_nanos)));
- })?;
+ llm.async_generate(
+ idx,
+ sampling_method,
+ Box::new(move |token| {
+ let now_nanos = batch_start_time.elapsed().as_nanos() as u64;
+ let prev_nanos = last_callback_nanos.swap(now_nanos, Ordering::SeqCst);
+ let _ = batch_sender.send((idx, token, now_nanos.saturating_sub(prev_nanos)));
+ }),
+ )?;
in_flight += 1;
}
next_to_submit = batch_end;
@@ -462,7 +466,7 @@ impl ChatSession {
in_flight -= 1;
let duration = nanos_to_secs(duration_nanos);
- llm.tokens.push(token);
+ llm.tokens_push(token);
results.push(GenerateResult {
tokens: vec![token],
forwardpass_duration: duration,
@@ -472,7 +476,7 @@ impl ChatSession {
let should_stop = if run_context.eos_tokens.contains(&token) {
finish_reason = FinishReason::Stop;
true
- } else if llm.tokens.len() >= run_context.context_length {
+ } else if llm.tokens_len() >= run_context.context_length {
finish_reason = FinishReason::ContextLimitReached;
true
} else if let Some(progress_fn) = progress {
@@ -496,10 +500,7 @@ impl ChatSession {
if let Some(slice) = cache_slice {
let accepted_in_batch = results.len().saturating_sub(accepted_before_batch);
if accepted_in_batch < batch_submitted {
- llm.context
- .cache_layers
- .borrow_mut()
- .apply_slice(&slice, Some(accepted_in_batch..batch_submitted));
+ llm.apply_slice(slice.as_ref(), accepted_in_batch..batch_submitted);
}
}
return Self::build_output(
@@ -520,78 +521,30 @@ impl ChatSession {
fn reconfigure_language_model_generator(
&mut self,
- context: Option<&Context<Metal>>,
+ context: Option<&Box<dyn Any>>,
) -> Result<(), Error> {
let language_model_generator = self.llm.as_mut().ok_or(Error::LanguageModelGeneratorNotLoaded)?;
language_model_generator.reset_state();
if let Some(ctx) = context {
- let mut llm_state = language_model_generator.context.cache_layers.borrow_mut();
- for (_layer_idx, (ctx_layer, gen_layer)) in
- ctx.cache_layers.data.iter().zip(llm_state.data.iter_mut()).enumerate()
- {
- match (ctx_layer, gen_layer) {
- (CacheLayer::Transformer(src), CacheLayer::Transformer(dst)) => {
- let copy_rows = src.prefix_segment_length();
- if copy_rows > 0 {
- {
- let mut dst_keys = dst.keys.borrow_mut();
- let src_keys = src.keys.borrow();
- dst_keys.copy_slice(&src_keys, 1, 0..copy_rows, 0);
- }
- {
- let mut dst_values = dst.values.borrow_mut();
- let src_values = src.values.borrow();
- dst_values.copy_slice(&src_values, 1, 0..copy_rows, 0);
- }
- }
- dst.state = src.state.clone();
- dst.prefix_token_positions = src.prefix_token_positions.clone();
- },
- (CacheLayer::StateSpace(src), CacheLayer::StateSpace(dst)) => {
- {
- let mut dst_conv = dst.conv_state.borrow_mut();
- let src_conv = src.conv_state.borrow();
- dst_conv.copy_from_array(&src_conv);
- }
- {
- let mut dst_ssm = dst.ssm_state.borrow_mut();
- let src_ssm = src.ssm_state.borrow();
- dst_ssm.copy_from_array(&src_ssm);
- }
- },
- _ => panic!("Layer type mismatch when reconfiguring language model generator cache"),
- }
- }
- drop(llm_state);
-
- language_model_generator.tokens = ctx.tokens.clone();
+ language_model_generator.reconfigure_from_context(ctx.as_ref());
}
Ok(())
}
- fn build_context_from_language_model_generator(&self) -> Result<Context<Metal>, Error> {
+ fn build_context_from_language_model_generator(&self) -> Result<Box<dyn Any>, Error> {
let language_model_generator = self.llm.as_ref().ok_or(Error::LanguageModelGeneratorNotLoaded)?;
- let cache_layers =
- language_model_generator.context.cache_layers.borrow().clone(&language_model_generator.context.context);
- let context = Context::new(
- language_model_generator.tokens.clone(),
- cache_layers,
- language_model_generator.decoding_config.clone(),
- );
- Ok(context)
+ Ok(language_model_generator.build_llm_context())
}
-}
-impl ChatSession {
fn check_finish_reason(
run_context: &RunContext,
- language_model_generator: &LanguageModelGenerator<Metal>,
+ language_model_generator: &dyn LanguageModelGeneratorTrait,
new_tokens: &[u64],
) -> Option<FinishReason> {
let start_idx = run_context.prefix_len_before + run_context.input_tokens_len;
- let total_new_tokens = language_model_generator.tokens.len().saturating_sub(start_idx);
+ let total_new_tokens = language_model_generator.tokens_len().saturating_sub(start_idx);
let has_eos = new_tokens.iter().any(|t| run_context.eos_tokens.contains(t));
- let context_limit = language_model_generator.tokens.len() >= run_context.context_length;
+ let context_limit = language_model_generator.tokens_len() >= run_context.context_length;
if has_eos {
Some(FinishReason::Stop)
@@ -621,7 +574,7 @@ impl ChatSession {
tokenizer: &Tokenizer,
output_parser: &OutputParser,
run_context: &RunContext,
- language_model_generator: &LanguageModelGenerator<Metal>,
+ language_model_generator: &dyn LanguageModelGeneratorTrait,
generate_results: &[GenerateResult],
generate_durations: &[f64],
finish_reason: Option<FinishReason>,
@@ -629,7 +582,7 @@ impl ChatSession {
let text = Self::decode_generated_tokens(tokenizer, &run_context.prefill_result.tokens, generate_results)?;
let parsed = output_parser.parse(text);
let start_idx = run_context.prefix_len_before + run_context.input_tokens_len;
- let output_tokens = language_model_generator.tokens.len().saturating_sub(start_idx);
+ let output_tokens = language_model_generator.tokens_len().saturating_sub(start_idx);
Ok(Output {
text: parsed,
@@ -639,7 +592,7 @@ impl ChatSession {
run_context.prefill_suffix_length,
generate_results.to_vec(),
generate_durations.to_vec(),
- language_model_generator.decoding_config.generate_suffix_length(),
+ language_model_generator.generate_suffix_length(),
run_context.run_start.elapsed().as_secs_f64(),
run_context.input_tokens_len,
output_tokens,
diff --git a/crates/uzu/src/session/classification_session.rs b/crates/uzu/src/session/classification_session.rs
index c1bd9e835..57fff930b 100644
--- a/crates/uzu/src/session/classification_session.rs
+++ b/crates/uzu/src/session/classification_session.rs
@@ -3,8 +3,8 @@ use std::path::PathBuf;
use tokenizers::Tokenizer;
use crate::{
- backends::metal::Metal,
- classifier::{ClassificationOutput, Classifier},
+ backends::{common::Backend, select_backend},
+ classifier::{ClassificationOutput, Classifier, ClassifierTrait},
config::ModelMetadata,
session::{
helpers::{InputProcessor, InputProcessorDefault},
@@ -18,12 +18,16 @@ pub struct ClassificationSession {
#[allow(dead_code)]
model_metadata: ModelMetadata,
tokenizer: Tokenizer,
- classifier: Classifier<Metal>,
+ classifier: Box<dyn ClassifierTrait>,
input_processor: Box<dyn InputProcessor>,
}
impl ClassificationSession {
pub fn new(model_path: PathBuf) -> Result<Self, Error> {
+ select_backend!(Self::new_with_backend::<B>(model_path), Error::UnableToCreateBackendContext)
+ }
+
+ pub fn new_with_backend<B: Backend>(model_path: PathBuf) -> Result<Self, Error> {
let config_path = model_path.join("config.json");
if !config_path.exists() {
return Err(Error::ModelFolderNotFound);
@@ -43,7 +47,7 @@ impl ClassificationSession {
let input_processor =
Box::new(InputProcessorDefault::new(classifier_model_config.message_processor_config.clone()))
as Box<dyn InputProcessor>;
- let classifier = Classifier::new(&model_path)?;
+ let classifier = Box::new(Classifier::<B>::new(&model_path)?);
Ok(Self {
model_path,
diff --git a/crates/uzu/src/tracer/trace_validator.rs b/crates/uzu/src/tracer/trace_validator.rs
index c3e3f5d43..7f24c91d5 100644
--- a/crates/uzu/src/tracer/trace_validator.rs
+++ b/crates/uzu/src/tracer/trace_validator.rs
@@ -18,10 +18,7 @@ use num_traits::NumCast;
use crate::{
ArrayElement, DataType,
array::Array,
- backends::common::{
- Backend, CommandBuffer, Context,
- kernel::{kv_cache_update::KVCacheUpdate, matmul::MatmulKernels},
- },
+ backends::common::{Backend, CommandBuffer, Context, kernel::kv_cache_update::KVCacheUpdate},
classifier::Classifier,
config::ModelMetadata,
encodable_block::{EncodableBlock, EncodingParameters, Sampling},
@@ -32,7 +29,7 @@ use crate::{
traces::ActivationTrace,
},
language_model::{
- LanguageModelGeneratorContext,
+ language_model_generator_context::LanguageModelGeneratorContext,
sampler::{ArgmaxSampler, LogitsSampler},
},
parameters::{ParameterLoader, ParameterTree, read_safetensors_metadata},
@@ -155,10 +152,7 @@ pub enum ArrayTransform {
// Model Context (internal)
// ============================================================================
-enum ModelContext<B: Backend + 'static>
-where
- B::Kernels: MatmulKernels,
-{
+enum ModelContext<B: Backend> {
LanguageModelGenerator(LanguageModelGeneratorContext<B>),
Classifier(Classifier<B>),
}
@@ -171,18 +165,12 @@ where
///
/// Automatically detects whether the model is an LLM or classifier and
/// runs the appropriate validation.
-pub struct TraceValidator<B: Backend + 'static>
-where
- B::Kernels: MatmulKernels,
-{
+pub struct TraceValidator<B: Backend> {
model_path: PathBuf,
context: ModelContext<B>,
}
-impl<B: Backend + 'static> TraceValidator<B>
-where
- B::Kernels: MatmulKernels,
-{
+impl<B: Backend> TraceValidator<B> {
/// Create a new trace validator for the given model path.
///
/// Automatically detects the model type from config.json.