A high-performance inference engine for AI models
Remove unused code
Aleksei Savin8ada78edaed5
diff --git a/uzu/src/backends/metal/executable_builder/attention_executable_provider.rs b/uzu/src/backends/metal/executable_builder/attention_executable_provider.rsdeleted file mode 100644index 081b0a1fe..000000000--- a/uzu/src/backends/metal/executable_builder/attention_executable_provider.rs+++ /dev/null@@ -1,142 +0,0 @@-use std::{collections::HashMap, rc::Rc};--use mpsgraph::Executable;-use objc2::rc::Retained;--use crate::{- backends::metal::{- MTLContext, compilation_parameters::CompilationConfig,- forward_pass::transformer_layer::attention_executable,- },- config::DecoderConfig,-};--pub struct AttentionExecutableProviderConfig {- pub prefix_length_step_by_suffix_length: HashMap<usize, usize>,-}--impl AttentionExecutableProviderConfig {- pub fn new(- prefix_length_step_by_suffix_length: HashMap<usize, usize>- ) -> Self {- Self {- prefix_length_step_by_suffix_length,- }- }-}--pub struct AttentionExecutableProvider {- mtl_context: Rc<MTLContext>,- decoder_config: Rc<DecoderConfig>,- compilation_config: Rc<CompilationConfig>,- config: AttentionExecutableProviderConfig,- pub executables: HashMap<usize, HashMap<usize, Retained<Executable>>>,-}--impl AttentionExecutableProvider {- pub fn new(- mtl_context: Rc<MTLContext>,- decoder_config: Rc<DecoderConfig>,- compilation_config: Rc<CompilationConfig>,- config: AttentionExecutableProviderConfig,- ) -> Self {- let max_prefix_length: usize;- if let Some(sliding_window_sizes) = &decoder_config.sliding_window_sizes- {- max_prefix_length = sliding_window_sizes- .iter()- .map(|size| size.unwrap_or(0))- .max()- .unwrap_or(0);- } else {- max_prefix_length = decoder_config.context_length;- }-- let mut instance = Self {- mtl_context: mtl_context,- decoder_config,- compilation_config,- config,- executables: HashMap::new(),- };- instance.prefetch(max_prefix_length);- instance- }-- fn prefetch(- &mut self,- max_prefix_length: usize,- ) {- let prefix_length_step_by_suffix_length =- self.config.prefix_length_step_by_suffix_length.clone();- for (suffix_length, prefix_length_step) in- prefix_length_step_by_suffix_length- {- let _ = self.executable(suffix_length, 0);- if prefix_length_step == 0 {- continue;- }-- for prefix_length in (prefix_length_step..=max_prefix_length)- .step_by(prefix_length_step)- {- let _ = self.executable(suffix_length, prefix_length);- }- }- }-- pub fn executable(- &mut self,- suffix_length: usize,- prefix_length: usize,- ) -> &Retained<Executable> {- if !self.executables.contains_key(&suffix_length) {- self.executables.insert(suffix_length, HashMap::new());- }-- let need_to_compile = {- let executables_for_suffix =- self.executables.get(&suffix_length).unwrap();- !executables_for_suffix.contains_key(&prefix_length)- };-- if need_to_compile {- let executable = self.compile(suffix_length, prefix_length);- let executables_for_suffix =- self.executables.get_mut(&suffix_length).unwrap();- executables_for_suffix.insert(prefix_length, executable);- }-- &self- .executables- .get(&suffix_length)- .unwrap()- .get(&prefix_length)- .unwrap()- }-- fn compile(- &self,- suffix_length: usize,- prefix_length: usize,- ) -> Retained<Executable> {- let config = &self.decoder_config;- let attention_executable = attention_executable(- config- .layer_config- .attention_config- .qkv_projection_config- .activation_precision()- .into(),- config.head_dim,- config.num_groups,- config.num_heads,- config.attention_scale,- &self.mtl_context,- suffix_length,- prefix_length,- &self.compilation_config.descriptor_general,- );- attention_executable- }-}diff --git a/uzu/src/backends/metal/executable_builder/decoder_executables.rs b/uzu/src/backends/metal/executable_builder/decoder_executables.rsindex 881a00b2a..b4c8196f7 100644--- a/uzu/src/backends/metal/executable_builder/decoder_executables.rs+++ b/uzu/src/backends/metal/executable_builder/decoder_executables.rs@@ -1,13 +1,8 @@-use std::{cell::RefCell, rc::Rc};+use std::rc::Rc;use mpsgraph::CommandBuffer as MPSCommandBuffer;-use super::{- attention_executable_provider::{- AttentionExecutableProvider, AttentionExecutableProviderConfig,- },- layer_executables::LayerExecutables,-};+use super::layer_executables::LayerExecutables;use crate::{DataType,backends::metal::{@@ -16,33 +11,14 @@ use crate::{forward_pass::{ArrayId, ForwardPassState, MPSGraphBlock, RopeType,encodable_with_state::{EncodableWithState, EncodingParameters},- transformer_layer::{- embed_block, readout_block, rms_norm_block, rotation_block,- rotation_executable,- },+ transformer_layer::{embed_block, readout_block},},kernel::{RMSNormKernelEncodable, RopeKernelEncodable},},- config::{RoPEConfig, decoder::DecoderConfig},+ config::decoder::DecoderConfig,parameters::ParameterTree,};-#[derive(Debug, Clone)]-pub struct KernelsConfig {- pub use_rope: bool,- pub use_attention: bool,- pub use_rms_norm: bool,-}--impl KernelsConfig {- pub fn default() -> Self {- Self {- use_rope: true,- use_attention: true,- use_rms_norm: true,- }- }-}pub struct DecoderExecutables {pub embed: MPSGraphBlock,pub layers: Box<[LayerExecutables]>,@@ -58,8 +34,6 @@ impl DecoderExecutables {decoder_config: Rc<DecoderConfig>,decoder_weight_loader: &ParameterTree<Rc<MTLContext>>,compilation_config: Rc<CompilationConfig>,- attention_executable_provider_config: AttentionExecutableProviderConfig,- kernels_config: KernelsConfig,) -> Self {let embed = embed_block(&decoder_config,@@ -76,45 +50,20 @@ impl DecoderExecutables {let global_rope = Self::create_rope_block(&mtl_context,&decoder_config,- &decoder_weight_loader,- compilation_config.clone(),- &kernels_config,- String::from("global_rope"),- &decoder_config.global_rope_config,RopeType::Global,);let local_rope: Option<Rc<Box<dyn EncodableWithState>>>;- if let Some(local_rope_config) = &decoder_config.local_rope_config {+ if let Some(_) = &decoder_config.local_rope_config {local_rope = Some(Self::create_rope_block(&mtl_context,&decoder_config,- &decoder_weight_loader,- compilation_config.clone(),- &kernels_config,- String::from("local_rope"),- local_rope_config,RopeType::Local,));} else {local_rope = None;}- let attention_executable_provider: Option<- Rc<RefCell<AttentionExecutableProvider>>,- >;- if kernels_config.use_attention {- attention_executable_provider = None;- } else {- attention_executable_provider =- Some(Rc::new(RefCell::new(AttentionExecutableProvider::new(- mtl_context.clone(),- decoder_config.clone(),- compilation_config.clone(),- attention_executable_provider_config,- ))));- }-let model_shape = ModelShape::from_decoder_config(&decoder_config);let sliding_window_sizes = model_shape.sliding_window_length_per_layer;@@ -148,37 +97,22 @@ impl DecoderExecutables {&decoder_weight_loader.subtree(&format!("layers.{}", layer_index)).unwrap(),- attention_executable_provider.clone(),rope,- kernels_config.clone(),)}).collect::<Vec<_>>();- let norm_block: Box<dyn EncodableWithState> =- if kernels_config.use_rms_norm {- Box::new(- RMSNormKernelEncodable::new(- &mtl_context,- intermediate_data_type,- decoder_config.output_norm_config.clone(),- ArrayId::Main,- ArrayId::Main,- &decoder_weight_loader.subtree("output_norm").unwrap(),- )- .expect("Failed to create output RMS norm kernel"),- )- } else {- Box::new(rms_norm_block(- &decoder_config.output_norm_config,- decoder_config.model_dim,- &mtl_context,- &decoder_weight_loader.subtree("output_norm").unwrap(),- ArrayId::Main,- ArrayId::Main,- &compilation_config.descriptor_general,- ))- };+ let norm_block: Box<dyn EncodableWithState> = Box::new(+ RMSNormKernelEncodable::new(+ &mtl_context,+ intermediate_data_type,+ decoder_config.output_norm_config.clone(),+ ArrayId::Main,+ ArrayId::Main,+ &decoder_weight_loader.subtree("output_norm").unwrap(),+ )+ .expect("Failed to create output RMS norm kernel"),+ );Self {embed: embed,@@ -193,11 +127,6 @@ impl DecoderExecutables {fn create_rope_block(mtl_context: &MTLContext,decoder_config: &DecoderConfig,- decoder_weight_loader: &ParameterTree<Rc<MTLContext>>,- compilation_config: Rc<CompilationConfig>,- kernels_config: &KernelsConfig,- rope_name: String,- rope_config: &RoPEConfig,rope_type: RopeType,) -> Rc<Box<dyn EncodableWithState>> {let intermediate_data_type: DataType = decoder_config@@ -208,30 +137,10 @@ impl DecoderExecutables {.into();let kernel_data_type: KernelDataType = intermediate_data_type.into();- let rotation: Box<dyn EncodableWithState>;- if kernels_config.use_rope {- rotation = Box::new(- RopeKernelEncodable::new(- mtl_context,- kernel_data_type,- rope_type,- )+ let rotation: Box<dyn EncodableWithState> = Box::new(+ RopeKernelEncodable::new(mtl_context, kernel_data_type, rope_type).expect("Failed to create RopeKernelEncodable"),- );- } else {- let rotation_executable = rotation_executable(- rope_name,- rope_config,- decoder_config.head_dim,- decoder_config.num_groups,- decoder_config.num_heads,- decoder_config.context_length,- &mtl_context,- decoder_weight_loader,- &compilation_config.descriptor_general,- );- rotation = Box::new(rotation_block(rotation_executable));- }+ );return Rc::new(rotation);}}diff --git a/uzu/src/backends/metal/executable_builder/layer_executables.rs b/uzu/src/backends/metal/executable_builder/layer_executables.rsindex b7b97810e..4bc0a74e9 100644--- a/uzu/src/backends/metal/executable_builder/layer_executables.rs+++ b/uzu/src/backends/metal/executable_builder/layer_executables.rs@@ -1,21 +1,17 @@-use std::{cell::RefCell, rc::Rc};+use std::rc::Rc;use mpsgraph::CommandBuffer as MPSCommandBuffer;use objc2::rc::autoreleasepool;-use super::{- attention_executable_provider::AttentionExecutableProvider,- decoder_executables::KernelsConfig,-};use crate::{- Array, DataType,+ DataType,backends::metal::{MTLContext,compilation_parameters::CompilationConfig,forward_pass::{ArrayId, ForwardPassState, MPSGraphBlock,encodable_with_state::{EncodableWithState, EncodingParameters},- transformer_layer::{self, attention_block},+ transformer_layer,},kernel::{AttentionKernelEncodable, KernelDataType, QKNormKernelEncodable,@@ -34,15 +30,12 @@ pub struct LayerExecutables {pub qk_norm: Option<Box<dyn EncodableWithState>>,pub rope: Rc<Box<dyn EncodableWithState>>,pub attention: Box<dyn EncodableWithState>,- pub attention_executable_provider:- Option<Rc<RefCell<AttentionExecutableProvider>>>,pub out_projection: MPSGraphBlock,pub post_attention_norm: Option<Box<dyn EncodableWithState>>,pub main_shortcut_add_swap: Box<dyn EncodableWithState>,pub pre_mlp_norm: Box<dyn EncodableWithState>,pub mlp: MPSGraphBlock,pub post_mlp_norm: Option<Box<dyn EncodableWithState>>,- pub kernels_config: KernelsConfig,}impl LayerExecutables {@@ -58,11 +51,7 @@ impl LayerExecutables {num_groups: usize,attention_scale: Option<f32>,decoder_layer_loader: &ParameterTree<Rc<MTLContext>>,- attention_executable_provider: Option<- Rc<RefCell<AttentionExecutableProvider>>,- >,rope: Rc<Box<dyn EncodableWithState>>,- kernels_config: KernelsConfig,) -> Self {autoreleasepool(|_| {let intermediate_data_type: DataType = layer_config@@ -82,34 +71,19 @@ impl LayerExecutables {.unwrap(),);- let pre_attention_norm: Box<dyn EncodableWithState> =- if kernels_config.use_rms_norm {- Box::new(- RMSNormKernelEncodable::new(- mtl_context,- intermediate_data_type,- layer_config.pre_attention_norm_config.clone(),- ArrayId::Main,- ArrayId::Main,- &decoder_layer_loader- .subtree("pre_attention_norm")- .unwrap(),- )- .expect("Failed to create RMS norm kernel"),- )- } else {- Box::new(transformer_layer::rms_norm_block(- &layer_config.pre_attention_norm_config,- model_dim,- mtl_context,- &decoder_layer_loader- .subtree("pre_attention_norm")- .unwrap(),- ArrayId::Main,- ArrayId::Main,- &compilation_config.descriptor_general,- ))- };+ let pre_attention_norm: Box<dyn EncodableWithState> = Box::new(+ RMSNormKernelEncodable::new(+ mtl_context,+ intermediate_data_type,+ layer_config.pre_attention_norm_config.clone(),+ ArrayId::Main,+ ArrayId::Main,+ &decoder_layer_loader+ .subtree("pre_attention_norm")+ .unwrap(),+ )+ .expect("Failed to create RMS norm kernel"),+ );let qkv_projection = transformer_layer::linear_block(&layer_config.attention_config.qkv_projection_config,@@ -174,33 +148,19 @@ impl LayerExecutables {if let Some(norm_config) =&layer_config.post_attention_norm_config{- if kernels_config.use_rms_norm {- Some(Box::new(- RMSNormKernelEncodable::new(- mtl_context,- intermediate_data_type,- norm_config.clone(),- ArrayId::Main,- ArrayId::Main,- &decoder_layer_loader- .subtree("post_attention_norm")- .unwrap(),- )- .expect("Failed to create RMS norm kernel"),- ))- } else {- Some(Box::new(transformer_layer::rms_norm_block(- norm_config,- model_dim,+ Some(Box::new(+ RMSNormKernelEncodable::new(mtl_context,+ intermediate_data_type,+ norm_config.clone(),+ ArrayId::Main,+ ArrayId::Main,&decoder_layer_loader.subtree("post_attention_norm").unwrap(),- ArrayId::Main,- ArrayId::Main,- &compilation_config.descriptor_general,- )))- }+ )+ .expect("Failed to create RMS norm kernel"),+ ))} else {None};@@ -214,31 +174,17 @@ impl LayerExecutables {.unwrap(),);- let pre_mlp_norm: Box<dyn EncodableWithState> = if kernels_config- .use_rms_norm- {- Box::new(- RMSNormKernelEncodable::new(- mtl_context,- intermediate_data_type,- layer_config.pre_mlp_norm_config.clone(),- ArrayId::Main,- ArrayId::Main,- &decoder_layer_loader.subtree("pre_mlp_norm").unwrap(),- )- .expect("Failed to create RMS norm kernel"),- )- } else {- Box::new(transformer_layer::rms_norm_block(- &layer_config.pre_mlp_norm_config,- model_dim,+ let pre_mlp_norm: Box<dyn EncodableWithState> = Box::new(+ RMSNormKernelEncodable::new(mtl_context,- &decoder_layer_loader.subtree("pre_mlp_norm").unwrap(),+ intermediate_data_type,+ layer_config.pre_mlp_norm_config.clone(),ArrayId::Main,ArrayId::Main,- &compilation_config.descriptor_general,- ))- };+ &decoder_layer_loader.subtree("pre_mlp_norm").unwrap(),+ )+ .expect("Failed to create RMS norm kernel"),+ );let mlp = transformer_layer::mlp_block(&layer_config.mlp_config,@@ -251,61 +197,32 @@ impl LayerExecutables {let post_mlp_norm: Option<Box<dyn EncodableWithState>> =if let Some(norm_config) = &layer_config.post_mlp_norm_config {- if kernels_config.use_rms_norm {- Some(Box::new(- RMSNormKernelEncodable::new(- mtl_context,- intermediate_data_type,- norm_config.clone(),- ArrayId::Main,- ArrayId::Main,- &decoder_layer_loader- .subtree("post_mlp_norm")- .unwrap(),- )- .expect("Failed to create RMS norm kernel"),- ))- } else {- Some(Box::new(transformer_layer::rms_norm_block(- norm_config,- model_dim,+ Some(Box::new(+ RMSNormKernelEncodable::new(mtl_context,+ intermediate_data_type,+ norm_config.clone(),+ ArrayId::Main,+ ArrayId::Main,&decoder_layer_loader.subtree("post_mlp_norm").unwrap(),- ArrayId::Main,- ArrayId::Main,- &compilation_config.descriptor_general,- )))- }+ )+ .expect("Failed to create RMS norm kernel"),+ ))} else {None};- let attention: Box<dyn EncodableWithState>;- if kernels_config.use_attention {- attention = Box::new(- AttentionKernelEncodable::new(- mtl_context,- kernel_data_type,- layer_index,- attention_scale,- )- .expect(- "Failed to create AttentionWrapper with Metal kernel",- ),- );- } else {- attention = Box::new(- AttentionKernelEncodable::new(- mtl_context,- kernel_data_type,- layer_index,- attention_scale,- )- .expect("Failed to create AttentionWrapper for MPS Graph"),- );- }+ let attention: Box<dyn EncodableWithState> = Box::new(+ AttentionKernelEncodable::new(+ mtl_context,+ kernel_data_type,+ layer_index,+ attention_scale,+ )+ .expect("Failed to create AttentionWrapper with Metal kernel"),+ );Self {layer_index,@@ -315,48 +232,15 @@ impl LayerExecutables {qk_norm,rope,attention,- attention_executable_provider,out_projection,post_attention_norm,main_shortcut_add_swap,pre_mlp_norm,mlp,post_mlp_norm,- kernels_config,}})}-- fn get_suffix_length(- &self,- state: &ForwardPassState,- ) -> usize {- let shape: Vec<usize> = (*state.arrays(&[ArrayId::TokenIds]))[0]- .borrow()- .shape()- .iter()- .map(|value| *value)- .collect();- *shape.first().unwrap()- }-- pub fn attention_block(- &self,- state: &ForwardPassState,- ) -> MPSGraphBlock {- let suffix_length = self.get_suffix_length(state);- let prefix_length: usize = state.kv_cache.borrow().data- [self.layer_index]- .effective_prefix_length();-- let mut attention_executable_provider =- self.attention_executable_provider.as_ref().unwrap().borrow_mut();- let attention_executable = attention_executable_provider- .executable(suffix_length, prefix_length);- let attention =- attention_block(attention_executable.clone(), self.layer_index);- attention- }}impl EncodableWithState for LayerExecutables {@@ -379,8 +263,6 @@ impl EncodableWithState for LayerExecutables {);}- let first_layer = self.layer_index == 0;-self.copy_main_to_shortcut.encode(state, command_buffer, parameters);// shortcut = input@@ -397,40 +279,7 @@ impl EncodableWithState for LayerExecutables {qk_norm.encode(state, command_buffer, parameters);}self.rope.encode(state, command_buffer, parameters);-- if parameters.warmup {- if first_layer {- if self.kernels_config.use_attention {- self.attention.encode(state, command_buffer, parameters);- } else {- let suffix_length = self.get_suffix_length(state);- if let Some(executables) = self- .attention_executable_provider- .as_ref()- .unwrap()- .borrow()- .executables- .get(&suffix_length)- {- for (_, executable) in executables {- let attention = attention_block(- executable.clone(),- self.layer_index,- );- attention.encode(state, command_buffer, parameters);- }- }- }- }- } else {- if self.kernels_config.use_attention {- self.attention.encode(state, command_buffer, parameters);- } else {- let attention = self.attention_block(state);- attention.encode(state, command_buffer, parameters);- }- }-+ self.attention.encode(state, command_buffer, parameters);self.out_projection.encode(state, command_buffer, parameters);if let Some(layer_traces) = layer_traces.clone() {state.copy_array(diff --git a/uzu/src/backends/metal/executable_builder/mod.rs b/uzu/src/backends/metal/executable_builder/mod.rsindex 35402e34d..84cfd0fbc 100644--- a/uzu/src/backends/metal/executable_builder/mod.rs+++ b/uzu/src/backends/metal/executable_builder/mod.rs@@ -1,6 +1,4 @@-pub mod attention_executable_provider;pub mod decoder_executables;pub mod layer_executables;-pub use decoder_executables::{DecoderExecutables, KernelsConfig};-// LayerExecutables likely internal to this module+pub use decoder_executables::DecoderExecutables;diff --git a/uzu/src/backends/metal/forward_pass/transformer_layer.rs b/uzu/src/backends/metal/forward_pass/transformer_layer.rsindex 469937775..3182518ef 100644--- a/uzu/src/backends/metal/forward_pass/transformer_layer.rs+++ b/uzu/src/backends/metal/forward_pass/transformer_layer.rs@@ -1,18 +1,15 @@use std::{collections::HashMap, rc::Rc};use mpsgraph::{- CompilationDescriptor, Device as MPSDevice, Executable,- ExecutableExecutionDescriptor, Graph, GraphTensorShapeOps,+ CompilationDescriptor, Device as MPSDevice, ExecutableExecutionDescriptor,+ Graph, GraphTensorShapeOps,};use objc2::rc::{Retained, autoreleasepool};use super::{super::{MTLContext,- graph::{- attention_subgraph, linear_subgraph, mlp_subgraph, placeholder,- rms_norm_subgraph, shaped_type,- },+ graph::{linear_subgraph, mlp_subgraph, placeholder, shaped_type},},io_arrays::IOArrays,mpsgraph_block::MPSGraphBlock,@@ -22,12 +19,9 @@ use crate::{DataType,backends::metal::graph::{embeddings_dequantize_weights_subgraph, embeddings_embed_subgraph,- embeddings_readout_subgraph, rotation_subgraph,- },- config::{- DecoderConfig, EmbeddingConfig, LinearConfig, MLPConfig, RMSNormConfig,- RoPEConfig,+ embeddings_readout_subgraph,},+ config::{DecoderConfig, EmbeddingConfig, LinearConfig, MLPConfig},parameters::ParameterTree,};@@ -134,226 +128,6 @@ pub fn mlp_block(})}-pub fn rms_norm_block(- config: &RMSNormConfig,- model_dim: usize,- context: &MTLContext,- parameter_tree: &ParameterTree<Rc<MTLContext>>,- input_array_id: ArrayId,- output_array_id: ArrayId,- compilation_descriptor: &CompilationDescriptor,-) -> MPSGraphBlock {- autoreleasepool(|_| {- let graph = Graph::new();-- let input_shape = [-1, model_dim as isize];- let input_data_type: DataType = config.scale_precision.into();- let input_placeholder =- placeholder(&graph, &input_shape, input_data_type);-- let output = rms_norm_subgraph(- &graph,- config,- &input_placeholder,- parameter_tree,- )- .unwrap();-- let retained_shaped_type = shaped_type(&input_shape, input_data_type);- let feeds =- HashMap::from([(&*input_placeholder, &*retained_shaped_type)]);-- let executable = graph.compile(- &MPSDevice::with_device(&context.device),- &feeds,- &[&output],- Some(compilation_descriptor),- );-- let arguments = IOArrays::new(- vec![input_array_id].into_boxed_slice(),- vec![output_array_id].into_boxed_slice(),- );-- MPSGraphBlock::new(executable, make_execution_descriptor(), arguments)- })-}--pub fn rotation_executable(- rope_name: String,- rope_config: &RoPEConfig,- head_dim: usize,- num_groups: usize,- num_heads: usize,- context_length: usize,- context: &MTLContext,- root_parameter_tree: &ParameterTree<Rc<MTLContext>>,- compilation_descriptor: &Retained<CompilationDescriptor>,-) -> Retained<Executable> {- autoreleasepool(|_| {- let graph = Graph::new();-- let activation_data_type: DataType =- rope_config.common().precision.into();-- let qkv_shape =- [-1, ((num_heads + 2 * num_groups) * head_dim) as isize];- let qkv_placeholder =- placeholder(&graph, &qkv_shape, activation_data_type);- let qkv_shaped_type = shaped_type(&qkv_shape, activation_data_type);-- let token_positions_placeholder =- placeholder(&graph, &[-1 as isize], DataType::I32);- let token_positions_shaped_type =- shaped_type(&[-1 as isize], DataType::I32);-- let (output_queries, output_keys) = rotation_subgraph(- &graph,- rope_name,- rope_config,- head_dim,- num_groups,- num_heads,- context_length,- &qkv_placeholder,- &token_positions_placeholder,- root_parameter_tree,- )- .unwrap();-- let feeds = HashMap::from([- (&*qkv_placeholder, &*qkv_shaped_type),- (&*token_positions_placeholder, &*token_positions_shaped_type),- ]);-- graph.compile(- &MPSDevice::with_device(&context.device),- &feeds,- &[&output_queries, &output_keys],- Some(compilation_descriptor),- )- })-}--pub fn attention_executable(- activation_data_type: DataType,- head_dim: usize,- num_groups: usize,- num_heads: usize,- attention_scale: Option<f32>,- context: &MTLContext,- suffix_length: usize,- prefix_length: usize,- compilation_descriptor: &CompilationDescriptor,-) -> Retained<Executable> {- autoreleasepool(|_| {- let graph = Graph::new();-- let qkv_shape =- [-1, ((num_heads + 2 * num_groups) * head_dim) as isize];- let qkv_placeholder =- placeholder(&graph, &qkv_shape, activation_data_type);- let retained_qkv_shaped_type =- shaped_type(&qkv_shape, activation_data_type);-- let kv_cache_shape = [num_groups as isize, -1, head_dim as isize];- let key_cache_placeholder =- placeholder(&graph, &kv_cache_shape, activation_data_type);- let value_cache_placeholder =- placeholder(&graph, &kv_cache_shape, activation_data_type);- let retained_kv_cache_shaped_type =- shaped_type(&kv_cache_shape, activation_data_type);-- let rotated_queries_shape = [num_heads as isize, -1, head_dim as isize];- let rotated_queries_placeholder =- placeholder(&graph, &rotated_queries_shape, activation_data_type);- let rotated_queries_shaped_type =- shaped_type(&rotated_queries_shape, activation_data_type);-- let rotated_keys_shape = [num_groups as isize, -1, head_dim as isize];- let rotated_keys_placeholder =- placeholder(&graph, &rotated_keys_shape, activation_data_type);- let rotated_keys_shaped_type =- shaped_type(&rotated_keys_shape, activation_data_type);-- let bias_placeholder =- placeholder(&graph, &[-1, -1], activation_data_type);- let retained_bias_shaped_type =- shaped_type(&[-1, -1], activation_data_type);-- let output = attention_subgraph(- &graph,- head_dim,- num_groups,- num_heads,- attention_scale,- &qkv_placeholder,- &key_cache_placeholder,- &value_cache_placeholder,- &rotated_queries_placeholder,- &rotated_keys_placeholder,- &bias_placeholder,- suffix_length,- prefix_length,- )- .unwrap();-- let feeds = HashMap::from([- (&*qkv_placeholder, &*retained_qkv_shaped_type),- (&*key_cache_placeholder, &*retained_kv_cache_shaped_type),- (&*value_cache_placeholder, &*retained_kv_cache_shaped_type),- (&*rotated_queries_placeholder, &*rotated_queries_shaped_type),- (&*rotated_keys_placeholder, &*rotated_keys_shaped_type),- (&bias_placeholder, &retained_bias_shaped_type),- ]);-- graph.compile(- &MPSDevice::with_device(&context.device),- &feeds,- &[&output],- Some(compilation_descriptor),- )- })-}--pub fn rotation_block(executable: Retained<Executable>) -> MPSGraphBlock {- autoreleasepool(|_| {- let input_array_ids = vec![ArrayId::QKV, ArrayId::TokenPositions];- let output_array_ids =- vec![ArrayId::RotatedQueries, ArrayId::RotatedKeys];-- let arguments = IOArrays::new(- input_array_ids.into_boxed_slice(),- output_array_ids.into_boxed_slice(),- );-- MPSGraphBlock::new(executable, make_execution_descriptor(), arguments)- })-}--pub fn attention_block(- executable: Retained<Executable>,- layer_index: usize,-) -> MPSGraphBlock {- autoreleasepool(|_| {- let input_array_ids = vec![- ArrayId::QKV,- ArrayId::Keys(layer_index),- ArrayId::Values(layer_index),- ArrayId::RotatedQueries,- ArrayId::RotatedKeys,- ];- let output_array_ids = vec![ArrayId::AttentionOutput];-- let arguments = IOArrays::new(- input_array_ids.into_boxed_slice(),- output_array_ids.into_boxed_slice(),- );-- MPSGraphBlock::new(executable, make_execution_descriptor(), arguments)- })-}-pub fn embed_block(config: &DecoderConfig,context: &MTLContext,diff --git a/uzu/src/backends/metal/graph/attention.rs b/uzu/src/backends/metal/graph/attention.rsdeleted file mode 100644index c0d2510f4..000000000--- a/uzu/src/backends/metal/graph/attention.rs+++ /dev/null@@ -1,212 +0,0 @@-use std::rc::Rc;--use mpsgraph::{- Graph, GraphControlFlowOps, GraphLinearAlgebraOps, GraphMemoryOps,- GraphTensorShapeOps, Tensor,-};-use objc2::{Message, rc::Retained};--use super::{- common::GraphConstructionError,- rope::{apply_rope_subgraph, positional_embeddings_subgraph},-};-use crate::{- backends::metal::{MTLContext, utils::mps_shape},- config::RoPEConfig,- parameters::ParameterTree,-};--fn expand_keys_values(- graph: &Graph,- keys_or_values: &Tensor,- num_groups: usize,- head_dim: usize,-) -> Retained<Tensor> {- graph.reshape(- keys_or_values,- &mps_shape(&[1, (num_groups) as i64, -1_i64, head_dim as i64]).into(),- None,- )-}--pub fn rotation_subgraph(- graph: &Graph,- rope_name: String,- rope_config: &RoPEConfig,- head_dim: usize,- num_groups: usize,- num_heads: usize,- context_length: usize,- qkv: &Tensor, // [suffix_length, (num_heads + 2 * num_groups) * head_dim]- token_positions: &Tensor, // u64 [suffix_length]- root_parameter_tree: &ParameterTree<Rc<MTLContext>>,-) -> Result<(Retained<Tensor>, Retained<Tensor>), GraphConstructionError> {- let positional_embeddings = positional_embeddings_subgraph(- graph,- rope_config,- head_dim,- context_length,- token_positions,- &root_parameter_tree.subtree(rope_name.as_str())?,- )?;-- let total_query_dim = num_heads * head_dim;- let total_key_value_dim = num_groups * head_dim;-- let query_slice = graph.slice(qkv, 1, 0, total_query_dim as i64, None);- let new_key_slice = graph.slice(- qkv,- 1,- total_query_dim as i64,- total_key_value_dim as i64,- None,- );-- let queries = graph.reshape(- &query_slice,- &mps_shape(&[-1, num_heads as isize, head_dim as isize]),- None,- );- let new_keys = graph.reshape(- &new_key_slice,- &mps_shape(&[-1, num_groups as isize, head_dim as isize]),- None,- );-- let rotated_queries =- apply_rope_subgraph(graph, &positional_embeddings, &*queries);- let rotated_new_keys =- apply_rope_subgraph(graph, &positional_embeddings, &*new_keys);-- let permuted_queries = graph.transpose(&rotated_queries, &[1, 0, 2], None);- let permuted_keys = graph.transpose(&rotated_new_keys, &[1, 0, 2], None);-- Ok((permuted_queries, permuted_keys))-}--pub fn attention_subgraph(- graph: &Retained<Graph>,- head_dim: usize,- num_groups: usize,- num_heads: usize,- attention_scale: Option<f32>,- qkv: &Retained<Tensor>, // [suffix_length, (num_heads + 2 * num_groups) * head_dim]- key_cache: &Retained<Tensor>, // [max_sequence_length, num_groups * head_dim]- value_cache: &Retained<Tensor>, // [max_sequence_length, num_groups * head_dim]- permuted_queries: &Retained<Tensor>,- permuted_keys: &Retained<Tensor>,- bias: &Retained<Tensor>, // [suffix_length, prefix_length]- suffix_length: usize,- prefix_length: usize,-) -> Result<Retained<Tensor>, GraphConstructionError> {- let total_query_dim = num_heads * head_dim;- let total_key_value_dim = num_groups * head_dim;-- let new_value_slice = graph.slice(- qkv,- 1,- (total_query_dim + total_key_value_dim) as i64,- total_key_value_dim as i64,- None,- );-- let new_values = graph.reshape(- &new_value_slice,- &mps_shape(&[-1, num_groups as isize, head_dim as isize]),- None,- );-- let permuted_values = graph.transpose(&new_values, &[1, 0, 2], None);-- let key_cache_slice_to_update = graph.slice(- key_cache,- 1,- prefix_length as i64,- suffix_length as i64,- None,- );- let value_cache_slice_to_update = graph.slice(- &value_cache,- 1,- prefix_length as i64,- suffix_length as i64,- None,- );-- let assign_keys =- graph.assign_variable(&key_cache_slice_to_update, &permuted_keys, None);- let assign_values = graph.assign_variable(- &value_cache_slice_to_update,- &permuted_values,- None,- );-- let retained_graph_for_keys = graph.retain();- let retained_key_cache = key_cache.retain();- let keys = graph- .control_dependency(- &[&assign_keys],- move || {- vec![retained_graph_for_keys.slice(- &retained_key_cache,- 1,- 0,- (prefix_length + suffix_length) as i64,- None,- )]- },- None,- )- .first()- .unwrap()- .clone();-- let retained_graph_for_values = graph.retain();- let retained_value_cache = value_cache.retain();- let values = graph- .control_dependency(- &[&assign_values],- move || {- vec![retained_graph_for_values.slice(- &retained_value_cache,- 1,- 0,- (prefix_length + suffix_length) as i64,- None,- )]- },- None,- )- .first()- .unwrap()- .clone();-- let group_size = num_heads / num_groups;-- let expanded_keys = expand_keys_values(graph, &*keys, num_groups, head_dim);- let expanded_values =- expand_keys_values(graph, &*values, num_groups, head_dim);- let expanded_queries = graph.expand_dims(&permuted_queries, 0, None);-- let scale = attention_scale.unwrap_or(1.0 / (head_dim as f32).sqrt());-- let expanded_attention_output = graph- .masked_scaled_dot_product_attention_with_scalar(- &expanded_queries,- &expanded_keys,- &expanded_values,- bias,- scale,- None,- );-- let attention_output =- graph.squeeze_axis(&expanded_attention_output, 0, None);- let permuted_attention_output =- graph.transpose(&attention_output, &[1, 0, 2], None);-- let output_shape =- mps_shape(&[-1_i64, (num_groups * group_size * head_dim) as i64]);-- Ok(graph.reshape(&permuted_attention_output, &output_shape, None))-}diff --git a/uzu/src/backends/metal/graph/mod.rs b/uzu/src/backends/metal/graph/mod.rsindex 27a626eb1..ddb1f870f 100644--- a/uzu/src/backends/metal/graph/mod.rs+++ b/uzu/src/backends/metal/graph/mod.rs@@ -1,17 +1,12 @@pub mod common;-use common::{GraphConstructionError, data_type_of, load_constant, shape_of};+use common::{GraphConstructionError, load_constant};pub use common::{placeholder, shaped_type};-mod attention;-pub use attention::{attention_subgraph, rotation_subgraph};mod linear;pub use linear::linear_subgraph;mod mlp;pub use mlp::mlp_subgraph;-mod rms_norm;-pub use rms_norm::rms_norm_subgraph;mod embedding;pub use embedding::{embeddings_dequantize_weights_subgraph, embeddings_embed_subgraph,embeddings_readout_subgraph,};-mod rope;diff --git a/uzu/src/backends/metal/graph/rms_norm.rs b/uzu/src/backends/metal/graph/rms_norm.rsdeleted file mode 100644index ec582b38d..000000000--- a/uzu/src/backends/metal/graph/rms_norm.rs+++ /dev/null@@ -1,108 +0,0 @@-use std::rc::Rc;--use mpsgraph::{- Graph, GraphArithmeticOps, GraphNormalizationOps, Tensor,- tensor_shape_ops::*,-};-use objc2::rc::Retained;--use super::{- super::MTLContext, GraphConstructionError, data_type_of, load_constant,- shape_of,-};-use crate::{- DataType,- config::{RMSNormConfig, UpcastMode},- parameters::ParameterTree,-};--fn cast_if_needed(- graph: &Graph,- input: &Retained<Tensor>,- data_type: DataType,-) -> Retained<Tensor> {- if data_type_of(input) != data_type {- graph.cast(input, data_type.into(), None)- } else {- input.clone()- }-}--fn adjusted_variance(- graph: &Graph,- input: &Retained<Tensor>,- epsilon: f32,-) -> Retained<Tensor> {- let epsilon_constant =- graph.constant_with_scalar(epsilon as f64, input.data_type());-- let input_squared = graph.square(&input, None);- let input_squared_mean = graph.mean(&input_squared, &[1], None);- let input_squared_mean_adjusted =- graph.add(&input_squared_mean, &epsilon_constant, None);-- input_squared_mean_adjusted-}--fn normalization(- graph: &Graph,- input: &Retained<Tensor>,- adjusted_variance: &Retained<Tensor>,-) -> Retained<Tensor> {- let inverse_rms_norm = graph.rsqrt(&adjusted_variance, None);- let normalized_input = graph.multiply(input, &inverse_rms_norm, None);- normalized_input-}--pub fn rms_norm_subgraph(- graph: &Graph,- config: &RMSNormConfig,- input: &Retained<Tensor>,- parameter_tree: &ParameterTree<Rc<MTLContext>>,-) -> Result<Retained<Tensor>, GraphConstructionError> {- let [_, model_dim] = *shape_of(input) else {- panic!(- "Expected two-dimensional input, received {0:?}",- shape_of(input)- );- };- assert_ne!(model_dim, -1);-- let accumulation_data_type: DataType = config.accumulation_precision.into();- let scale_data_type: DataType = config.scale_precision.into();-- let x = cast_if_needed(graph, input, accumulation_data_type);- let adjusted_variance = adjusted_variance(graph, &x, config.epsilon);- let mut normalized_x = normalization(graph, &x, &adjusted_variance);- match config.upcast_mode {- UpcastMode::OnlyNormalization => {- normalized_x =- cast_if_needed(graph, &normalized_x, scale_data_type);- },- UpcastMode::FullLayer => {},- }-- let mut scales = load_constant(- graph,- parameter_tree,- "scales",- &[model_dim as usize],- scale_data_type,- )?;- match config.upcast_mode {- UpcastMode::OnlyNormalization => {},- UpcastMode::FullLayer => {- scales = cast_if_needed(graph, &scales, accumulation_data_type);- },- }- if let Some(scale_offset) = config.scale_offset {- let scale_offset_constant =- graph.constant_with_scalar(scale_offset as f64, scales.data_type());- scales = graph.add(&scales, &scale_offset_constant, None);- }-- let mut result = graph.multiply(&normalized_x, &scales, None);- result = cast_if_needed(graph, &result, scale_data_type);- assert_eq!(shape_of(&*result), shape_of(input));- Ok(result)-}diff --git a/uzu/src/backends/metal/graph/rope.rs b/uzu/src/backends/metal/graph/rope.rsdeleted file mode 100644index 73cf62ed6..000000000--- a/uzu/src/backends/metal/graph/rope.rs+++ /dev/null@@ -1,121 +0,0 @@-use std::rc::Rc;--use mpsgraph::{Graph, GraphGatherOps, Tensor, tensor_shape_ops::*};-use objc2::rc::Retained;--use super::{- super::MTLContext, GraphConstructionError, common::data_type_of,- load_constant, shape_of,-};-use crate::{DataType, config::RoPEConfig, parameters::ParameterTree};--pub struct PositionalEmbeddings {- sines: Retained<Tensor>,- cosines: Retained<Tensor>,-}--impl PositionalEmbeddings {- pub fn new(- sines: Retained<Tensor>,- cosines: Retained<Tensor>,- ) -> Self {- let cosines_shape = shape_of(&*cosines);- let sines_shape = shape_of(&*sines);- assert_eq!(cosines_shape, sines_shape);- Self {- sines,- cosines,- }- }-}--pub fn positional_embeddings_subgraph(- graph: &Graph,- rope_config: &RoPEConfig,- head_dim: usize,- context_length: usize,- indices: &Tensor, // u64: [max_sequence_length]- parameter_tree: &ParameterTree<Rc<MTLContext>>,-) -> Result<PositionalEmbeddings, GraphConstructionError> {- assert_eq!(data_type_of(indices), DataType::I32);- let expected_shape = [context_length, head_dim];- let expected_data_type: DataType = rope_config.common().precision.into();- let all_cosines = load_constant(- graph,- parameter_tree,- "cosines",- &expected_shape,- expected_data_type,- )?;- let all_sines = load_constant(- graph,- parameter_tree,- "sines",- &expected_shape,- expected_data_type,- )?;-- let cosine_slice = graph.gather(&all_cosines, indices, 0, 0, None);- let sine_slice = graph.gather(&all_sines, indices, 0, 0, None);- Ok(PositionalEmbeddings::new(sine_slice, cosine_slice))-}--pub fn apply_rope_subgraph(- graph: &Graph,- positional_embeddings: &PositionalEmbeddings,- input: &Tensor,-) -> Retained<Tensor> {- let [suffix_length, _, head_dim] = *shape_of(input) else {- panic!(- "Expected three-dimensional input tensor, got {0:?}",- shape_of(input)- )- };- let pe_shape = shape_of(&*positional_embeddings.cosines);- let [pe_suffix_length, pe_head_dim] = *pe_shape else {- panic!(- "Expected two-dimensional positional embeddings, got {0:?}",- pe_shape- )- };- assert_eq!(head_dim, pe_head_dim);- assert_eq!(suffix_length, pe_suffix_length);-- let x = graph.slice(input, 2, 0, (head_dim / 2) as i64, None);- let y = graph.slice(- input,- 2,- (head_dim / 2) as i64,- (head_dim / 2) as i64,- None,- );- // [suffix_length, num_heads, head_dim / 2]- let neg_y = graph.multiply(- &y,- &graph.constant_with_scalar(-1.0_f64, y.data_type()),- None,- );- // [suffix_length, num_heads, head_dim]- let half_rotated_heads = graph.concat(&neg_y, &x, 2, None);- // [suffix_length, 1, head_dim]- let unsqueezed_cosines = graph.expand_dims(- &positional_embeddings.cosines,- 1,- Some("unsqueezed_cosines_expand_dims"),- );- // [suffix_length, 1, head_dim]- let unsqueezed_sines = graph.expand_dims(- &positional_embeddings.sines,- 1,- Some("unsqueezed_sines_expand_dims"),- );- // [suffix_length, num_heads, head_dim]- let mul_cos = graph.multiply(input, &unsqueezed_cosines, None);- let mul_sin = graph.multiply(&half_rotated_heads, &unsqueezed_sines, None);- // [suffix_length, num_heads, head_dim]- let result = graph.add(&mul_cos, &mul_sin, None);-- assert_eq!(shape_of(input), shape_of(&*result));-- result-}diff --git a/uzu/src/backends/metal/mod.rs b/uzu/src/backends/metal/mod.rsindex 61d765aee..f6ea05d42 100644--- a/uzu/src/backends/metal/mod.rs+++ b/uzu/src/backends/metal/mod.rs@@ -16,10 +16,7 @@ pub use array::MetalArray;pub use buffer_allocator::BufferAllocator;pub use context::MTLContext;pub use error::MTLError;-pub use executable_builder::{- DecoderExecutables, KernelsConfig,- attention_executable_provider::AttentionExecutableProviderConfig,-};+pub use executable_builder::DecoderExecutables;pub use forward_pass::{ForwardPassState, KVCache, ModelShape};pub use kernel::{KVCacheUpdate, KernelDataType, RopeKernel};// pub use kernel::{diff --git a/uzu/src/config/rope.rs b/uzu/src/config/rope.rsindex c6265256d..93d3f675d 100644--- a/uzu/src/config/rope.rs+++ b/uzu/src/config/rope.rs@@ -32,6 +32,11 @@ pub enum RoPEConfig {low_frequency_factor: f32,high_frequency_factor: f32,},+ LinearScalingRoPEConfig {+ #[serde(flatten)]+ common: RopeConfigCommon,+ scaling_factor: f32,+ },}impl RoPEConfig {@@ -46,6 +51,10 @@ impl RoPEConfig {common,..} => common,+ RoPEConfig::LinearScalingRoPEConfig {+ common,+ ..+ } => common,}}}diff --git a/uzu/src/decoder_runner.rs b/uzu/src/decoder_runner.rsindex f2a3010cf..def6bcf60 100644--- a/uzu/src/decoder_runner.rs+++ b/uzu/src/decoder_runner.rs@@ -1,7 +1,6 @@#![allow(dead_code)]use std::{- cell::RefCell, collections::HashMap, fs::File, io::BufReader, path::Path,- rc::Rc, time::Instant,+ cell::RefCell, fs::File, io::BufReader, path::Path, rc::Rc, time::Instant,};use mpsgraph::CommandBuffer as MPSCommandBuffer;@@ -9,8 +8,7 @@ use serde::{Deserialize, Serialize};use crate::{backends::metal::{- AttentionExecutableProviderConfig, DecoderExecutables,- ForwardPassState, KVCache, KernelsConfig, MTLContext, ModelShape,+ DecoderExecutables, ForwardPassState, KVCache, MTLContext, ModelShape,compilation_parameters::CompilationConfig,forward_pass::{ForwardPassBuffers, SharedBuffers,@@ -133,8 +131,6 @@ impl DecoderTestContext {decoder_config.clone(),&root_loader_view,compilation_config.clone(),- AttentionExecutableProviderConfig::new(HashMap::from([(1, 0)])),- KernelsConfig::default(),);Ok((Self {diff --git a/uzu/src/generator/config.rs b/uzu/src/generator/config.rsindex 81b822e5e..a892a9372 100644--- a/uzu/src/generator/config.rs+++ b/uzu/src/generator/config.rs@@ -76,7 +76,6 @@ pub trait GeneratorConfigProvider {pub struct GeneratorConfig {pub prefill_step_size: usize,- pub prefix_length_step: Option<usize>,pub speculator_config: SpeculatorConfig,pub allow_pre_encode: bool,pub sampling_seed: u64,@@ -86,7 +85,6 @@ pub struct GeneratorConfig {impl GeneratorConfig {pub fn new(prefill_step_size: usize,- prefix_length_step: Option<usize>,speculator_config: SpeculatorConfig,allow_pre_encode: bool,sampling_seed: SamplingSeed,@@ -94,7 +92,6 @@ impl GeneratorConfig {) -> Self {Self {prefill_step_size,- prefix_length_step,speculator_config,allow_pre_encode,sampling_seed: sampling_seed.get_value(),diff --git a/uzu/src/generator/context.rs b/uzu/src/generator/context.rsindex 0c78a8356..f4ad3c1f8 100644--- a/uzu/src/generator/context.rs+++ b/uzu/src/generator/context.rs@@ -1,7 +1,4 @@-use std::{- cell::RefCell, collections::HashMap, fs::File, io::BufReader, path::Path,- rc::Rc,-};+use std::{cell::RefCell, fs::File, io::BufReader, path::Path, rc::Rc};use mpsgraph::CommandBuffer as MPSCommandBuffer;use objc2::rc::Retained;@@ -10,8 +7,8 @@ use super::{config::GeneratorConfig, error::GeneratorError};use crate::{DataType,backends::metal::{- AttentionExecutableProviderConfig, DecoderExecutables, KVCache,- KVCacheUpdate, KernelDataType, KernelsConfig, MTLContext, ModelShape,+ DecoderExecutables, KVCache, KVCacheUpdate, KernelDataType, MTLContext,+ ModelShape,compilation_parameters::CompilationConfig,forward_pass::{ForwardPassBuffers, SharedBuffers},kernel::SamplingKernelEncodable,@@ -21,7 +18,6 @@ use crate::{};pub struct GeneratorContext {- pub kernels_config: KernelsConfig,pub mtl_context: Rc<MTLContext>,pub command_buffer: Retained<MPSCommandBuffer>,@@ -40,8 +36,6 @@ impl GeneratorContext {model_path: &Path,config: &GeneratorConfig,) -> Result<Self, GeneratorError> {- let kernels_config = KernelsConfig::default();-let mtl_device = metal::Device::system_default().ok_or(GeneratorError::UnableToCreateMetalContext)?;let mtl_command_queue =@@ -99,17 +93,11 @@ impl GeneratorContext {max_suffix_length,);- let prefix_length_step = config.prefix_length_step.unwrap_or(0);let executables = DecoderExecutables::new(mtl_context.clone(),decoder_config.clone(),&root_loader_view,compilation_config.clone(),- AttentionExecutableProviderConfig::new(HashMap::from([- (config.prefill_step_size, 0),- (1, prefix_length_step),- ])),- kernels_config.clone(),);let kv_cache = Rc::new(RefCell::new(KVCache::new(@@ -141,7 +129,6 @@ impl GeneratorContext {.map_err(|_| GeneratorError::UnableToCreateMetalContext)?;let context = Self {- kernels_config,mtl_context,command_buffer,kv_cache,diff --git a/uzu/src/generator/generator.rs b/uzu/src/generator/generator.rsindex 25aab8354..49597ee06 100644--- a/uzu/src/generator/generator.rs+++ b/uzu/src/generator/generator.rs@@ -408,9 +408,7 @@ impl Generator {fn allow_pre_encode(&self) -> bool {let metal_debug_active = MetalEnvVar::DeviceWrapperType.is_enabled();- let result = self.config.allow_pre_encode- && self.context.kernels_config.use_attention- && !metal_debug_active;+ let result = self.config.allow_pre_encode && !metal_debug_active;result}diff --git a/uzu/src/session/session.rs b/uzu/src/session/session.rsindex 0eebec23e..bbb2caa24 100644--- a/uzu/src/session/session.rs+++ b/uzu/src/session/session.rs@@ -36,7 +36,7 @@ impl Session {// Load tokenizer JSONlet tokenizer_path = model_path.join("tokenizer.json");let tokenizer = Tokenizer::from_file(&tokenizer_path)- .map_err(|_| SessionError::UnableToLoadTokenizerConfig)?;+ .map_err(|_| SessionError::UnableToLoadTokenizer)?;// Load tokenizer configurationlet tokenizer_config =diff --git a/uzu/src/session/session_config.rs b/uzu/src/session/session_config.rsindex f01149e12..e01b1ea5e 100644--- a/uzu/src/session/session_config.rs+++ b/uzu/src/session/session_config.rs@@ -9,7 +9,6 @@ use crate::generator::config::{#[derive(Clone)]pub struct SessionConfig {pub prefill_step_size: usize,- pub prefix_length_step: Option<usize>,pub speculator_config: SpeculatorConfig,pub allow_pre_encode: bool,pub sampling_seed: SamplingSeed,@@ -19,7 +18,6 @@ pub struct SessionConfig {impl SessionConfig {pub fn new(prefill_step_size: usize,- prefix_length_step: Option<usize>,speculator_config: SpeculatorConfig,allow_pre_encode: bool,sampling_seed: SamplingSeed,@@ -27,7 +25,6 @@ impl SessionConfig {) -> Self {Self {prefill_step_size,- prefix_length_step,speculator_config,allow_pre_encode,sampling_seed,@@ -38,7 +35,6 @@ impl SessionConfig {pub fn to_generator_config(&self) -> GeneratorConfig {GeneratorConfig::new(self.prefill_step_size,- self.prefix_length_step,self.speculator_config.clone(),self.allow_pre_encode,self.sampling_seed,@@ -51,7 +47,6 @@ impl Default for SessionConfig {fn default() -> Self {Self::new(8,- None,SpeculatorConfig::default(),true,SamplingSeed::default(),diff --git a/uzu/src/session/session_error.rs b/uzu/src/session/session_error.rsindex bc76c2103..feb489cd9 100644--- a/uzu/src/session/session_error.rs+++ b/uzu/src/session/session_error.rs@@ -8,6 +8,8 @@ pub enum SessionError {UnableToLoadConfig,#[error("Unable to load model weights")]UnableToLoadWeights,+ #[error("Unable to load tokenizer")]+ UnableToLoadTokenizer,#[error("Unable to load tokenizer configuration")]UnableToLoadTokenizerConfig,}diff --git a/uzu/src/tracer/tracer.rs b/uzu/src/tracer/tracer.rsindex fc9ae6785..a0fb050f8 100644--- a/uzu/src/tracer/tracer.rs+++ b/uzu/src/tracer/tracer.rs@@ -143,7 +143,6 @@ impl Tracer {pub fn new(model_path: &Path) -> Self {let generator_config = GeneratorConfig::new(1024,- None,SpeculatorConfig {number_of_speculated_tokens: 0,speculator: Arc::new(EmptySpeculator {}),@@ -168,7 +167,6 @@ impl Tracer {) -> Self {let generator_config = GeneratorConfig::new(1024,- None,SpeculatorConfig {number_of_speculated_tokens: 0,speculator: Arc::new(EmptySpeculator {}),@@ -219,9 +217,8 @@ impl Tracer {.map(|chunk| chunk.to_vec()).collect();- let external_bias_fn = |row: usize, col: usize| -> bool {- !mask[row][col]- };+ let external_bias_fn =+ |row: usize, col: usize| -> bool { !mask[row][col] };let mut state = ForwardPassState::new(self.generator_context.mtl_context.clone(),