Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Remove unused code

Aleksei Savin8ada78edaed5
diff --git a/uzu/src/backends/metal/executable_builder/attention_executable_provider.rs b/uzu/src/backends/metal/executable_builder/attention_executable_provider.rs
deleted file mode 100644
index 081b0a1fe..000000000
--- a/uzu/src/backends/metal/executable_builder/attention_executable_provider.rs
+++ /dev/null
@@ -1,142 +0,0 @@
-use std::{collections::HashMap, rc::Rc};
-
-use mpsgraph::Executable;
-use objc2::rc::Retained;
-
-use crate::{
- backends::metal::{
- MTLContext, compilation_parameters::CompilationConfig,
- forward_pass::transformer_layer::attention_executable,
- },
- config::DecoderConfig,
-};
-
-pub struct AttentionExecutableProviderConfig {
- pub prefix_length_step_by_suffix_length: HashMap<usize, usize>,
-}
-
-impl AttentionExecutableProviderConfig {
- pub fn new(
- prefix_length_step_by_suffix_length: HashMap<usize, usize>
- ) -> Self {
- Self {
- prefix_length_step_by_suffix_length,
- }
- }
-}
-
-pub struct AttentionExecutableProvider {
- mtl_context: Rc<MTLContext>,
- decoder_config: Rc<DecoderConfig>,
- compilation_config: Rc<CompilationConfig>,
- config: AttentionExecutableProviderConfig,
- pub executables: HashMap<usize, HashMap<usize, Retained<Executable>>>,
-}
-
-impl AttentionExecutableProvider {
- pub fn new(
- mtl_context: Rc<MTLContext>,
- decoder_config: Rc<DecoderConfig>,
- compilation_config: Rc<CompilationConfig>,
- config: AttentionExecutableProviderConfig,
- ) -> Self {
- let max_prefix_length: usize;
- if let Some(sliding_window_sizes) = &decoder_config.sliding_window_sizes
- {
- max_prefix_length = sliding_window_sizes
- .iter()
- .map(|size| size.unwrap_or(0))
- .max()
- .unwrap_or(0);
- } else {
- max_prefix_length = decoder_config.context_length;
- }
-
- let mut instance = Self {
- mtl_context: mtl_context,
- decoder_config,
- compilation_config,
- config,
- executables: HashMap::new(),
- };
- instance.prefetch(max_prefix_length);
- instance
- }
-
- fn prefetch(
- &mut self,
- max_prefix_length: usize,
- ) {
- let prefix_length_step_by_suffix_length =
- self.config.prefix_length_step_by_suffix_length.clone();
- for (suffix_length, prefix_length_step) in
- prefix_length_step_by_suffix_length
- {
- let _ = self.executable(suffix_length, 0);
- if prefix_length_step == 0 {
- continue;
- }
-
- for prefix_length in (prefix_length_step..=max_prefix_length)
- .step_by(prefix_length_step)
- {
- let _ = self.executable(suffix_length, prefix_length);
- }
- }
- }
-
- pub fn executable(
- &mut self,
- suffix_length: usize,
- prefix_length: usize,
- ) -> &Retained<Executable> {
- if !self.executables.contains_key(&suffix_length) {
- self.executables.insert(suffix_length, HashMap::new());
- }
-
- let need_to_compile = {
- let executables_for_suffix =
- self.executables.get(&suffix_length).unwrap();
- !executables_for_suffix.contains_key(&prefix_length)
- };
-
- if need_to_compile {
- let executable = self.compile(suffix_length, prefix_length);
- let executables_for_suffix =
- self.executables.get_mut(&suffix_length).unwrap();
- executables_for_suffix.insert(prefix_length, executable);
- }
-
- &self
- .executables
- .get(&suffix_length)
- .unwrap()
- .get(&prefix_length)
- .unwrap()
- }
-
- fn compile(
- &self,
- suffix_length: usize,
- prefix_length: usize,
- ) -> Retained<Executable> {
- let config = &self.decoder_config;
- let attention_executable = attention_executable(
- config
- .layer_config
- .attention_config
- .qkv_projection_config
- .activation_precision()
- .into(),
- config.head_dim,
- config.num_groups,
- config.num_heads,
- config.attention_scale,
- &self.mtl_context,
- suffix_length,
- prefix_length,
- &self.compilation_config.descriptor_general,
- );
- attention_executable
- }
-}
diff --git a/uzu/src/backends/metal/executable_builder/decoder_executables.rs b/uzu/src/backends/metal/executable_builder/decoder_executables.rs
index 881a00b2a..b4c8196f7 100644
--- a/uzu/src/backends/metal/executable_builder/decoder_executables.rs
+++ b/uzu/src/backends/metal/executable_builder/decoder_executables.rs
@@ -1,13 +1,8 @@
-use std::{cell::RefCell, rc::Rc};
+use std::rc::Rc;
use mpsgraph::CommandBuffer as MPSCommandBuffer;
-use super::{
- attention_executable_provider::{
- AttentionExecutableProvider, AttentionExecutableProviderConfig,
- },
- layer_executables::LayerExecutables,
-};
+use super::layer_executables::LayerExecutables;
use crate::{
DataType,
backends::metal::{
@@ -16,33 +11,14 @@ use crate::{
forward_pass::{
ArrayId, ForwardPassState, MPSGraphBlock, RopeType,
encodable_with_state::{EncodableWithState, EncodingParameters},
- transformer_layer::{
- embed_block, readout_block, rms_norm_block, rotation_block,
- rotation_executable,
- },
+ transformer_layer::{embed_block, readout_block},
},
kernel::{RMSNormKernelEncodable, RopeKernelEncodable},
},
- config::{RoPEConfig, decoder::DecoderConfig},
+ config::decoder::DecoderConfig,
parameters::ParameterTree,
};
-#[derive(Debug, Clone)]
-pub struct KernelsConfig {
- pub use_rope: bool,
- pub use_attention: bool,
- pub use_rms_norm: bool,
-}
-
-impl KernelsConfig {
- pub fn default() -> Self {
- Self {
- use_rope: true,
- use_attention: true,
- use_rms_norm: true,
- }
- }
-}
pub struct DecoderExecutables {
pub embed: MPSGraphBlock,
pub layers: Box<[LayerExecutables]>,
@@ -58,8 +34,6 @@ impl DecoderExecutables {
decoder_config: Rc<DecoderConfig>,
decoder_weight_loader: &ParameterTree<Rc<MTLContext>>,
compilation_config: Rc<CompilationConfig>,
- attention_executable_provider_config: AttentionExecutableProviderConfig,
- kernels_config: KernelsConfig,
) -> Self {
let embed = embed_block(
&decoder_config,
@@ -76,45 +50,20 @@ impl DecoderExecutables {
let global_rope = Self::create_rope_block(
&mtl_context,
&decoder_config,
- &decoder_weight_loader,
- compilation_config.clone(),
- &kernels_config,
- String::from("global_rope"),
- &decoder_config.global_rope_config,
RopeType::Global,
);
let local_rope: Option<Rc<Box<dyn EncodableWithState>>>;
- if let Some(local_rope_config) = &decoder_config.local_rope_config {
+ if let Some(_) = &decoder_config.local_rope_config {
local_rope = Some(Self::create_rope_block(
&mtl_context,
&decoder_config,
- &decoder_weight_loader,
- compilation_config.clone(),
- &kernels_config,
- String::from("local_rope"),
- local_rope_config,
RopeType::Local,
));
} else {
local_rope = None;
}
- let attention_executable_provider: Option<
- Rc<RefCell<AttentionExecutableProvider>>,
- >;
- if kernels_config.use_attention {
- attention_executable_provider = None;
- } else {
- attention_executable_provider =
- Some(Rc::new(RefCell::new(AttentionExecutableProvider::new(
- mtl_context.clone(),
- decoder_config.clone(),
- compilation_config.clone(),
- attention_executable_provider_config,
- ))));
- }
-
let model_shape = ModelShape::from_decoder_config(&decoder_config);
let sliding_window_sizes = model_shape.sliding_window_length_per_layer;
@@ -148,37 +97,22 @@ impl DecoderExecutables {
&decoder_weight_loader
.subtree(&format!("layers.{}", layer_index))
.unwrap(),
- attention_executable_provider.clone(),
rope,
- kernels_config.clone(),
)
})
.collect::<Vec<_>>();
- let norm_block: Box<dyn EncodableWithState> =
- if kernels_config.use_rms_norm {
- Box::new(
- RMSNormKernelEncodable::new(
- &mtl_context,
- intermediate_data_type,
- decoder_config.output_norm_config.clone(),
- ArrayId::Main,
- ArrayId::Main,
- &decoder_weight_loader.subtree("output_norm").unwrap(),
- )
- .expect("Failed to create output RMS norm kernel"),
- )
- } else {
- Box::new(rms_norm_block(
- &decoder_config.output_norm_config,
- decoder_config.model_dim,
- &mtl_context,
- &decoder_weight_loader.subtree("output_norm").unwrap(),
- ArrayId::Main,
- ArrayId::Main,
- &compilation_config.descriptor_general,
- ))
- };
+ let norm_block: Box<dyn EncodableWithState> = Box::new(
+ RMSNormKernelEncodable::new(
+ &mtl_context,
+ intermediate_data_type,
+ decoder_config.output_norm_config.clone(),
+ ArrayId::Main,
+ ArrayId::Main,
+ &decoder_weight_loader.subtree("output_norm").unwrap(),
+ )
+ .expect("Failed to create output RMS norm kernel"),
+ );
Self {
embed: embed,
@@ -193,11 +127,6 @@ impl DecoderExecutables {
fn create_rope_block(
mtl_context: &MTLContext,
decoder_config: &DecoderConfig,
- decoder_weight_loader: &ParameterTree<Rc<MTLContext>>,
- compilation_config: Rc<CompilationConfig>,
- kernels_config: &KernelsConfig,
- rope_name: String,
- rope_config: &RoPEConfig,
rope_type: RopeType,
) -> Rc<Box<dyn EncodableWithState>> {
let intermediate_data_type: DataType = decoder_config
@@ -208,30 +137,10 @@ impl DecoderExecutables {
.into();
let kernel_data_type: KernelDataType = intermediate_data_type.into();
- let rotation: Box<dyn EncodableWithState>;
- if kernels_config.use_rope {
- rotation = Box::new(
- RopeKernelEncodable::new(
- mtl_context,
- kernel_data_type,
- rope_type,
- )
+ let rotation: Box<dyn EncodableWithState> = Box::new(
+ RopeKernelEncodable::new(mtl_context, kernel_data_type, rope_type)
.expect("Failed to create RopeKernelEncodable"),
- );
- } else {
- let rotation_executable = rotation_executable(
- rope_name,
- rope_config,
- decoder_config.head_dim,
- decoder_config.num_groups,
- decoder_config.num_heads,
- decoder_config.context_length,
- &mtl_context,
- decoder_weight_loader,
- &compilation_config.descriptor_general,
- );
- rotation = Box::new(rotation_block(rotation_executable));
- }
+ );
return Rc::new(rotation);
}
}
diff --git a/uzu/src/backends/metal/executable_builder/layer_executables.rs b/uzu/src/backends/metal/executable_builder/layer_executables.rs
index b7b97810e..4bc0a74e9 100644
--- a/uzu/src/backends/metal/executable_builder/layer_executables.rs
+++ b/uzu/src/backends/metal/executable_builder/layer_executables.rs
@@ -1,21 +1,17 @@
-use std::{cell::RefCell, rc::Rc};
+use std::rc::Rc;
use mpsgraph::CommandBuffer as MPSCommandBuffer;
use objc2::rc::autoreleasepool;
-use super::{
- attention_executable_provider::AttentionExecutableProvider,
- decoder_executables::KernelsConfig,
-};
use crate::{
- Array, DataType,
+ DataType,
backends::metal::{
MTLContext,
compilation_parameters::CompilationConfig,
forward_pass::{
ArrayId, ForwardPassState, MPSGraphBlock,
encodable_with_state::{EncodableWithState, EncodingParameters},
- transformer_layer::{self, attention_block},
+ transformer_layer,
},
kernel::{
AttentionKernelEncodable, KernelDataType, QKNormKernelEncodable,
@@ -34,15 +30,12 @@ pub struct LayerExecutables {
pub qk_norm: Option<Box<dyn EncodableWithState>>,
pub rope: Rc<Box<dyn EncodableWithState>>,
pub attention: Box<dyn EncodableWithState>,
- pub attention_executable_provider:
- Option<Rc<RefCell<AttentionExecutableProvider>>>,
pub out_projection: MPSGraphBlock,
pub post_attention_norm: Option<Box<dyn EncodableWithState>>,
pub main_shortcut_add_swap: Box<dyn EncodableWithState>,
pub pre_mlp_norm: Box<dyn EncodableWithState>,
pub mlp: MPSGraphBlock,
pub post_mlp_norm: Option<Box<dyn EncodableWithState>>,
- pub kernels_config: KernelsConfig,
}
impl LayerExecutables {
@@ -58,11 +51,7 @@ impl LayerExecutables {
num_groups: usize,
attention_scale: Option<f32>,
decoder_layer_loader: &ParameterTree<Rc<MTLContext>>,
- attention_executable_provider: Option<
- Rc<RefCell<AttentionExecutableProvider>>,
- >,
rope: Rc<Box<dyn EncodableWithState>>,
- kernels_config: KernelsConfig,
) -> Self {
autoreleasepool(|_| {
let intermediate_data_type: DataType = layer_config
@@ -82,34 +71,19 @@ impl LayerExecutables {
.unwrap(),
);
- let pre_attention_norm: Box<dyn EncodableWithState> =
- if kernels_config.use_rms_norm {
- Box::new(
- RMSNormKernelEncodable::new(
- mtl_context,
- intermediate_data_type,
- layer_config.pre_attention_norm_config.clone(),
- ArrayId::Main,
- ArrayId::Main,
- &decoder_layer_loader
- .subtree("pre_attention_norm")
- .unwrap(),
- )
- .expect("Failed to create RMS norm kernel"),
- )
- } else {
- Box::new(transformer_layer::rms_norm_block(
- &layer_config.pre_attention_norm_config,
- model_dim,
- mtl_context,
- &decoder_layer_loader
- .subtree("pre_attention_norm")
- .unwrap(),
- ArrayId::Main,
- ArrayId::Main,
- &compilation_config.descriptor_general,
- ))
- };
+ let pre_attention_norm: Box<dyn EncodableWithState> = Box::new(
+ RMSNormKernelEncodable::new(
+ mtl_context,
+ intermediate_data_type,
+ layer_config.pre_attention_norm_config.clone(),
+ ArrayId::Main,
+ ArrayId::Main,
+ &decoder_layer_loader
+ .subtree("pre_attention_norm")
+ .unwrap(),
+ )
+ .expect("Failed to create RMS norm kernel"),
+ );
let qkv_projection = transformer_layer::linear_block(
&layer_config.attention_config.qkv_projection_config,
@@ -174,33 +148,19 @@ impl LayerExecutables {
if let Some(norm_config) =
&layer_config.post_attention_norm_config
{
- if kernels_config.use_rms_norm {
- Some(Box::new(
- RMSNormKernelEncodable::new(
- mtl_context,
- intermediate_data_type,
- norm_config.clone(),
- ArrayId::Main,
- ArrayId::Main,
- &decoder_layer_loader
- .subtree("post_attention_norm")
- .unwrap(),
- )
- .expect("Failed to create RMS norm kernel"),
- ))
- } else {
- Some(Box::new(transformer_layer::rms_norm_block(
- norm_config,
- model_dim,
+ Some(Box::new(
+ RMSNormKernelEncodable::new(
mtl_context,
+ intermediate_data_type,
+ norm_config.clone(),
+ ArrayId::Main,
+ ArrayId::Main,
&decoder_layer_loader
.subtree("post_attention_norm")
.unwrap(),
- ArrayId::Main,
- ArrayId::Main,
- &compilation_config.descriptor_general,
- )))
- }
+ )
+ .expect("Failed to create RMS norm kernel"),
+ ))
} else {
None
};
@@ -214,31 +174,17 @@ impl LayerExecutables {
.unwrap(),
);
- let pre_mlp_norm: Box<dyn EncodableWithState> = if kernels_config
- .use_rms_norm
- {
- Box::new(
- RMSNormKernelEncodable::new(
- mtl_context,
- intermediate_data_type,
- layer_config.pre_mlp_norm_config.clone(),
- ArrayId::Main,
- ArrayId::Main,
- &decoder_layer_loader.subtree("pre_mlp_norm").unwrap(),
- )
- .expect("Failed to create RMS norm kernel"),
- )
- } else {
- Box::new(transformer_layer::rms_norm_block(
- &layer_config.pre_mlp_norm_config,
- model_dim,
+ let pre_mlp_norm: Box<dyn EncodableWithState> = Box::new(
+ RMSNormKernelEncodable::new(
mtl_context,
- &decoder_layer_loader.subtree("pre_mlp_norm").unwrap(),
+ intermediate_data_type,
+ layer_config.pre_mlp_norm_config.clone(),
ArrayId::Main,
ArrayId::Main,
- &compilation_config.descriptor_general,
- ))
- };
+ &decoder_layer_loader.subtree("pre_mlp_norm").unwrap(),
+ )
+ .expect("Failed to create RMS norm kernel"),
+ );
let mlp = transformer_layer::mlp_block(
&layer_config.mlp_config,
@@ -251,61 +197,32 @@ impl LayerExecutables {
let post_mlp_norm: Option<Box<dyn EncodableWithState>> =
if let Some(norm_config) = &layer_config.post_mlp_norm_config {
- if kernels_config.use_rms_norm {
- Some(Box::new(
- RMSNormKernelEncodable::new(
- mtl_context,
- intermediate_data_type,
- norm_config.clone(),
- ArrayId::Main,
- ArrayId::Main,
- &decoder_layer_loader
- .subtree("post_mlp_norm")
- .unwrap(),
- )
- .expect("Failed to create RMS norm kernel"),
- ))
- } else {
- Some(Box::new(transformer_layer::rms_norm_block(
- norm_config,
- model_dim,
+ Some(Box::new(
+ RMSNormKernelEncodable::new(
mtl_context,
+ intermediate_data_type,
+ norm_config.clone(),
+ ArrayId::Main,
+ ArrayId::Main,
&decoder_layer_loader
.subtree("post_mlp_norm")
.unwrap(),
- ArrayId::Main,
- ArrayId::Main,
- &compilation_config.descriptor_general,
- )))
- }
+ )
+ .expect("Failed to create RMS norm kernel"),
+ ))
} else {
None
};
- let attention: Box<dyn EncodableWithState>;
- if kernels_config.use_attention {
- attention = Box::new(
- AttentionKernelEncodable::new(
- mtl_context,
- kernel_data_type,
- layer_index,
- attention_scale,
- )
- .expect(
- "Failed to create AttentionWrapper with Metal kernel",
- ),
- );
- } else {
- attention = Box::new(
- AttentionKernelEncodable::new(
- mtl_context,
- kernel_data_type,
- layer_index,
- attention_scale,
- )
- .expect("Failed to create AttentionWrapper for MPS Graph"),
- );
- }
+ let attention: Box<dyn EncodableWithState> = Box::new(
+ AttentionKernelEncodable::new(
+ mtl_context,
+ kernel_data_type,
+ layer_index,
+ attention_scale,
+ )
+ .expect("Failed to create AttentionWrapper with Metal kernel"),
+ );
Self {
layer_index,
@@ -315,48 +232,15 @@ impl LayerExecutables {
qk_norm,
rope,
attention,
- attention_executable_provider,
out_projection,
post_attention_norm,
main_shortcut_add_swap,
pre_mlp_norm,
mlp,
post_mlp_norm,
- kernels_config,
}
})
}
-
- fn get_suffix_length(
- &self,
- state: &ForwardPassState,
- ) -> usize {
- let shape: Vec<usize> = (*state.arrays(&[ArrayId::TokenIds]))[0]
- .borrow()
- .shape()
- .iter()
- .map(|value| *value)
- .collect();
- *shape.first().unwrap()
- }
-
- pub fn attention_block(
- &self,
- state: &ForwardPassState,
- ) -> MPSGraphBlock {
- let suffix_length = self.get_suffix_length(state);
- let prefix_length: usize = state.kv_cache.borrow().data
- [self.layer_index]
- .effective_prefix_length();
-
- let mut attention_executable_provider =
- self.attention_executable_provider.as_ref().unwrap().borrow_mut();
- let attention_executable = attention_executable_provider
- .executable(suffix_length, prefix_length);
- let attention =
- attention_block(attention_executable.clone(), self.layer_index);
- attention
- }
}
impl EncodableWithState for LayerExecutables {
@@ -379,8 +263,6 @@ impl EncodableWithState for LayerExecutables {
);
}
- let first_layer = self.layer_index == 0;
-
self.copy_main_to_shortcut.encode(state, command_buffer, parameters);
// shortcut = input
@@ -397,40 +279,7 @@ impl EncodableWithState for LayerExecutables {
qk_norm.encode(state, command_buffer, parameters);
}
self.rope.encode(state, command_buffer, parameters);
-
- if parameters.warmup {
- if first_layer {
- if self.kernels_config.use_attention {
- self.attention.encode(state, command_buffer, parameters);
- } else {
- let suffix_length = self.get_suffix_length(state);
- if let Some(executables) = self
- .attention_executable_provider
- .as_ref()
- .unwrap()
- .borrow()
- .executables
- .get(&suffix_length)
- {
- for (_, executable) in executables {
- let attention = attention_block(
- executable.clone(),
- self.layer_index,
- );
- attention.encode(state, command_buffer, parameters);
- }
- }
- }
- }
- } else {
- if self.kernels_config.use_attention {
- self.attention.encode(state, command_buffer, parameters);
- } else {
- let attention = self.attention_block(state);
- attention.encode(state, command_buffer, parameters);
- }
- }
-
+ self.attention.encode(state, command_buffer, parameters);
self.out_projection.encode(state, command_buffer, parameters);
if let Some(layer_traces) = layer_traces.clone() {
state.copy_array(
diff --git a/uzu/src/backends/metal/executable_builder/mod.rs b/uzu/src/backends/metal/executable_builder/mod.rs
index 35402e34d..84cfd0fbc 100644
--- a/uzu/src/backends/metal/executable_builder/mod.rs
+++ b/uzu/src/backends/metal/executable_builder/mod.rs
@@ -1,6 +1,4 @@
-pub mod attention_executable_provider;
pub mod decoder_executables;
pub mod layer_executables;
-pub use decoder_executables::{DecoderExecutables, KernelsConfig};
-// LayerExecutables likely internal to this module
+pub use decoder_executables::DecoderExecutables;
diff --git a/uzu/src/backends/metal/forward_pass/transformer_layer.rs b/uzu/src/backends/metal/forward_pass/transformer_layer.rs
index 469937775..3182518ef 100644
--- a/uzu/src/backends/metal/forward_pass/transformer_layer.rs
+++ b/uzu/src/backends/metal/forward_pass/transformer_layer.rs
@@ -1,18 +1,15 @@
use std::{collections::HashMap, rc::Rc};
use mpsgraph::{
- CompilationDescriptor, Device as MPSDevice, Executable,
- ExecutableExecutionDescriptor, Graph, GraphTensorShapeOps,
+ CompilationDescriptor, Device as MPSDevice, ExecutableExecutionDescriptor,
+ Graph, GraphTensorShapeOps,
};
use objc2::rc::{Retained, autoreleasepool};
use super::{
super::{
MTLContext,
- graph::{
- attention_subgraph, linear_subgraph, mlp_subgraph, placeholder,
- rms_norm_subgraph, shaped_type,
- },
+ graph::{linear_subgraph, mlp_subgraph, placeholder, shaped_type},
},
io_arrays::IOArrays,
mpsgraph_block::MPSGraphBlock,
@@ -22,12 +19,9 @@ use crate::{
DataType,
backends::metal::graph::{
embeddings_dequantize_weights_subgraph, embeddings_embed_subgraph,
- embeddings_readout_subgraph, rotation_subgraph,
- },
- config::{
- DecoderConfig, EmbeddingConfig, LinearConfig, MLPConfig, RMSNormConfig,
- RoPEConfig,
+ embeddings_readout_subgraph,
},
+ config::{DecoderConfig, EmbeddingConfig, LinearConfig, MLPConfig},
parameters::ParameterTree,
};
@@ -134,226 +128,6 @@ pub fn mlp_block(
})
}
-pub fn rms_norm_block(
- config: &RMSNormConfig,
- model_dim: usize,
- context: &MTLContext,
- parameter_tree: &ParameterTree<Rc<MTLContext>>,
- input_array_id: ArrayId,
- output_array_id: ArrayId,
- compilation_descriptor: &CompilationDescriptor,
-) -> MPSGraphBlock {
- autoreleasepool(|_| {
- let graph = Graph::new();
-
- let input_shape = [-1, model_dim as isize];
- let input_data_type: DataType = config.scale_precision.into();
- let input_placeholder =
- placeholder(&graph, &input_shape, input_data_type);
-
- let output = rms_norm_subgraph(
- &graph,
- config,
- &input_placeholder,
- parameter_tree,
- )
- .unwrap();
-
- let retained_shaped_type = shaped_type(&input_shape, input_data_type);
- let feeds =
- HashMap::from([(&*input_placeholder, &*retained_shaped_type)]);
-
- let executable = graph.compile(
- &MPSDevice::with_device(&context.device),
- &feeds,
- &[&output],
- Some(compilation_descriptor),
- );
-
- let arguments = IOArrays::new(
- vec![input_array_id].into_boxed_slice(),
- vec![output_array_id].into_boxed_slice(),
- );
-
- MPSGraphBlock::new(executable, make_execution_descriptor(), arguments)
- })
-}
-
-pub fn rotation_executable(
- rope_name: String,
- rope_config: &RoPEConfig,
- head_dim: usize,
- num_groups: usize,
- num_heads: usize,
- context_length: usize,
- context: &MTLContext,
- root_parameter_tree: &ParameterTree<Rc<MTLContext>>,
- compilation_descriptor: &Retained<CompilationDescriptor>,
-) -> Retained<Executable> {
- autoreleasepool(|_| {
- let graph = Graph::new();
-
- let activation_data_type: DataType =
- rope_config.common().precision.into();
-
- let qkv_shape =
- [-1, ((num_heads + 2 * num_groups) * head_dim) as isize];
- let qkv_placeholder =
- placeholder(&graph, &qkv_shape, activation_data_type);
- let qkv_shaped_type = shaped_type(&qkv_shape, activation_data_type);
-
- let token_positions_placeholder =
- placeholder(&graph, &[-1 as isize], DataType::I32);
- let token_positions_shaped_type =
- shaped_type(&[-1 as isize], DataType::I32);
-
- let (output_queries, output_keys) = rotation_subgraph(
- &graph,
- rope_name,
- rope_config,
- head_dim,
- num_groups,
- num_heads,
- context_length,
- &qkv_placeholder,
- &token_positions_placeholder,
- root_parameter_tree,
- )
- .unwrap();
-
- let feeds = HashMap::from([
- (&*qkv_placeholder, &*qkv_shaped_type),
- (&*token_positions_placeholder, &*token_positions_shaped_type),
- ]);
-
- graph.compile(
- &MPSDevice::with_device(&context.device),
- &feeds,
- &[&output_queries, &output_keys],
- Some(compilation_descriptor),
- )
- })
-}
-
-pub fn attention_executable(
- activation_data_type: DataType,
- head_dim: usize,
- num_groups: usize,
- num_heads: usize,
- attention_scale: Option<f32>,
- context: &MTLContext,
- suffix_length: usize,
- prefix_length: usize,
- compilation_descriptor: &CompilationDescriptor,
-) -> Retained<Executable> {
- autoreleasepool(|_| {
- let graph = Graph::new();
-
- let qkv_shape =
- [-1, ((num_heads + 2 * num_groups) * head_dim) as isize];
- let qkv_placeholder =
- placeholder(&graph, &qkv_shape, activation_data_type);
- let retained_qkv_shaped_type =
- shaped_type(&qkv_shape, activation_data_type);
-
- let kv_cache_shape = [num_groups as isize, -1, head_dim as isize];
- let key_cache_placeholder =
- placeholder(&graph, &kv_cache_shape, activation_data_type);
- let value_cache_placeholder =
- placeholder(&graph, &kv_cache_shape, activation_data_type);
- let retained_kv_cache_shaped_type =
- shaped_type(&kv_cache_shape, activation_data_type);
-
- let rotated_queries_shape = [num_heads as isize, -1, head_dim as isize];
- let rotated_queries_placeholder =
- placeholder(&graph, &rotated_queries_shape, activation_data_type);
- let rotated_queries_shaped_type =
- shaped_type(&rotated_queries_shape, activation_data_type);
-
- let rotated_keys_shape = [num_groups as isize, -1, head_dim as isize];
- let rotated_keys_placeholder =
- placeholder(&graph, &rotated_keys_shape, activation_data_type);
- let rotated_keys_shaped_type =
- shaped_type(&rotated_keys_shape, activation_data_type);
-
- let bias_placeholder =
- placeholder(&graph, &[-1, -1], activation_data_type);
- let retained_bias_shaped_type =
- shaped_type(&[-1, -1], activation_data_type);
-
- let output = attention_subgraph(
- &graph,
- head_dim,
- num_groups,
- num_heads,
- attention_scale,
- &qkv_placeholder,
- &key_cache_placeholder,
- &value_cache_placeholder,
- &rotated_queries_placeholder,
- &rotated_keys_placeholder,
- &bias_placeholder,
- suffix_length,
- prefix_length,
- )
- .unwrap();
-
- let feeds = HashMap::from([
- (&*qkv_placeholder, &*retained_qkv_shaped_type),
- (&*key_cache_placeholder, &*retained_kv_cache_shaped_type),
- (&*value_cache_placeholder, &*retained_kv_cache_shaped_type),
- (&*rotated_queries_placeholder, &*rotated_queries_shaped_type),
- (&*rotated_keys_placeholder, &*rotated_keys_shaped_type),
- (&bias_placeholder, &retained_bias_shaped_type),
- ]);
-
- graph.compile(
- &MPSDevice::with_device(&context.device),
- &feeds,
- &[&output],
- Some(compilation_descriptor),
- )
- })
-}
-
-pub fn rotation_block(executable: Retained<Executable>) -> MPSGraphBlock {
- autoreleasepool(|_| {
- let input_array_ids = vec![ArrayId::QKV, ArrayId::TokenPositions];
- let output_array_ids =
- vec![ArrayId::RotatedQueries, ArrayId::RotatedKeys];
-
- let arguments = IOArrays::new(
- input_array_ids.into_boxed_slice(),
- output_array_ids.into_boxed_slice(),
- );
-
- MPSGraphBlock::new(executable, make_execution_descriptor(), arguments)
- })
-}
-
-pub fn attention_block(
- executable: Retained<Executable>,
- layer_index: usize,
-) -> MPSGraphBlock {
- autoreleasepool(|_| {
- let input_array_ids = vec![
- ArrayId::QKV,
- ArrayId::Keys(layer_index),
- ArrayId::Values(layer_index),
- ArrayId::RotatedQueries,
- ArrayId::RotatedKeys,
- ];
- let output_array_ids = vec![ArrayId::AttentionOutput];
-
- let arguments = IOArrays::new(
- input_array_ids.into_boxed_slice(),
- output_array_ids.into_boxed_slice(),
- );
-
- MPSGraphBlock::new(executable, make_execution_descriptor(), arguments)
- })
-}
-
pub fn embed_block(
config: &DecoderConfig,
context: &MTLContext,
diff --git a/uzu/src/backends/metal/graph/attention.rs b/uzu/src/backends/metal/graph/attention.rs
deleted file mode 100644
index c0d2510f4..000000000
--- a/uzu/src/backends/metal/graph/attention.rs
+++ /dev/null
@@ -1,212 +0,0 @@
-use std::rc::Rc;
-
-use mpsgraph::{
- Graph, GraphControlFlowOps, GraphLinearAlgebraOps, GraphMemoryOps,
- GraphTensorShapeOps, Tensor,
-};
-use objc2::{Message, rc::Retained};
-
-use super::{
- common::GraphConstructionError,
- rope::{apply_rope_subgraph, positional_embeddings_subgraph},
-};
-use crate::{
- backends::metal::{MTLContext, utils::mps_shape},
- config::RoPEConfig,
- parameters::ParameterTree,
-};
-
-fn expand_keys_values(
- graph: &Graph,
- keys_or_values: &Tensor,
- num_groups: usize,
- head_dim: usize,
-) -> Retained<Tensor> {
- graph.reshape(
- keys_or_values,
- &mps_shape(&[1, (num_groups) as i64, -1_i64, head_dim as i64]).into(),
- None,
- )
-}
-
-pub fn rotation_subgraph(
- graph: &Graph,
- rope_name: String,
- rope_config: &RoPEConfig,
- head_dim: usize,
- num_groups: usize,
- num_heads: usize,
- context_length: usize,
- qkv: &Tensor, // [suffix_length, (num_heads + 2 * num_groups) * head_dim]
- token_positions: &Tensor, // u64 [suffix_length]
- root_parameter_tree: &ParameterTree<Rc<MTLContext>>,
-) -> Result<(Retained<Tensor>, Retained<Tensor>), GraphConstructionError> {
- let positional_embeddings = positional_embeddings_subgraph(
- graph,
- rope_config,
- head_dim,
- context_length,
- token_positions,
- &root_parameter_tree.subtree(rope_name.as_str())?,
- )?;
-
- let total_query_dim = num_heads * head_dim;
- let total_key_value_dim = num_groups * head_dim;
-
- let query_slice = graph.slice(qkv, 1, 0, total_query_dim as i64, None);
- let new_key_slice = graph.slice(
- qkv,
- 1,
- total_query_dim as i64,
- total_key_value_dim as i64,
- None,
- );
-
- let queries = graph.reshape(
- &query_slice,
- &mps_shape(&[-1, num_heads as isize, head_dim as isize]),
- None,
- );
- let new_keys = graph.reshape(
- &new_key_slice,
- &mps_shape(&[-1, num_groups as isize, head_dim as isize]),
- None,
- );
-
- let rotated_queries =
- apply_rope_subgraph(graph, &positional_embeddings, &*queries);
- let rotated_new_keys =
- apply_rope_subgraph(graph, &positional_embeddings, &*new_keys);
-
- let permuted_queries = graph.transpose(&rotated_queries, &[1, 0, 2], None);
- let permuted_keys = graph.transpose(&rotated_new_keys, &[1, 0, 2], None);
-
- Ok((permuted_queries, permuted_keys))
-}
-
-pub fn attention_subgraph(
- graph: &Retained<Graph>,
- head_dim: usize,
- num_groups: usize,
- num_heads: usize,
- attention_scale: Option<f32>,
- qkv: &Retained<Tensor>, // [suffix_length, (num_heads + 2 * num_groups) * head_dim]
- key_cache: &Retained<Tensor>, // [max_sequence_length, num_groups * head_dim]
- value_cache: &Retained<Tensor>, // [max_sequence_length, num_groups * head_dim]
- permuted_queries: &Retained<Tensor>,
- permuted_keys: &Retained<Tensor>,
- bias: &Retained<Tensor>, // [suffix_length, prefix_length]
- suffix_length: usize,
- prefix_length: usize,
-) -> Result<Retained<Tensor>, GraphConstructionError> {
- let total_query_dim = num_heads * head_dim;
- let total_key_value_dim = num_groups * head_dim;
-
- let new_value_slice = graph.slice(
- qkv,
- 1,
- (total_query_dim + total_key_value_dim) as i64,
- total_key_value_dim as i64,
- None,
- );
-
- let new_values = graph.reshape(
- &new_value_slice,
- &mps_shape(&[-1, num_groups as isize, head_dim as isize]),
- None,
- );
-
- let permuted_values = graph.transpose(&new_values, &[1, 0, 2], None);
-
- let key_cache_slice_to_update = graph.slice(
- key_cache,
- 1,
- prefix_length as i64,
- suffix_length as i64,
- None,
- );
- let value_cache_slice_to_update = graph.slice(
- &value_cache,
- 1,
- prefix_length as i64,
- suffix_length as i64,
- None,
- );
-
- let assign_keys =
- graph.assign_variable(&key_cache_slice_to_update, &permuted_keys, None);
- let assign_values = graph.assign_variable(
- &value_cache_slice_to_update,
- &permuted_values,
- None,
- );
-
- let retained_graph_for_keys = graph.retain();
- let retained_key_cache = key_cache.retain();
- let keys = graph
- .control_dependency(
- &[&assign_keys],
- move || {
- vec![retained_graph_for_keys.slice(
- &retained_key_cache,
- 1,
- 0,
- (prefix_length + suffix_length) as i64,
- None,
- )]
- },
- None,
- )
- .first()
- .unwrap()
- .clone();
-
- let retained_graph_for_values = graph.retain();
- let retained_value_cache = value_cache.retain();
- let values = graph
- .control_dependency(
- &[&assign_values],
- move || {
- vec![retained_graph_for_values.slice(
- &retained_value_cache,
- 1,
- 0,
- (prefix_length + suffix_length) as i64,
- None,
- )]
- },
- None,
- )
- .first()
- .unwrap()
- .clone();
-
- let group_size = num_heads / num_groups;
-
- let expanded_keys = expand_keys_values(graph, &*keys, num_groups, head_dim);
- let expanded_values =
- expand_keys_values(graph, &*values, num_groups, head_dim);
- let expanded_queries = graph.expand_dims(&permuted_queries, 0, None);
-
- let scale = attention_scale.unwrap_or(1.0 / (head_dim as f32).sqrt());
-
- let expanded_attention_output = graph
- .masked_scaled_dot_product_attention_with_scalar(
- &expanded_queries,
- &expanded_keys,
- &expanded_values,
- bias,
- scale,
- None,
- );
-
- let attention_output =
- graph.squeeze_axis(&expanded_attention_output, 0, None);
- let permuted_attention_output =
- graph.transpose(&attention_output, &[1, 0, 2], None);
-
- let output_shape =
- mps_shape(&[-1_i64, (num_groups * group_size * head_dim) as i64]);
-
- Ok(graph.reshape(&permuted_attention_output, &output_shape, None))
-}
diff --git a/uzu/src/backends/metal/graph/mod.rs b/uzu/src/backends/metal/graph/mod.rs
index 27a626eb1..ddb1f870f 100644
--- a/uzu/src/backends/metal/graph/mod.rs
+++ b/uzu/src/backends/metal/graph/mod.rs
@@ -1,17 +1,12 @@
pub mod common;
-use common::{GraphConstructionError, data_type_of, load_constant, shape_of};
+use common::{GraphConstructionError, load_constant};
pub use common::{placeholder, shaped_type};
-mod attention;
-pub use attention::{attention_subgraph, rotation_subgraph};
mod linear;
pub use linear::linear_subgraph;
mod mlp;
pub use mlp::mlp_subgraph;
-mod rms_norm;
-pub use rms_norm::rms_norm_subgraph;
mod embedding;
pub use embedding::{
embeddings_dequantize_weights_subgraph, embeddings_embed_subgraph,
embeddings_readout_subgraph,
};
-mod rope;
diff --git a/uzu/src/backends/metal/graph/rms_norm.rs b/uzu/src/backends/metal/graph/rms_norm.rs
deleted file mode 100644
index ec582b38d..000000000
--- a/uzu/src/backends/metal/graph/rms_norm.rs
+++ /dev/null
@@ -1,108 +0,0 @@
-use std::rc::Rc;
-
-use mpsgraph::{
- Graph, GraphArithmeticOps, GraphNormalizationOps, Tensor,
- tensor_shape_ops::*,
-};
-use objc2::rc::Retained;
-
-use super::{
- super::MTLContext, GraphConstructionError, data_type_of, load_constant,
- shape_of,
-};
-use crate::{
- DataType,
- config::{RMSNormConfig, UpcastMode},
- parameters::ParameterTree,
-};
-
-fn cast_if_needed(
- graph: &Graph,
- input: &Retained<Tensor>,
- data_type: DataType,
-) -> Retained<Tensor> {
- if data_type_of(input) != data_type {
- graph.cast(input, data_type.into(), None)
- } else {
- input.clone()
- }
-}
-
-fn adjusted_variance(
- graph: &Graph,
- input: &Retained<Tensor>,
- epsilon: f32,
-) -> Retained<Tensor> {
- let epsilon_constant =
- graph.constant_with_scalar(epsilon as f64, input.data_type());
-
- let input_squared = graph.square(&input, None);
- let input_squared_mean = graph.mean(&input_squared, &[1], None);
- let input_squared_mean_adjusted =
- graph.add(&input_squared_mean, &epsilon_constant, None);
-
- input_squared_mean_adjusted
-}
-
-fn normalization(
- graph: &Graph,
- input: &Retained<Tensor>,
- adjusted_variance: &Retained<Tensor>,
-) -> Retained<Tensor> {
- let inverse_rms_norm = graph.rsqrt(&adjusted_variance, None);
- let normalized_input = graph.multiply(input, &inverse_rms_norm, None);
- normalized_input
-}
-
-pub fn rms_norm_subgraph(
- graph: &Graph,
- config: &RMSNormConfig,
- input: &Retained<Tensor>,
- parameter_tree: &ParameterTree<Rc<MTLContext>>,
-) -> Result<Retained<Tensor>, GraphConstructionError> {
- let [_, model_dim] = *shape_of(input) else {
- panic!(
- "Expected two-dimensional input, received {0:?}",
- shape_of(input)
- );
- };
- assert_ne!(model_dim, -1);
-
- let accumulation_data_type: DataType = config.accumulation_precision.into();
- let scale_data_type: DataType = config.scale_precision.into();
-
- let x = cast_if_needed(graph, input, accumulation_data_type);
- let adjusted_variance = adjusted_variance(graph, &x, config.epsilon);
- let mut normalized_x = normalization(graph, &x, &adjusted_variance);
- match config.upcast_mode {
- UpcastMode::OnlyNormalization => {
- normalized_x =
- cast_if_needed(graph, &normalized_x, scale_data_type);
- },
- UpcastMode::FullLayer => {},
- }
-
- let mut scales = load_constant(
- graph,
- parameter_tree,
- "scales",
- &[model_dim as usize],
- scale_data_type,
- )?;
- match config.upcast_mode {
- UpcastMode::OnlyNormalization => {},
- UpcastMode::FullLayer => {
- scales = cast_if_needed(graph, &scales, accumulation_data_type);
- },
- }
- if let Some(scale_offset) = config.scale_offset {
- let scale_offset_constant =
- graph.constant_with_scalar(scale_offset as f64, scales.data_type());
- scales = graph.add(&scales, &scale_offset_constant, None);
- }
-
- let mut result = graph.multiply(&normalized_x, &scales, None);
- result = cast_if_needed(graph, &result, scale_data_type);
- assert_eq!(shape_of(&*result), shape_of(input));
- Ok(result)
-}
diff --git a/uzu/src/backends/metal/graph/rope.rs b/uzu/src/backends/metal/graph/rope.rs
deleted file mode 100644
index 73cf62ed6..000000000
--- a/uzu/src/backends/metal/graph/rope.rs
+++ /dev/null
@@ -1,121 +0,0 @@
-use std::rc::Rc;
-
-use mpsgraph::{Graph, GraphGatherOps, Tensor, tensor_shape_ops::*};
-use objc2::rc::Retained;
-
-use super::{
- super::MTLContext, GraphConstructionError, common::data_type_of,
- load_constant, shape_of,
-};
-use crate::{DataType, config::RoPEConfig, parameters::ParameterTree};
-
-pub struct PositionalEmbeddings {
- sines: Retained<Tensor>,
- cosines: Retained<Tensor>,
-}
-
-impl PositionalEmbeddings {
- pub fn new(
- sines: Retained<Tensor>,
- cosines: Retained<Tensor>,
- ) -> Self {
- let cosines_shape = shape_of(&*cosines);
- let sines_shape = shape_of(&*sines);
- assert_eq!(cosines_shape, sines_shape);
- Self {
- sines,
- cosines,
- }
- }
-}
-
-pub fn positional_embeddings_subgraph(
- graph: &Graph,
- rope_config: &RoPEConfig,
- head_dim: usize,
- context_length: usize,
- indices: &Tensor, // u64: [max_sequence_length]
- parameter_tree: &ParameterTree<Rc<MTLContext>>,
-) -> Result<PositionalEmbeddings, GraphConstructionError> {
- assert_eq!(data_type_of(indices), DataType::I32);
- let expected_shape = [context_length, head_dim];
- let expected_data_type: DataType = rope_config.common().precision.into();
- let all_cosines = load_constant(
- graph,
- parameter_tree,
- "cosines",
- &expected_shape,
- expected_data_type,
- )?;
- let all_sines = load_constant(
- graph,
- parameter_tree,
- "sines",
- &expected_shape,
- expected_data_type,
- )?;
-
- let cosine_slice = graph.gather(&all_cosines, indices, 0, 0, None);
- let sine_slice = graph.gather(&all_sines, indices, 0, 0, None);
- Ok(PositionalEmbeddings::new(sine_slice, cosine_slice))
-}
-
-pub fn apply_rope_subgraph(
- graph: &Graph,
- positional_embeddings: &PositionalEmbeddings,
- input: &Tensor,
-) -> Retained<Tensor> {
- let [suffix_length, _, head_dim] = *shape_of(input) else {
- panic!(
- "Expected three-dimensional input tensor, got {0:?}",
- shape_of(input)
- )
- };
- let pe_shape = shape_of(&*positional_embeddings.cosines);
- let [pe_suffix_length, pe_head_dim] = *pe_shape else {
- panic!(
- "Expected two-dimensional positional embeddings, got {0:?}",
- pe_shape
- )
- };
- assert_eq!(head_dim, pe_head_dim);
- assert_eq!(suffix_length, pe_suffix_length);
-
- let x = graph.slice(input, 2, 0, (head_dim / 2) as i64, None);
- let y = graph.slice(
- input,
- 2,
- (head_dim / 2) as i64,
- (head_dim / 2) as i64,
- None,
- );
- // [suffix_length, num_heads, head_dim / 2]
- let neg_y = graph.multiply(
- &y,
- &graph.constant_with_scalar(-1.0_f64, y.data_type()),
- None,
- );
- // [suffix_length, num_heads, head_dim]
- let half_rotated_heads = graph.concat(&neg_y, &x, 2, None);
- // [suffix_length, 1, head_dim]
- let unsqueezed_cosines = graph.expand_dims(
- &positional_embeddings.cosines,
- 1,
- Some("unsqueezed_cosines_expand_dims"),
- );
- // [suffix_length, 1, head_dim]
- let unsqueezed_sines = graph.expand_dims(
- &positional_embeddings.sines,
- 1,
- Some("unsqueezed_sines_expand_dims"),
- );
- // [suffix_length, num_heads, head_dim]
- let mul_cos = graph.multiply(input, &unsqueezed_cosines, None);
- let mul_sin = graph.multiply(&half_rotated_heads, &unsqueezed_sines, None);
- // [suffix_length, num_heads, head_dim]
- let result = graph.add(&mul_cos, &mul_sin, None);
-
- assert_eq!(shape_of(input), shape_of(&*result));
-
- result
-}
diff --git a/uzu/src/backends/metal/mod.rs b/uzu/src/backends/metal/mod.rs
index 61d765aee..f6ea05d42 100644
--- a/uzu/src/backends/metal/mod.rs
+++ b/uzu/src/backends/metal/mod.rs
@@ -16,10 +16,7 @@ pub use array::MetalArray;
pub use buffer_allocator::BufferAllocator;
pub use context::MTLContext;
pub use error::MTLError;
-pub use executable_builder::{
- DecoderExecutables, KernelsConfig,
- attention_executable_provider::AttentionExecutableProviderConfig,
-};
+pub use executable_builder::DecoderExecutables;
pub use forward_pass::{ForwardPassState, KVCache, ModelShape};
pub use kernel::{KVCacheUpdate, KernelDataType, RopeKernel};
// pub use kernel::{
diff --git a/uzu/src/config/rope.rs b/uzu/src/config/rope.rs
index c6265256d..93d3f675d 100644
--- a/uzu/src/config/rope.rs
+++ b/uzu/src/config/rope.rs
@@ -32,6 +32,11 @@ pub enum RoPEConfig {
low_frequency_factor: f32,
high_frequency_factor: f32,
},
+ LinearScalingRoPEConfig {
+ #[serde(flatten)]
+ common: RopeConfigCommon,
+ scaling_factor: f32,
+ },
}
impl RoPEConfig {
@@ -46,6 +51,10 @@ impl RoPEConfig {
common,
..
} => common,
+ RoPEConfig::LinearScalingRoPEConfig {
+ common,
+ ..
+ } => common,
}
}
}
diff --git a/uzu/src/decoder_runner.rs b/uzu/src/decoder_runner.rs
index f2a3010cf..def6bcf60 100644
--- a/uzu/src/decoder_runner.rs
+++ b/uzu/src/decoder_runner.rs
@@ -1,7 +1,6 @@
#![allow(dead_code)]
use std::{
- cell::RefCell, collections::HashMap, fs::File, io::BufReader, path::Path,
- rc::Rc, time::Instant,
+ cell::RefCell, fs::File, io::BufReader, path::Path, rc::Rc, time::Instant,
};
use mpsgraph::CommandBuffer as MPSCommandBuffer;
@@ -9,8 +8,7 @@ use serde::{Deserialize, Serialize};
use crate::{
backends::metal::{
- AttentionExecutableProviderConfig, DecoderExecutables,
- ForwardPassState, KVCache, KernelsConfig, MTLContext, ModelShape,
+ DecoderExecutables, ForwardPassState, KVCache, MTLContext, ModelShape,
compilation_parameters::CompilationConfig,
forward_pass::{
ForwardPassBuffers, SharedBuffers,
@@ -133,8 +131,6 @@ impl DecoderTestContext {
decoder_config.clone(),
&root_loader_view,
compilation_config.clone(),
- AttentionExecutableProviderConfig::new(HashMap::from([(1, 0)])),
- KernelsConfig::default(),
);
Ok((
Self {
diff --git a/uzu/src/generator/config.rs b/uzu/src/generator/config.rs
index 81b822e5e..a892a9372 100644
--- a/uzu/src/generator/config.rs
+++ b/uzu/src/generator/config.rs
@@ -76,7 +76,6 @@ pub trait GeneratorConfigProvider {
pub struct GeneratorConfig {
pub prefill_step_size: usize,
- pub prefix_length_step: Option<usize>,
pub speculator_config: SpeculatorConfig,
pub allow_pre_encode: bool,
pub sampling_seed: u64,
@@ -86,7 +85,6 @@ pub struct GeneratorConfig {
impl GeneratorConfig {
pub fn new(
prefill_step_size: usize,
- prefix_length_step: Option<usize>,
speculator_config: SpeculatorConfig,
allow_pre_encode: bool,
sampling_seed: SamplingSeed,
@@ -94,7 +92,6 @@ impl GeneratorConfig {
) -> Self {
Self {
prefill_step_size,
- prefix_length_step,
speculator_config,
allow_pre_encode,
sampling_seed: sampling_seed.get_value(),
diff --git a/uzu/src/generator/context.rs b/uzu/src/generator/context.rs
index 0c78a8356..f4ad3c1f8 100644
--- a/uzu/src/generator/context.rs
+++ b/uzu/src/generator/context.rs
@@ -1,7 +1,4 @@
-use std::{
- cell::RefCell, collections::HashMap, fs::File, io::BufReader, path::Path,
- rc::Rc,
-};
+use std::{cell::RefCell, fs::File, io::BufReader, path::Path, rc::Rc};
use mpsgraph::CommandBuffer as MPSCommandBuffer;
use objc2::rc::Retained;
@@ -10,8 +7,8 @@ use super::{config::GeneratorConfig, error::GeneratorError};
use crate::{
DataType,
backends::metal::{
- AttentionExecutableProviderConfig, DecoderExecutables, KVCache,
- KVCacheUpdate, KernelDataType, KernelsConfig, MTLContext, ModelShape,
+ DecoderExecutables, KVCache, KVCacheUpdate, KernelDataType, MTLContext,
+ ModelShape,
compilation_parameters::CompilationConfig,
forward_pass::{ForwardPassBuffers, SharedBuffers},
kernel::SamplingKernelEncodable,
@@ -21,7 +18,6 @@ use crate::{
};
pub struct GeneratorContext {
- pub kernels_config: KernelsConfig,
pub mtl_context: Rc<MTLContext>,
pub command_buffer: Retained<MPSCommandBuffer>,
@@ -40,8 +36,6 @@ impl GeneratorContext {
model_path: &Path,
config: &GeneratorConfig,
) -> Result<Self, GeneratorError> {
- let kernels_config = KernelsConfig::default();
-
let mtl_device = metal::Device::system_default()
.ok_or(GeneratorError::UnableToCreateMetalContext)?;
let mtl_command_queue =
@@ -99,17 +93,11 @@ impl GeneratorContext {
max_suffix_length,
);
- let prefix_length_step = config.prefix_length_step.unwrap_or(0);
let executables = DecoderExecutables::new(
mtl_context.clone(),
decoder_config.clone(),
&root_loader_view,
compilation_config.clone(),
- AttentionExecutableProviderConfig::new(HashMap::from([
- (config.prefill_step_size, 0),
- (1, prefix_length_step),
- ])),
- kernels_config.clone(),
);
let kv_cache = Rc::new(RefCell::new(KVCache::new(
@@ -141,7 +129,6 @@ impl GeneratorContext {
.map_err(|_| GeneratorError::UnableToCreateMetalContext)?;
let context = Self {
- kernels_config,
mtl_context,
command_buffer,
kv_cache,
diff --git a/uzu/src/generator/generator.rs b/uzu/src/generator/generator.rs
index 25aab8354..49597ee06 100644
--- a/uzu/src/generator/generator.rs
+++ b/uzu/src/generator/generator.rs
@@ -408,9 +408,7 @@ impl Generator {
fn allow_pre_encode(&self) -> bool {
let metal_debug_active = MetalEnvVar::DeviceWrapperType.is_enabled();
- let result = self.config.allow_pre_encode
- && self.context.kernels_config.use_attention
- && !metal_debug_active;
+ let result = self.config.allow_pre_encode && !metal_debug_active;
result
}
diff --git a/uzu/src/session/session.rs b/uzu/src/session/session.rs
index 0eebec23e..bbb2caa24 100644
--- a/uzu/src/session/session.rs
+++ b/uzu/src/session/session.rs
@@ -36,7 +36,7 @@ impl Session {
// Load tokenizer JSON
let tokenizer_path = model_path.join("tokenizer.json");
let tokenizer = Tokenizer::from_file(&tokenizer_path)
- .map_err(|_| SessionError::UnableToLoadTokenizerConfig)?;
+ .map_err(|_| SessionError::UnableToLoadTokenizer)?;
// Load tokenizer configuration
let tokenizer_config =
diff --git a/uzu/src/session/session_config.rs b/uzu/src/session/session_config.rs
index f01149e12..e01b1ea5e 100644
--- a/uzu/src/session/session_config.rs
+++ b/uzu/src/session/session_config.rs
@@ -9,7 +9,6 @@ use crate::generator::config::{
#[derive(Clone)]
pub struct SessionConfig {
pub prefill_step_size: usize,
- pub prefix_length_step: Option<usize>,
pub speculator_config: SpeculatorConfig,
pub allow_pre_encode: bool,
pub sampling_seed: SamplingSeed,
@@ -19,7 +18,6 @@ pub struct SessionConfig {
impl SessionConfig {
pub fn new(
prefill_step_size: usize,
- prefix_length_step: Option<usize>,
speculator_config: SpeculatorConfig,
allow_pre_encode: bool,
sampling_seed: SamplingSeed,
@@ -27,7 +25,6 @@ impl SessionConfig {
) -> Self {
Self {
prefill_step_size,
- prefix_length_step,
speculator_config,
allow_pre_encode,
sampling_seed,
@@ -38,7 +35,6 @@ impl SessionConfig {
pub fn to_generator_config(&self) -> GeneratorConfig {
GeneratorConfig::new(
self.prefill_step_size,
- self.prefix_length_step,
self.speculator_config.clone(),
self.allow_pre_encode,
self.sampling_seed,
@@ -51,7 +47,6 @@ impl Default for SessionConfig {
fn default() -> Self {
Self::new(
8,
- None,
SpeculatorConfig::default(),
true,
SamplingSeed::default(),
diff --git a/uzu/src/session/session_error.rs b/uzu/src/session/session_error.rs
index bc76c2103..feb489cd9 100644
--- a/uzu/src/session/session_error.rs
+++ b/uzu/src/session/session_error.rs
@@ -8,6 +8,8 @@ pub enum SessionError {
UnableToLoadConfig,
#[error("Unable to load model weights")]
UnableToLoadWeights,
+ #[error("Unable to load tokenizer")]
+ UnableToLoadTokenizer,
#[error("Unable to load tokenizer configuration")]
UnableToLoadTokenizerConfig,
}
diff --git a/uzu/src/tracer/tracer.rs b/uzu/src/tracer/tracer.rs
index fc9ae6785..a0fb050f8 100644
--- a/uzu/src/tracer/tracer.rs
+++ b/uzu/src/tracer/tracer.rs
@@ -143,7 +143,6 @@ impl Tracer {
pub fn new(model_path: &Path) -> Self {
let generator_config = GeneratorConfig::new(
1024,
- None,
SpeculatorConfig {
number_of_speculated_tokens: 0,
speculator: Arc::new(EmptySpeculator {}),
@@ -168,7 +167,6 @@ impl Tracer {
) -> Self {
let generator_config = GeneratorConfig::new(
1024,
- None,
SpeculatorConfig {
number_of_speculated_tokens: 0,
speculator: Arc::new(EmptySpeculator {}),
@@ -219,9 +217,8 @@ impl Tracer {
.map(|chunk| chunk.to_vec())
.collect();
- let external_bias_fn = |row: usize, col: usize| -> bool {
- !mask[row][col]
- };
+ let external_bias_fn =
+ |row: usize, col: usize| -> bool { !mask[row][col] };
let mut state = ForwardPassState::new(
self.generator_context.mtl_context.clone(),