A high-performance inference engine for AI models
remove qlora
eugene1bf7f04539f1
diff --git a/crates/uzu/src/backends/metal/classifier_context.rs b/crates/uzu/src/backends/metal/classifier_context.rsindex adf8f17c2..c48875e70 100644--- a/crates/uzu/src/backends/metal/classifier_context.rs+++ b/crates/uzu/src/backends/metal/classifier_context.rs@@ -397,7 +397,13 @@ impl ClassifierContext {ArrayId::ClassifierPredictionHeadNorm,ArrayId::ClassifierPredictionHeadLogits,&compilation_config.descriptor_general,- );+ )+ .map_err(|e| {+ Error::Classifier(ClassifierError::KernelCreationFailed(format!(+ "prediction head readout: {:?}",+ e+ )))+ })?;let pooling_kernel = PoolingKernel::new(&mtl_context, data_type).map_err(|e| {@@ -417,7 +423,11 @@ impl ClassifierContext {));let prediction_head = Box::new(ClassifierPredictionHead::new(- prediction_head_dense,+ prediction_head_dense.map_err(|e| {+ Error::Classifier(ClassifierError::KernelCreationFailed(+ format!("prediction head dense: {:?}", e),+ ))+ })?,prediction_head_activation,Box::new(prediction_head_norm),prediction_head_final_linear,diff --git a/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs b/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rsindex 289e205e1..fb9ab8e9f 100644--- a/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs+++ b/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs@@ -113,7 +113,8 @@ impl ClassifierLayer {ArrayId::Main,ArrayId::QKV,&compilation_config.descriptor_mlp,- );+ )+ .expect("Failed to create qkv projection");let qk_norm: Option<Box<dyn EncodableBlock>> = if attention_config.query_norm_config@@ -151,7 +152,8 @@ impl ClassifierLayer {ArrayId::AttentionOutput,ArrayId::Main,&compilation_config.descriptor_mlp,- );+ )+ .expect("Failed to create out projection");let post_attention_norm: Option<Box<dyn EncodableBlock>> =if let Some(norm_config) =@@ -209,7 +211,8 @@ impl ClassifierLayer {mtl_context,&layer_loader.subtree("mlp").unwrap(),&compilation_config.descriptor_mlp,- );+ )+ .expect("Failed to create mlp block");let post_mlp_norm: Option<Box<dyn EncodableBlock>> =if let Some(norm_config) = &layer_config.post_mlp_norm_config {diff --git a/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs b/crates/uzu/src/backends/metal/encodable_block/layer/executables.rsindex 09b9fa29d..a2cd5dc3a 100644--- a/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs+++ b/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs@@ -115,7 +115,8 @@ impl LayerExecutables {ArrayId::Main,ArrayId::QKV,&compilation_config.descriptor_mlp,- );+ )+ .expect("Failed to create qkv projection");let qk_norm: Option<Box<dyn EncodableBlock>> =if attention_config.query_norm_config.is_some()@@ -155,7 +156,8 @@ impl LayerExecutables {ArrayId::AttentionOutput,ArrayId::Main,&compilation_config.descriptor_mlp,- );+ )+ .expect("Failed to create out projection");let attention = Box::new(Attention::new(@@ -262,7 +264,8 @@ impl LayerExecutables {mtl_context,&decoder_layer_loader.subtree("mlp").unwrap(),&compilation_config.descriptor_mlp,- );+ )+ .expect("Failed to create mlp block");let post_mlp_norm: Option<Box<dyn EncodableBlock>> =if let Some(norm_config) = &layer_config.post_mlp_norm_config {diff --git a/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs b/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rsindex 65209d626..047656f42 100644--- a/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs+++ b/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs@@ -83,7 +83,8 @@ impl MambaMixer {ArrayId::Main,ArrayId::SsmInProj,&compilation_config.descriptor_mlp,- );+ )+ .expect("Failed to create in-projection kernel");let out_projection = transformer_layer::linear_block(&mamba_config.out_projection_config,@@ -98,7 +99,8 @@ impl MambaMixer {ArrayId::AttentionOutput,ArrayId::Main,&compilation_config.descriptor_mlp,- );+ )+ .expect("Failed to create out-projection kernel");let conv_weight = conv_tree.leaf("weights").unwrap().clone();let conv_bias = if mamba_config.conv_config.has_biases {diff --git a/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs b/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rsindex 7d2057a80..8823cf39b 100644--- a/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs+++ b/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs@@ -78,7 +78,8 @@ impl ShortConvMixer {ArrayId::Main,ArrayId::SsmInProj,&compilation_config.descriptor_mlp,- );+ )+ .expect("Failed to create in-projection kernel");let out_projection = transformer_layer::linear_block(&short_conv_config.out_projection_config,@@ -90,7 +91,8 @@ impl ShortConvMixer {ArrayId::AttentionOutput,ArrayId::Main,&compilation_config.descriptor_mlp,- );+ )+ .expect("Failed to create out-projection kernel");let conv_weight = conv_tree.leaf("weights").unwrap().clone();let conv_bias = if short_conv_config.conv_config.has_biases {diff --git a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rsindex c4957425c..4eba859c7 100644--- a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs+++ b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs@@ -14,12 +14,11 @@ use super::{use crate::{DataType,backends::metal::{- MTLContext,+ MTLContext, MTLError,forward_pass::{ArrayId, IOArrays, MPSGraphBlock},graph::{embeddings_dequantize_weights_subgraph, embeddings_embed_subgraph,- embeddings_readout_subgraph, linear_subgraph, mlp_subgraph,- placeholder, shaped_type,+ embeddings_readout_subgraph, placeholder, shaped_type,},kernel::mlp::MlpGateActMulEncodable,},@@ -42,8 +41,8 @@ pub fn linear_block<const N: usize>(parameter_tree: &ParameterTree<Rc<MTLContext>>,input_array_id: ArrayId,output_array_id: ArrayId,- compilation_descriptor: &CompilationDescriptor,-) -> Box<dyn EncodableBlock> {+ _compilation_descriptor: &CompilationDescriptor,+) -> Result<Box<dyn EncodableBlock>, MTLError> {let out_sum: usize = output_dims.iter().sum();match config {LinearConfig::Quantized(quant_config)@@ -56,9 +55,8 @@ pub fn linear_block<const N: usize>(parameter_tree,input_array_id,output_array_id,- )- .expect("Failed to create quantized linear kernel block");- return Box::new(block);+ )?;+ Ok(Box::new(block))},LinearConfig::FullPrecision {precision,@@ -71,57 +69,15 @@ pub fn linear_block<const N: usize>(parameter_tree,input_array_id,output_array_id,- )- .expect("Failed to create full precision linear kernel block");- return Box::new(block);+ )?;+ Ok(Box::new(block))},LinearConfig::QLoRA {..- } => {},+ } => Err(MTLError::Generic(+ "QLoRA linear layer not supported".to_string(),+ )),}-- autoreleasepool(|_| {- let graph = Graph::new();-- let input_shape = [-1, input_dim as isize];- let input_data_type: DataType = config.activation_precision().into();- let input_placeholder =- placeholder(&graph, &input_shape, input_data_type);-- let output = linear_subgraph(- &graph,- config,- input_dim,- output_dims,- true,- &input_placeholder,- parameter_tree,- )- .unwrap();-- let retained_shaped_type = shaped_type(&input_shape, input_data_type);- let feeds =- HashMap::from([(&*input_placeholder, &*retained_shaped_type)]);-- let executable = graph.compile(- &MPSDevice::with_device(&context.device),- &feeds,- &[&output],- None,- Some(&compilation_descriptor),- );-- let arguments = IOArrays::new(- vec![input_array_id].into_boxed_slice(),- vec![output_array_id].into_boxed_slice(),- );-- Box::new(MPSGraphBlock::new(- executable,- make_execution_descriptor(),- arguments,- ))- })}pub fn mlp_block(@@ -130,8 +86,8 @@ pub fn mlp_block(hidden_dim: usize,context: &MTLContext,parameter_tree: &ParameterTree<Rc<MTLContext>>,- compilation_descriptor: &CompilationDescriptor,-) -> Box<dyn EncodableBlock> {+ _compilation_descriptor: &CompilationDescriptor,+) -> Result<Box<dyn EncodableBlock>, MTLError> {if let crate::config::MLPConfig::Dense(dense) = config {match &dense.linear_config {LinearConfig::Quantized(quant_config)@@ -144,33 +100,34 @@ pub fn mlp_block(quant_config,model_dim,2 * hidden_dim,- ¶meter_tree.subtree("up_projection").unwrap(),+ ¶meter_tree+ .subtree("up_projection")+ .map_err(|e| MTLError::Generic(format!("{:?}", e)))?,ArrayId::Main,ArrayId::MlpFusedUp,- )- .expect("Failed to build MLP up quantized block");+ )?;let gate_op = MlpGateActMulEncodable::new(context,dtype,dense.activation,hidden_dim,- )- .expect("Failed to build MLP gate activation kernel");+ )?;let down = QuantizedLinear::new(context,quant_config,hidden_dim,model_dim,- ¶meter_tree.subtree("down_projection").unwrap(),+ ¶meter_tree+ .subtree("down_projection")+ .map_err(|e| MTLError::Generic(format!("{:?}", e)))?,ArrayId::MlpHidden,ArrayId::Main,- )- .expect("Failed to build MLP down quantized block");+ )?;let enc = MlpBlock::new(Box::new(up), gate_op, Box::new(down));- return Box::new(enc);+ return Ok(Box::new(enc));},LinearConfig::FullPrecision {precision,@@ -182,98 +139,52 @@ pub fn mlp_block(dtype,model_dim,2 * hidden_dim,- ¶meter_tree.subtree("up_projection").unwrap(),+ ¶meter_tree+ .subtree("up_projection")+ .map_err(|e| MTLError::Generic(format!("{:?}", e)))?,ArrayId::Main,ArrayId::MlpFusedUp,- )- .expect("Failed to build MLP up full precision block");+ )?;let gate_op = MlpGateActMulEncodable::new(context,dtype,dense.activation,hidden_dim,- )- .expect("Failed to build MLP gate activation kernel");+ )?;let down = FullPrecisionLinear::new(context,dtype,hidden_dim,model_dim,- ¶meter_tree.subtree("down_projection").unwrap(),+ ¶meter_tree+ .subtree("down_projection")+ .map_err(|e| MTLError::Generic(format!("{:?}", e)))?,ArrayId::MlpHidden,ArrayId::Main,- )- .expect("Failed to build MLP down full precision block");+ )?;let enc = MlpBlock::new(Box::new(up), gate_op, Box::new(down));- return Box::new(enc);+ return Ok(Box::new(enc));+ },+ LinearConfig::QLoRA {+ ..+ } => {+ return Err(MTLError::Generic(+ "QLoRA MLP not supported".to_string(),+ ));},- _ => {},}}if let crate::config::MLPConfig::MixtureOfExperts(moe) = config {let moe_block =- MoeBlock::new(context, moe, model_dim, hidden_dim, parameter_tree)- .expect("Failed to build MoE block");- return Box::new(moe_block);+ MoeBlock::new(context, moe, model_dim, hidden_dim, parameter_tree)?;+ return Ok(Box::new(moe_block));}- autoreleasepool(|_| {- let graph = Graph::new();-- let input_shape = [-1, model_dim as isize];- let input_data_type: DataType = match config {- crate::config::MLPConfig::Dense(dense) => {- dense.linear_config.activation_precision().into()- },- crate::config::MLPConfig::MixtureOfExperts(moe) => {- moe.expert_config.linear_config.activation_precision().into()- },- };- let input_placeholder =- placeholder(&graph, &input_shape, input_data_type);-- let output = match config {- crate::config::MLPConfig::Dense(dense) => mlp_subgraph(- &graph,- &MLPConfig::Dense(dense.clone()),- model_dim,- hidden_dim,- &input_placeholder,- parameter_tree,- )- .unwrap(),- crate::config::MLPConfig::MixtureOfExperts(_moe) => {- unreachable!("MoE uses MoeBlock, not MPSGraph")- },- };-- let retained_shaped_type = shaped_type(&input_shape, input_data_type);- let feeds =- HashMap::from([(&*input_placeholder, &*retained_shaped_type)]);-- let executable = graph.compile(- &MPSDevice::with_device(&context.device),- &feeds,- &[&output],- None,- Some(&compilation_descriptor),- );-- let arguments = IOArrays::new(- vec![ArrayId::Main].into_boxed_slice(),- vec![ArrayId::Main].into_boxed_slice(),- );-- Box::new(MPSGraphBlock::new(- executable,- make_execution_descriptor(),- arguments,- ))- })+ unreachable!("Unknown MLP config")}pub fn embed_block(