Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

remove qlora

eugene1bf7f04539f1
diff --git a/crates/uzu/src/backends/metal/classifier_context.rs b/crates/uzu/src/backends/metal/classifier_context.rs
index adf8f17c2..c48875e70 100644
--- a/crates/uzu/src/backends/metal/classifier_context.rs
+++ b/crates/uzu/src/backends/metal/classifier_context.rs
@@ -397,7 +397,13 @@ impl ClassifierContext {
ArrayId::ClassifierPredictionHeadNorm,
ArrayId::ClassifierPredictionHeadLogits,
&compilation_config.descriptor_general,
- );
+ )
+ .map_err(|e| {
+ Error::Classifier(ClassifierError::KernelCreationFailed(format!(
+ "prediction head readout: {:?}",
+ e
+ )))
+ })?;
let pooling_kernel = PoolingKernel::new(&mtl_context, data_type)
.map_err(|e| {
@@ -417,7 +423,11 @@ impl ClassifierContext {
));
let prediction_head = Box::new(ClassifierPredictionHead::new(
- prediction_head_dense,
+ prediction_head_dense.map_err(|e| {
+ Error::Classifier(ClassifierError::KernelCreationFailed(
+ format!("prediction head dense: {:?}", e),
+ ))
+ })?,
prediction_head_activation,
Box::new(prediction_head_norm),
prediction_head_final_linear,
diff --git a/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs b/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs
index 289e205e1..fb9ab8e9f 100644
--- a/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs
@@ -113,7 +113,8 @@ impl ClassifierLayer {
ArrayId::Main,
ArrayId::QKV,
&compilation_config.descriptor_mlp,
- );
+ )
+ .expect("Failed to create qkv projection");
let qk_norm: Option<Box<dyn EncodableBlock>> = if attention_config
.query_norm_config
@@ -151,7 +152,8 @@ impl ClassifierLayer {
ArrayId::AttentionOutput,
ArrayId::Main,
&compilation_config.descriptor_mlp,
- );
+ )
+ .expect("Failed to create out projection");
let post_attention_norm: Option<Box<dyn EncodableBlock>> =
if let Some(norm_config) =
@@ -209,7 +211,8 @@ impl ClassifierLayer {
mtl_context,
&layer_loader.subtree("mlp").unwrap(),
&compilation_config.descriptor_mlp,
- );
+ )
+ .expect("Failed to create mlp block");
let post_mlp_norm: Option<Box<dyn EncodableBlock>> =
if let Some(norm_config) = &layer_config.post_mlp_norm_config {
diff --git a/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs b/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs
index 09b9fa29d..a2cd5dc3a 100644
--- a/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs
@@ -115,7 +115,8 @@ impl LayerExecutables {
ArrayId::Main,
ArrayId::QKV,
&compilation_config.descriptor_mlp,
- );
+ )
+ .expect("Failed to create qkv projection");
let qk_norm: Option<Box<dyn EncodableBlock>> =
if attention_config.query_norm_config.is_some()
@@ -155,7 +156,8 @@ impl LayerExecutables {
ArrayId::AttentionOutput,
ArrayId::Main,
&compilation_config.descriptor_mlp,
- );
+ )
+ .expect("Failed to create out projection");
let attention = Box::new(
Attention::new(
@@ -262,7 +264,8 @@ impl LayerExecutables {
mtl_context,
&decoder_layer_loader.subtree("mlp").unwrap(),
&compilation_config.descriptor_mlp,
- );
+ )
+ .expect("Failed to create mlp block");
let post_mlp_norm: Option<Box<dyn EncodableBlock>> =
if let Some(norm_config) = &layer_config.post_mlp_norm_config {
diff --git a/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs b/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs
index 65209d626..047656f42 100644
--- a/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs
@@ -83,7 +83,8 @@ impl MambaMixer {
ArrayId::Main,
ArrayId::SsmInProj,
&compilation_config.descriptor_mlp,
- );
+ )
+ .expect("Failed to create in-projection kernel");
let out_projection = transformer_layer::linear_block(
&mamba_config.out_projection_config,
@@ -98,7 +99,8 @@ impl MambaMixer {
ArrayId::AttentionOutput,
ArrayId::Main,
&compilation_config.descriptor_mlp,
- );
+ )
+ .expect("Failed to create out-projection kernel");
let conv_weight = conv_tree.leaf("weights").unwrap().clone();
let conv_bias = if mamba_config.conv_config.has_biases {
diff --git a/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs b/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs
index 7d2057a80..8823cf39b 100644
--- a/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs
@@ -78,7 +78,8 @@ impl ShortConvMixer {
ArrayId::Main,
ArrayId::SsmInProj,
&compilation_config.descriptor_mlp,
- );
+ )
+ .expect("Failed to create in-projection kernel");
let out_projection = transformer_layer::linear_block(
&short_conv_config.out_projection_config,
@@ -90,7 +91,8 @@ impl ShortConvMixer {
ArrayId::AttentionOutput,
ArrayId::Main,
&compilation_config.descriptor_mlp,
- );
+ )
+ .expect("Failed to create out-projection kernel");
let conv_weight = conv_tree.leaf("weights").unwrap().clone();
let conv_bias = if short_conv_config.conv_config.has_biases {
diff --git a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
index c4957425c..4eba859c7 100644
--- a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
@@ -14,12 +14,11 @@ use super::{
use crate::{
DataType,
backends::metal::{
- MTLContext,
+ MTLContext, MTLError,
forward_pass::{ArrayId, IOArrays, MPSGraphBlock},
graph::{
embeddings_dequantize_weights_subgraph, embeddings_embed_subgraph,
- embeddings_readout_subgraph, linear_subgraph, mlp_subgraph,
- placeholder, shaped_type,
+ embeddings_readout_subgraph, placeholder, shaped_type,
},
kernel::mlp::MlpGateActMulEncodable,
},
@@ -42,8 +41,8 @@ pub fn linear_block<const N: usize>(
parameter_tree: &ParameterTree<Rc<MTLContext>>,
input_array_id: ArrayId,
output_array_id: ArrayId,
- compilation_descriptor: &CompilationDescriptor,
-) -> Box<dyn EncodableBlock> {
+ _compilation_descriptor: &CompilationDescriptor,
+) -> Result<Box<dyn EncodableBlock>, MTLError> {
let out_sum: usize = output_dims.iter().sum();
match config {
LinearConfig::Quantized(quant_config)
@@ -56,9 +55,8 @@ pub fn linear_block<const N: usize>(
parameter_tree,
input_array_id,
output_array_id,
- )
- .expect("Failed to create quantized linear kernel block");
- return Box::new(block);
+ )?;
+ Ok(Box::new(block))
},
LinearConfig::FullPrecision {
precision,
@@ -71,57 +69,15 @@ pub fn linear_block<const N: usize>(
parameter_tree,
input_array_id,
output_array_id,
- )
- .expect("Failed to create full precision linear kernel block");
- return Box::new(block);
+ )?;
+ Ok(Box::new(block))
},
LinearConfig::QLoRA {
..
- } => {},
+ } => Err(MTLError::Generic(
+ "QLoRA linear layer not supported".to_string(),
+ )),
}
-
- autoreleasepool(|_| {
- let graph = Graph::new();
-
- let input_shape = [-1, input_dim as isize];
- let input_data_type: DataType = config.activation_precision().into();
- let input_placeholder =
- placeholder(&graph, &input_shape, input_data_type);
-
- let output = linear_subgraph(
- &graph,
- config,
- input_dim,
- output_dims,
- true,
- &input_placeholder,
- parameter_tree,
- )
- .unwrap();
-
- let retained_shaped_type = shaped_type(&input_shape, input_data_type);
- let feeds =
- HashMap::from([(&*input_placeholder, &*retained_shaped_type)]);
-
- let executable = graph.compile(
- &MPSDevice::with_device(&context.device),
- &feeds,
- &[&output],
- None,
- Some(&compilation_descriptor),
- );
-
- let arguments = IOArrays::new(
- vec![input_array_id].into_boxed_slice(),
- vec![output_array_id].into_boxed_slice(),
- );
-
- Box::new(MPSGraphBlock::new(
- executable,
- make_execution_descriptor(),
- arguments,
- ))
- })
}
pub fn mlp_block(
@@ -130,8 +86,8 @@ pub fn mlp_block(
hidden_dim: usize,
context: &MTLContext,
parameter_tree: &ParameterTree<Rc<MTLContext>>,
- compilation_descriptor: &CompilationDescriptor,
-) -> Box<dyn EncodableBlock> {
+ _compilation_descriptor: &CompilationDescriptor,
+) -> Result<Box<dyn EncodableBlock>, MTLError> {
if let crate::config::MLPConfig::Dense(dense) = config {
match &dense.linear_config {
LinearConfig::Quantized(quant_config)
@@ -144,33 +100,34 @@ pub fn mlp_block(
quant_config,
model_dim,
2 * hidden_dim,
- &parameter_tree.subtree("up_projection").unwrap(),
+ &parameter_tree
+ .subtree("up_projection")
+ .map_err(|e| MTLError::Generic(format!("{:?}", e)))?,
ArrayId::Main,
ArrayId::MlpFusedUp,
- )
- .expect("Failed to build MLP up quantized block");
+ )?;
let gate_op = MlpGateActMulEncodable::new(
context,
dtype,
dense.activation,
hidden_dim,
- )
- .expect("Failed to build MLP gate activation kernel");
+ )?;
let down = QuantizedLinear::new(
context,
quant_config,
hidden_dim,
model_dim,
- &parameter_tree.subtree("down_projection").unwrap(),
+ &parameter_tree
+ .subtree("down_projection")
+ .map_err(|e| MTLError::Generic(format!("{:?}", e)))?,
ArrayId::MlpHidden,
ArrayId::Main,
- )
- .expect("Failed to build MLP down quantized block");
+ )?;
let enc = MlpBlock::new(Box::new(up), gate_op, Box::new(down));
- return Box::new(enc);
+ return Ok(Box::new(enc));
},
LinearConfig::FullPrecision {
precision,
@@ -182,98 +139,52 @@ pub fn mlp_block(
dtype,
model_dim,
2 * hidden_dim,
- &parameter_tree.subtree("up_projection").unwrap(),
+ &parameter_tree
+ .subtree("up_projection")
+ .map_err(|e| MTLError::Generic(format!("{:?}", e)))?,
ArrayId::Main,
ArrayId::MlpFusedUp,
- )
- .expect("Failed to build MLP up full precision block");
+ )?;
let gate_op = MlpGateActMulEncodable::new(
context,
dtype,
dense.activation,
hidden_dim,
- )
- .expect("Failed to build MLP gate activation kernel");
+ )?;
let down = FullPrecisionLinear::new(
context,
dtype,
hidden_dim,
model_dim,
- &parameter_tree.subtree("down_projection").unwrap(),
+ &parameter_tree
+ .subtree("down_projection")
+ .map_err(|e| MTLError::Generic(format!("{:?}", e)))?,
ArrayId::MlpHidden,
ArrayId::Main,
- )
- .expect("Failed to build MLP down full precision block");
+ )?;
let enc = MlpBlock::new(Box::new(up), gate_op, Box::new(down));
- return Box::new(enc);
+ return Ok(Box::new(enc));
+ },
+ LinearConfig::QLoRA {
+ ..
+ } => {
+ return Err(MTLError::Generic(
+ "QLoRA MLP not supported".to_string(),
+ ));
},
- _ => {},
}
}
if let crate::config::MLPConfig::MixtureOfExperts(moe) = config {
let moe_block =
- MoeBlock::new(context, moe, model_dim, hidden_dim, parameter_tree)
- .expect("Failed to build MoE block");
- return Box::new(moe_block);
+ MoeBlock::new(context, moe, model_dim, hidden_dim, parameter_tree)?;
+ return Ok(Box::new(moe_block));
}
- autoreleasepool(|_| {
- let graph = Graph::new();
-
- let input_shape = [-1, model_dim as isize];
- let input_data_type: DataType = match config {
- crate::config::MLPConfig::Dense(dense) => {
- dense.linear_config.activation_precision().into()
- },
- crate::config::MLPConfig::MixtureOfExperts(moe) => {
- moe.expert_config.linear_config.activation_precision().into()
- },
- };
- let input_placeholder =
- placeholder(&graph, &input_shape, input_data_type);
-
- let output = match config {
- crate::config::MLPConfig::Dense(dense) => mlp_subgraph(
- &graph,
- &MLPConfig::Dense(dense.clone()),
- model_dim,
- hidden_dim,
- &input_placeholder,
- parameter_tree,
- )
- .unwrap(),
- crate::config::MLPConfig::MixtureOfExperts(_moe) => {
- unreachable!("MoE uses MoeBlock, not MPSGraph")
- },
- };
-
- let retained_shaped_type = shaped_type(&input_shape, input_data_type);
- let feeds =
- HashMap::from([(&*input_placeholder, &*retained_shaped_type)]);
-
- let executable = graph.compile(
- &MPSDevice::with_device(&context.device),
- &feeds,
- &[&output],
- None,
- Some(&compilation_descriptor),
- );
-
- let arguments = IOArrays::new(
- vec![ArrayId::Main].into_boxed_slice(),
- vec![ArrayId::Main].into_boxed_slice(),
- );
-
- Box::new(MPSGraphBlock::new(
- executable,
- make_execution_descriptor(),
- arguments,
- ))
- })
+ unreachable!("Unknown MLP config")
}
pub fn embed_block(