A high-performance inference engine for AI models
update mlp
eugene21fa8fee37b6
diff --git a/crates/uzu/src/backends/metal/encodable_block/mlp.rs b/crates/uzu/src/backends/metal/encodable_block/mlp.rsindex 63acec5a3..3b9c104fe 100644--- a/crates/uzu/src/backends/metal/encodable_block/mlp.rs+++ b/crates/uzu/src/backends/metal/encodable_block/mlp.rs@@ -9,16 +9,16 @@ use crate::backends::metal::{};pub struct MlpBlock {- up: QuantizedLinear,+ up: Box<dyn EncodableBlock>,gate: MlpGateActMulEncodable,- down: QuantizedLinear,+ down: Box<dyn EncodableBlock>,}impl MlpBlock {pub fn new(- up: QuantizedLinear,+ up: Box<dyn EncodableBlock>,gate: MlpGateActMulEncodable,- down: QuantizedLinear,+ down: Box<dyn EncodableBlock>,) -> Self {Self {up,diff --git a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rsindex 9ef459df6..7c7115d50 100644--- a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs+++ b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs@@ -133,45 +133,84 @@ pub fn mlp_block(compilation_descriptor: &CompilationDescriptor,) -> Box<dyn EncodableBlock> {if let crate::config::MLPConfig::Dense(dense) = config {- if let crate::config::LinearConfig::Quantized(ref quant_config)- | crate::config::LinearConfig::MLXQuantized(ref quant_config) =- dense.linear_config- {- let dtype: DataType =- dense.linear_config.activation_precision().into();-- let up = QuantizedLinear::new(- context,- quant_config,- model_dim,- 2 * hidden_dim,- ¶meter_tree.subtree("up_projection").unwrap(),- ArrayId::Main,- ArrayId::MlpFusedUp,- )- .expect("Failed to build MLP up quantized block");-- let gate_op = MlpGateActMulEncodable::new(- context,- dtype,- dense.activation,- hidden_dim,- )- .expect("Failed to build MLP gate activation kernel");-- let down = QuantizedLinear::new(- context,- quant_config,- hidden_dim,- model_dim,- ¶meter_tree.subtree("down_projection").unwrap(),- ArrayId::MlpHidden,- ArrayId::Main,- )- .expect("Failed to build MLP down quantized block");-- let enc = MlpBlock::new(up, gate_op, down);- return Box::new(enc);+ match &dense.linear_config {+ LinearConfig::Quantized(quant_config)+ | LinearConfig::MLXQuantized(quant_config) => {+ let dtype: DataType =+ dense.linear_config.activation_precision().into();++ let up = QuantizedLinear::new(+ context,+ quant_config,+ model_dim,+ 2 * hidden_dim,+ ¶meter_tree.subtree("up_projection").unwrap(),+ ArrayId::Main,+ ArrayId::MlpFusedUp,+ )+ .expect("Failed to build MLP up quantized block");++ let gate_op = MlpGateActMulEncodable::new(+ context,+ dtype,+ dense.activation,+ hidden_dim,+ )+ .expect("Failed to build MLP gate activation kernel");++ let down = QuantizedLinear::new(+ context,+ quant_config,+ hidden_dim,+ model_dim,+ ¶meter_tree.subtree("down_projection").unwrap(),+ ArrayId::MlpHidden,+ ArrayId::Main,+ )+ .expect("Failed to build MLP down quantized block");++ let enc = MlpBlock::new(Box::new(up), gate_op, Box::new(down));+ return Box::new(enc);+ },+ LinearConfig::FullPrecision {+ precision,+ } => {+ let dtype: DataType = (*precision).into();++ let up = FullPrecisionLinear::new(+ context,+ dtype,+ model_dim,+ 2 * hidden_dim,+ ¶meter_tree.subtree("up_projection").unwrap(),+ ArrayId::Main,+ ArrayId::MlpFusedUp,+ )+ .expect("Failed to build MLP up full precision block");++ let gate_op = MlpGateActMulEncodable::new(+ context,+ dtype,+ dense.activation,+ hidden_dim,+ )+ .expect("Failed to build MLP gate activation kernel");++ let down = FullPrecisionLinear::new(+ context,+ dtype,+ hidden_dim,+ model_dim,+ ¶meter_tree.subtree("down_projection").unwrap(),+ ArrayId::MlpHidden,+ ArrayId::Main,+ )+ .expect("Failed to build MLP down full precision block");++ let enc = MlpBlock::new(Box::new(up), gate_op, Box::new(down));+ return Box::new(enc);+ },+ _ => {},}}