A high-performance inference engine for AI models
fix after merge
eugene75a81beabf6b
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_lookup.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_lookup.rsindex 9f4099dc3..753c4bb00 100644--- a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_lookup.rs+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_lookup.rs@@ -24,6 +24,7 @@ pub struct QuantizedEmbeddingLookup {weights_buffer: MTLBuffer,scales_buffer: MTLBuffer,biases_buffer: MTLBuffer,+ input_scale: f32,vocab_size: u32,model_dim: u32,group_size: u32,@@ -37,6 +38,7 @@ impl QuantizedEmbeddingLookup {model_dim: usize,group_size: usize,mode: QuantizationMode,+ input_scale: f32,parameter_tree: &ParameterTree<Rc<MTLContext>>,) -> Result<Self, EmbeddingError> {Self::new_with_names(@@ -46,6 +48,7 @@ impl QuantizedEmbeddingLookup {model_dim,group_size,mode,+ input_scale,"weights","scales","biases",@@ -60,6 +63,7 @@ impl QuantizedEmbeddingLookup {model_dim: usize,group_size: usize,mode: QuantizationMode,+ input_scale: f32,parameter_tree: &ParameterTree<Rc<MTLContext>>,) -> Result<Self, EmbeddingError> {Self::new_with_names(@@ -69,6 +73,7 @@ impl QuantizedEmbeddingLookup {model_dim,group_size,mode,+ input_scale,"input_weights","input_scales","input_biases",@@ -83,6 +88,7 @@ impl QuantizedEmbeddingLookup {model_dim: usize,group_size: usize,mode: QuantizationMode,+ input_scale: f32,weights_name: &str,scales_name: &str,biases_name: &str,@@ -102,13 +108,13 @@ impl QuantizedEmbeddingLookup {})?;if weights.data_type() != mode.storage_type() {- return Err(EmbeddingError::MetalError(- MTLError::Generic(format!(+ return Err(EmbeddingError::MetalError(MTLError::Generic(+ format!("Expected packed weights of type {:?}, got {:?}",mode.storage_type(),weights.data_type()- )),- ));+ ),+ )));}// Load scales [vocab_size, num_groups]@@ -122,24 +128,24 @@ impl QuantizedEmbeddingLookup {// Validate shapes and typeslet num_groups = (model_dim + group_size - 1) / group_size;if weights.shape() != [vocab_size, model_dim / packing_divisor] {- return Err(EmbeddingError::MetalError(- MTLError::Generic(format!(+ return Err(EmbeddingError::MetalError(MTLError::Generic(+ format!("Embedding lookup weights shape mismatch: got {:?}, expected [{}, {}]",weights.shape(),vocab_size,model_dim / packing_divisor- )),- ));+ ),+ )));}if scales.shape() != [vocab_size, num_groups] {- return Err(EmbeddingError::MetalError(- MTLError::Generic(format!(+ return Err(EmbeddingError::MetalError(MTLError::Generic(+ format!("Embedding lookup scales shape mismatch: got {:?}, expected [{}, {}]",scales.shape(),vocab_size,num_groups- )),- ));+ ),+ )));}if scales.data_type() != data_type {return Err(EmbeddingError::UnsupportedDataType(@@ -151,14 +157,14 @@ impl QuantizedEmbeddingLookup {let biases_buffer: MTLBuffer = match parameter_tree.leaf(biases_name) {Ok(mut deq_biases) => {if deq_biases.shape() != [vocab_size, num_groups] {- return Err(EmbeddingError::MetalError(- MTLError::Generic(format!(+ return Err(EmbeddingError::MetalError(MTLError::Generic(+ format!("Embedding lookup deq_biases shape mismatch: got {:?}, expected [{}, {}]",deq_biases.shape(),vocab_size,num_groups- )),- ));+ ),+ )));}if deq_biases.data_type() != data_type {return Err(EmbeddingError::UnsupportedDataType(@@ -172,9 +178,7 @@ impl QuantizedEmbeddingLookup {DataType::F16 | DataType::BF16 => 2,DataType::F32 => 4,other => {- return Err(- EmbeddingError::UnsupportedDataType(other),- );+ return Err(EmbeddingError::UnsupportedDataType(other));},};let size_bytes = (vocab_size * num_groups * elem_size) as u64;@@ -201,6 +205,7 @@ impl QuantizedEmbeddingLookup {weights_buffer,scales_buffer,biases_buffer,+ input_scale,vocab_size: vocab_size as u32,model_dim: model_dim as u32,group_size: group_size as u32,@@ -235,6 +240,7 @@ impl EncodableBlock for QuantizedEmbeddingLookup {vocab_size: self.vocab_size,model_dim: self.model_dim,group_size: self.group_size,+ input_scale: self.input_scale,};self.kerneldiff --git a/crates/uzu/src/backends/metal/encodable_block/mlp.rs b/crates/uzu/src/backends/metal/encodable_block/mlp.rsindex e3cbf1a28..76baf7024 100644--- a/crates/uzu/src/backends/metal/encodable_block/mlp.rs+++ b/crates/uzu/src/backends/metal/encodable_block/mlp.rs@@ -2,7 +2,7 @@use metal::CommandBufferRef;-use super::{EncodableBlock, EncodingParameters, QuantizedLinear};+use super::{EncodableBlock, EncodingParameters};use crate::backends::metal::{forward_pass::{ArrayId, ForwardPassState},kernel::mlp::MlpGateActMulEncodable,diff --git a/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs b/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rsindex f3c280e2e..f8a6f3229 100644--- a/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs+++ b/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs@@ -148,21 +148,15 @@ impl ShortConvMixer {command_buffer: &CommandBufferRef,suffix_length: usize,) {- let padded = state- .short_conv_padded_buffer()- .expect("short_conv_padded scratch buffer not initialized");-let arrays = state.arrays(&[ArrayId::SsmInProj,ArrayId::ShortConvState(self.layer_index),ArrayId::AttentionOutput,]);- let mut padded = padded.borrow_mut();let mut in_proj = arrays[0].borrow_mut();let mut conv_state = arrays[1].borrow_mut();let mut out = arrays[2].borrow_mut();- let padded_buf = unsafe { padded.mtl_buffer().to_owned() };let in_proj_buf = unsafe { in_proj.mtl_buffer().to_owned() };let state_buf = unsafe { conv_state.mtl_buffer().to_owned() };let out_buf = unsafe { out.mtl_buffer().to_owned() };@@ -189,9 +183,7 @@ impl ShortConvMixer {metal::MTLResourceOptions::StorageModePrivate,);- let mtl_command_buffer =- command_buffer.root_command_buffer().to_owned();- let compute = mtl_command_buffer.new_compute_command_encoder();+ let compute = command_buffer.new_compute_command_encoder();self.short_conv_kernel.encode_pack(diff --git a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rsindex d26cf38b1..ea855f24a 100644--- a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs+++ b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs@@ -242,6 +242,8 @@ pub fn embed_block(..} => {let data_type: DataType = (*activation_precision).into();+ let input_scale =+ config.embedding_config.common().input_scale.unwrap_or(1.0);let embeddings_tree = parameter_tree.subtree("embedding")@@ -254,6 +256,7 @@ pub fn embed_block(config.model_dim,*group_size,*embedding_quantization_mode,+ input_scale,&embeddings_tree,).expect("Failed to create quantized embedding lookup kernel");@@ -273,40 +276,15 @@ pub fn embed_block(// For QuantizedTied, group_size is implicit (per-row quantization), so group_size == model_dimlet group_size = config.model_dim;-- let block = QuantizedEmbeddingLookup::new_tied(- context,- data_type,- config.vocab_size,- config.model_dim,- group_size,- *embedding_quantization_mode,- &embeddings_tree,- )- .expect("Failed to create quantized embedding lookup kernel");-- Box::new(block)- },- EmbeddingConfig::MLXQuantizedTied {- group_size,- embedding_quantization_mode,- ..- } => {- let data_type: DataType =- config.output_norm_config.scale_precision.into();let input_scale =config.embedding_config.common().input_scale.unwrap_or(1.0);- let embeddings_tree = parameter_tree- .subtree("embedding")- .expect("Failed to get embedding subtree");-let block = QuantizedEmbeddingLookup::new_tied(context,data_type,config.vocab_size,config.model_dim,- *group_size,+ group_size,*embedding_quantization_mode,input_scale,&embeddings_tree,@@ -315,7 +293,7 @@ pub fn embed_block(Box::new(block)},- EmbeddingConfig::MLXQuantizedUntied {+ EmbeddingConfig::MLXQuantizedTied {group_size,embedding_quantization_mode,..@@ -329,7 +307,7 @@ pub fn embed_block(.subtree("embedding").expect("Failed to get embedding subtree");- let block = QuantizedEmbeddingLookup::new_untied_input(+ let block = QuantizedEmbeddingLookup::new_tied(context,data_type,config.vocab_size,@@ -483,8 +461,8 @@ pub fn readout_block(data_type,config.vocab_size,config.model_dim,- group_size,- embedding_quantization_mode,+ *group_size,+ *embedding_quantization_mode,&embeddings_tree,).expect("Failed to create quantized embedding readout kernel");diff --git a/crates/uzu/src/config/embedding.rs b/crates/uzu/src/config/embedding.rsindex 9ed78fd5b..490d76e37 100644--- a/crates/uzu/src/config/embedding.rs+++ b/crates/uzu/src/config/embedding.rs@@ -58,15 +58,6 @@ pub enum EmbeddingConfig {activation_quantization_mode: Option<QuantizationMode>,activation_precision: ConfigDataType,},- #[serde(rename = "MLXQuantizedUntiedEmbeddingConfig")]- MLXQuantizedUntied {- #[serde(flatten)]- common: EmbeddingConfigCommon,- group_size: usize,- embedding_quantization_mode: QuantizationMode,- activation_quantization_mode: Option<QuantizationMode>,- activation_precision: ConfigDataType,- },}impl EmbeddingConfig {@@ -96,10 +87,6 @@ impl EmbeddingConfig {common,..} => common,- EmbeddingConfig::MLXQuantizedUntied {- common,- ..- } => common,}}}