A high-performance inference engine for AI models
add mlx quantized untied
eugeneeb3eafbc3975
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_lookup.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_lookup.rsindex 1e4945d46..4c1510b2a 100644--- a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_lookup.rs+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_lookup.rs@@ -3,7 +3,10 @@ use std::rc::Rc;use metal::Buffer as MTLBuffer;use mpsgraph::CommandBuffer;-use super::QuantizedEmbeddingError;+use super::{+ super::{EncodableBlock, EncodingParameters},+ QuantizedEmbeddingError,+};use crate::{Array, DataType,backends::metal::{@@ -17,8 +20,6 @@ use crate::{parameters::ParameterTree,};-use super::super::{EncodableBlock, EncodingParameters};-pub struct QuantizedEmbeddingLookup {kernel: QuantizedEmbeddingLookupKernel,weights_buffer: MTLBuffer,@@ -30,7 +31,7 @@ pub struct QuantizedEmbeddingLookup {}impl QuantizedEmbeddingLookup {- pub fn new(+ pub fn new_tied(mtl_context: &MTLContext,data_type: DataType,vocab_size: usize,@@ -38,6 +39,55 @@ impl QuantizedEmbeddingLookup {group_size: usize,mode: QuantizationMode,parameter_tree: &ParameterTree<Rc<MTLContext>>,+ ) -> Result<Self, QuantizedEmbeddingError> {+ Self::new_with_names(+ mtl_context,+ data_type,+ vocab_size,+ model_dim,+ group_size,+ mode,+ "weights",+ "scales",+ "biases",+ parameter_tree,+ )+ }++ pub fn new_untied_input(+ mtl_context: &MTLContext,+ data_type: DataType,+ vocab_size: usize,+ model_dim: usize,+ group_size: usize,+ mode: QuantizationMode,+ parameter_tree: &ParameterTree<Rc<MTLContext>>,+ ) -> Result<Self, QuantizedEmbeddingError> {+ Self::new_with_names(+ mtl_context,+ data_type,+ vocab_size,+ model_dim,+ group_size,+ mode,+ "input_weights",+ "input_scales",+ "input_biases",+ parameter_tree,+ )+ }++ fn new_with_names(+ mtl_context: &MTLContext,+ data_type: DataType,+ vocab_size: usize,+ model_dim: usize,+ group_size: usize,+ mode: QuantizationMode,+ weights_name: &str,+ scales_name: &str,+ biases_name: &str,+ parameter_tree: &ParameterTree<Rc<MTLContext>>,) -> Result<Self, QuantizedEmbeddingError> {let packing_divisor = mode.packing_divisor();@@ -45,15 +95,12 @@ impl QuantizedEmbeddingLookup {QuantizedEmbeddingLookupKernel::new(mtl_context, data_type, mode)?;// Load weights [vocab_size, model_dim/packing_divisor] as storage_type- let mut weights = match parameter_tree.leaf("weights") {- Ok(weights) => weights,- Err(_) => parameter_tree.leaf("output_weights").map_err(|e| {- QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(- "Failed to load weights: {:?}",- e- )))- })?,- };+ let mut weights = parameter_tree.leaf(weights_name).map_err(|e| {+ QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(+ "Failed to load weights: {:?}",+ e+ )))+ })?;if weights.data_type() != mode.storage_type() {return Err(QuantizedEmbeddingError::MetalError(@@ -66,15 +113,12 @@ impl QuantizedEmbeddingLookup {}// Load scales [vocab_size, num_groups]- let mut scales = match parameter_tree.leaf("scales") {- Ok(scales) => scales,- Err(_) => parameter_tree.leaf("output_scales").map_err(|e| {- QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(- "Failed to load scales: {:?}",- e- )))- })?,- };+ let mut scales = parameter_tree.leaf(scales_name).map_err(|e| {+ QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(+ "Failed to load scales: {:?}",+ e+ )))+ })?;// Validate shapes and typeslet num_groups = (model_dim + group_size - 1) / group_size;@@ -105,10 +149,7 @@ impl QuantizedEmbeddingLookup {}// Load or create biases buffer [vocab_size, num_groups] (MLX key: "biases")- let biases_buffer: MTLBuffer = match parameter_tree- .leaf("biases")- .or_else(|_| parameter_tree.leaf("output_biases"))- {+ let biases_buffer: MTLBuffer = match parameter_tree.leaf(biases_name) {Ok(mut deq_biases) => {if deq_biases.shape() != [vocab_size, num_groups] {return Err(QuantizedEmbeddingError::MetalError(@@ -212,4 +253,3 @@ impl EncodableBlock for QuantizedEmbeddingLookup {}}}-diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rsindex 2b11f8a27..6b619ae93 100644--- a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs@@ -3,7 +3,10 @@ use std::rc::Rc;use metal::Buffer as MTLBuffer;use mpsgraph::CommandBuffer;-use super::QuantizedEmbeddingError;+use super::{+ super::{EncodableBlock, EncodingParameters},+ QuantizedEmbeddingError,+};use crate::{Array, DataType,backends::metal::{@@ -17,8 +20,6 @@ use crate::{parameters::ParameterTree,};-use super::super::{EncodableBlock, EncodingParameters};-pub struct QuantizedEmbeddingReadout {kernel: QuantizedMatmulKernel,weights_buffer: MTLBuffer,@@ -29,7 +30,30 @@ pub struct QuantizedEmbeddingReadout {}impl QuantizedEmbeddingReadout {- pub fn new(+ pub fn new_tied(+ mtl_context: &MTLContext,+ data_type: DataType,+ vocab_size: usize,+ model_dim: usize,+ group_size: usize,+ mode: QuantizationMode,+ parameter_tree: &ParameterTree<Rc<MTLContext>>,+ ) -> Result<Self, QuantizedEmbeddingError> {+ Self::new_with_names(+ mtl_context,+ data_type,+ vocab_size,+ model_dim,+ group_size,+ mode,+ "weights",+ "scales",+ "biases",+ parameter_tree,+ )+ }++ pub fn new_untied_output(mtl_context: &MTLContext,data_type: DataType,vocab_size: usize,@@ -37,28 +61,48 @@ impl QuantizedEmbeddingReadout {group_size: usize,mode: QuantizationMode,parameter_tree: &ParameterTree<Rc<MTLContext>>,+ ) -> Result<Self, QuantizedEmbeddingError> {+ Self::new_with_names(+ mtl_context,+ data_type,+ vocab_size,+ model_dim,+ group_size,+ mode,+ "output_weights",+ "output_scales",+ "output_biases",+ parameter_tree,+ )+ }++ fn new_with_names(+ mtl_context: &MTLContext,+ data_type: DataType,+ vocab_size: usize,+ model_dim: usize,+ group_size: usize,+ mode: QuantizationMode,+ weights_name: &str,+ scales_name: &str,+ biases_name: &str,+ parameter_tree: &ParameterTree<Rc<MTLContext>>,) -> Result<Self, QuantizedEmbeddingError> {// Load weights [vocab_size, model_dim/2] as U8- let mut weights = match parameter_tree.leaf("weights") {- Ok(weights) => weights,- Err(_) => parameter_tree.leaf("output_weights").map_err(|e| {- QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(- "Failed to load weights: {:?}",- e- )))- })?,- };+ let mut weights = parameter_tree.leaf(weights_name).map_err(|e| {+ QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(+ "Failed to load weights: {:?}",+ e+ )))+ })?;// Load scales [vocab_size, num_groups]- let mut scales = match parameter_tree.leaf("scales") {- Ok(scales) => scales,- Err(_) => parameter_tree.leaf("output_scales").map_err(|e| {- QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(- "Failed to load scales: {:?}",- e- )))- })?,- };+ let mut scales = parameter_tree.leaf(scales_name).map_err(|e| {+ QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(+ "Failed to load scales: {:?}",+ e+ )))+ })?;// Validate shapeslet num_groups = (model_dim + group_size - 1) / group_size;@@ -94,10 +138,7 @@ impl QuantizedEmbeddingReadout {}// MLX requires per-group biases; if missing, create a zero buffer of shape [vocab_size, num_groups]- let biases_buffer: MTLBuffer = match parameter_tree- .leaf("biases")- .or_else(|_| parameter_tree.leaf("output_biases"))- {+ let biases_buffer: MTLBuffer = match parameter_tree.leaf(biases_name) {Ok(mut deq_biases) => {if deq_biases.shape() != [vocab_size, num_groups] {return Err(QuantizedEmbeddingError::MetalError(@@ -218,4 +259,3 @@ impl EncodableBlock for QuantizedEmbeddingReadout {}}}-diff --git a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rsindex 7c7115d50..c4957425c 100644--- a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs+++ b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs@@ -344,6 +344,31 @@ pub fn embed_block(Box::new(block)},+ EmbeddingConfig::MLXQuantizedUntied {+ group_size,+ embedding_quantization_mode,+ activation_precision,+ ..+ } => {+ let data_type: DataType = (*activation_precision).into();++ let embeddings_tree = parameter_tree+ .subtree("embedding")+ .expect("Failed to get embedding subtree");++ let block = QuantizedEmbeddingLookup::new_untied_input(+ context,+ data_type,+ config.vocab_size,+ config.model_dim,+ *group_size,+ *embedding_quantization_mode,+ &embeddings_tree,+ )+ .expect("Failed to create quantized embedding lookup kernel");++ Box::new(block)+ },EmbeddingConfig::QuantizedTied {embedding_quantization_mode: _,..diff --git a/crates/uzu/src/config/embedding.rs b/crates/uzu/src/config/embedding.rsindex 490d76e37..9ed78fd5b 100644--- a/crates/uzu/src/config/embedding.rs+++ b/crates/uzu/src/config/embedding.rs@@ -58,6 +58,15 @@ pub enum EmbeddingConfig {activation_quantization_mode: Option<QuantizationMode>,activation_precision: ConfigDataType,},+ #[serde(rename = "MLXQuantizedUntiedEmbeddingConfig")]+ MLXQuantizedUntied {+ #[serde(flatten)]+ common: EmbeddingConfigCommon,+ group_size: usize,+ embedding_quantization_mode: QuantizationMode,+ activation_quantization_mode: Option<QuantizationMode>,+ activation_precision: ConfigDataType,+ },}impl EmbeddingConfig {@@ -87,6 +96,10 @@ impl EmbeddingConfig {common,..} => common,+ EmbeddingConfig::MLXQuantizedUntied {+ common,+ ..+ } => common,}}}