Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

add mlx quantized untied

eugeneeb3eafbc3975
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_lookup.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_lookup.rs
index 1e4945d46..4c1510b2a 100644
--- a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_lookup.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_lookup.rs
@@ -3,7 +3,10 @@ use std::rc::Rc;
use metal::Buffer as MTLBuffer;
use mpsgraph::CommandBuffer;
-use super::QuantizedEmbeddingError;
+use super::{
+ super::{EncodableBlock, EncodingParameters},
+ QuantizedEmbeddingError,
+};
use crate::{
Array, DataType,
backends::metal::{
@@ -17,8 +20,6 @@ use crate::{
parameters::ParameterTree,
};
-use super::super::{EncodableBlock, EncodingParameters};
-
pub struct QuantizedEmbeddingLookup {
kernel: QuantizedEmbeddingLookupKernel,
weights_buffer: MTLBuffer,
@@ -30,7 +31,7 @@ pub struct QuantizedEmbeddingLookup {
}
impl QuantizedEmbeddingLookup {
- pub fn new(
+ pub fn new_tied(
mtl_context: &MTLContext,
data_type: DataType,
vocab_size: usize,
@@ -38,6 +39,55 @@ impl QuantizedEmbeddingLookup {
group_size: usize,
mode: QuantizationMode,
parameter_tree: &ParameterTree<Rc<MTLContext>>,
+ ) -> Result<Self, QuantizedEmbeddingError> {
+ Self::new_with_names(
+ mtl_context,
+ data_type,
+ vocab_size,
+ model_dim,
+ group_size,
+ mode,
+ "weights",
+ "scales",
+ "biases",
+ parameter_tree,
+ )
+ }
+
+ pub fn new_untied_input(
+ mtl_context: &MTLContext,
+ data_type: DataType,
+ vocab_size: usize,
+ model_dim: usize,
+ group_size: usize,
+ mode: QuantizationMode,
+ parameter_tree: &ParameterTree<Rc<MTLContext>>,
+ ) -> Result<Self, QuantizedEmbeddingError> {
+ Self::new_with_names(
+ mtl_context,
+ data_type,
+ vocab_size,
+ model_dim,
+ group_size,
+ mode,
+ "input_weights",
+ "input_scales",
+ "input_biases",
+ parameter_tree,
+ )
+ }
+
+ fn new_with_names(
+ mtl_context: &MTLContext,
+ data_type: DataType,
+ vocab_size: usize,
+ model_dim: usize,
+ group_size: usize,
+ mode: QuantizationMode,
+ weights_name: &str,
+ scales_name: &str,
+ biases_name: &str,
+ parameter_tree: &ParameterTree<Rc<MTLContext>>,
) -> Result<Self, QuantizedEmbeddingError> {
let packing_divisor = mode.packing_divisor();
@@ -45,15 +95,12 @@ impl QuantizedEmbeddingLookup {
QuantizedEmbeddingLookupKernel::new(mtl_context, data_type, mode)?;
// Load weights [vocab_size, model_dim/packing_divisor] as storage_type
- let mut weights = match parameter_tree.leaf("weights") {
- Ok(weights) => weights,
- Err(_) => parameter_tree.leaf("output_weights").map_err(|e| {
- QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(
- "Failed to load weights: {:?}",
- e
- )))
- })?,
- };
+ let mut weights = parameter_tree.leaf(weights_name).map_err(|e| {
+ QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(
+ "Failed to load weights: {:?}",
+ e
+ )))
+ })?;
if weights.data_type() != mode.storage_type() {
return Err(QuantizedEmbeddingError::MetalError(
@@ -66,15 +113,12 @@ impl QuantizedEmbeddingLookup {
}
// Load scales [vocab_size, num_groups]
- let mut scales = match parameter_tree.leaf("scales") {
- Ok(scales) => scales,
- Err(_) => parameter_tree.leaf("output_scales").map_err(|e| {
- QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(
- "Failed to load scales: {:?}",
- e
- )))
- })?,
- };
+ let mut scales = parameter_tree.leaf(scales_name).map_err(|e| {
+ QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(
+ "Failed to load scales: {:?}",
+ e
+ )))
+ })?;
// Validate shapes and types
let num_groups = (model_dim + group_size - 1) / group_size;
@@ -105,10 +149,7 @@ impl QuantizedEmbeddingLookup {
}
// Load or create biases buffer [vocab_size, num_groups] (MLX key: "biases")
- let biases_buffer: MTLBuffer = match parameter_tree
- .leaf("biases")
- .or_else(|_| parameter_tree.leaf("output_biases"))
- {
+ let biases_buffer: MTLBuffer = match parameter_tree.leaf(biases_name) {
Ok(mut deq_biases) => {
if deq_biases.shape() != [vocab_size, num_groups] {
return Err(QuantizedEmbeddingError::MetalError(
@@ -212,4 +253,3 @@ impl EncodableBlock for QuantizedEmbeddingLookup {
}
}
}
-
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs
index 2b11f8a27..6b619ae93 100644
--- a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs
@@ -3,7 +3,10 @@ use std::rc::Rc;
use metal::Buffer as MTLBuffer;
use mpsgraph::CommandBuffer;
-use super::QuantizedEmbeddingError;
+use super::{
+ super::{EncodableBlock, EncodingParameters},
+ QuantizedEmbeddingError,
+};
use crate::{
Array, DataType,
backends::metal::{
@@ -17,8 +20,6 @@ use crate::{
parameters::ParameterTree,
};
-use super::super::{EncodableBlock, EncodingParameters};
-
pub struct QuantizedEmbeddingReadout {
kernel: QuantizedMatmulKernel,
weights_buffer: MTLBuffer,
@@ -29,7 +30,30 @@ pub struct QuantizedEmbeddingReadout {
}
impl QuantizedEmbeddingReadout {
- pub fn new(
+ pub fn new_tied(
+ mtl_context: &MTLContext,
+ data_type: DataType,
+ vocab_size: usize,
+ model_dim: usize,
+ group_size: usize,
+ mode: QuantizationMode,
+ parameter_tree: &ParameterTree<Rc<MTLContext>>,
+ ) -> Result<Self, QuantizedEmbeddingError> {
+ Self::new_with_names(
+ mtl_context,
+ data_type,
+ vocab_size,
+ model_dim,
+ group_size,
+ mode,
+ "weights",
+ "scales",
+ "biases",
+ parameter_tree,
+ )
+ }
+
+ pub fn new_untied_output(
mtl_context: &MTLContext,
data_type: DataType,
vocab_size: usize,
@@ -37,28 +61,48 @@ impl QuantizedEmbeddingReadout {
group_size: usize,
mode: QuantizationMode,
parameter_tree: &ParameterTree<Rc<MTLContext>>,
+ ) -> Result<Self, QuantizedEmbeddingError> {
+ Self::new_with_names(
+ mtl_context,
+ data_type,
+ vocab_size,
+ model_dim,
+ group_size,
+ mode,
+ "output_weights",
+ "output_scales",
+ "output_biases",
+ parameter_tree,
+ )
+ }
+
+ fn new_with_names(
+ mtl_context: &MTLContext,
+ data_type: DataType,
+ vocab_size: usize,
+ model_dim: usize,
+ group_size: usize,
+ mode: QuantizationMode,
+ weights_name: &str,
+ scales_name: &str,
+ biases_name: &str,
+ parameter_tree: &ParameterTree<Rc<MTLContext>>,
) -> Result<Self, QuantizedEmbeddingError> {
// Load weights [vocab_size, model_dim/2] as U8
- let mut weights = match parameter_tree.leaf("weights") {
- Ok(weights) => weights,
- Err(_) => parameter_tree.leaf("output_weights").map_err(|e| {
- QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(
- "Failed to load weights: {:?}",
- e
- )))
- })?,
- };
+ let mut weights = parameter_tree.leaf(weights_name).map_err(|e| {
+ QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(
+ "Failed to load weights: {:?}",
+ e
+ )))
+ })?;
// Load scales [vocab_size, num_groups]
- let mut scales = match parameter_tree.leaf("scales") {
- Ok(scales) => scales,
- Err(_) => parameter_tree.leaf("output_scales").map_err(|e| {
- QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(
- "Failed to load scales: {:?}",
- e
- )))
- })?,
- };
+ let mut scales = parameter_tree.leaf(scales_name).map_err(|e| {
+ QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(
+ "Failed to load scales: {:?}",
+ e
+ )))
+ })?;
// Validate shapes
let num_groups = (model_dim + group_size - 1) / group_size;
@@ -94,10 +138,7 @@ impl QuantizedEmbeddingReadout {
}
// MLX requires per-group biases; if missing, create a zero buffer of shape [vocab_size, num_groups]
- let biases_buffer: MTLBuffer = match parameter_tree
- .leaf("biases")
- .or_else(|_| parameter_tree.leaf("output_biases"))
- {
+ let biases_buffer: MTLBuffer = match parameter_tree.leaf(biases_name) {
Ok(mut deq_biases) => {
if deq_biases.shape() != [vocab_size, num_groups] {
return Err(QuantizedEmbeddingError::MetalError(
@@ -218,4 +259,3 @@ impl EncodableBlock for QuantizedEmbeddingReadout {
}
}
}
-
diff --git a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
index 7c7115d50..c4957425c 100644
--- a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
@@ -344,6 +344,31 @@ pub fn embed_block(
Box::new(block)
},
+ EmbeddingConfig::MLXQuantizedUntied {
+ group_size,
+ embedding_quantization_mode,
+ activation_precision,
+ ..
+ } => {
+ let data_type: DataType = (*activation_precision).into();
+
+ let embeddings_tree = parameter_tree
+ .subtree("embedding")
+ .expect("Failed to get embedding subtree");
+
+ let block = QuantizedEmbeddingLookup::new_untied_input(
+ context,
+ data_type,
+ config.vocab_size,
+ config.model_dim,
+ *group_size,
+ *embedding_quantization_mode,
+ &embeddings_tree,
+ )
+ .expect("Failed to create quantized embedding lookup kernel");
+
+ Box::new(block)
+ },
EmbeddingConfig::QuantizedTied {
embedding_quantization_mode: _,
..
diff --git a/crates/uzu/src/config/embedding.rs b/crates/uzu/src/config/embedding.rs
index 490d76e37..9ed78fd5b 100644
--- a/crates/uzu/src/config/embedding.rs
+++ b/crates/uzu/src/config/embedding.rs
@@ -58,6 +58,15 @@ pub enum EmbeddingConfig {
activation_quantization_mode: Option<QuantizationMode>,
activation_precision: ConfigDataType,
},
+ #[serde(rename = "MLXQuantizedUntiedEmbeddingConfig")]
+ MLXQuantizedUntied {
+ #[serde(flatten)]
+ common: EmbeddingConfigCommon,
+ group_size: usize,
+ embedding_quantization_mode: QuantizationMode,
+ activation_quantization_mode: Option<QuantizationMode>,
+ activation_precision: ConfigDataType,
+ },
}
impl EmbeddingConfig {
@@ -87,6 +96,10 @@ impl EmbeddingConfig {
common,
..
} => common,
+ EmbeddingConfig::MLXQuantizedUntied {
+ common,
+ ..
+ } => common,
}
}
}