Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

update embedding

eugene240f418e50f7
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding.rs b/crates/uzu/src/backends/metal/encodable_block/embedding.rs
deleted file mode 100644
index f80562f35..000000000
--- a/crates/uzu/src/backends/metal/encodable_block/embedding.rs
+++ /dev/null
@@ -1,516 +0,0 @@
-//! Quantized embedding encodables.
-
-use std::rc::Rc;
-
-use metal::Buffer as MTLBuffer;
-use mpsgraph::CommandBuffer;
-
-use super::{EncodableBlock, EncodingParameters};
-use crate::{
- Array, DataType,
- backends::metal::{
- MTLContext, MTLError,
- forward_pass::{ArrayId, ForwardPassState},
- kernel::{
- embedding::{
- QuantizedEmbeddingLookupArguments,
- QuantizedEmbeddingLookupKernel,
- },
- quant_matmul::{
- QuantizationType, QuantizedMatmulArguments,
- QuantizedMatmulKernel,
- },
- },
- },
- config::QuantizationMode,
- parameters::ParameterTree,
-};
-
-#[derive(Debug, thiserror::Error)]
-pub enum QuantizedEmbeddingError {
- #[error("Metal error: {0}")]
- MetalError(#[from] MTLError),
- #[error("Unsupported data type: {0:?}")]
- UnsupportedDataType(DataType),
-}
-
-pub struct QuantizedEmbeddingLookup {
- kernel: QuantizedEmbeddingLookupKernel,
- weights_buffer: MTLBuffer,
- scales_buffer: MTLBuffer,
- biases_buffer: MTLBuffer,
- vocab_size: u32,
- model_dim: u32,
- group_size: u32,
- input_scale: f32,
-}
-
-impl QuantizedEmbeddingLookup {
- pub fn new_tied(
- mtl_context: &MTLContext,
- data_type: DataType,
- vocab_size: usize,
- model_dim: usize,
- group_size: usize,
- mode: QuantizationMode,
- input_scale: f32,
- parameter_tree: &ParameterTree<Rc<MTLContext>>,
- ) -> Result<Self, QuantizedEmbeddingError> {
- Self::new_with_names(
- mtl_context,
- data_type,
- vocab_size,
- model_dim,
- group_size,
- mode,
- input_scale,
- "weights",
- "scales",
- "biases",
- parameter_tree,
- )
- }
-
- pub fn new_untied_input(
- mtl_context: &MTLContext,
- data_type: DataType,
- vocab_size: usize,
- model_dim: usize,
- group_size: usize,
- mode: QuantizationMode,
- input_scale: f32,
- parameter_tree: &ParameterTree<Rc<MTLContext>>,
- ) -> Result<Self, QuantizedEmbeddingError> {
- Self::new_with_names(
- mtl_context,
- data_type,
- vocab_size,
- model_dim,
- group_size,
- mode,
- input_scale,
- "input_weights",
- "input_scales",
- "input_biases",
- parameter_tree,
- )
- }
-
- fn new_with_names(
- mtl_context: &MTLContext,
- data_type: DataType,
- vocab_size: usize,
- model_dim: usize,
- group_size: usize,
- mode: QuantizationMode,
- input_scale: f32,
- weights_name: &str,
- scales_name: &str,
- biases_name: &str,
- parameter_tree: &ParameterTree<Rc<MTLContext>>,
- ) -> Result<Self, QuantizedEmbeddingError> {
- let packing_divisor = mode.packing_divisor();
-
- let kernel =
- QuantizedEmbeddingLookupKernel::new(mtl_context, data_type, mode)?;
-
- // Load weights [vocab_size, model_dim/packing_divisor] as storage_type
- let mut weights = parameter_tree.leaf(weights_name).map_err(|e| {
- QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(
- "Failed to load {weights_name}: {:?}",
- e
- )))
- })?;
-
- if weights.data_type() != mode.storage_type() {
- return Err(QuantizedEmbeddingError::MetalError(
- MTLError::Generic(format!(
- "Expected packed weights of type {:?}, got {:?}",
- mode.storage_type(),
- weights.data_type()
- )),
- ));
- }
-
- // Load scales [vocab_size, num_groups]
- let mut scales = parameter_tree.leaf(scales_name).map_err(|e| {
- QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(
- "Failed to load {scales_name}: {:?}",
- e
- )))
- })?;
-
- // Validate shapes and types
- let num_groups = (model_dim + group_size - 1) / group_size;
- if weights.shape() != [vocab_size, model_dim / packing_divisor] {
- return Err(QuantizedEmbeddingError::MetalError(
- MTLError::Generic(format!(
- "Embedding lookup weights shape mismatch: got {:?}, expected [{}, {}]",
- weights.shape(),
- vocab_size,
- model_dim / packing_divisor
- )),
- ));
- }
- if scales.shape() != [vocab_size, num_groups] {
- return Err(QuantizedEmbeddingError::MetalError(
- MTLError::Generic(format!(
- "Embedding lookup scales shape mismatch: got {:?}, expected [{}, {}]",
- scales.shape(),
- vocab_size,
- num_groups
- )),
- ));
- }
- if scales.data_type() != data_type {
- return Err(QuantizedEmbeddingError::UnsupportedDataType(
- scales.data_type(),
- ));
- }
-
- // Load or create biases buffer [vocab_size, num_groups]
- let biases_buffer: MTLBuffer = match parameter_tree.leaf(biases_name) {
- Ok(mut deq_biases) => {
- if deq_biases.shape() != [vocab_size, num_groups] {
- return Err(QuantizedEmbeddingError::MetalError(
- MTLError::Generic(format!(
- "Embedding lookup biases shape mismatch: got {:?}, expected [{}, {}]",
- deq_biases.shape(),
- vocab_size,
- num_groups
- )),
- ));
- }
- if deq_biases.data_type() != data_type {
- return Err(QuantizedEmbeddingError::UnsupportedDataType(
- deq_biases.data_type(),
- ));
- }
- unsafe { deq_biases.mtl_buffer().to_owned() }
- },
- Err(_) => {
- let elem_size: usize = match data_type {
- DataType::F16 | DataType::BF16 => 2,
- DataType::F32 => 4,
- other => {
- return Err(
- QuantizedEmbeddingError::UnsupportedDataType(other),
- );
- },
- };
- let size_bytes = (vocab_size * num_groups * elem_size) as u64;
- let buf = mtl_context.device.new_buffer(
- size_bytes,
- metal::MTLResourceOptions::StorageModeShared,
- );
- unsafe {
- std::ptr::write_bytes(
- buf.contents(),
- 0,
- size_bytes as usize,
- );
- }
- buf
- },
- };
-
- let weights_buffer = unsafe { weights.mtl_buffer().to_owned() };
- let scales_buffer = unsafe { scales.mtl_buffer().to_owned() };
-
- Ok(Self {
- kernel,
- weights_buffer,
- scales_buffer,
- biases_buffer,
- vocab_size: vocab_size as u32,
- model_dim: model_dim as u32,
- group_size: group_size as u32,
- input_scale,
- })
- }
-}
-
-impl EncodableBlock for QuantizedEmbeddingLookup {
- fn encode(
- &self,
- state: &mut ForwardPassState,
- command_buffer: &CommandBuffer,
- parameters: &EncodingParameters,
- ) {
- let arrays = state.arrays(&[ArrayId::TokenIds, ArrayId::Main]);
- let batch_size = state.active_suffix_length();
- let mut token_ids_array_mut = arrays[0].borrow_mut();
- let mut output_array_mut = arrays[1].borrow_mut();
-
- let token_ids_buffer = unsafe { token_ids_array_mut.mtl_buffer() };
- let output_buffer = unsafe { output_array_mut.mtl_buffer() };
-
- let root_command_buffer = command_buffer.root_command_buffer();
- let encoder = root_command_buffer.new_compute_command_encoder();
-
- let args = QuantizedEmbeddingLookupArguments {
- token_ids_buffer,
- weights_buffer: &self.weights_buffer,
- scales_buffer: &self.scales_buffer,
- biases_buffer: &self.biases_buffer,
- output_buffer,
- batch_size: batch_size as u32,
- vocab_size: self.vocab_size,
- model_dim: self.model_dim,
- group_size: self.group_size,
- input_scale: self.input_scale,
- };
-
- self.kernel
- .encode(encoder, args)
- .expect("Failed to encode quantized embedding lookup kernel");
-
- encoder.end_encoding();
-
- if parameters.wait_until_completed {
- let mtl_command_buffer =
- command_buffer.root_command_buffer().to_owned();
- command_buffer.commit_and_continue();
- mtl_command_buffer.wait_until_completed();
- }
- }
-}
-pub struct QuantizedEmbeddingReadout {
- kernel: QuantizedMatmulKernel,
- weights_buffer: MTLBuffer,
- scales_buffer: MTLBuffer,
- biases_buffer: MTLBuffer,
- vocab_size: usize,
- model_dim: usize,
-}
-
-impl QuantizedEmbeddingReadout {
- pub fn new_tied(
- mtl_context: &MTLContext,
- data_type: DataType,
- vocab_size: usize,
- model_dim: usize,
- group_size: usize,
- mode: QuantizationMode,
- parameter_tree: &ParameterTree<Rc<MTLContext>>,
- ) -> Result<Self, QuantizedEmbeddingError> {
- Self::new_with_names(
- mtl_context,
- data_type,
- vocab_size,
- model_dim,
- group_size,
- mode,
- "weights",
- "scales",
- "biases",
- parameter_tree,
- )
- }
-
- pub fn new_untied_output(
- mtl_context: &MTLContext,
- data_type: DataType,
- vocab_size: usize,
- model_dim: usize,
- group_size: usize,
- mode: QuantizationMode,
- parameter_tree: &ParameterTree<Rc<MTLContext>>,
- ) -> Result<Self, QuantizedEmbeddingError> {
- Self::new_with_names(
- mtl_context,
- data_type,
- vocab_size,
- model_dim,
- group_size,
- mode,
- "output_weights",
- "output_scales",
- "output_biases",
- parameter_tree,
- )
- }
-
- fn new_with_names(
- mtl_context: &MTLContext,
- data_type: DataType,
- vocab_size: usize,
- model_dim: usize,
- group_size: usize,
- mode: QuantizationMode,
- weights_name: &str,
- scales_name: &str,
- biases_name: &str,
- parameter_tree: &ParameterTree<Rc<MTLContext>>,
- ) -> Result<Self, QuantizedEmbeddingError> {
- // Load weights [vocab_size, model_dim/2] as U8
- let mut weights = parameter_tree.leaf(weights_name).map_err(|e| {
- QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(
- "Failed to load {weights_name}: {:?}",
- e
- )))
- })?;
-
- // Load scales [vocab_size, num_groups]
- let mut scales = parameter_tree.leaf(scales_name).map_err(|e| {
- QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(
- "Failed to load {scales_name}: {:?}",
- e
- )))
- })?;
-
- // Validate shapes
- let num_groups = (model_dim + group_size - 1) / group_size;
- let packing_divisor = mode.packing_divisor();
-
- // Determine if weights are transposed by checking shape
- let weights_transposed = weights.shape()[0] == vocab_size;
-
- if weights.shape() != [vocab_size, model_dim / packing_divisor] {
- return Err(QuantizedEmbeddingError::MetalError(
- MTLError::Generic(format!(
- "Embedding readout weights shape mismatch: got {:?}, expected [{}, {}]",
- weights.shape(),
- vocab_size,
- model_dim / packing_divisor
- )),
- ));
- }
- if scales.shape() != [vocab_size, num_groups] {
- return Err(QuantizedEmbeddingError::MetalError(
- MTLError::Generic(format!(
- "Embedding readout scales shape mismatch: got {:?}, expected [{}, {}]",
- scales.shape(),
- vocab_size,
- num_groups
- )),
- ));
- }
- if scales.data_type() != data_type {
- return Err(QuantizedEmbeddingError::UnsupportedDataType(
- scales.data_type(),
- ));
- }
-
- // MLX requires per-group biases; if missing, create a zero buffer of shape [vocab_size, num_groups]
- let biases_buffer: MTLBuffer = match parameter_tree.leaf(biases_name) {
- Ok(mut deq_biases) => {
- if deq_biases.shape() != [vocab_size, num_groups] {
- return Err(QuantizedEmbeddingError::MetalError(
- MTLError::Generic(format!(
- "Embedding readout deq_biases shape mismatch: got {:?}, expected [{}, {}]",
- deq_biases.shape(),
- vocab_size,
- num_groups
- )),
- ));
- }
- if deq_biases.data_type() != data_type {
- return Err(QuantizedEmbeddingError::UnsupportedDataType(
- deq_biases.data_type(),
- ));
- }
- unsafe { deq_biases.mtl_buffer().to_owned() }
- },
- Err(_) => {
- // Allocate zero-initialized biases buffer
- let elem_size: usize = match data_type {
- DataType::F16 | DataType::BF16 => 2,
- DataType::F32 => 4,
- other => {
- return Err(
- QuantizedEmbeddingError::UnsupportedDataType(other),
- );
- },
- };
- let size_bytes = (vocab_size * num_groups * elem_size) as u64;
- let buf = mtl_context.device.new_buffer(
- size_bytes,
- metal::MTLResourceOptions::StorageModeShared,
- );
- unsafe {
- std::ptr::write_bytes(
- buf.contents(),
- 0,
- size_bytes as usize,
- );
- }
- buf
- },
- };
-
- let weights_buffer = unsafe { weights.mtl_buffer().to_owned() };
- let scales_buffer = unsafe { scales.mtl_buffer().to_owned() };
-
- let kernel = QuantizedMatmulKernel::new(
- mtl_context,
- data_type,
- group_size,
- model_dim,
- vocab_size,
- mode,
- QuantizationType::Mlx,
- weights_transposed,
- )
- .map_err(|e| {
- QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(
- "Failed to create kernel: {:?}",
- e
- )))
- })?;
-
- Ok(Self {
- kernel,
- weights_buffer,
- scales_buffer,
- biases_buffer,
- vocab_size,
- model_dim,
- })
- }
-}
-
-impl EncodableBlock for QuantizedEmbeddingReadout {
- fn encode(
- &self,
- state: &mut ForwardPassState,
- command_buffer: &CommandBuffer,
- parameters: &EncodingParameters,
- ) {
- let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);
- let batch_size = state.active_suffix_length();
- let mut input_array_mut = arrays[0].borrow_mut();
- let mut output_array_mut = arrays[1].borrow_mut();
-
- let input_buffer = unsafe { input_array_mut.mtl_buffer() };
- let output_buffer = unsafe { output_array_mut.mtl_buffer() };
-
- let root_command_buffer = command_buffer.root_command_buffer();
- let encoder = root_command_buffer.new_compute_command_encoder();
-
- let args = QuantizedMatmulArguments {
- a_buffer: input_buffer,
- b_buffer: &self.weights_buffer,
- scales_buffer: &self.scales_buffer,
- zero_points_or_biases_buffer: &self.biases_buffer,
- output_buffer,
- batch: batch_size as i32,
- input_dim: self.model_dim as i32,
- output_dim: self.vocab_size as i32,
- quantization_type: QuantizationType::Mlx,
- };
-
- self.kernel
- .encode(encoder, args)
- .expect("Failed to encode quantized embedding readout kernel");
-
- encoder.end_encoding();
-
- if parameters.wait_until_completed {
- let mtl_command_buffer =
- command_buffer.root_command_buffer().to_owned();
- command_buffer.commit_and_continue();
- mtl_command_buffer.wait_until_completed();
- }
- }
-}
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/error.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/error.rs
new file mode 100644
index 000000000..c13db8c9a
--- /dev/null
+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/error.rs
@@ -0,0 +1,9 @@
+use crate::{DataType, backends::metal::MTLError};
+
+#[derive(Debug, thiserror::Error)]
+pub enum QuantizedEmbeddingError {
+ #[error("Metal error: {0}")]
+ MetalError(#[from] MTLError),
+ #[error("Unsupported data type: {0:?}")]
+ UnsupportedDataType(DataType),
+}
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs
new file mode 100644
index 000000000..b3c6c0860
--- /dev/null
+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs
@@ -0,0 +1,134 @@
+use std::{cell::RefCell, rc::Rc};
+
+use metal::Buffer as MTLBuffer;
+use mpsgraph::CommandBuffer;
+
+use super::{
+ super::{EncodableBlock, EncodingParameters},
+ QuantizedEmbeddingError,
+};
+use crate::{
+ Array, DataType,
+ backends::metal::{
+ MTLContext, MTLError,
+ forward_pass::{ArrayId, ForwardPassState},
+ kernel::matmul::{MatmulArguments, MatmulKernel},
+ },
+ parameters::ParameterTree,
+};
+
+pub struct FullPrecisionEmbeddingReadout {
+ kernel: RefCell<MatmulKernel>,
+ weights_buffer: MTLBuffer,
+ vocab_size: usize,
+ model_dim: usize,
+}
+
+impl FullPrecisionEmbeddingReadout {
+ pub fn new(
+ mtl_context: &MTLContext,
+ data_type: DataType,
+ vocab_size: usize,
+ model_dim: usize,
+ parameter_tree: &ParameterTree<Rc<MTLContext>>,
+ ) -> Result<Self, QuantizedEmbeddingError> {
+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)
+ {
+ return Err(QuantizedEmbeddingError::UnsupportedDataType(
+ data_type,
+ ));
+ }
+
+ let mut weights = match parameter_tree.leaf("weights") {
+ Ok(weights) => weights,
+ Err(_) => parameter_tree.leaf("output_weights").map_err(|e| {
+ QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(
+ "Failed to load weights: {:?}",
+ e
+ )))
+ })?,
+ };
+
+ if weights.shape() != [vocab_size, model_dim] {
+ return Err(QuantizedEmbeddingError::MetalError(
+ MTLError::Generic(format!(
+ "Embedding readout weights shape mismatch: got {:?}, expected [{}, {}]",
+ weights.shape(),
+ vocab_size,
+ model_dim
+ )),
+ ));
+ }
+
+ if weights.data_type() != data_type {
+ return Err(QuantizedEmbeddingError::MetalError(
+ MTLError::Generic(format!(
+ "Weights dtype mismatch: got {:?}, expected {:?}",
+ weights.data_type(),
+ data_type
+ )),
+ ));
+ }
+
+ let weights_buffer = unsafe { weights.mtl_buffer().to_owned() };
+
+ // Weights are [vocab_size, model_dim], we compute input @ weights^T
+ // MatmulKernel with transpose_b=true handles this
+ let kernel = MatmulKernel::new(mtl_context, data_type, false, true)
+ .map_err(QuantizedEmbeddingError::MetalError)?;
+
+ Ok(Self {
+ kernel: RefCell::new(kernel),
+ weights_buffer,
+ vocab_size,
+ model_dim,
+ })
+ }
+}
+
+impl EncodableBlock for FullPrecisionEmbeddingReadout {
+ fn encode(
+ &self,
+ state: &mut ForwardPassState,
+ command_buffer: &CommandBuffer,
+ parameters: &EncodingParameters,
+ ) {
+ let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);
+ let batch_size = state.active_suffix_length();
+ let mut input_array_mut = arrays[0].borrow_mut();
+ let mut output_array_mut = arrays[1].borrow_mut();
+
+ let input_buffer = unsafe { input_array_mut.mtl_buffer() };
+ let output_buffer = unsafe { output_array_mut.mtl_buffer() };
+
+ let root_command_buffer = command_buffer.root_command_buffer();
+ let encoder = root_command_buffer.new_compute_command_encoder();
+
+ let args = MatmulArguments {
+ a: input_buffer,
+ b: &self.weights_buffer,
+ d: output_buffer,
+ batch: batch_size as i32,
+ input_dim: self.model_dim as i32,
+ output_dim: self.vocab_size as i32,
+ lda: self.model_dim as i32,
+ ldb: self.model_dim as i32,
+ ldd: self.vocab_size as i32,
+ batch_count: 1,
+ };
+
+ self.kernel
+ .borrow_mut()
+ .encode(state.mtl_context(), encoder, args)
+ .expect("Failed to encode full precision embedding readout kernel");
+
+ encoder.end_encoding();
+
+ if parameters.wait_until_completed {
+ let mtl_command_buffer =
+ command_buffer.root_command_buffer().to_owned();
+ command_buffer.commit_and_continue();
+ mtl_command_buffer.wait_until_completed();
+ }
+ }
+}
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rs
new file mode 100644
index 000000000..619eca99e
--- /dev/null
+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rs
@@ -0,0 +1,12 @@
+//! Embedding encodables (quantized and full-precision).
+
+mod error;
+mod full_precision_readout;
+mod quantized_lookup;
+mod quantized_readout;
+
+pub use error::QuantizedEmbeddingError;
+pub use full_precision_readout::FullPrecisionEmbeddingReadout;
+pub use quantized_lookup::QuantizedEmbeddingLookup;
+pub use quantized_readout::QuantizedEmbeddingReadout;
+
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_lookup.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_lookup.rs
new file mode 100644
index 000000000..1e4945d46
--- /dev/null
+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_lookup.rs
@@ -0,0 +1,215 @@
+use std::rc::Rc;
+
+use metal::Buffer as MTLBuffer;
+use mpsgraph::CommandBuffer;
+
+use super::QuantizedEmbeddingError;
+use crate::{
+ Array, DataType,
+ backends::metal::{
+ MTLContext, MTLError,
+ forward_pass::{ArrayId, ForwardPassState},
+ kernel::embedding::{
+ QuantizedEmbeddingLookupArguments, QuantizedEmbeddingLookupKernel,
+ },
+ },
+ config::QuantizationMode,
+ parameters::ParameterTree,
+};
+
+use super::super::{EncodableBlock, EncodingParameters};
+
+pub struct QuantizedEmbeddingLookup {
+ kernel: QuantizedEmbeddingLookupKernel,
+ weights_buffer: MTLBuffer,
+ scales_buffer: MTLBuffer,
+ biases_buffer: MTLBuffer,
+ vocab_size: u32,
+ model_dim: u32,
+ group_size: u32,
+}
+
+impl QuantizedEmbeddingLookup {
+ pub fn new(
+ mtl_context: &MTLContext,
+ data_type: DataType,
+ vocab_size: usize,
+ model_dim: usize,
+ group_size: usize,
+ mode: QuantizationMode,
+ parameter_tree: &ParameterTree<Rc<MTLContext>>,
+ ) -> Result<Self, QuantizedEmbeddingError> {
+ let packing_divisor = mode.packing_divisor();
+
+ let kernel =
+ QuantizedEmbeddingLookupKernel::new(mtl_context, data_type, mode)?;
+
+ // Load weights [vocab_size, model_dim/packing_divisor] as storage_type
+ let mut weights = match parameter_tree.leaf("weights") {
+ Ok(weights) => weights,
+ Err(_) => parameter_tree.leaf("output_weights").map_err(|e| {
+ QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(
+ "Failed to load weights: {:?}",
+ e
+ )))
+ })?,
+ };
+
+ if weights.data_type() != mode.storage_type() {
+ return Err(QuantizedEmbeddingError::MetalError(
+ MTLError::Generic(format!(
+ "Expected packed weights of type {:?}, got {:?}",
+ mode.storage_type(),
+ weights.data_type()
+ )),
+ ));
+ }
+
+ // Load scales [vocab_size, num_groups]
+ let mut scales = match parameter_tree.leaf("scales") {
+ Ok(scales) => scales,
+ Err(_) => parameter_tree.leaf("output_scales").map_err(|e| {
+ QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(
+ "Failed to load scales: {:?}",
+ e
+ )))
+ })?,
+ };
+
+ // Validate shapes and types
+ let num_groups = (model_dim + group_size - 1) / group_size;
+ if weights.shape() != [vocab_size, model_dim / packing_divisor] {
+ return Err(QuantizedEmbeddingError::MetalError(
+ MTLError::Generic(format!(
+ "Embedding lookup weights shape mismatch: got {:?}, expected [{}, {}]",
+ weights.shape(),
+ vocab_size,
+ model_dim / packing_divisor
+ )),
+ ));
+ }
+ if scales.shape() != [vocab_size, num_groups] {
+ return Err(QuantizedEmbeddingError::MetalError(
+ MTLError::Generic(format!(
+ "Embedding lookup scales shape mismatch: got {:?}, expected [{}, {}]",
+ scales.shape(),
+ vocab_size,
+ num_groups
+ )),
+ ));
+ }
+ if scales.data_type() != data_type {
+ return Err(QuantizedEmbeddingError::UnsupportedDataType(
+ scales.data_type(),
+ ));
+ }
+
+ // Load or create biases buffer [vocab_size, num_groups] (MLX key: "biases")
+ let biases_buffer: MTLBuffer = match parameter_tree
+ .leaf("biases")
+ .or_else(|_| parameter_tree.leaf("output_biases"))
+ {
+ Ok(mut deq_biases) => {
+ if deq_biases.shape() != [vocab_size, num_groups] {
+ return Err(QuantizedEmbeddingError::MetalError(
+ MTLError::Generic(format!(
+ "Embedding lookup deq_biases shape mismatch: got {:?}, expected [{}, {}]",
+ deq_biases.shape(),
+ vocab_size,
+ num_groups
+ )),
+ ));
+ }
+ if deq_biases.data_type() != data_type {
+ return Err(QuantizedEmbeddingError::UnsupportedDataType(
+ deq_biases.data_type(),
+ ));
+ }
+ unsafe { deq_biases.mtl_buffer().to_owned() }
+ },
+ Err(_) => {
+ let elem_size: usize = match data_type {
+ DataType::F16 | DataType::BF16 => 2,
+ DataType::F32 => 4,
+ other => {
+ return Err(
+ QuantizedEmbeddingError::UnsupportedDataType(other),
+ );
+ },
+ };
+ let size_bytes = (vocab_size * num_groups * elem_size) as u64;
+ let buf = mtl_context.device.new_buffer(
+ size_bytes,
+ metal::MTLResourceOptions::StorageModeShared,
+ );
+ unsafe {
+ std::ptr::write_bytes(
+ buf.contents(),
+ 0,
+ size_bytes as usize,
+ );
+ }
+ buf
+ },
+ };
+
+ let weights_buffer = unsafe { weights.mtl_buffer().to_owned() };
+ let scales_buffer = unsafe { scales.mtl_buffer().to_owned() };
+
+ Ok(Self {
+ kernel,
+ weights_buffer,
+ scales_buffer,
+ biases_buffer,
+ vocab_size: vocab_size as u32,
+ model_dim: model_dim as u32,
+ group_size: group_size as u32,
+ })
+ }
+}
+
+impl EncodableBlock for QuantizedEmbeddingLookup {
+ fn encode(
+ &self,
+ state: &mut ForwardPassState,
+ command_buffer: &CommandBuffer,
+ parameters: &EncodingParameters,
+ ) {
+ let arrays = state.arrays(&[ArrayId::TokenIds, ArrayId::Main]);
+ let batch_size = state.active_suffix_length();
+ let mut token_ids_array_mut = arrays[0].borrow_mut();
+ let mut output_array_mut = arrays[1].borrow_mut();
+
+ let token_ids_buffer = unsafe { token_ids_array_mut.mtl_buffer() };
+ let output_buffer = unsafe { output_array_mut.mtl_buffer() };
+
+ let root_command_buffer = command_buffer.root_command_buffer();
+ let encoder = root_command_buffer.new_compute_command_encoder();
+
+ let args = QuantizedEmbeddingLookupArguments {
+ token_ids_buffer,
+ weights_buffer: &self.weights_buffer,
+ scales_buffer: &self.scales_buffer,
+ biases_buffer: &self.biases_buffer,
+ output_buffer,
+ batch_size: batch_size as u32,
+ vocab_size: self.vocab_size,
+ model_dim: self.model_dim,
+ group_size: self.group_size,
+ };
+
+ self.kernel
+ .encode(encoder, args)
+ .expect("Failed to encode quantized embedding lookup kernel");
+
+ encoder.end_encoding();
+
+ if parameters.wait_until_completed {
+ let mtl_command_buffer =
+ command_buffer.root_command_buffer().to_owned();
+ command_buffer.commit_and_continue();
+ mtl_command_buffer.wait_until_completed();
+ }
+ }
+}
+
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs
new file mode 100644
index 000000000..2b11f8a27
--- /dev/null
+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs
@@ -0,0 +1,221 @@
+use std::rc::Rc;
+
+use metal::Buffer as MTLBuffer;
+use mpsgraph::CommandBuffer;
+
+use super::QuantizedEmbeddingError;
+use crate::{
+ Array, DataType,
+ backends::metal::{
+ MTLContext, MTLError,
+ forward_pass::{ArrayId, ForwardPassState},
+ kernel::quant_matmul::{
+ QuantizationType, QuantizedMatmulArguments, QuantizedMatmulKernel,
+ },
+ },
+ config::QuantizationMode,
+ parameters::ParameterTree,
+};
+
+use super::super::{EncodableBlock, EncodingParameters};
+
+pub struct QuantizedEmbeddingReadout {
+ kernel: QuantizedMatmulKernel,
+ weights_buffer: MTLBuffer,
+ scales_buffer: MTLBuffer,
+ biases_buffer: MTLBuffer,
+ vocab_size: usize,
+ model_dim: usize,
+}
+
+impl QuantizedEmbeddingReadout {
+ pub fn new(
+ mtl_context: &MTLContext,
+ data_type: DataType,
+ vocab_size: usize,
+ model_dim: usize,
+ group_size: usize,
+ mode: QuantizationMode,
+ parameter_tree: &ParameterTree<Rc<MTLContext>>,
+ ) -> Result<Self, QuantizedEmbeddingError> {
+ // Load weights [vocab_size, model_dim/2] as U8
+ let mut weights = match parameter_tree.leaf("weights") {
+ Ok(weights) => weights,
+ Err(_) => parameter_tree.leaf("output_weights").map_err(|e| {
+ QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(
+ "Failed to load weights: {:?}",
+ e
+ )))
+ })?,
+ };
+
+ // Load scales [vocab_size, num_groups]
+ let mut scales = match parameter_tree.leaf("scales") {
+ Ok(scales) => scales,
+ Err(_) => parameter_tree.leaf("output_scales").map_err(|e| {
+ QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(
+ "Failed to load scales: {:?}",
+ e
+ )))
+ })?,
+ };
+
+ // Validate shapes
+ let num_groups = (model_dim + group_size - 1) / group_size;
+ let packing_divisor = mode.packing_divisor();
+
+ // Determine if weights are transposed by checking shape
+ let weights_transposed = weights.shape()[0] == vocab_size;
+
+ if weights.shape() != [vocab_size, model_dim / packing_divisor] {
+ return Err(QuantizedEmbeddingError::MetalError(
+ MTLError::Generic(format!(
+ "Embedding readout weights shape mismatch: got {:?}, expected [{}, {}]",
+ weights.shape(),
+ vocab_size,
+ model_dim / packing_divisor
+ )),
+ ));
+ }
+ if scales.shape() != [vocab_size, num_groups] {
+ return Err(QuantizedEmbeddingError::MetalError(
+ MTLError::Generic(format!(
+ "Embedding readout scales shape mismatch: got {:?}, expected [{}, {}]",
+ scales.shape(),
+ vocab_size,
+ num_groups
+ )),
+ ));
+ }
+ if scales.data_type() != data_type {
+ return Err(QuantizedEmbeddingError::UnsupportedDataType(
+ scales.data_type(),
+ ));
+ }
+
+ // MLX requires per-group biases; if missing, create a zero buffer of shape [vocab_size, num_groups]
+ let biases_buffer: MTLBuffer = match parameter_tree
+ .leaf("biases")
+ .or_else(|_| parameter_tree.leaf("output_biases"))
+ {
+ Ok(mut deq_biases) => {
+ if deq_biases.shape() != [vocab_size, num_groups] {
+ return Err(QuantizedEmbeddingError::MetalError(
+ MTLError::Generic(format!(
+ "Embedding readout deq_biases shape mismatch: got {:?}, expected [{}, {}]",
+ deq_biases.shape(),
+ vocab_size,
+ num_groups
+ )),
+ ));
+ }
+ if deq_biases.data_type() != data_type {
+ return Err(QuantizedEmbeddingError::UnsupportedDataType(
+ deq_biases.data_type(),
+ ));
+ }
+ unsafe { deq_biases.mtl_buffer().to_owned() }
+ },
+ Err(_) => {
+ // Allocate zero-initialized biases buffer
+ let elem_size: usize = match data_type {
+ DataType::F16 | DataType::BF16 => 2,
+ DataType::F32 => 4,
+ other => {
+ return Err(
+ QuantizedEmbeddingError::UnsupportedDataType(other),
+ );
+ },
+ };
+ let size_bytes = (vocab_size * num_groups * elem_size) as u64;
+ let buf = mtl_context.device.new_buffer(
+ size_bytes,
+ metal::MTLResourceOptions::StorageModeShared,
+ );
+ unsafe {
+ std::ptr::write_bytes(
+ buf.contents(),
+ 0,
+ size_bytes as usize,
+ );
+ }
+ buf
+ },
+ };
+
+ let weights_buffer = unsafe { weights.mtl_buffer().to_owned() };
+ let scales_buffer = unsafe { scales.mtl_buffer().to_owned() };
+
+ let kernel = QuantizedMatmulKernel::new(
+ mtl_context,
+ data_type,
+ group_size,
+ model_dim,
+ vocab_size,
+ mode,
+ QuantizationType::Mlx,
+ weights_transposed,
+ )
+ .map_err(|e| {
+ QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(
+ "Failed to create kernel: {:?}",
+ e
+ )))
+ })?;
+
+ Ok(Self {
+ kernel,
+ weights_buffer,
+ scales_buffer,
+ biases_buffer,
+ vocab_size,
+ model_dim,
+ })
+ }
+}
+
+impl EncodableBlock for QuantizedEmbeddingReadout {
+ fn encode(
+ &self,
+ state: &mut ForwardPassState,
+ command_buffer: &CommandBuffer,
+ parameters: &EncodingParameters,
+ ) {
+ let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);
+ let batch_size = state.active_suffix_length();
+ let mut input_array_mut = arrays[0].borrow_mut();
+ let mut output_array_mut = arrays[1].borrow_mut();
+
+ let input_buffer = unsafe { input_array_mut.mtl_buffer() };
+ let output_buffer = unsafe { output_array_mut.mtl_buffer() };
+
+ let root_command_buffer = command_buffer.root_command_buffer();
+ let encoder = root_command_buffer.new_compute_command_encoder();
+
+ let args = QuantizedMatmulArguments {
+ a_buffer: input_buffer,
+ b_buffer: &self.weights_buffer,
+ scales_buffer: &self.scales_buffer,
+ zero_points_or_biases_buffer: &self.biases_buffer,
+ output_buffer,
+ batch: batch_size as i32,
+ input_dim: self.model_dim as i32,
+ output_dim: self.vocab_size as i32,
+ quantization_type: QuantizationType::Mlx,
+ };
+
+ self.kernel
+ .encode(encoder, args)
+ .expect("Failed to encode quantized embedding readout kernel");
+
+ encoder.end_encoding();
+
+ if parameters.wait_until_completed {
+ let mtl_command_buffer =
+ command_buffer.root_command_buffer().to_owned();
+ command_buffer.commit_and_continue();
+ mtl_command_buffer.wait_until_completed();
+ }
+ }
+}
+
diff --git a/crates/uzu/src/backends/metal/encodable_block/mod.rs b/crates/uzu/src/backends/metal/encodable_block/mod.rs
index 5577febb6..2acd7b814 100644
--- a/crates/uzu/src/backends/metal/encodable_block/mod.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/mod.rs
@@ -35,8 +35,8 @@ pub use attention::Attention;
pub use classifier_layer::ClassifierLayer;
pub use decoder::Decoder;
pub use embedding::{
- QuantizedEmbeddingError, QuantizedEmbeddingLookup,
- QuantizedEmbeddingReadout,
+ FullPrecisionEmbeddingReadout, QuantizedEmbeddingError,
+ QuantizedEmbeddingLookup, QuantizedEmbeddingReadout,
};
pub use encoding_parameters::EncodingParameters;
pub use layer::LayerExecutables;
diff --git a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
index 013620942..53ed69431 100644
--- a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
@@ -7,8 +7,9 @@ use mpsgraph::{
use objc2::rc::{Retained, autoreleasepool};
use super::{
- EncodableBlock, FullPrecisionLinear, MlpBlock, MoeBlock,
- QuantizedEmbeddingLookup, QuantizedEmbeddingReadout, QuantizedLinear,
+ EncodableBlock, FullPrecisionEmbeddingReadout, FullPrecisionLinear,
+ MlpBlock, MoeBlock, QuantizedEmbeddingLookup, QuantizedEmbeddingReadout,
+ QuantizedLinear,
};
use crate::{
DataType,
@@ -538,14 +539,6 @@ pub fn readout_block(
compilation_descriptor: &CompilationDescriptor,
parameter_tree: &ParameterTree<Rc<MTLContext>>,
) -> Box<dyn EncodableBlock> {
- let graph = Graph::new();
-
- let input_shape = [-1 as isize, config.model_dim as isize];
- let input_data_type: DataType =
- config.output_norm_config.scale_precision.into();
- let input_placeholder = placeholder(&graph, &input_shape, input_data_type);
- let input_shaped_type = shaped_type(&input_shape, input_data_type);
-
match config.embedding_config {
EmbeddingConfig::Tied {
precision,
@@ -555,52 +548,14 @@ pub fn readout_block(
.subtree("embedding")
.expect("Failed to get embedding subtree");
- let weights_shape =
- [config.vocab_size as isize, config.model_dim as isize];
- let weights_data_type: DataType = precision.into();
- let weights_shaped_type =
- shaped_type(&weights_shape, weights_data_type);
- let weights_placeholder =
- placeholder(&graph, &weights_shape, weights_data_type);
-
- let weights_transposed =
- graph.transpose(&weights_placeholder, &[1, 0], None);
- let input_shape = [-1 as isize, config.model_dim as isize];
- let input_data_type: DataType =
- config.output_norm_config.scale_precision.into();
- let input_placeholder =
- placeholder(&graph, &input_shape, input_data_type);
- let input_shaped_type = shaped_type(&input_shape, input_data_type);
-
- let output = embeddings_readout_subgraph(
- &graph,
- &input_placeholder,
- &weights_transposed,
+ let block = FullPrecisionEmbeddingReadout::new(
+ context,
+ precision.into(),
+ config.vocab_size,
+ config.model_dim,
+ &embeddings_tree,
)
- .unwrap();
-
- let feeds = HashMap::from([
- (&*input_placeholder, &*input_shaped_type),
- (&*weights_placeholder, &*weights_shaped_type),
- ]);
-
- let executable = graph.compile(
- &MPSDevice::with_device(&context.device),
- &feeds,
- &[&output],
- None,
- Some(compilation_descriptor),
- );
-
- let block = MPSGraphBlock::new(
- executable,
- make_execution_descriptor(),
- IOArrays::new(
- vec![ArrayId::Main, ArrayId::EmbeddingsOutputWeights]
- .into_boxed_slice(),
- vec![ArrayId::Logits].into_boxed_slice(),
- ),
- );
+ .expect("Failed to create full precision embedding readout");
Box::new(block)
},
@@ -611,53 +566,15 @@ pub fn readout_block(
let embeddings_tree = parameter_tree
.subtree("embedding")
.expect("Failed to get embedding subtree");
- // Use MPSGraph readout path
- let weights_shape =
- [config.vocab_size as isize, config.model_dim as isize];
- let weights_data_type: DataType = precision.into();
- let weights_shaped_type =
- shaped_type(&weights_shape, weights_data_type);
- let weights_placeholder =
- placeholder(&graph, &weights_shape, weights_data_type);
-
- let weights_transposed =
- graph.transpose(&weights_placeholder, &[1, 0], None);
- let input_shape = [-1 as isize, config.model_dim as isize];
- let input_data_type: DataType =
- config.output_norm_config.scale_precision.into();
- let input_placeholder =
- placeholder(&graph, &input_shape, input_data_type);
- let input_shaped_type = shaped_type(&input_shape, input_data_type);
- let output = embeddings_readout_subgraph(
- &graph,
- &input_placeholder,
- &weights_transposed,
+ let block = FullPrecisionEmbeddingReadout::new(
+ context,
+ precision.into(),
+ config.vocab_size,
+ config.model_dim,
+ &embeddings_tree,
)
- .unwrap();
-
- let feeds = HashMap::from([
- (&*input_placeholder, &*input_shaped_type),
- (&*weights_placeholder, &*weights_shaped_type),
- ]);
-
- let executable = graph.compile(
- &MPSDevice::with_device(&context.device),
- &feeds,
- &[&output],
- None,
- Some(compilation_descriptor),
- );
-
- let block = MPSGraphBlock::new(
- executable,
- make_execution_descriptor(),
- IOArrays::new(
- vec![ArrayId::Main, ArrayId::EmbeddingsOutputWeights]
- .into_boxed_slice(),
- vec![ArrayId::Logits].into_boxed_slice(),
- ),
- );
+ .expect("Failed to create full precision embedding readout");
Box::new(block)
},
diff --git a/crates/uzu/src/backends/metal/kernel/embedding/mod.rs b/crates/uzu/src/backends/metal/kernel/embedding/mod.rs
index f5513f2af..4adbc459a 100644
--- a/crates/uzu/src/backends/metal/kernel/embedding/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/embedding/mod.rs
@@ -1,20 +1,14 @@
-use std::{mem::size_of, rc::Rc};
+use std::mem::size_of;
use metal::{
Buffer as MTLBuffer, ComputeCommandEncoderRef,
ComputePipelineState as MTLComputePipelineState, MTLSize,
};
-use super::super::{MTLContext, kernel::matmul::MatmulArguments};
+use super::super::MTLContext;
use crate::{
- DataType,
- backends::metal::{
- ForwardPassState, MTLError, encodable_block::QuantizedEmbeddingError,
- forward_pass::ArrayId,
- },
+ DataType, backends::metal::encodable_block::QuantizedEmbeddingError,
config::QuantizationMode,
- device::array::Array,
- parameters::ParameterTree,
};
pub struct QuantizedEmbeddingLookupKernel {
@@ -125,102 +119,3 @@ impl QuantizedEmbeddingLookupKernel {
Ok(())
}
}
-
-pub struct FullPrecisionReadoutKernelBlock {
- kernel: std::cell::RefCell<super::super::kernel::matmul::MatmulKernel>,
- weights_buffer: MTLBuffer,
- vocab_size: usize,
- model_dim: usize,
-}
-
-impl FullPrecisionReadoutKernelBlock {
- pub fn new(
- mtl_context: &MTLContext,
- data_type: DataType,
- vocab_size: usize,
- model_dim: usize,
- parameter_tree: &ParameterTree<Rc<MTLContext>>,
- ) -> Result<Self, MTLError> {
- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)
- {
- return Err(MTLError::Generic(format!(
- "Unsupported data type for full precision readout: {:?}",
- data_type
- )));
- }
-
- let mut weights = match parameter_tree.leaf("weights") {
- Ok(weights) => weights,
- Err(_) => parameter_tree.leaf("output_weights").map_err(|e| {
- MTLError::Generic(format!("Failed to load weights: {:?}", e))
- })?,
- };
-
- if weights.shape() != [vocab_size, model_dim] {
- return Err(MTLError::Generic(format!(
- "Embedding readout weights shape mismatch: got {:?}, expected [{}, {}]",
- weights.shape(),
- vocab_size,
- model_dim
- )));
- }
-
- if weights.data_type() != data_type {
- return Err(MTLError::Generic(format!(
- "Weights dtype mismatch: got {:?}, expected {:?}",
- weights.data_type(),
- data_type
- )));
- }
-
- let weights_buffer = unsafe { weights.mtl_buffer().to_owned() };
-
- let kernel = super::super::kernel::matmul::MatmulKernel::new(
- mtl_context,
- data_type,
- false,
- true,
- )?;
-
- Ok(Self {
- kernel: std::cell::RefCell::new(kernel),
- weights_buffer,
- vocab_size,
- model_dim,
- })
- }
-}
-
-impl FullPrecisionReadoutKernelBlock {
- fn encode_impl(
- &self,
- state: &mut ForwardPassState,
- encoder: &metal::ComputeCommandEncoderRef,
- ) {
- let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);
- let batch_size = state.active_suffix_length();
- let mut input_array_mut = arrays[0].borrow_mut();
- let mut output_array_mut = arrays[1].borrow_mut();
-
- let input_buffer = unsafe { input_array_mut.mtl_buffer() };
- let output_buffer = unsafe { output_array_mut.mtl_buffer() };
-
- let args = MatmulArguments {
- a: input_buffer,
- b: &self.weights_buffer,
- d: output_buffer,
- batch: batch_size as i32,
- input_dim: self.model_dim as i32,
- output_dim: self.vocab_size as i32,
- lda: self.model_dim as i32,
- ldb: self.model_dim as i32, // B is [vocab_size, model_dim], stride is model_dim
- ldd: self.vocab_size as i32,
- batch_count: 1,
- };
-
- self.kernel
- .borrow_mut()
- .encode(state.mtl_context(), encoder, args)
- .expect("Failed to encode full precision readout kernel");
- }
-}