A high-performance inference engine for AI models
update embedding
eugene240f418e50f7
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding.rs b/crates/uzu/src/backends/metal/encodable_block/embedding.rsdeleted file mode 100644index f80562f35..000000000--- a/crates/uzu/src/backends/metal/encodable_block/embedding.rs+++ /dev/null@@ -1,516 +0,0 @@-//! Quantized embedding encodables.--use std::rc::Rc;--use metal::Buffer as MTLBuffer;-use mpsgraph::CommandBuffer;--use super::{EncodableBlock, EncodingParameters};-use crate::{- Array, DataType,- backends::metal::{- MTLContext, MTLError,- forward_pass::{ArrayId, ForwardPassState},- kernel::{- embedding::{- QuantizedEmbeddingLookupArguments,- QuantizedEmbeddingLookupKernel,- },- quant_matmul::{- QuantizationType, QuantizedMatmulArguments,- QuantizedMatmulKernel,- },- },- },- config::QuantizationMode,- parameters::ParameterTree,-};--#[derive(Debug, thiserror::Error)]-pub enum QuantizedEmbeddingError {- #[error("Metal error: {0}")]- MetalError(#[from] MTLError),- #[error("Unsupported data type: {0:?}")]- UnsupportedDataType(DataType),-}--pub struct QuantizedEmbeddingLookup {- kernel: QuantizedEmbeddingLookupKernel,- weights_buffer: MTLBuffer,- scales_buffer: MTLBuffer,- biases_buffer: MTLBuffer,- vocab_size: u32,- model_dim: u32,- group_size: u32,- input_scale: f32,-}--impl QuantizedEmbeddingLookup {- pub fn new_tied(- mtl_context: &MTLContext,- data_type: DataType,- vocab_size: usize,- model_dim: usize,- group_size: usize,- mode: QuantizationMode,- input_scale: f32,- parameter_tree: &ParameterTree<Rc<MTLContext>>,- ) -> Result<Self, QuantizedEmbeddingError> {- Self::new_with_names(- mtl_context,- data_type,- vocab_size,- model_dim,- group_size,- mode,- input_scale,- "weights",- "scales",- "biases",- parameter_tree,- )- }-- pub fn new_untied_input(- mtl_context: &MTLContext,- data_type: DataType,- vocab_size: usize,- model_dim: usize,- group_size: usize,- mode: QuantizationMode,- input_scale: f32,- parameter_tree: &ParameterTree<Rc<MTLContext>>,- ) -> Result<Self, QuantizedEmbeddingError> {- Self::new_with_names(- mtl_context,- data_type,- vocab_size,- model_dim,- group_size,- mode,- input_scale,- "input_weights",- "input_scales",- "input_biases",- parameter_tree,- )- }-- fn new_with_names(- mtl_context: &MTLContext,- data_type: DataType,- vocab_size: usize,- model_dim: usize,- group_size: usize,- mode: QuantizationMode,- input_scale: f32,- weights_name: &str,- scales_name: &str,- biases_name: &str,- parameter_tree: &ParameterTree<Rc<MTLContext>>,- ) -> Result<Self, QuantizedEmbeddingError> {- let packing_divisor = mode.packing_divisor();-- let kernel =- QuantizedEmbeddingLookupKernel::new(mtl_context, data_type, mode)?;-- // Load weights [vocab_size, model_dim/packing_divisor] as storage_type- let mut weights = parameter_tree.leaf(weights_name).map_err(|e| {- QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(- "Failed to load {weights_name}: {:?}",- e- )))- })?;-- if weights.data_type() != mode.storage_type() {- return Err(QuantizedEmbeddingError::MetalError(- MTLError::Generic(format!(- "Expected packed weights of type {:?}, got {:?}",- mode.storage_type(),- weights.data_type()- )),- ));- }-- // Load scales [vocab_size, num_groups]- let mut scales = parameter_tree.leaf(scales_name).map_err(|e| {- QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(- "Failed to load {scales_name}: {:?}",- e- )))- })?;-- // Validate shapes and types- let num_groups = (model_dim + group_size - 1) / group_size;- if weights.shape() != [vocab_size, model_dim / packing_divisor] {- return Err(QuantizedEmbeddingError::MetalError(- MTLError::Generic(format!(- "Embedding lookup weights shape mismatch: got {:?}, expected [{}, {}]",- weights.shape(),- vocab_size,- model_dim / packing_divisor- )),- ));- }- if scales.shape() != [vocab_size, num_groups] {- return Err(QuantizedEmbeddingError::MetalError(- MTLError::Generic(format!(- "Embedding lookup scales shape mismatch: got {:?}, expected [{}, {}]",- scales.shape(),- vocab_size,- num_groups- )),- ));- }- if scales.data_type() != data_type {- return Err(QuantizedEmbeddingError::UnsupportedDataType(- scales.data_type(),- ));- }-- // Load or create biases buffer [vocab_size, num_groups]- let biases_buffer: MTLBuffer = match parameter_tree.leaf(biases_name) {- Ok(mut deq_biases) => {- if deq_biases.shape() != [vocab_size, num_groups] {- return Err(QuantizedEmbeddingError::MetalError(- MTLError::Generic(format!(- "Embedding lookup biases shape mismatch: got {:?}, expected [{}, {}]",- deq_biases.shape(),- vocab_size,- num_groups- )),- ));- }- if deq_biases.data_type() != data_type {- return Err(QuantizedEmbeddingError::UnsupportedDataType(- deq_biases.data_type(),- ));- }- unsafe { deq_biases.mtl_buffer().to_owned() }- },- Err(_) => {- let elem_size: usize = match data_type {- DataType::F16 | DataType::BF16 => 2,- DataType::F32 => 4,- other => {- return Err(- QuantizedEmbeddingError::UnsupportedDataType(other),- );- },- };- let size_bytes = (vocab_size * num_groups * elem_size) as u64;- let buf = mtl_context.device.new_buffer(- size_bytes,- metal::MTLResourceOptions::StorageModeShared,- );- unsafe {- std::ptr::write_bytes(- buf.contents(),- 0,- size_bytes as usize,- );- }- buf- },- };-- let weights_buffer = unsafe { weights.mtl_buffer().to_owned() };- let scales_buffer = unsafe { scales.mtl_buffer().to_owned() };-- Ok(Self {- kernel,- weights_buffer,- scales_buffer,- biases_buffer,- vocab_size: vocab_size as u32,- model_dim: model_dim as u32,- group_size: group_size as u32,- input_scale,- })- }-}--impl EncodableBlock for QuantizedEmbeddingLookup {- fn encode(- &self,- state: &mut ForwardPassState,- command_buffer: &CommandBuffer,- parameters: &EncodingParameters,- ) {- let arrays = state.arrays(&[ArrayId::TokenIds, ArrayId::Main]);- let batch_size = state.active_suffix_length();- let mut token_ids_array_mut = arrays[0].borrow_mut();- let mut output_array_mut = arrays[1].borrow_mut();-- let token_ids_buffer = unsafe { token_ids_array_mut.mtl_buffer() };- let output_buffer = unsafe { output_array_mut.mtl_buffer() };-- let root_command_buffer = command_buffer.root_command_buffer();- let encoder = root_command_buffer.new_compute_command_encoder();-- let args = QuantizedEmbeddingLookupArguments {- token_ids_buffer,- weights_buffer: &self.weights_buffer,- scales_buffer: &self.scales_buffer,- biases_buffer: &self.biases_buffer,- output_buffer,- batch_size: batch_size as u32,- vocab_size: self.vocab_size,- model_dim: self.model_dim,- group_size: self.group_size,- input_scale: self.input_scale,- };-- self.kernel- .encode(encoder, args)- .expect("Failed to encode quantized embedding lookup kernel");-- encoder.end_encoding();-- if parameters.wait_until_completed {- let mtl_command_buffer =- command_buffer.root_command_buffer().to_owned();- command_buffer.commit_and_continue();- mtl_command_buffer.wait_until_completed();- }- }-}-pub struct QuantizedEmbeddingReadout {- kernel: QuantizedMatmulKernel,- weights_buffer: MTLBuffer,- scales_buffer: MTLBuffer,- biases_buffer: MTLBuffer,- vocab_size: usize,- model_dim: usize,-}--impl QuantizedEmbeddingReadout {- pub fn new_tied(- mtl_context: &MTLContext,- data_type: DataType,- vocab_size: usize,- model_dim: usize,- group_size: usize,- mode: QuantizationMode,- parameter_tree: &ParameterTree<Rc<MTLContext>>,- ) -> Result<Self, QuantizedEmbeddingError> {- Self::new_with_names(- mtl_context,- data_type,- vocab_size,- model_dim,- group_size,- mode,- "weights",- "scales",- "biases",- parameter_tree,- )- }-- pub fn new_untied_output(- mtl_context: &MTLContext,- data_type: DataType,- vocab_size: usize,- model_dim: usize,- group_size: usize,- mode: QuantizationMode,- parameter_tree: &ParameterTree<Rc<MTLContext>>,- ) -> Result<Self, QuantizedEmbeddingError> {- Self::new_with_names(- mtl_context,- data_type,- vocab_size,- model_dim,- group_size,- mode,- "output_weights",- "output_scales",- "output_biases",- parameter_tree,- )- }-- fn new_with_names(- mtl_context: &MTLContext,- data_type: DataType,- vocab_size: usize,- model_dim: usize,- group_size: usize,- mode: QuantizationMode,- weights_name: &str,- scales_name: &str,- biases_name: &str,- parameter_tree: &ParameterTree<Rc<MTLContext>>,- ) -> Result<Self, QuantizedEmbeddingError> {- // Load weights [vocab_size, model_dim/2] as U8- let mut weights = parameter_tree.leaf(weights_name).map_err(|e| {- QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(- "Failed to load {weights_name}: {:?}",- e- )))- })?;-- // Load scales [vocab_size, num_groups]- let mut scales = parameter_tree.leaf(scales_name).map_err(|e| {- QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(- "Failed to load {scales_name}: {:?}",- e- )))- })?;-- // Validate shapes- let num_groups = (model_dim + group_size - 1) / group_size;- let packing_divisor = mode.packing_divisor();-- // Determine if weights are transposed by checking shape- let weights_transposed = weights.shape()[0] == vocab_size;-- if weights.shape() != [vocab_size, model_dim / packing_divisor] {- return Err(QuantizedEmbeddingError::MetalError(- MTLError::Generic(format!(- "Embedding readout weights shape mismatch: got {:?}, expected [{}, {}]",- weights.shape(),- vocab_size,- model_dim / packing_divisor- )),- ));- }- if scales.shape() != [vocab_size, num_groups] {- return Err(QuantizedEmbeddingError::MetalError(- MTLError::Generic(format!(- "Embedding readout scales shape mismatch: got {:?}, expected [{}, {}]",- scales.shape(),- vocab_size,- num_groups- )),- ));- }- if scales.data_type() != data_type {- return Err(QuantizedEmbeddingError::UnsupportedDataType(- scales.data_type(),- ));- }-- // MLX requires per-group biases; if missing, create a zero buffer of shape [vocab_size, num_groups]- let biases_buffer: MTLBuffer = match parameter_tree.leaf(biases_name) {- Ok(mut deq_biases) => {- if deq_biases.shape() != [vocab_size, num_groups] {- return Err(QuantizedEmbeddingError::MetalError(- MTLError::Generic(format!(- "Embedding readout deq_biases shape mismatch: got {:?}, expected [{}, {}]",- deq_biases.shape(),- vocab_size,- num_groups- )),- ));- }- if deq_biases.data_type() != data_type {- return Err(QuantizedEmbeddingError::UnsupportedDataType(- deq_biases.data_type(),- ));- }- unsafe { deq_biases.mtl_buffer().to_owned() }- },- Err(_) => {- // Allocate zero-initialized biases buffer- let elem_size: usize = match data_type {- DataType::F16 | DataType::BF16 => 2,- DataType::F32 => 4,- other => {- return Err(- QuantizedEmbeddingError::UnsupportedDataType(other),- );- },- };- let size_bytes = (vocab_size * num_groups * elem_size) as u64;- let buf = mtl_context.device.new_buffer(- size_bytes,- metal::MTLResourceOptions::StorageModeShared,- );- unsafe {- std::ptr::write_bytes(- buf.contents(),- 0,- size_bytes as usize,- );- }- buf- },- };-- let weights_buffer = unsafe { weights.mtl_buffer().to_owned() };- let scales_buffer = unsafe { scales.mtl_buffer().to_owned() };-- let kernel = QuantizedMatmulKernel::new(- mtl_context,- data_type,- group_size,- model_dim,- vocab_size,- mode,- QuantizationType::Mlx,- weights_transposed,- )- .map_err(|e| {- QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(- "Failed to create kernel: {:?}",- e- )))- })?;-- Ok(Self {- kernel,- weights_buffer,- scales_buffer,- biases_buffer,- vocab_size,- model_dim,- })- }-}--impl EncodableBlock for QuantizedEmbeddingReadout {- fn encode(- &self,- state: &mut ForwardPassState,- command_buffer: &CommandBuffer,- parameters: &EncodingParameters,- ) {- let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);- let batch_size = state.active_suffix_length();- let mut input_array_mut = arrays[0].borrow_mut();- let mut output_array_mut = arrays[1].borrow_mut();-- let input_buffer = unsafe { input_array_mut.mtl_buffer() };- let output_buffer = unsafe { output_array_mut.mtl_buffer() };-- let root_command_buffer = command_buffer.root_command_buffer();- let encoder = root_command_buffer.new_compute_command_encoder();-- let args = QuantizedMatmulArguments {- a_buffer: input_buffer,- b_buffer: &self.weights_buffer,- scales_buffer: &self.scales_buffer,- zero_points_or_biases_buffer: &self.biases_buffer,- output_buffer,- batch: batch_size as i32,- input_dim: self.model_dim as i32,- output_dim: self.vocab_size as i32,- quantization_type: QuantizationType::Mlx,- };-- self.kernel- .encode(encoder, args)- .expect("Failed to encode quantized embedding readout kernel");-- encoder.end_encoding();-- if parameters.wait_until_completed {- let mtl_command_buffer =- command_buffer.root_command_buffer().to_owned();- command_buffer.commit_and_continue();- mtl_command_buffer.wait_until_completed();- }- }-}diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/error.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/error.rsnew file mode 100644index 000000000..c13db8c9a--- /dev/null+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/error.rs@@ -0,0 +1,9 @@+use crate::{DataType, backends::metal::MTLError};++#[derive(Debug, thiserror::Error)]+pub enum QuantizedEmbeddingError {+ #[error("Metal error: {0}")]+ MetalError(#[from] MTLError),+ #[error("Unsupported data type: {0:?}")]+ UnsupportedDataType(DataType),+}diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rsnew file mode 100644index 000000000..b3c6c0860--- /dev/null+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs@@ -0,0 +1,134 @@+use std::{cell::RefCell, rc::Rc};++use metal::Buffer as MTLBuffer;+use mpsgraph::CommandBuffer;++use super::{+ super::{EncodableBlock, EncodingParameters},+ QuantizedEmbeddingError,+};+use crate::{+ Array, DataType,+ backends::metal::{+ MTLContext, MTLError,+ forward_pass::{ArrayId, ForwardPassState},+ kernel::matmul::{MatmulArguments, MatmulKernel},+ },+ parameters::ParameterTree,+};++pub struct FullPrecisionEmbeddingReadout {+ kernel: RefCell<MatmulKernel>,+ weights_buffer: MTLBuffer,+ vocab_size: usize,+ model_dim: usize,+}++impl FullPrecisionEmbeddingReadout {+ pub fn new(+ mtl_context: &MTLContext,+ data_type: DataType,+ vocab_size: usize,+ model_dim: usize,+ parameter_tree: &ParameterTree<Rc<MTLContext>>,+ ) -> Result<Self, QuantizedEmbeddingError> {+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)+ {+ return Err(QuantizedEmbeddingError::UnsupportedDataType(+ data_type,+ ));+ }++ let mut weights = match parameter_tree.leaf("weights") {+ Ok(weights) => weights,+ Err(_) => parameter_tree.leaf("output_weights").map_err(|e| {+ QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(+ "Failed to load weights: {:?}",+ e+ )))+ })?,+ };++ if weights.shape() != [vocab_size, model_dim] {+ return Err(QuantizedEmbeddingError::MetalError(+ MTLError::Generic(format!(+ "Embedding readout weights shape mismatch: got {:?}, expected [{}, {}]",+ weights.shape(),+ vocab_size,+ model_dim+ )),+ ));+ }++ if weights.data_type() != data_type {+ return Err(QuantizedEmbeddingError::MetalError(+ MTLError::Generic(format!(+ "Weights dtype mismatch: got {:?}, expected {:?}",+ weights.data_type(),+ data_type+ )),+ ));+ }++ let weights_buffer = unsafe { weights.mtl_buffer().to_owned() };++ // Weights are [vocab_size, model_dim], we compute input @ weights^T+ // MatmulKernel with transpose_b=true handles this+ let kernel = MatmulKernel::new(mtl_context, data_type, false, true)+ .map_err(QuantizedEmbeddingError::MetalError)?;++ Ok(Self {+ kernel: RefCell::new(kernel),+ weights_buffer,+ vocab_size,+ model_dim,+ })+ }+}++impl EncodableBlock for FullPrecisionEmbeddingReadout {+ fn encode(+ &self,+ state: &mut ForwardPassState,+ command_buffer: &CommandBuffer,+ parameters: &EncodingParameters,+ ) {+ let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);+ let batch_size = state.active_suffix_length();+ let mut input_array_mut = arrays[0].borrow_mut();+ let mut output_array_mut = arrays[1].borrow_mut();++ let input_buffer = unsafe { input_array_mut.mtl_buffer() };+ let output_buffer = unsafe { output_array_mut.mtl_buffer() };++ let root_command_buffer = command_buffer.root_command_buffer();+ let encoder = root_command_buffer.new_compute_command_encoder();++ let args = MatmulArguments {+ a: input_buffer,+ b: &self.weights_buffer,+ d: output_buffer,+ batch: batch_size as i32,+ input_dim: self.model_dim as i32,+ output_dim: self.vocab_size as i32,+ lda: self.model_dim as i32,+ ldb: self.model_dim as i32,+ ldd: self.vocab_size as i32,+ batch_count: 1,+ };++ self.kernel+ .borrow_mut()+ .encode(state.mtl_context(), encoder, args)+ .expect("Failed to encode full precision embedding readout kernel");++ encoder.end_encoding();++ if parameters.wait_until_completed {+ let mtl_command_buffer =+ command_buffer.root_command_buffer().to_owned();+ command_buffer.commit_and_continue();+ mtl_command_buffer.wait_until_completed();+ }+ }+}diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rsnew file mode 100644index 000000000..619eca99e--- /dev/null+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rs@@ -0,0 +1,12 @@+//! Embedding encodables (quantized and full-precision).++mod error;+mod full_precision_readout;+mod quantized_lookup;+mod quantized_readout;++pub use error::QuantizedEmbeddingError;+pub use full_precision_readout::FullPrecisionEmbeddingReadout;+pub use quantized_lookup::QuantizedEmbeddingLookup;+pub use quantized_readout::QuantizedEmbeddingReadout;+diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_lookup.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_lookup.rsnew file mode 100644index 000000000..1e4945d46--- /dev/null+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_lookup.rs@@ -0,0 +1,215 @@+use std::rc::Rc;++use metal::Buffer as MTLBuffer;+use mpsgraph::CommandBuffer;++use super::QuantizedEmbeddingError;+use crate::{+ Array, DataType,+ backends::metal::{+ MTLContext, MTLError,+ forward_pass::{ArrayId, ForwardPassState},+ kernel::embedding::{+ QuantizedEmbeddingLookupArguments, QuantizedEmbeddingLookupKernel,+ },+ },+ config::QuantizationMode,+ parameters::ParameterTree,+};++use super::super::{EncodableBlock, EncodingParameters};++pub struct QuantizedEmbeddingLookup {+ kernel: QuantizedEmbeddingLookupKernel,+ weights_buffer: MTLBuffer,+ scales_buffer: MTLBuffer,+ biases_buffer: MTLBuffer,+ vocab_size: u32,+ model_dim: u32,+ group_size: u32,+}++impl QuantizedEmbeddingLookup {+ pub fn new(+ mtl_context: &MTLContext,+ data_type: DataType,+ vocab_size: usize,+ model_dim: usize,+ group_size: usize,+ mode: QuantizationMode,+ parameter_tree: &ParameterTree<Rc<MTLContext>>,+ ) -> Result<Self, QuantizedEmbeddingError> {+ let packing_divisor = mode.packing_divisor();++ let kernel =+ QuantizedEmbeddingLookupKernel::new(mtl_context, data_type, mode)?;++ // Load weights [vocab_size, model_dim/packing_divisor] as storage_type+ let mut weights = match parameter_tree.leaf("weights") {+ Ok(weights) => weights,+ Err(_) => parameter_tree.leaf("output_weights").map_err(|e| {+ QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(+ "Failed to load weights: {:?}",+ e+ )))+ })?,+ };++ if weights.data_type() != mode.storage_type() {+ return Err(QuantizedEmbeddingError::MetalError(+ MTLError::Generic(format!(+ "Expected packed weights of type {:?}, got {:?}",+ mode.storage_type(),+ weights.data_type()+ )),+ ));+ }++ // Load scales [vocab_size, num_groups]+ let mut scales = match parameter_tree.leaf("scales") {+ Ok(scales) => scales,+ Err(_) => parameter_tree.leaf("output_scales").map_err(|e| {+ QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(+ "Failed to load scales: {:?}",+ e+ )))+ })?,+ };++ // Validate shapes and types+ let num_groups = (model_dim + group_size - 1) / group_size;+ if weights.shape() != [vocab_size, model_dim / packing_divisor] {+ return Err(QuantizedEmbeddingError::MetalError(+ MTLError::Generic(format!(+ "Embedding lookup weights shape mismatch: got {:?}, expected [{}, {}]",+ weights.shape(),+ vocab_size,+ model_dim / packing_divisor+ )),+ ));+ }+ if scales.shape() != [vocab_size, num_groups] {+ return Err(QuantizedEmbeddingError::MetalError(+ MTLError::Generic(format!(+ "Embedding lookup scales shape mismatch: got {:?}, expected [{}, {}]",+ scales.shape(),+ vocab_size,+ num_groups+ )),+ ));+ }+ if scales.data_type() != data_type {+ return Err(QuantizedEmbeddingError::UnsupportedDataType(+ scales.data_type(),+ ));+ }++ // Load or create biases buffer [vocab_size, num_groups] (MLX key: "biases")+ let biases_buffer: MTLBuffer = match parameter_tree+ .leaf("biases")+ .or_else(|_| parameter_tree.leaf("output_biases"))+ {+ Ok(mut deq_biases) => {+ if deq_biases.shape() != [vocab_size, num_groups] {+ return Err(QuantizedEmbeddingError::MetalError(+ MTLError::Generic(format!(+ "Embedding lookup deq_biases shape mismatch: got {:?}, expected [{}, {}]",+ deq_biases.shape(),+ vocab_size,+ num_groups+ )),+ ));+ }+ if deq_biases.data_type() != data_type {+ return Err(QuantizedEmbeddingError::UnsupportedDataType(+ deq_biases.data_type(),+ ));+ }+ unsafe { deq_biases.mtl_buffer().to_owned() }+ },+ Err(_) => {+ let elem_size: usize = match data_type {+ DataType::F16 | DataType::BF16 => 2,+ DataType::F32 => 4,+ other => {+ return Err(+ QuantizedEmbeddingError::UnsupportedDataType(other),+ );+ },+ };+ let size_bytes = (vocab_size * num_groups * elem_size) as u64;+ let buf = mtl_context.device.new_buffer(+ size_bytes,+ metal::MTLResourceOptions::StorageModeShared,+ );+ unsafe {+ std::ptr::write_bytes(+ buf.contents(),+ 0,+ size_bytes as usize,+ );+ }+ buf+ },+ };++ let weights_buffer = unsafe { weights.mtl_buffer().to_owned() };+ let scales_buffer = unsafe { scales.mtl_buffer().to_owned() };++ Ok(Self {+ kernel,+ weights_buffer,+ scales_buffer,+ biases_buffer,+ vocab_size: vocab_size as u32,+ model_dim: model_dim as u32,+ group_size: group_size as u32,+ })+ }+}++impl EncodableBlock for QuantizedEmbeddingLookup {+ fn encode(+ &self,+ state: &mut ForwardPassState,+ command_buffer: &CommandBuffer,+ parameters: &EncodingParameters,+ ) {+ let arrays = state.arrays(&[ArrayId::TokenIds, ArrayId::Main]);+ let batch_size = state.active_suffix_length();+ let mut token_ids_array_mut = arrays[0].borrow_mut();+ let mut output_array_mut = arrays[1].borrow_mut();++ let token_ids_buffer = unsafe { token_ids_array_mut.mtl_buffer() };+ let output_buffer = unsafe { output_array_mut.mtl_buffer() };++ let root_command_buffer = command_buffer.root_command_buffer();+ let encoder = root_command_buffer.new_compute_command_encoder();++ let args = QuantizedEmbeddingLookupArguments {+ token_ids_buffer,+ weights_buffer: &self.weights_buffer,+ scales_buffer: &self.scales_buffer,+ biases_buffer: &self.biases_buffer,+ output_buffer,+ batch_size: batch_size as u32,+ vocab_size: self.vocab_size,+ model_dim: self.model_dim,+ group_size: self.group_size,+ };++ self.kernel+ .encode(encoder, args)+ .expect("Failed to encode quantized embedding lookup kernel");++ encoder.end_encoding();++ if parameters.wait_until_completed {+ let mtl_command_buffer =+ command_buffer.root_command_buffer().to_owned();+ command_buffer.commit_and_continue();+ mtl_command_buffer.wait_until_completed();+ }+ }+}+diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rsnew file mode 100644index 000000000..2b11f8a27--- /dev/null+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs@@ -0,0 +1,221 @@+use std::rc::Rc;++use metal::Buffer as MTLBuffer;+use mpsgraph::CommandBuffer;++use super::QuantizedEmbeddingError;+use crate::{+ Array, DataType,+ backends::metal::{+ MTLContext, MTLError,+ forward_pass::{ArrayId, ForwardPassState},+ kernel::quant_matmul::{+ QuantizationType, QuantizedMatmulArguments, QuantizedMatmulKernel,+ },+ },+ config::QuantizationMode,+ parameters::ParameterTree,+};++use super::super::{EncodableBlock, EncodingParameters};++pub struct QuantizedEmbeddingReadout {+ kernel: QuantizedMatmulKernel,+ weights_buffer: MTLBuffer,+ scales_buffer: MTLBuffer,+ biases_buffer: MTLBuffer,+ vocab_size: usize,+ model_dim: usize,+}++impl QuantizedEmbeddingReadout {+ pub fn new(+ mtl_context: &MTLContext,+ data_type: DataType,+ vocab_size: usize,+ model_dim: usize,+ group_size: usize,+ mode: QuantizationMode,+ parameter_tree: &ParameterTree<Rc<MTLContext>>,+ ) -> Result<Self, QuantizedEmbeddingError> {+ // Load weights [vocab_size, model_dim/2] as U8+ let mut weights = match parameter_tree.leaf("weights") {+ Ok(weights) => weights,+ Err(_) => parameter_tree.leaf("output_weights").map_err(|e| {+ QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(+ "Failed to load weights: {:?}",+ e+ )))+ })?,+ };++ // Load scales [vocab_size, num_groups]+ let mut scales = match parameter_tree.leaf("scales") {+ Ok(scales) => scales,+ Err(_) => parameter_tree.leaf("output_scales").map_err(|e| {+ QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(+ "Failed to load scales: {:?}",+ e+ )))+ })?,+ };++ // Validate shapes+ let num_groups = (model_dim + group_size - 1) / group_size;+ let packing_divisor = mode.packing_divisor();++ // Determine if weights are transposed by checking shape+ let weights_transposed = weights.shape()[0] == vocab_size;++ if weights.shape() != [vocab_size, model_dim / packing_divisor] {+ return Err(QuantizedEmbeddingError::MetalError(+ MTLError::Generic(format!(+ "Embedding readout weights shape mismatch: got {:?}, expected [{}, {}]",+ weights.shape(),+ vocab_size,+ model_dim / packing_divisor+ )),+ ));+ }+ if scales.shape() != [vocab_size, num_groups] {+ return Err(QuantizedEmbeddingError::MetalError(+ MTLError::Generic(format!(+ "Embedding readout scales shape mismatch: got {:?}, expected [{}, {}]",+ scales.shape(),+ vocab_size,+ num_groups+ )),+ ));+ }+ if scales.data_type() != data_type {+ return Err(QuantizedEmbeddingError::UnsupportedDataType(+ scales.data_type(),+ ));+ }++ // MLX requires per-group biases; if missing, create a zero buffer of shape [vocab_size, num_groups]+ let biases_buffer: MTLBuffer = match parameter_tree+ .leaf("biases")+ .or_else(|_| parameter_tree.leaf("output_biases"))+ {+ Ok(mut deq_biases) => {+ if deq_biases.shape() != [vocab_size, num_groups] {+ return Err(QuantizedEmbeddingError::MetalError(+ MTLError::Generic(format!(+ "Embedding readout deq_biases shape mismatch: got {:?}, expected [{}, {}]",+ deq_biases.shape(),+ vocab_size,+ num_groups+ )),+ ));+ }+ if deq_biases.data_type() != data_type {+ return Err(QuantizedEmbeddingError::UnsupportedDataType(+ deq_biases.data_type(),+ ));+ }+ unsafe { deq_biases.mtl_buffer().to_owned() }+ },+ Err(_) => {+ // Allocate zero-initialized biases buffer+ let elem_size: usize = match data_type {+ DataType::F16 | DataType::BF16 => 2,+ DataType::F32 => 4,+ other => {+ return Err(+ QuantizedEmbeddingError::UnsupportedDataType(other),+ );+ },+ };+ let size_bytes = (vocab_size * num_groups * elem_size) as u64;+ let buf = mtl_context.device.new_buffer(+ size_bytes,+ metal::MTLResourceOptions::StorageModeShared,+ );+ unsafe {+ std::ptr::write_bytes(+ buf.contents(),+ 0,+ size_bytes as usize,+ );+ }+ buf+ },+ };++ let weights_buffer = unsafe { weights.mtl_buffer().to_owned() };+ let scales_buffer = unsafe { scales.mtl_buffer().to_owned() };++ let kernel = QuantizedMatmulKernel::new(+ mtl_context,+ data_type,+ group_size,+ model_dim,+ vocab_size,+ mode,+ QuantizationType::Mlx,+ weights_transposed,+ )+ .map_err(|e| {+ QuantizedEmbeddingError::MetalError(MTLError::Generic(format!(+ "Failed to create kernel: {:?}",+ e+ )))+ })?;++ Ok(Self {+ kernel,+ weights_buffer,+ scales_buffer,+ biases_buffer,+ vocab_size,+ model_dim,+ })+ }+}++impl EncodableBlock for QuantizedEmbeddingReadout {+ fn encode(+ &self,+ state: &mut ForwardPassState,+ command_buffer: &CommandBuffer,+ parameters: &EncodingParameters,+ ) {+ let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);+ let batch_size = state.active_suffix_length();+ let mut input_array_mut = arrays[0].borrow_mut();+ let mut output_array_mut = arrays[1].borrow_mut();++ let input_buffer = unsafe { input_array_mut.mtl_buffer() };+ let output_buffer = unsafe { output_array_mut.mtl_buffer() };++ let root_command_buffer = command_buffer.root_command_buffer();+ let encoder = root_command_buffer.new_compute_command_encoder();++ let args = QuantizedMatmulArguments {+ a_buffer: input_buffer,+ b_buffer: &self.weights_buffer,+ scales_buffer: &self.scales_buffer,+ zero_points_or_biases_buffer: &self.biases_buffer,+ output_buffer,+ batch: batch_size as i32,+ input_dim: self.model_dim as i32,+ output_dim: self.vocab_size as i32,+ quantization_type: QuantizationType::Mlx,+ };++ self.kernel+ .encode(encoder, args)+ .expect("Failed to encode quantized embedding readout kernel");++ encoder.end_encoding();++ if parameters.wait_until_completed {+ let mtl_command_buffer =+ command_buffer.root_command_buffer().to_owned();+ command_buffer.commit_and_continue();+ mtl_command_buffer.wait_until_completed();+ }+ }+}+diff --git a/crates/uzu/src/backends/metal/encodable_block/mod.rs b/crates/uzu/src/backends/metal/encodable_block/mod.rsindex 5577febb6..2acd7b814 100644--- a/crates/uzu/src/backends/metal/encodable_block/mod.rs+++ b/crates/uzu/src/backends/metal/encodable_block/mod.rs@@ -35,8 +35,8 @@ pub use attention::Attention;pub use classifier_layer::ClassifierLayer;pub use decoder::Decoder;pub use embedding::{- QuantizedEmbeddingError, QuantizedEmbeddingLookup,- QuantizedEmbeddingReadout,+ FullPrecisionEmbeddingReadout, QuantizedEmbeddingError,+ QuantizedEmbeddingLookup, QuantizedEmbeddingReadout,};pub use encoding_parameters::EncodingParameters;pub use layer::LayerExecutables;diff --git a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rsindex 013620942..53ed69431 100644--- a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs+++ b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs@@ -7,8 +7,9 @@ use mpsgraph::{use objc2::rc::{Retained, autoreleasepool};use super::{- EncodableBlock, FullPrecisionLinear, MlpBlock, MoeBlock,- QuantizedEmbeddingLookup, QuantizedEmbeddingReadout, QuantizedLinear,+ EncodableBlock, FullPrecisionEmbeddingReadout, FullPrecisionLinear,+ MlpBlock, MoeBlock, QuantizedEmbeddingLookup, QuantizedEmbeddingReadout,+ QuantizedLinear,};use crate::{DataType,@@ -538,14 +539,6 @@ pub fn readout_block(compilation_descriptor: &CompilationDescriptor,parameter_tree: &ParameterTree<Rc<MTLContext>>,) -> Box<dyn EncodableBlock> {- let graph = Graph::new();-- let input_shape = [-1 as isize, config.model_dim as isize];- let input_data_type: DataType =- config.output_norm_config.scale_precision.into();- let input_placeholder = placeholder(&graph, &input_shape, input_data_type);- let input_shaped_type = shaped_type(&input_shape, input_data_type);-match config.embedding_config {EmbeddingConfig::Tied {precision,@@ -555,52 +548,14 @@ pub fn readout_block(.subtree("embedding").expect("Failed to get embedding subtree");- let weights_shape =- [config.vocab_size as isize, config.model_dim as isize];- let weights_data_type: DataType = precision.into();- let weights_shaped_type =- shaped_type(&weights_shape, weights_data_type);- let weights_placeholder =- placeholder(&graph, &weights_shape, weights_data_type);-- let weights_transposed =- graph.transpose(&weights_placeholder, &[1, 0], None);- let input_shape = [-1 as isize, config.model_dim as isize];- let input_data_type: DataType =- config.output_norm_config.scale_precision.into();- let input_placeholder =- placeholder(&graph, &input_shape, input_data_type);- let input_shaped_type = shaped_type(&input_shape, input_data_type);-- let output = embeddings_readout_subgraph(- &graph,- &input_placeholder,- &weights_transposed,+ let block = FullPrecisionEmbeddingReadout::new(+ context,+ precision.into(),+ config.vocab_size,+ config.model_dim,+ &embeddings_tree,)- .unwrap();-- let feeds = HashMap::from([- (&*input_placeholder, &*input_shaped_type),- (&*weights_placeholder, &*weights_shaped_type),- ]);-- let executable = graph.compile(- &MPSDevice::with_device(&context.device),- &feeds,- &[&output],- None,- Some(compilation_descriptor),- );-- let block = MPSGraphBlock::new(- executable,- make_execution_descriptor(),- IOArrays::new(- vec![ArrayId::Main, ArrayId::EmbeddingsOutputWeights]- .into_boxed_slice(),- vec![ArrayId::Logits].into_boxed_slice(),- ),- );+ .expect("Failed to create full precision embedding readout");Box::new(block)},@@ -611,53 +566,15 @@ pub fn readout_block(let embeddings_tree = parameter_tree.subtree("embedding").expect("Failed to get embedding subtree");- // Use MPSGraph readout path- let weights_shape =- [config.vocab_size as isize, config.model_dim as isize];- let weights_data_type: DataType = precision.into();- let weights_shaped_type =- shaped_type(&weights_shape, weights_data_type);- let weights_placeholder =- placeholder(&graph, &weights_shape, weights_data_type);-- let weights_transposed =- graph.transpose(&weights_placeholder, &[1, 0], None);- let input_shape = [-1 as isize, config.model_dim as isize];- let input_data_type: DataType =- config.output_norm_config.scale_precision.into();- let input_placeholder =- placeholder(&graph, &input_shape, input_data_type);- let input_shaped_type = shaped_type(&input_shape, input_data_type);- let output = embeddings_readout_subgraph(- &graph,- &input_placeholder,- &weights_transposed,+ let block = FullPrecisionEmbeddingReadout::new(+ context,+ precision.into(),+ config.vocab_size,+ config.model_dim,+ &embeddings_tree,)- .unwrap();-- let feeds = HashMap::from([- (&*input_placeholder, &*input_shaped_type),- (&*weights_placeholder, &*weights_shaped_type),- ]);-- let executable = graph.compile(- &MPSDevice::with_device(&context.device),- &feeds,- &[&output],- None,- Some(compilation_descriptor),- );-- let block = MPSGraphBlock::new(- executable,- make_execution_descriptor(),- IOArrays::new(- vec![ArrayId::Main, ArrayId::EmbeddingsOutputWeights]- .into_boxed_slice(),- vec![ArrayId::Logits].into_boxed_slice(),- ),- );+ .expect("Failed to create full precision embedding readout");Box::new(block)},diff --git a/crates/uzu/src/backends/metal/kernel/embedding/mod.rs b/crates/uzu/src/backends/metal/kernel/embedding/mod.rsindex f5513f2af..4adbc459a 100644--- a/crates/uzu/src/backends/metal/kernel/embedding/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/embedding/mod.rs@@ -1,20 +1,14 @@-use std::{mem::size_of, rc::Rc};+use std::mem::size_of;use metal::{Buffer as MTLBuffer, ComputeCommandEncoderRef,ComputePipelineState as MTLComputePipelineState, MTLSize,};-use super::super::{MTLContext, kernel::matmul::MatmulArguments};+use super::super::MTLContext;use crate::{- DataType,- backends::metal::{- ForwardPassState, MTLError, encodable_block::QuantizedEmbeddingError,- forward_pass::ArrayId,- },+ DataType, backends::metal::encodable_block::QuantizedEmbeddingError,config::QuantizationMode,- device::array::Array,- parameters::ParameterTree,};pub struct QuantizedEmbeddingLookupKernel {@@ -125,102 +119,3 @@ impl QuantizedEmbeddingLookupKernel {Ok(())}}--pub struct FullPrecisionReadoutKernelBlock {- kernel: std::cell::RefCell<super::super::kernel::matmul::MatmulKernel>,- weights_buffer: MTLBuffer,- vocab_size: usize,- model_dim: usize,-}--impl FullPrecisionReadoutKernelBlock {- pub fn new(- mtl_context: &MTLContext,- data_type: DataType,- vocab_size: usize,- model_dim: usize,- parameter_tree: &ParameterTree<Rc<MTLContext>>,- ) -> Result<Self, MTLError> {- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)- {- return Err(MTLError::Generic(format!(- "Unsupported data type for full precision readout: {:?}",- data_type- )));- }-- let mut weights = match parameter_tree.leaf("weights") {- Ok(weights) => weights,- Err(_) => parameter_tree.leaf("output_weights").map_err(|e| {- MTLError::Generic(format!("Failed to load weights: {:?}", e))- })?,- };-- if weights.shape() != [vocab_size, model_dim] {- return Err(MTLError::Generic(format!(- "Embedding readout weights shape mismatch: got {:?}, expected [{}, {}]",- weights.shape(),- vocab_size,- model_dim- )));- }-- if weights.data_type() != data_type {- return Err(MTLError::Generic(format!(- "Weights dtype mismatch: got {:?}, expected {:?}",- weights.data_type(),- data_type- )));- }-- let weights_buffer = unsafe { weights.mtl_buffer().to_owned() };-- let kernel = super::super::kernel::matmul::MatmulKernel::new(- mtl_context,- data_type,- false,- true,- )?;-- Ok(Self {- kernel: std::cell::RefCell::new(kernel),- weights_buffer,- vocab_size,- model_dim,- })- }-}--impl FullPrecisionReadoutKernelBlock {- fn encode_impl(- &self,- state: &mut ForwardPassState,- encoder: &metal::ComputeCommandEncoderRef,- ) {- let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);- let batch_size = state.active_suffix_length();- let mut input_array_mut = arrays[0].borrow_mut();- let mut output_array_mut = arrays[1].borrow_mut();-- let input_buffer = unsafe { input_array_mut.mtl_buffer() };- let output_buffer = unsafe { output_array_mut.mtl_buffer() };-- let args = MatmulArguments {- a: input_buffer,- b: &self.weights_buffer,- d: output_buffer,- batch: batch_size as i32,- input_dim: self.model_dim as i32,- output_dim: self.vocab_size as i32,- lda: self.model_dim as i32,- ldb: self.model_dim as i32, // B is [vocab_size, model_dim], stride is model_dim- ldd: self.vocab_size as i32,- batch_count: 1,- };-- self.kernel- .borrow_mut()- .encode(state.mtl_context(), encoder, args)- .expect("Failed to encode full precision readout kernel");- }-}