A high-performance inference engine for AI models
generalize embedding (#203)
eugenee903f8324e6b
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/error.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/error.rsdeleted file mode 100644index 34f566354..000000000--- a/crates/uzu/src/backends/metal/encodable_block/embedding/error.rs+++ /dev/null@@ -1,9 +0,0 @@-use crate::{DataType, backends::metal::MetalError};--#[derive(Debug, thiserror::Error)]-pub enum EmbeddingError {- #[error("Metal error: {0}")]- MetalError(#[from] MetalError),- #[error("Unsupported data type: {0:?}")]- UnsupportedDataType(DataType),-}diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rsindex f90626864..296eda3f1 100644--- a/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rs+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rs@@ -1,18 +1,16 @@//! Embedding encodables (quantized and full-precision).-mod error;-mod quantized_readout;--pub use error::EmbeddingError;-pub use quantized_readout::QuantizedEmbeddingReadout;-use super::super::Metal;use crate::encodable_block::{FullPrecisionEmbeddingLookup as GenericFullPrecisionEmbeddingLookup,FullPrecisionEmbeddingReadout as GenericFullPrecisionEmbeddingReadout,QuantizedEmbeddingLookup as GenericQuantizedEmbeddingLookup,+ QuantizedEmbeddingReadout as GenericQuantizedEmbeddingReadout,+ QuantizedEmbeddingReadoutError as GenericQuantizedEmbeddingReadoutError,};+pub type EmbeddingError = GenericQuantizedEmbeddingReadoutError<Metal>;+pub type QuantizedEmbeddingReadout = GenericQuantizedEmbeddingReadout<Metal>;pub type FullPrecisionEmbeddingReadout = GenericFullPrecisionEmbeddingReadout<Metal>;pub type FullPrecisionEmbeddingLookup = GenericFullPrecisionEmbeddingLookup<Metal>;pub type QuantizedEmbeddingLookup = GenericQuantizedEmbeddingLookup<Metal>;diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rsdeleted file mode 100644index ea479efc0..000000000--- a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs+++ /dev/null@@ -1,246 +0,0 @@-use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder};-use objc2::{rc::Retained, runtime::ProtocolObject};--use super::EmbeddingError;-use crate::{- DataType,- backends::{- common::{- Context,- kernel::quant_matmul::{- QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulKernelEncodable,- QuantizedMatmulType,- },- },- metal::{Metal, MetalContext, MetalError},- },- config::QuantizationMode,- encodable_block::{EncodableBlock, EncodingParameters},- forward_pass::state::{ArrayId, ForwardPassState},- parameters::ParameterTree,-};--pub struct QuantizedEmbeddingReadout {- kernel: QuantizedMatmulKernelEncodable<Metal>,- weights_buffer: Retained<ProtocolObject<dyn MTLBuffer>>,- scales_buffer: Retained<ProtocolObject<dyn MTLBuffer>>,- biases_buffer: Retained<ProtocolObject<dyn MTLBuffer>>,- vocab_size: usize,- model_dim: usize,-}--impl QuantizedEmbeddingReadout {- pub fn new_tied(- mtl_context: &MetalContext,- data_type: DataType,- vocab_size: usize,- model_dim: usize,- group_size: usize,- mode: QuantizationMode,- parameter_tree: &ParameterTree<MetalContext>,- ) -> Result<Self, EmbeddingError> {- Self::new_with_names(- mtl_context,- data_type,- vocab_size,- model_dim,- group_size,- mode,- "weights",- "scales",- "biases",- parameter_tree,- )- }-- pub fn new_untied_output(- mtl_context: &MetalContext,- data_type: DataType,- vocab_size: usize,- model_dim: usize,- group_size: usize,- mode: QuantizationMode,- parameter_tree: &ParameterTree<MetalContext>,- ) -> Result<Self, EmbeddingError> {- Self::new_with_names(- mtl_context,- data_type,- vocab_size,- model_dim,- group_size,- mode,- "output_weights",- "output_scales",- "output_biases",- parameter_tree,- )- }-- fn new_with_names(- mtl_context: &MetalContext,- data_type: DataType,- vocab_size: usize,- model_dim: usize,- group_size: usize,- mode: QuantizationMode,- weights_name: &str,- scales_name: &str,- biases_name: &str,- parameter_tree: &ParameterTree<MetalContext>,- ) -> Result<Self, EmbeddingError> {- // Load weights [vocab_size, model_dim/2] as U8- let weights = parameter_tree- .leaf(weights_name)- .map_err(|e| EmbeddingError::MetalError(MetalError::Generic(format!("Failed to load weights: {:?}", e))))?;-- // Load scales [vocab_size, num_groups]- let scales = parameter_tree- .leaf(scales_name)- .map_err(|e| EmbeddingError::MetalError(MetalError::Generic(format!("Failed to load scales: {:?}", e))))?;-- // Validate shapes- let num_groups = (model_dim + group_size - 1) / group_size;- let packing_divisor = mode.packing_divisor();-- // Determine if weights are transposed by checking shape- let weights_transposed = weights.shape()[0] == vocab_size;-- if weights.shape() != [vocab_size, model_dim / packing_divisor] {- return Err(EmbeddingError::MetalError(MetalError::Generic(format!(- "Embedding readout weights shape mismatch: got {:?}, expected [{}, {}]",- weights.shape(),- vocab_size,- model_dim / packing_divisor- ))));- }- if scales.shape() != [vocab_size, num_groups] {- return Err(EmbeddingError::MetalError(MetalError::Generic(format!(- "Embedding readout scales shape mismatch: got {:?}, expected [{}, {}]",- scales.shape(),- vocab_size,- num_groups- ))));- }- if scales.data_type() != data_type {- return Err(EmbeddingError::UnsupportedDataType(scales.data_type()));- }-- // MLX requires per-group biases; if missing, create a zero buffer of shape [vocab_size, num_groups]- let biases_buffer: Retained<ProtocolObject<dyn MTLBuffer>> = match parameter_tree.leaf(biases_name) {- Ok(deq_biases) => {- if deq_biases.shape() != [vocab_size, num_groups] {- return Err(EmbeddingError::MetalError(MetalError::Generic(format!(- "Embedding readout deq_biases shape mismatch: got {:?}, expected [{}, {}]",- deq_biases.shape(),- vocab_size,- num_groups- ))));- }- if deq_biases.data_type() != data_type {- return Err(EmbeddingError::UnsupportedDataType(deq_biases.data_type()));- }- deq_biases.buffer().to_owned().into()- },- Err(_) => {- // Allocate zero-initialized biases buffer- let elem_size: usize = match data_type {- DataType::F16 | DataType::BF16 => 2,- DataType::F32 => 4,- other => {- return Err(EmbeddingError::UnsupportedDataType(other));- },- };- let size_bytes = vocab_size * num_groups * elem_size;- let buf = mtl_context.create_buffer(size_bytes).expect("Failed to allocate buffer");- unsafe {- std::ptr::write_bytes(metal::MTLBuffer::contents(&*buf).as_ptr(), 0, size_bytes as usize);- }- buf- },- };-- let weights_buffer = weights.buffer().to_owned().into();- let scales_buffer = scales.buffer().to_owned().into();-- let kernel = QuantizedMatmulKernelEncodable::new(- mtl_context,- QuantizedMatmulConfiguration {- data_type,- group_size,- input_dim: model_dim,- output_dim: vocab_size,- mode,- quantization_type: QuantizedMatmulType::Mlx,- weights_transposed,- },- )- .map_err(|e| EmbeddingError::MetalError(MetalError::Generic(format!("Failed to create kernel: {:?}", e))))?;-- Ok(Self {- kernel,- weights_buffer,- scales_buffer,- biases_buffer,- vocab_size,- model_dim,- })- }-}--impl EncodableBlock<Metal> for QuantizedEmbeddingReadout {- fn encode(- &self,- state: &mut ForwardPassState<Metal>,- parameters: &EncodingParameters<Metal>,- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- ) {- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");- self.encode_with_shared_encoder(state, parameters, &encoder);- encoder.end_encoding();-- if parameters.wait_until_completed {- command_buffer.commit();- command_buffer.wait_until_completed();- }- }-- fn supports_shared_encoder(&self) -> bool {- true- }-- fn encode_with_shared_encoder(- &self,- state: &mut ForwardPassState<Metal>,- _parameters: &EncodingParameters<Metal>,- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- ) {- let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);- let batch_size = state.sampling_length();- if batch_size == 0 {- return;- }- let sampling_start = state.sampling_start();- let input_array_mut = arrays[0].borrow_mut();- let output_array_mut = arrays[1].borrow_mut();-- let elem_size = input_array_mut.data_type().size_in_bytes();- let input_buffer = input_array_mut.buffer();- let output_buffer = output_array_mut.buffer();- let a_offset = (sampling_start * self.model_dim * elem_size) as u64;-- let args = QuantizedMatmulArguments {- a_buffer: input_buffer,- a_offset: a_offset as usize,- b_buffer: &self.weights_buffer,- scales_buffer: &self.scales_buffer,- zero_points_or_biases_buffer: &self.biases_buffer,- output_buffer,- batch: batch_size,- input_dim: self.model_dim,- output_dim: self.vocab_size,- quantization_type: QuantizedMatmulType::Mlx,- };-- self.kernel.encode(encoder, args).expect("Failed to encode quantized embedding readout kernel");- }-}diff --git a/crates/uzu/src/encodable_block/mod.rs b/crates/uzu/src/encodable_block/mod.rsindex 050224bc2..1aaf72743 100644--- a/crates/uzu/src/encodable_block/mod.rs+++ b/crates/uzu/src/encodable_block/mod.rs@@ -16,6 +16,7 @@ mod pooling;mod prediction_head;mod qk_norm;mod quantized_embedding_lookup;+mod quantized_embedding_readout;mod quantized_linear;mod rms_norm;mod rope;@@ -37,6 +38,7 @@ pub use pooling::Pooling;pub use prediction_head::ClassifierPredictionHead;pub use qk_norm::{QKNorm, QKNormError};pub use quantized_embedding_lookup::{QuantizedEmbeddingLookup, QuantizedEmbeddingLookupError};+pub use quantized_embedding_readout::{QuantizedEmbeddingReadout, QuantizedEmbeddingReadoutError};pub use quantized_linear::{QuantizedLinear, QuantizedLinearError};pub use rms_norm::{RMSNorm, RMSNormError};pub use rope::Rope;diff --git a/crates/uzu/src/encodable_block/quantized_embedding_readout.rs b/crates/uzu/src/encodable_block/quantized_embedding_readout.rsnew file mode 100644index 000000000..bee6ab076--- /dev/null+++ b/crates/uzu/src/encodable_block/quantized_embedding_readout.rs@@ -0,0 +1,267 @@+//! Quantized embedding readout encodable.++use thiserror::Error;++use super::{EncodableBlock, EncodingParameters};+use crate::{+ DataType,+ backends::common::{+ Backend, Context, NativeBuffer,+ kernel::quant_matmul::{+ QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulError,+ QuantizedMatmulKernelEncodable, QuantizedMatmulType,+ },+ },+ config::QuantizationMode,+ forward_pass::state::{ArrayId, ForwardPassState},+ parameters::{ParameterLoaderError, ParameterTree},+};++#[derive(Debug, Error)]+pub enum QuantizedEmbeddingReadoutError<B: Backend> {+ #[error("Backend error: {0}")]+ BackendError(#[source] B::Error),+ #[error("Matmul error: {0}")]+ MatmulError(#[source] QuantizedMatmulError<B>),+ #[error("Parameter loading error: {0}")]+ ParameterError(ParameterLoaderError),+ #[error("Unsupported data type: {0:?}")]+ UnsupportedDataType(DataType),+ #[error(+ "Embedding readout weights shape mismatch: got {got:?}, expected [{expected_vocab_size}, {expected_packed_dim}]"+ )]+ InvalidWeightsShape {+ got: Box<[usize]>,+ expected_vocab_size: usize,+ expected_packed_dim: usize,+ },+ #[error(+ "Embedding readout scales shape mismatch: got {got:?}, expected [{expected_vocab_size}, {expected_num_groups}]"+ )]+ InvalidScalesShape {+ got: Box<[usize]>,+ expected_vocab_size: usize,+ expected_num_groups: usize,+ },+ #[error("Scales dtype mismatch: got {got:?}, expected {expected:?}")]+ InvalidScalesDataType {+ expected: DataType,+ got: DataType,+ },+ #[error(+ "Embedding readout deq_biases shape mismatch: got {got:?}, expected [{expected_vocab_size}, {expected_num_groups}]"+ )]+ InvalidBiasesShape {+ got: Box<[usize]>,+ expected_vocab_size: usize,+ expected_num_groups: usize,+ },+ #[error("Biases dtype mismatch: got {got:?}, expected {expected:?}")]+ InvalidBiasesDataType {+ expected: DataType,+ got: DataType,+ },+}++pub struct QuantizedEmbeddingReadout<B: Backend> {+ kernel: QuantizedMatmulKernelEncodable<B>,+ weights_buffer: B::NativeBuffer,+ scales_buffer: B::NativeBuffer,+ biases_buffer: B::NativeBuffer,+ vocab_size: usize,+ model_dim: usize,+}++impl<B: Backend> QuantizedEmbeddingReadout<B> {+ pub fn new_tied(+ context: &B::Context,+ data_type: DataType,+ vocab_size: usize,+ model_dim: usize,+ group_size: usize,+ mode: QuantizationMode,+ parameter_tree: &ParameterTree<B::Context>,+ ) -> Result<Self, QuantizedEmbeddingReadoutError<B>> {+ Self::new_with_names(+ context,+ data_type,+ vocab_size,+ model_dim,+ group_size,+ mode,+ "weights",+ "scales",+ "biases",+ parameter_tree,+ )+ }++ pub fn new_untied_output(+ context: &B::Context,+ data_type: DataType,+ vocab_size: usize,+ model_dim: usize,+ group_size: usize,+ mode: QuantizationMode,+ parameter_tree: &ParameterTree<B::Context>,+ ) -> Result<Self, QuantizedEmbeddingReadoutError<B>> {+ Self::new_with_names(+ context,+ data_type,+ vocab_size,+ model_dim,+ group_size,+ mode,+ "output_weights",+ "output_scales",+ "output_biases",+ parameter_tree,+ )+ }++ fn new_with_names(+ context: &B::Context,+ data_type: DataType,+ vocab_size: usize,+ model_dim: usize,+ group_size: usize,+ mode: QuantizationMode,+ weights_name: &str,+ scales_name: &str,+ biases_name: &str,+ parameter_tree: &ParameterTree<B::Context>,+ ) -> Result<Self, QuantizedEmbeddingReadoutError<B>> {+ let weights = parameter_tree.leaf(weights_name).map_err(QuantizedEmbeddingReadoutError::ParameterError)?;+ let scales = parameter_tree.leaf(scales_name).map_err(QuantizedEmbeddingReadoutError::ParameterError)?;++ let num_groups = (model_dim + group_size - 1) / group_size;+ let packing_divisor = mode.packing_divisor();+ let weights_transposed = weights.shape()[0] == vocab_size;++ if weights.shape() != [vocab_size, model_dim / packing_divisor] {+ return Err(QuantizedEmbeddingReadoutError::InvalidWeightsShape {+ got: weights.shape().to_vec().into_boxed_slice(),+ expected_vocab_size: vocab_size,+ expected_packed_dim: model_dim / packing_divisor,+ });+ }++ if scales.shape() != [vocab_size, num_groups] {+ return Err(QuantizedEmbeddingReadoutError::InvalidScalesShape {+ got: scales.shape().to_vec().into_boxed_slice(),+ expected_vocab_size: vocab_size,+ expected_num_groups: num_groups,+ });+ }++ if scales.data_type() != data_type {+ return Err(QuantizedEmbeddingReadoutError::InvalidScalesDataType {+ expected: data_type,+ got: scales.data_type(),+ });+ }++ let biases_buffer = match parameter_tree.leaf(biases_name) {+ Ok(deq_biases) => {+ if deq_biases.shape() != [vocab_size, num_groups] {+ return Err(QuantizedEmbeddingReadoutError::InvalidBiasesShape {+ got: deq_biases.shape().to_vec().into_boxed_slice(),+ expected_vocab_size: vocab_size,+ expected_num_groups: num_groups,+ });+ }+ if deq_biases.data_type() != data_type {+ return Err(QuantizedEmbeddingReadoutError::InvalidBiasesDataType {+ expected: data_type,+ got: deq_biases.data_type(),+ });+ }+ deq_biases.buffer().clone()+ },+ Err(_) => {+ let element_size = match data_type {+ DataType::F16 | DataType::BF16 => 2,+ DataType::F32 => 4,+ other => {+ return Err(QuantizedEmbeddingReadoutError::UnsupportedDataType(other));+ },+ };+ let size_bytes = vocab_size * num_groups * element_size;+ let buffer = context.create_buffer(size_bytes).map_err(QuantizedEmbeddingReadoutError::BackendError)?;++ unsafe {+ std::ptr::write_bytes(buffer.cpu_ptr().as_ptr().cast::<u8>(), 0, size_bytes);+ }++ buffer+ },+ };++ let kernel = QuantizedMatmulKernelEncodable::new(+ context,+ QuantizedMatmulConfiguration {+ data_type,+ group_size,+ input_dim: model_dim,+ output_dim: vocab_size,+ mode,+ quantization_type: QuantizedMatmulType::Mlx,+ weights_transposed,+ },+ )+ .map_err(QuantizedEmbeddingReadoutError::MatmulError)?;++ Ok(Self {+ kernel,+ weights_buffer: weights.buffer().clone(),+ scales_buffer: scales.buffer().clone(),+ biases_buffer,+ vocab_size,+ model_dim,+ })+ }+}++impl<B: Backend> EncodableBlock<B> for QuantizedEmbeddingReadout<B> {+ fn supports_shared_encoder(&self) -> bool {+ true+ }++ fn encode_with_shared_encoder(+ &self,+ state: &mut ForwardPassState<B>,+ _parameters: &EncodingParameters<B>,+ encoder: &B::ComputeEncoder,+ ) {+ let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);+ let batch_size = state.sampling_length();+ if batch_size == 0 {+ return;+ }++ let sampling_start = state.sampling_start();+ let input_array = arrays[0].borrow_mut();+ let output_array = arrays[1].borrow_mut();++ let element_size = input_array.data_type().size_in_bytes();+ let a_offset = sampling_start * self.model_dim * element_size;++ self.kernel+ .encode(+ encoder,+ QuantizedMatmulArguments {+ a_buffer: input_array.buffer(),+ a_offset,+ b_buffer: &self.weights_buffer,+ scales_buffer: &self.scales_buffer,+ zero_points_or_biases_buffer: &self.biases_buffer,+ output_buffer: output_array.buffer(),+ batch: batch_size,+ input_dim: self.model_dim,+ output_dim: self.vocab_size,+ quantization_type: QuantizedMatmulType::Mlx,+ },+ )+ .expect("Failed to encode quantized embedding readout kernel");+ }+}