Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

generalize embedding (#203)

eugenee903f8324e6b
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/error.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/error.rs
deleted file mode 100644
index 34f566354..000000000
--- a/crates/uzu/src/backends/metal/encodable_block/embedding/error.rs
+++ /dev/null
@@ -1,9 +0,0 @@
-use crate::{DataType, backends::metal::MetalError};
-
-#[derive(Debug, thiserror::Error)]
-pub enum EmbeddingError {
- #[error("Metal error: {0}")]
- MetalError(#[from] MetalError),
- #[error("Unsupported data type: {0:?}")]
- UnsupportedDataType(DataType),
-}
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rs
index f90626864..296eda3f1 100644
--- a/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rs
@@ -1,18 +1,16 @@
//! Embedding encodables (quantized and full-precision).
-mod error;
-mod quantized_readout;
-
-pub use error::EmbeddingError;
-pub use quantized_readout::QuantizedEmbeddingReadout;
-
use super::super::Metal;
use crate::encodable_block::{
FullPrecisionEmbeddingLookup as GenericFullPrecisionEmbeddingLookup,
FullPrecisionEmbeddingReadout as GenericFullPrecisionEmbeddingReadout,
QuantizedEmbeddingLookup as GenericQuantizedEmbeddingLookup,
+ QuantizedEmbeddingReadout as GenericQuantizedEmbeddingReadout,
+ QuantizedEmbeddingReadoutError as GenericQuantizedEmbeddingReadoutError,
};
+pub type EmbeddingError = GenericQuantizedEmbeddingReadoutError<Metal>;
+pub type QuantizedEmbeddingReadout = GenericQuantizedEmbeddingReadout<Metal>;
pub type FullPrecisionEmbeddingReadout = GenericFullPrecisionEmbeddingReadout<Metal>;
pub type FullPrecisionEmbeddingLookup = GenericFullPrecisionEmbeddingLookup<Metal>;
pub type QuantizedEmbeddingLookup = GenericQuantizedEmbeddingLookup<Metal>;
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs
deleted file mode 100644
index ea479efc0..000000000
--- a/crates/uzu/src/backends/metal/encodable_block/embedding/quantized_readout.rs
+++ /dev/null
@@ -1,246 +0,0 @@
-use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder};
-use objc2::{rc::Retained, runtime::ProtocolObject};
-
-use super::EmbeddingError;
-use crate::{
- DataType,
- backends::{
- common::{
- Context,
- kernel::quant_matmul::{
- QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulKernelEncodable,
- QuantizedMatmulType,
- },
- },
- metal::{Metal, MetalContext, MetalError},
- },
- config::QuantizationMode,
- encodable_block::{EncodableBlock, EncodingParameters},
- forward_pass::state::{ArrayId, ForwardPassState},
- parameters::ParameterTree,
-};
-
-pub struct QuantizedEmbeddingReadout {
- kernel: QuantizedMatmulKernelEncodable<Metal>,
- weights_buffer: Retained<ProtocolObject<dyn MTLBuffer>>,
- scales_buffer: Retained<ProtocolObject<dyn MTLBuffer>>,
- biases_buffer: Retained<ProtocolObject<dyn MTLBuffer>>,
- vocab_size: usize,
- model_dim: usize,
-}
-
-impl QuantizedEmbeddingReadout {
- pub fn new_tied(
- mtl_context: &MetalContext,
- data_type: DataType,
- vocab_size: usize,
- model_dim: usize,
- group_size: usize,
- mode: QuantizationMode,
- parameter_tree: &ParameterTree<MetalContext>,
- ) -> Result<Self, EmbeddingError> {
- Self::new_with_names(
- mtl_context,
- data_type,
- vocab_size,
- model_dim,
- group_size,
- mode,
- "weights",
- "scales",
- "biases",
- parameter_tree,
- )
- }
-
- pub fn new_untied_output(
- mtl_context: &MetalContext,
- data_type: DataType,
- vocab_size: usize,
- model_dim: usize,
- group_size: usize,
- mode: QuantizationMode,
- parameter_tree: &ParameterTree<MetalContext>,
- ) -> Result<Self, EmbeddingError> {
- Self::new_with_names(
- mtl_context,
- data_type,
- vocab_size,
- model_dim,
- group_size,
- mode,
- "output_weights",
- "output_scales",
- "output_biases",
- parameter_tree,
- )
- }
-
- fn new_with_names(
- mtl_context: &MetalContext,
- data_type: DataType,
- vocab_size: usize,
- model_dim: usize,
- group_size: usize,
- mode: QuantizationMode,
- weights_name: &str,
- scales_name: &str,
- biases_name: &str,
- parameter_tree: &ParameterTree<MetalContext>,
- ) -> Result<Self, EmbeddingError> {
- // Load weights [vocab_size, model_dim/2] as U8
- let weights = parameter_tree
- .leaf(weights_name)
- .map_err(|e| EmbeddingError::MetalError(MetalError::Generic(format!("Failed to load weights: {:?}", e))))?;
-
- // Load scales [vocab_size, num_groups]
- let scales = parameter_tree
- .leaf(scales_name)
- .map_err(|e| EmbeddingError::MetalError(MetalError::Generic(format!("Failed to load scales: {:?}", e))))?;
-
- // Validate shapes
- let num_groups = (model_dim + group_size - 1) / group_size;
- let packing_divisor = mode.packing_divisor();
-
- // Determine if weights are transposed by checking shape
- let weights_transposed = weights.shape()[0] == vocab_size;
-
- if weights.shape() != [vocab_size, model_dim / packing_divisor] {
- return Err(EmbeddingError::MetalError(MetalError::Generic(format!(
- "Embedding readout weights shape mismatch: got {:?}, expected [{}, {}]",
- weights.shape(),
- vocab_size,
- model_dim / packing_divisor
- ))));
- }
- if scales.shape() != [vocab_size, num_groups] {
- return Err(EmbeddingError::MetalError(MetalError::Generic(format!(
- "Embedding readout scales shape mismatch: got {:?}, expected [{}, {}]",
- scales.shape(),
- vocab_size,
- num_groups
- ))));
- }
- if scales.data_type() != data_type {
- return Err(EmbeddingError::UnsupportedDataType(scales.data_type()));
- }
-
- // MLX requires per-group biases; if missing, create a zero buffer of shape [vocab_size, num_groups]
- let biases_buffer: Retained<ProtocolObject<dyn MTLBuffer>> = match parameter_tree.leaf(biases_name) {
- Ok(deq_biases) => {
- if deq_biases.shape() != [vocab_size, num_groups] {
- return Err(EmbeddingError::MetalError(MetalError::Generic(format!(
- "Embedding readout deq_biases shape mismatch: got {:?}, expected [{}, {}]",
- deq_biases.shape(),
- vocab_size,
- num_groups
- ))));
- }
- if deq_biases.data_type() != data_type {
- return Err(EmbeddingError::UnsupportedDataType(deq_biases.data_type()));
- }
- deq_biases.buffer().to_owned().into()
- },
- Err(_) => {
- // Allocate zero-initialized biases buffer
- let elem_size: usize = match data_type {
- DataType::F16 | DataType::BF16 => 2,
- DataType::F32 => 4,
- other => {
- return Err(EmbeddingError::UnsupportedDataType(other));
- },
- };
- let size_bytes = vocab_size * num_groups * elem_size;
- let buf = mtl_context.create_buffer(size_bytes).expect("Failed to allocate buffer");
- unsafe {
- std::ptr::write_bytes(metal::MTLBuffer::contents(&*buf).as_ptr(), 0, size_bytes as usize);
- }
- buf
- },
- };
-
- let weights_buffer = weights.buffer().to_owned().into();
- let scales_buffer = scales.buffer().to_owned().into();
-
- let kernel = QuantizedMatmulKernelEncodable::new(
- mtl_context,
- QuantizedMatmulConfiguration {
- data_type,
- group_size,
- input_dim: model_dim,
- output_dim: vocab_size,
- mode,
- quantization_type: QuantizedMatmulType::Mlx,
- weights_transposed,
- },
- )
- .map_err(|e| EmbeddingError::MetalError(MetalError::Generic(format!("Failed to create kernel: {:?}", e))))?;
-
- Ok(Self {
- kernel,
- weights_buffer,
- scales_buffer,
- biases_buffer,
- vocab_size,
- model_dim,
- })
- }
-}
-
-impl EncodableBlock<Metal> for QuantizedEmbeddingReadout {
- fn encode(
- &self,
- state: &mut ForwardPassState<Metal>,
- parameters: &EncodingParameters<Metal>,
- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- ) {
- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
- self.encode_with_shared_encoder(state, parameters, &encoder);
- encoder.end_encoding();
-
- if parameters.wait_until_completed {
- command_buffer.commit();
- command_buffer.wait_until_completed();
- }
- }
-
- fn supports_shared_encoder(&self) -> bool {
- true
- }
-
- fn encode_with_shared_encoder(
- &self,
- state: &mut ForwardPassState<Metal>,
- _parameters: &EncodingParameters<Metal>,
- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- ) {
- let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);
- let batch_size = state.sampling_length();
- if batch_size == 0 {
- return;
- }
- let sampling_start = state.sampling_start();
- let input_array_mut = arrays[0].borrow_mut();
- let output_array_mut = arrays[1].borrow_mut();
-
- let elem_size = input_array_mut.data_type().size_in_bytes();
- let input_buffer = input_array_mut.buffer();
- let output_buffer = output_array_mut.buffer();
- let a_offset = (sampling_start * self.model_dim * elem_size) as u64;
-
- let args = QuantizedMatmulArguments {
- a_buffer: input_buffer,
- a_offset: a_offset as usize,
- b_buffer: &self.weights_buffer,
- scales_buffer: &self.scales_buffer,
- zero_points_or_biases_buffer: &self.biases_buffer,
- output_buffer,
- batch: batch_size,
- input_dim: self.model_dim,
- output_dim: self.vocab_size,
- quantization_type: QuantizedMatmulType::Mlx,
- };
-
- self.kernel.encode(encoder, args).expect("Failed to encode quantized embedding readout kernel");
- }
-}
diff --git a/crates/uzu/src/encodable_block/mod.rs b/crates/uzu/src/encodable_block/mod.rs
index 050224bc2..1aaf72743 100644
--- a/crates/uzu/src/encodable_block/mod.rs
+++ b/crates/uzu/src/encodable_block/mod.rs
@@ -16,6 +16,7 @@ mod pooling;
mod prediction_head;
mod qk_norm;
mod quantized_embedding_lookup;
+mod quantized_embedding_readout;
mod quantized_linear;
mod rms_norm;
mod rope;
@@ -37,6 +38,7 @@ pub use pooling::Pooling;
pub use prediction_head::ClassifierPredictionHead;
pub use qk_norm::{QKNorm, QKNormError};
pub use quantized_embedding_lookup::{QuantizedEmbeddingLookup, QuantizedEmbeddingLookupError};
+pub use quantized_embedding_readout::{QuantizedEmbeddingReadout, QuantizedEmbeddingReadoutError};
pub use quantized_linear::{QuantizedLinear, QuantizedLinearError};
pub use rms_norm::{RMSNorm, RMSNormError};
pub use rope::Rope;
diff --git a/crates/uzu/src/encodable_block/quantized_embedding_readout.rs b/crates/uzu/src/encodable_block/quantized_embedding_readout.rs
new file mode 100644
index 000000000..bee6ab076
--- /dev/null
+++ b/crates/uzu/src/encodable_block/quantized_embedding_readout.rs
@@ -0,0 +1,267 @@
+//! Quantized embedding readout encodable.
+
+use thiserror::Error;
+
+use super::{EncodableBlock, EncodingParameters};
+use crate::{
+ DataType,
+ backends::common::{
+ Backend, Context, NativeBuffer,
+ kernel::quant_matmul::{
+ QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulError,
+ QuantizedMatmulKernelEncodable, QuantizedMatmulType,
+ },
+ },
+ config::QuantizationMode,
+ forward_pass::state::{ArrayId, ForwardPassState},
+ parameters::{ParameterLoaderError, ParameterTree},
+};
+
+#[derive(Debug, Error)]
+pub enum QuantizedEmbeddingReadoutError<B: Backend> {
+ #[error("Backend error: {0}")]
+ BackendError(#[source] B::Error),
+ #[error("Matmul error: {0}")]
+ MatmulError(#[source] QuantizedMatmulError<B>),
+ #[error("Parameter loading error: {0}")]
+ ParameterError(ParameterLoaderError),
+ #[error("Unsupported data type: {0:?}")]
+ UnsupportedDataType(DataType),
+ #[error(
+ "Embedding readout weights shape mismatch: got {got:?}, expected [{expected_vocab_size}, {expected_packed_dim}]"
+ )]
+ InvalidWeightsShape {
+ got: Box<[usize]>,
+ expected_vocab_size: usize,
+ expected_packed_dim: usize,
+ },
+ #[error(
+ "Embedding readout scales shape mismatch: got {got:?}, expected [{expected_vocab_size}, {expected_num_groups}]"
+ )]
+ InvalidScalesShape {
+ got: Box<[usize]>,
+ expected_vocab_size: usize,
+ expected_num_groups: usize,
+ },
+ #[error("Scales dtype mismatch: got {got:?}, expected {expected:?}")]
+ InvalidScalesDataType {
+ expected: DataType,
+ got: DataType,
+ },
+ #[error(
+ "Embedding readout deq_biases shape mismatch: got {got:?}, expected [{expected_vocab_size}, {expected_num_groups}]"
+ )]
+ InvalidBiasesShape {
+ got: Box<[usize]>,
+ expected_vocab_size: usize,
+ expected_num_groups: usize,
+ },
+ #[error("Biases dtype mismatch: got {got:?}, expected {expected:?}")]
+ InvalidBiasesDataType {
+ expected: DataType,
+ got: DataType,
+ },
+}
+
+pub struct QuantizedEmbeddingReadout<B: Backend> {
+ kernel: QuantizedMatmulKernelEncodable<B>,
+ weights_buffer: B::NativeBuffer,
+ scales_buffer: B::NativeBuffer,
+ biases_buffer: B::NativeBuffer,
+ vocab_size: usize,
+ model_dim: usize,
+}
+
+impl<B: Backend> QuantizedEmbeddingReadout<B> {
+ pub fn new_tied(
+ context: &B::Context,
+ data_type: DataType,
+ vocab_size: usize,
+ model_dim: usize,
+ group_size: usize,
+ mode: QuantizationMode,
+ parameter_tree: &ParameterTree<B::Context>,
+ ) -> Result<Self, QuantizedEmbeddingReadoutError<B>> {
+ Self::new_with_names(
+ context,
+ data_type,
+ vocab_size,
+ model_dim,
+ group_size,
+ mode,
+ "weights",
+ "scales",
+ "biases",
+ parameter_tree,
+ )
+ }
+
+ pub fn new_untied_output(
+ context: &B::Context,
+ data_type: DataType,
+ vocab_size: usize,
+ model_dim: usize,
+ group_size: usize,
+ mode: QuantizationMode,
+ parameter_tree: &ParameterTree<B::Context>,
+ ) -> Result<Self, QuantizedEmbeddingReadoutError<B>> {
+ Self::new_with_names(
+ context,
+ data_type,
+ vocab_size,
+ model_dim,
+ group_size,
+ mode,
+ "output_weights",
+ "output_scales",
+ "output_biases",
+ parameter_tree,
+ )
+ }
+
+ fn new_with_names(
+ context: &B::Context,
+ data_type: DataType,
+ vocab_size: usize,
+ model_dim: usize,
+ group_size: usize,
+ mode: QuantizationMode,
+ weights_name: &str,
+ scales_name: &str,
+ biases_name: &str,
+ parameter_tree: &ParameterTree<B::Context>,
+ ) -> Result<Self, QuantizedEmbeddingReadoutError<B>> {
+ let weights = parameter_tree.leaf(weights_name).map_err(QuantizedEmbeddingReadoutError::ParameterError)?;
+ let scales = parameter_tree.leaf(scales_name).map_err(QuantizedEmbeddingReadoutError::ParameterError)?;
+
+ let num_groups = (model_dim + group_size - 1) / group_size;
+ let packing_divisor = mode.packing_divisor();
+ let weights_transposed = weights.shape()[0] == vocab_size;
+
+ if weights.shape() != [vocab_size, model_dim / packing_divisor] {
+ return Err(QuantizedEmbeddingReadoutError::InvalidWeightsShape {
+ got: weights.shape().to_vec().into_boxed_slice(),
+ expected_vocab_size: vocab_size,
+ expected_packed_dim: model_dim / packing_divisor,
+ });
+ }
+
+ if scales.shape() != [vocab_size, num_groups] {
+ return Err(QuantizedEmbeddingReadoutError::InvalidScalesShape {
+ got: scales.shape().to_vec().into_boxed_slice(),
+ expected_vocab_size: vocab_size,
+ expected_num_groups: num_groups,
+ });
+ }
+
+ if scales.data_type() != data_type {
+ return Err(QuantizedEmbeddingReadoutError::InvalidScalesDataType {
+ expected: data_type,
+ got: scales.data_type(),
+ });
+ }
+
+ let biases_buffer = match parameter_tree.leaf(biases_name) {
+ Ok(deq_biases) => {
+ if deq_biases.shape() != [vocab_size, num_groups] {
+ return Err(QuantizedEmbeddingReadoutError::InvalidBiasesShape {
+ got: deq_biases.shape().to_vec().into_boxed_slice(),
+ expected_vocab_size: vocab_size,
+ expected_num_groups: num_groups,
+ });
+ }
+ if deq_biases.data_type() != data_type {
+ return Err(QuantizedEmbeddingReadoutError::InvalidBiasesDataType {
+ expected: data_type,
+ got: deq_biases.data_type(),
+ });
+ }
+ deq_biases.buffer().clone()
+ },
+ Err(_) => {
+ let element_size = match data_type {
+ DataType::F16 | DataType::BF16 => 2,
+ DataType::F32 => 4,
+ other => {
+ return Err(QuantizedEmbeddingReadoutError::UnsupportedDataType(other));
+ },
+ };
+ let size_bytes = vocab_size * num_groups * element_size;
+ let buffer = context.create_buffer(size_bytes).map_err(QuantizedEmbeddingReadoutError::BackendError)?;
+
+ unsafe {
+ std::ptr::write_bytes(buffer.cpu_ptr().as_ptr().cast::<u8>(), 0, size_bytes);
+ }
+
+ buffer
+ },
+ };
+
+ let kernel = QuantizedMatmulKernelEncodable::new(
+ context,
+ QuantizedMatmulConfiguration {
+ data_type,
+ group_size,
+ input_dim: model_dim,
+ output_dim: vocab_size,
+ mode,
+ quantization_type: QuantizedMatmulType::Mlx,
+ weights_transposed,
+ },
+ )
+ .map_err(QuantizedEmbeddingReadoutError::MatmulError)?;
+
+ Ok(Self {
+ kernel,
+ weights_buffer: weights.buffer().clone(),
+ scales_buffer: scales.buffer().clone(),
+ biases_buffer,
+ vocab_size,
+ model_dim,
+ })
+ }
+}
+
+impl<B: Backend> EncodableBlock<B> for QuantizedEmbeddingReadout<B> {
+ fn supports_shared_encoder(&self) -> bool {
+ true
+ }
+
+ fn encode_with_shared_encoder(
+ &self,
+ state: &mut ForwardPassState<B>,
+ _parameters: &EncodingParameters<B>,
+ encoder: &B::ComputeEncoder,
+ ) {
+ let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);
+ let batch_size = state.sampling_length();
+ if batch_size == 0 {
+ return;
+ }
+
+ let sampling_start = state.sampling_start();
+ let input_array = arrays[0].borrow_mut();
+ let output_array = arrays[1].borrow_mut();
+
+ let element_size = input_array.data_type().size_in_bytes();
+ let a_offset = sampling_start * self.model_dim * element_size;
+
+ self.kernel
+ .encode(
+ encoder,
+ QuantizedMatmulArguments {
+ a_buffer: input_array.buffer(),
+ a_offset,
+ b_buffer: &self.weights_buffer,
+ scales_buffer: &self.scales_buffer,
+ zero_points_or_biases_buffer: &self.biases_buffer,
+ output_buffer: output_array.buffer(),
+ batch: batch_size,
+ input_dim: self.model_dim,
+ output_dim: self.vocab_size,
+ quantization_type: QuantizedMatmulType::Mlx,
+ },
+ )
+ .expect("Failed to encode quantized embedding readout kernel");
+ }
+}