Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

generalize linear encodables

eugene0b58815e402d
diff --git a/crates/uzu/src/backends/common/kernel/matmul/full_precision.rs b/crates/uzu/src/backends/common/kernel/matmul/full_precision.rs
new file mode 100644
index 000000000..22476608f
--- /dev/null
+++ b/crates/uzu/src/backends/common/kernel/matmul/full_precision.rs
@@ -0,0 +1,29 @@
+use super::MatmulKernels;
+use crate::{DataType, backends::common::Backend};
+
+pub struct FullPrecisionMatmulArguments<'a, B: Backend> {
+ pub a: &'a B::NativeBuffer,
+ pub a_offset: usize,
+ pub b: &'a B::NativeBuffer,
+ pub output: &'a B::NativeBuffer,
+ pub bias: Option<&'a B::NativeBuffer>,
+ pub batch: usize,
+ pub input_dim: usize,
+ pub output_dim: usize,
+}
+
+pub trait FullPrecisionMatmulKernel: Sized {
+ type Backend: Backend<Kernels: MatmulKernels<FullPrecisionMatmulKernel = Self>>;
+
+ fn new(
+ context: &<Self::Backend as Backend>::Context,
+ data_type: DataType,
+ ) -> Result<Self, <Self::Backend as Backend>::Error>;
+
+ fn encode(
+ &mut self,
+ context: &<Self::Backend as Backend>::Context,
+ encoder: &<Self::Backend as Backend>::ComputeEncoder,
+ arguments: FullPrecisionMatmulArguments<Self::Backend>,
+ );
+}
diff --git a/crates/uzu/src/backends/common/kernel/matmul/mod.rs b/crates/uzu/src/backends/common/kernel/matmul/mod.rs
new file mode 100644
index 000000000..d3f0d7c44
--- /dev/null
+++ b/crates/uzu/src/backends/common/kernel/matmul/mod.rs
@@ -0,0 +1,14 @@
+mod full_precision;
+mod quantized;
+
+pub use full_precision::{FullPrecisionMatmulArguments, FullPrecisionMatmulKernel};
+pub use quantized::{
+ QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulKernel, QuantizedMatmulType,
+};
+
+use super::Kernels;
+
+pub trait MatmulKernels: Kernels {
+ type FullPrecisionMatmulKernel: FullPrecisionMatmulKernel<Backend = Self::Backend>;
+ type QuantizedMatmulKernel: QuantizedMatmulKernel<Backend = Self::Backend>;
+}
diff --git a/crates/uzu/src/backends/common/kernel/matmul/quantized.rs b/crates/uzu/src/backends/common/kernel/matmul/quantized.rs
new file mode 100644
index 000000000..88da264dd
--- /dev/null
+++ b/crates/uzu/src/backends/common/kernel/matmul/quantized.rs
@@ -0,0 +1,47 @@
+use super::MatmulKernels;
+use crate::{DataType, backends::common::Backend, config::QuantizationMode};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum QuantizedMatmulType {
+ ZeroPoint,
+ Mlx,
+}
+
+#[derive(Debug, Clone, Copy)]
+pub struct QuantizedMatmulConfiguration {
+ pub data_type: DataType,
+ pub group_size: usize,
+ pub input_dim: usize,
+ pub output_dim: usize,
+ pub mode: QuantizationMode,
+ pub quantization_type: QuantizedMatmulType,
+ pub weights_transposed: bool,
+}
+
+pub struct QuantizedMatmulArguments<'a, B: Backend> {
+ pub a_buffer: &'a B::NativeBuffer,
+ pub a_offset: usize,
+ pub b_buffer: &'a B::NativeBuffer,
+ pub scales_buffer: &'a B::NativeBuffer,
+ pub zero_points_or_biases_buffer: &'a B::NativeBuffer,
+ pub output_buffer: &'a B::NativeBuffer,
+ pub batch: usize,
+ pub input_dim: usize,
+ pub output_dim: usize,
+ pub quantization_type: QuantizedMatmulType,
+}
+
+pub trait QuantizedMatmulKernel: Sized {
+ type Backend: Backend<Kernels: MatmulKernels<QuantizedMatmulKernel = Self>>;
+
+ fn new(
+ context: &<Self::Backend as Backend>::Context,
+ configuration: QuantizedMatmulConfiguration,
+ ) -> Result<Self, <Self::Backend as Backend>::Error>;
+
+ fn encode(
+ &self,
+ encoder: &<Self::Backend as Backend>::ComputeEncoder,
+ arguments: QuantizedMatmulArguments<Self::Backend>,
+ );
+}
diff --git a/crates/uzu/src/backends/common/kernel/mod.rs b/crates/uzu/src/backends/common/kernel/mod.rs
index fb7523be0..ca042ab4e 100644
--- a/crates/uzu/src/backends/common/kernel/mod.rs
+++ b/crates/uzu/src/backends/common/kernel/mod.rs
@@ -3,4 +3,5 @@
include!(concat!(env!("OUT_DIR"), "/traits.rs"));
pub mod kv_cache_update;
+pub mod matmul;
pub mod sampling;
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs
deleted file mode 100644
index 2a1212e09..000000000
--- a/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs
+++ /dev/null
@@ -1,142 +0,0 @@
-use std::cell::RefCell;
-
-use super::{
- super::{EncodableBlock, Metal},
- EmbeddingError,
-};
-use crate::{
- DataType,
- backends::metal::{
- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, MTLError, ProtocolObject,
- Retained,
- kernel::matmul::{MatmulArguments, MatmulKernel},
- },
- encodable_block::EncodingParameters,
- forward_pass::state::{ArrayId, ForwardPassState},
- parameters::ParameterTree,
-};
-
-pub struct FullPrecisionEmbeddingReadout {
- kernel: RefCell<MatmulKernel>,
- weights_buffer: Retained<ProtocolObject<dyn MTLBuffer>>,
- vocab_size: usize,
- model_dim: usize,
-}
-
-impl FullPrecisionEmbeddingReadout {
- pub fn new(
- _mtl_context: &MTLContext,
- data_type: DataType,
- vocab_size: usize,
- model_dim: usize,
- parameter_tree: &ParameterTree<MTLContext>,
- ) -> Result<Self, EmbeddingError> {
- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
- return Err(EmbeddingError::UnsupportedDataType(data_type));
- }
-
- let weights = match parameter_tree.leaf("weights") {
- Ok(weights) => weights,
- Err(_) => parameter_tree.leaf("output_weights").map_err(|e| {
- EmbeddingError::MetalError(MTLError::Generic(format!("Failed to load weights: {:?}", e)))
- })?,
- };
-
- if weights.shape() != [vocab_size, model_dim] {
- return Err(EmbeddingError::MetalError(MTLError::Generic(format!(
- "Embedding readout weights shape mismatch: got {:?}, expected [{}, {}]",
- weights.shape(),
- vocab_size,
- model_dim
- ))));
- }
-
- if weights.data_type() != data_type {
- return Err(EmbeddingError::MetalError(MTLError::Generic(format!(
- "Weights dtype mismatch: got {:?}, expected {:?}",
- weights.data_type(),
- data_type
- ))));
- }
-
- let weights_buffer = weights.buffer().to_owned();
-
- let mut kernel = MatmulKernel::new(data_type).map_err(EmbeddingError::MetalError)?;
- kernel.precompile(_mtl_context).map_err(EmbeddingError::MetalError)?;
-
- Ok(Self {
- kernel: RefCell::new(kernel),
- weights_buffer: weights_buffer.into(),
- vocab_size,
- model_dim,
- })
- }
-}
-
-impl EncodableBlock<Metal> for FullPrecisionEmbeddingReadout {
- fn encode(
- &self,
- state: &mut ForwardPassState<Metal>,
- parameters: &EncodingParameters<Metal>,
- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- ) {
- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
- self.encode_with_shared_encoder(state, parameters, &encoder);
- encoder.end_encoding();
-
- if parameters.wait_until_completed {
- command_buffer.commit();
- command_buffer.wait_until_completed();
- }
- }
-
- fn supports_shared_encoder(&self) -> bool {
- true
- }
-
- fn encode_with_shared_encoder(
- &self,
- state: &mut ForwardPassState<Metal>,
- _parameters: &EncodingParameters<Metal>,
- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- ) {
- let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);
- let batch_size = state.sampling_length();
- if batch_size == 0 {
- return;
- }
- let sampling_start = state.sampling_start();
- let input_array_mut = arrays[0].borrow_mut();
- let output_array_mut = arrays[1].borrow_mut();
-
- let elem_size = input_array_mut.data_type().size_in_bytes();
- let input_buffer = input_array_mut.buffer();
- let output_buffer = output_array_mut.buffer();
- let a_offset = (sampling_start * self.model_dim * elem_size) as u64;
-
- let args = MatmulArguments {
- a: input_buffer,
- a_offset,
- b: &self.weights_buffer,
- c: None,
- d: output_buffer,
- bias: None,
- batch: batch_size as i32,
- input_dim: self.model_dim as i32,
- output_dim: self.vocab_size as i32,
- lda: self.model_dim as i32,
- ldb: self.model_dim as i32,
- ldd: self.vocab_size as i32,
- batch_count: 1,
- alpha: 1.0,
- beta: 0.0,
- transpose_a: false,
- transpose_b: true,
- };
-
- self.kernel
- .borrow_mut()
- .encode(state.mtl_context(), encoder, args)
- .expect("Failed to encode full precision embedding readout kernel");
- }
-}
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rs
index 836bb9c2c..f90626864 100644
--- a/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rs
@@ -1,18 +1,18 @@
//! Embedding encodables (quantized and full-precision).
mod error;
-mod full_precision_readout;
mod quantized_readout;
pub use error::EmbeddingError;
-pub use full_precision_readout::FullPrecisionEmbeddingReadout;
pub use quantized_readout::QuantizedEmbeddingReadout;
use super::super::Metal;
use crate::encodable_block::{
FullPrecisionEmbeddingLookup as GenericFullPrecisionEmbeddingLookup,
+ FullPrecisionEmbeddingReadout as GenericFullPrecisionEmbeddingReadout,
QuantizedEmbeddingLookup as GenericQuantizedEmbeddingLookup,
};
+pub type FullPrecisionEmbeddingReadout = GenericFullPrecisionEmbeddingReadout<Metal>;
pub type FullPrecisionEmbeddingLookup = GenericFullPrecisionEmbeddingLookup<Metal>;
pub type QuantizedEmbeddingLookup = GenericQuantizedEmbeddingLookup<Metal>;
diff --git a/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs b/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs
deleted file mode 100644
index a04831507..000000000
--- a/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs
+++ /dev/null
@@ -1,193 +0,0 @@
-use std::cell::RefCell;
-
-use crate::{
- DataType,
- backends::metal::{
- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, MTLError, Metal,
- ProtocolObject, Retained,
- encodable_block::EncodableBlock,
- kernel::matmul::{MatmulArguments, MatmulKernel},
- },
- encodable_block::EncodingParameters,
- forward_pass::state::{ArrayId, ForwardPassState},
- parameters::ParameterTree,
-};
-
-pub struct FullPrecisionLinear {
- kernel: RefCell<MatmulKernel>,
- bias_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>>,
- weights_buffer: Retained<ProtocolObject<dyn MTLBuffer>>,
- input_dim: usize,
- output_dim: usize,
- input_array_id: ArrayId,
- output_array_id: ArrayId,
-}
-
-impl FullPrecisionLinear {
- pub fn new(
- _mtl_context: &MTLContext,
- precision: DataType,
- input_dim: usize,
- output_dim: usize,
- parameter_tree: &ParameterTree<MTLContext>,
- input_array_id: ArrayId,
- output_array_id: ArrayId,
- ) -> Result<Self, MTLError> {
- if !matches!(precision, DataType::F16 | DataType::BF16 | DataType::F32) {
- return Err(MTLError::Generic(format!(
- "Unsupported data type for full precision linear kernel: {:?}",
- precision
- )));
- }
-
- let weights = parameter_tree
- .leaf("weights")
- .map_err(|e| MTLError::Generic(format!("Failed to load weights: {:?}", e)))?;
-
- let w_shape = weights.shape();
- if w_shape != [output_dim, input_dim] {
- return Err(MTLError::Generic(format!(
- "Unexpected weights shape: got {:?}, expected [{}, {}]",
- w_shape, output_dim, input_dim
- )));
- }
-
- if weights.data_type() != precision {
- return Err(MTLError::Generic(format!(
- "Weights dtype mismatch: got {:?}, expected {:?}",
- weights.data_type(),
- precision
- )));
- }
-
- let weights_buffer: Retained<ProtocolObject<dyn MTLBuffer>> = weights.buffer().to_owned().into();
-
- let bias_buffer = match parameter_tree.leaf("biases") {
- Ok(biases) => {
- if biases.shape() != [output_dim] {
- return Err(MTLError::Generic(format!(
- "Bias shape mismatch: got {:?}, expected [{:?}]",
- biases.shape(),
- output_dim
- )));
- }
- if biases.data_type() != precision {
- return Err(MTLError::Generic(format!(
- "Bias dtype mismatch: got {:?}, expected {:?}",
- biases.data_type(),
- precision
- )));
- }
- let bias_buffer: Retained<ProtocolObject<dyn MTLBuffer>> = biases.buffer().to_owned().into();
- Some(bias_buffer)
- },
- Err(_) => None,
- };
-
- let mut kernel = MatmulKernel::new(precision)?;
- kernel.precompile(_mtl_context)?;
-
- Ok(Self {
- kernel: RefCell::new(kernel),
- bias_buffer,
- weights_buffer,
- input_dim,
- output_dim,
- input_array_id,
- output_array_id,
- })
- }
-}
-
-impl EncodableBlock<Metal> for FullPrecisionLinear {
- fn encode(
- &self,
- state: &mut ForwardPassState<Metal>,
- parameters: &EncodingParameters<Metal>,
- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- ) {
- let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);
- let batch_size = state.active_suffix_length();
-
- let input_array_mut = arrays[0].borrow_mut();
- let output_array_mut = arrays[1].borrow_mut();
-
- let input_buffer = input_array_mut.buffer();
- let output_buffer = output_array_mut.buffer();
-
- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
-
- let args = MatmulArguments {
- a: input_buffer,
- a_offset: 0,
- b: &self.weights_buffer,
- c: None,
- d: output_buffer,
- bias: self.bias_buffer.as_ref(),
- batch: batch_size as i32,
- input_dim: self.input_dim as i32,
- output_dim: self.output_dim as i32,
- lda: self.input_dim as i32,
- ldb: self.input_dim as i32,
- ldd: self.output_dim as i32,
- batch_count: 1,
- alpha: 1.0,
- beta: 0.0,
- transpose_a: false,
- transpose_b: true,
- };
-
- let mut kernel = self.kernel.borrow_mut();
- kernel.encode(state.mtl_context(), &encoder, args).expect("Failed to encode matmul kernel");
-
- encoder.end_encoding();
-
- if parameters.wait_until_completed {
- command_buffer.commit();
- command_buffer.wait_until_completed();
- }
- }
-
- fn supports_shared_encoder(&self) -> bool {
- true
- }
-
- fn encode_with_shared_encoder(
- &self,
- state: &mut ForwardPassState<Metal>,
- _parameters: &EncodingParameters<Metal>,
- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- ) {
- let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);
- let batch_size = state.active_suffix_length();
-
- let input_array_mut = arrays[0].borrow_mut();
- let output_array_mut = arrays[1].borrow_mut();
-
- let input_buffer = input_array_mut.buffer();
- let output_buffer = output_array_mut.buffer();
-
- let args = MatmulArguments {
- a: input_buffer,
- a_offset: 0,
- b: &self.weights_buffer,
- c: None,
- d: output_buffer,
- bias: self.bias_buffer.as_ref(),
- batch: batch_size as i32,
- input_dim: self.input_dim as i32,
- output_dim: self.output_dim as i32,
- lda: self.input_dim as i32,
- ldb: self.input_dim as i32,
- ldd: self.output_dim as i32,
- batch_count: 1,
- alpha: 1.0,
- beta: 0.0,
- transpose_a: false,
- transpose_b: true,
- };
-
- let mut kernel = self.kernel.borrow_mut();
- kernel.encode(state.mtl_context(), encoder, args).expect("Failed to encode matmul kernel");
- }
-}
diff --git a/crates/uzu/src/backends/metal/encodable_block/linear/mod.rs b/crates/uzu/src/backends/metal/encodable_block/linear/mod.rs
index 139ecef36..f393ef8ba 100644
--- a/crates/uzu/src/backends/metal/encodable_block/linear/mod.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/linear/mod.rs
@@ -1,5 +1,7 @@
-mod full_precision;
-mod quantized;
+use super::super::Metal;
+use crate::encodable_block::{
+ FullPrecisionLinear as GenericFullPrecisionLinear, QuantizedLinear as GenericQuantizedLinear,
+};
-pub use full_precision::FullPrecisionLinear;
-pub use quantized::QuantizedLinear;
+pub type FullPrecisionLinear = GenericFullPrecisionLinear<Metal>;
+pub type QuantizedLinear = GenericQuantizedLinear<Metal>;
diff --git a/crates/uzu/src/backends/metal/encodable_block/linear/quantized.rs b/crates/uzu/src/backends/metal/encodable_block/linear/quantized.rs
deleted file mode 100644
index 51bd932a8..000000000
--- a/crates/uzu/src/backends/metal/encodable_block/linear/quantized.rs
+++ /dev/null
@@ -1,303 +0,0 @@
-use crate::{
- DataType,
- backends::{
- common::kernel::TensorAddBiasKernel,
- metal::{
- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, MTLError, Metal,
- ProtocolObject, Retained,
- encodable_block::EncodableBlock,
- kernel::{
- dsl::TensorAddBiasMetalKernel,
- quant_matmul::{QuantizationType, QuantizedMatmulArguments, QuantizedMatmulKernel},
- },
- },
- },
- config::QuantizationConfig,
- encodable_block::EncodingParameters,
- forward_pass::state::{ArrayId, ForwardPassState},
- parameters::ParameterTree,
-};
-
-pub struct QuantizedLinear {
- kernel: QuantizedMatmulKernel,
- bias_add_kernel: Option<TensorAddBiasMetalKernel>,
- biases_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>>,
- weights_buffer: Retained<ProtocolObject<dyn MTLBuffer>>,
- scales_buffer: Retained<ProtocolObject<dyn MTLBuffer>>,
- zero_points_or_biases_buffer: Retained<ProtocolObject<dyn MTLBuffer>>,
- quantization_type: QuantizationType,
- input_dim: usize,
- output_dim: usize,
- input_array_id: ArrayId,
- output_array_id: ArrayId,
-}
-
-impl QuantizedLinear {
- pub fn new(
- mtl_context: &MTLContext,
- config: &QuantizationConfig,
- input_dim: usize,
- output_dim: usize,
- parameter_tree: &ParameterTree<MTLContext>,
- input_array_id: ArrayId,
- output_array_id: ArrayId,
- ) -> Result<Self, MTLError> {
- let kernel_data_type: DataType = config.activation_precision.into();
-
- if !matches!(kernel_data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
- return Err(MTLError::Generic(format!(
- "Unsupported data type for quantized kernel: {:?}",
- kernel_data_type
- )));
- }
-
- let weights = parameter_tree
- .leaf("weights")
- .map_err(|e| MTLError::Generic(format!("Failed to load weights: {:?}", e)))?;
-
- let packing_divisor = config.weight_quantization_mode.packing_divisor();
- let storage_type = config.weight_quantization_mode.storage_type();
-
- if weights.data_type() != storage_type {
- return Err(MTLError::Generic(format!(
- "Expected weights of type {:?}, got {:?}",
- storage_type,
- weights.data_type()
- )));
- }
-
- let scales =
- parameter_tree.leaf("scales").map_err(|e| MTLError::Generic(format!("Failed to load scales: {:?}", e)))?;
- let k_g = (input_dim + config.group_size - 1) / config.group_size;
-
- let w_shape = weights.shape();
- let s_shape = scales.shape();
-
- // Determine if weights are transposed by checking shape
- let weights_transposed = w_shape[0] == output_dim;
-
- // Determine quantization style: prefer MLX (deq_biases), else AWQ (zero_points)
- let (quantization_type, zero_points_or_biases_buffer, scales_buffer) = {
- if scales.data_type() != kernel_data_type {
- return Err(MTLError::Generic(format!(
- "Scales dtype mismatch: got {:?}, expected {:?}",
- scales.data_type(),
- kernel_data_type
- )));
- }
- match parameter_tree.leaf("deq_biases") {
- Ok(deq_biases) => {
- let db_shape = deq_biases.shape();
- if !(w_shape == [output_dim, input_dim / packing_divisor]
- && s_shape == [output_dim, k_g]
- && db_shape == [output_dim, k_g])
- {
- return Err(MTLError::Generic(format!(
- "Unexpected MLX shapes. weights={:?}, scales={:?}, deq_biases={:?}; expected [N,K/{}],[N,K_g],[N,K_g]",
- w_shape, s_shape, db_shape, packing_divisor
- )));
- }
- if deq_biases.data_type() != kernel_data_type {
- return Err(MTLError::Generic(format!(
- "deq_biases dtype mismatch: got {:?}, expected {:?}",
- deq_biases.data_type(),
- kernel_data_type
- )));
- }
- let scales_buffer: Retained<ProtocolObject<dyn MTLBuffer>> = scales.buffer().clone();
- let biases_buf: Retained<ProtocolObject<dyn MTLBuffer>> = deq_biases.buffer().clone();
- (QuantizationType::Mlx, biases_buf, scales_buffer)
- },
- Err(_) => {
- let zero_points = parameter_tree
- .leaf("zero_points")
- .map_err(|e| MTLError::Generic(format!("Failed to load zero_points: {:?}", e)))?;
- let zp_shape = zero_points.shape();
- let expected_zp_entries = (k_g + packing_divisor - 1) / packing_divisor;
- if !(w_shape == [output_dim, input_dim / packing_divisor]
- && s_shape == [output_dim, k_g]
- && zp_shape == [output_dim, expected_zp_entries])
- {
- return Err(MTLError::Generic(format!(
- "Unexpected AWQ shapes. weights={:?}, scales={:?}, zero_points={:?}; expected [N,K/{}],[N,K_g],[N,(K_g+{})/{}]",
- w_shape,
- s_shape,
- zp_shape,
- packing_divisor,
- packing_divisor - 1,
- packing_divisor
- )));
- }
- if zero_points.data_type() != storage_type {
- return Err(MTLError::Generic(format!(
- "Zero-points dtype mismatch: got {:?}, expected {:?}",
- zero_points.data_type(),
- storage_type
- )));
- }
- let scales_buffer: Retained<ProtocolObject<dyn MTLBuffer>> = scales.buffer().clone();
- let zps_buf: Retained<ProtocolObject<dyn MTLBuffer>> = zero_points.buffer().clone();
- (QuantizationType::ZeroPoint, zps_buf, scales_buffer)
- },
- }
- };
-
- let weights_buffer: Retained<ProtocolObject<dyn MTLBuffer>> = weights.buffer().to_owned().into();
-
- // Optional trainable bias support
- let (bias_add_kernel, biases_buffer) = match parameter_tree.leaf("biases") {
- Ok(biases) => {
- if biases.shape() != [output_dim] {
- return Err(MTLError::Generic(format!(
- "Bias shape mismatch: got {:?}, expected [{:?}]",
- biases.shape(),
- output_dim
- )));
- }
- if biases.data_type() != kernel_data_type {
- return Err(MTLError::Generic(format!(
- "Bias dtype mismatch: got {:?}, expected {:?}",
- biases.data_type(),
- kernel_data_type
- )));
- }
- let bias_add_kernel = Some(TensorAddBiasMetalKernel::new(mtl_context, kernel_data_type)?);
- let biases_buffer: Retained<ProtocolObject<dyn MTLBuffer>> = biases.buffer().to_owned().into();
- (bias_add_kernel, Some(biases_buffer))
- },
- Err(_) => (None, None),
- };
-
- let g = config.group_size;
- let mode = config.weight_quantization_mode;
-
- let kernel = QuantizedMatmulKernel::new(
- mtl_context,
- kernel_data_type,
- g,
- input_dim,
- output_dim,
- mode,
- quantization_type,
- weights_transposed,
- )
- .map_err(|e| MTLError::Generic(format!("{:?}", e)))?;
-
- Ok(Self {
- kernel,
- bias_add_kernel,
- biases_buffer,
- weights_buffer,
- scales_buffer: scales_buffer.into(),
- zero_points_or_biases_buffer: zero_points_or_biases_buffer.into(),
- quantization_type,
- input_dim,
- output_dim,
- input_array_id,
- output_array_id,
- })
- }
-}
-
-impl EncodableBlock<Metal> for QuantizedLinear {
- fn encode(
- &self,
- state: &mut ForwardPassState<Metal>,
- parameters: &EncodingParameters<Metal>,
- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- ) {
- let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);
- let batch_size = state.active_suffix_length();
-
- let input_array_mut = arrays[0].borrow_mut();
- let output_array_mut = arrays[1].borrow_mut();
-
- let input_buffer = input_array_mut.buffer();
- let output_buffer = output_array_mut.buffer();
-
- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
-
- let args = QuantizedMatmulArguments {
- a_buffer: input_buffer,
- a_offset: 0,
- b_buffer: &self.weights_buffer,
- scales_buffer: &self.scales_buffer,
- zero_points_or_biases_buffer: &self.zero_points_or_biases_buffer,
- output_buffer: output_buffer,
- batch: batch_size as i32,
- input_dim: self.input_dim as i32,
- output_dim: self.output_dim as i32,
- quantization_type: self.quantization_type,
- };
-
- self.kernel.encode(&encoder, args).expect("Failed to encode quantized matmul kernel");
-
- encoder.end_encoding();
-
- if let (Some(bias_add), Some(bias_buf)) = (&self.bias_add_kernel, &self.biases_buffer) {
- let total_len = batch_size * self.output_dim;
- bias_add.encode_if(
- output_buffer,
- bias_buf,
- output_buffer,
- self.output_dim as u32,
- total_len as u32,
- &encoder,
- parameters.predicate,
- );
- }
-
- if parameters.wait_until_completed {
- command_buffer.commit();
- command_buffer.wait_until_completed();
- }
- }
-
- fn supports_shared_encoder(&self) -> bool {
- true
- }
-
- fn encode_with_shared_encoder(
- &self,
- state: &mut ForwardPassState<Metal>,
- parameters: &EncodingParameters<Metal>,
- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- ) {
- let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);
- let batch_size = state.active_suffix_length();
-
- let input_array_mut = arrays[0].borrow_mut();
- let output_array_mut = arrays[1].borrow_mut();
-
- let input_buffer = input_array_mut.buffer();
- let output_buffer = output_array_mut.buffer();
-
- let args = QuantizedMatmulArguments {
- a_buffer: input_buffer,
- a_offset: 0,
- b_buffer: &self.weights_buffer,
- scales_buffer: &self.scales_buffer,
- zero_points_or_biases_buffer: &self.zero_points_or_biases_buffer,
- output_buffer: output_buffer,
- batch: batch_size as i32,
- input_dim: self.input_dim as i32,
- output_dim: self.output_dim as i32,
- quantization_type: self.quantization_type,
- };
-
- self.kernel.encode(encoder, args).expect("Failed to encode quantized matmul kernel");
-
- if let (Some(bias_add), Some(bias_buf)) = (&self.bias_add_kernel, &self.biases_buffer) {
- let total_len = batch_size * self.output_dim;
- bias_add.encode_if(
- output_buffer,
- bias_buf,
- output_buffer,
- self.output_dim as u32,
- total_len as u32,
- encoder,
- parameters.predicate,
- );
- }
- }
-}
diff --git a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
index 30685df8b..ac2bfa1c9 100644
--- a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
@@ -36,7 +36,8 @@ pub fn linear_block<const N: usize>(
parameter_tree,
input_array_id,
output_array_id,
- )?;
+ )
+ .map_err(|error| MTLError::Generic(format!("{:?}", error)))?;
Ok(Box::new(block))
},
LinearConfig::FullPrecision {
@@ -50,7 +51,8 @@ pub fn linear_block<const N: usize>(
parameter_tree,
input_array_id,
output_array_id,
- )?;
+ )
+ .map_err(|error| MTLError::Generic(format!("{:?}", error)))?;
Ok(Box::new(block))
},
LinearConfig::QLoRA {
@@ -162,7 +164,8 @@ pub fn mlp_fused_block(
.map_err(|error| MTLError::Generic(format!("{:?}", error)))?,
ArrayId::MlpHidden,
ArrayId::Main,
- )?;
+ )
+ .map_err(|error| MTLError::Generic(format!("{:?}", error)))?;
let fused_block = MlpFusedBlock::new_quantized(
context,
@@ -208,7 +211,8 @@ pub fn mlp_fused_block(
.map_err(|error| MTLError::Generic(format!("{:?}", error)))?,
ArrayId::MlpHidden,
ArrayId::Main,
- )?;
+ )
+ .map_err(|error| MTLError::Generic(format!("{:?}", error)))?;
let fused_block = MlpFusedBlock::new_full_precision(
context,
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/mod.rs
index fcf105d50..8c3b55834 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/mod.rs
@@ -10,3 +10,56 @@ pub use dispatch_descriptor::{MatmulKernelVariant, determine_kernel_variant};
pub use gemv::GemvKernel;
pub use kernel::MatmulKernel;
pub use split_k::SplitKGemm;
+
+use crate::{
+ DataType,
+ backends::{
+ common::kernel::matmul::{
+ FullPrecisionMatmulArguments, FullPrecisionMatmulKernel as FullPrecisionMatmulKernelTrait,
+ },
+ metal::{MTLComputeCommandEncoder, MTLContext, MTLError, Metal, ProtocolObject},
+ },
+};
+
+impl FullPrecisionMatmulKernelTrait for MatmulKernel {
+ type Backend = Metal;
+
+ fn new(
+ context: &MTLContext,
+ data_type: DataType,
+ ) -> Result<Self, MTLError> {
+ let mut kernel = MatmulKernel::new(data_type)?;
+ kernel.precompile(context)?;
+ Ok(kernel)
+ }
+
+ fn encode(
+ &mut self,
+ context: &MTLContext,
+ encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
+ arguments: FullPrecisionMatmulArguments<Metal>,
+ ) {
+ let matmul_arguments = MatmulArguments {
+ a: arguments.a,
+ a_offset: arguments.a_offset as u64,
+ b: arguments.b,
+ c: None,
+ d: arguments.output,
+ bias: arguments.bias,
+ batch: arguments.batch as i32,
+ input_dim: arguments.input_dim as i32,
+ output_dim: arguments.output_dim as i32,
+ lda: arguments.input_dim as i32,
+ ldb: arguments.input_dim as i32,
+ ldd: arguments.output_dim as i32,
+ batch_count: 1,
+ alpha: 1.0,
+ beta: 0.0,
+ transpose_a: false,
+ transpose_b: true,
+ };
+
+ MatmulKernel::encode(self, context, encoder, matmul_arguments)
+ .expect("Failed to encode full precision matmul kernel");
+ }
+}
diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rs
index 17f263a99..202e9b4d7 100644
--- a/crates/uzu/src/backends/metal/kernel/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/mod.rs
@@ -1,5 +1,6 @@
pub mod attention;
mod data_type;
+use crate::backends::common::kernel::matmul::MatmulKernels;
pub mod dsl {
include!(concat!(env!("OUT_DIR"), "/dsl.rs"));
}
@@ -33,3 +34,8 @@ pub use short_conv::{ShortConvDecodeArguments, ShortConvKernel, ShortConvKernelE
pub use ssm::{
Conv1dPackArguments, Conv1dScanArguments, Conv1dScanKernel, SSDPrefillArguments, SSDPrefillMode, SSMKernelError,
};
+
+impl MatmulKernels for dsl::MetalKernels {
+ type FullPrecisionMatmulKernel = MatmulKernel;
+ type QuantizedMatmulKernel = quant_matmul::QuantizedMatmulKernel;
+}
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs
index 0311dd7bc..ace98f02d 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs
@@ -2,10 +2,16 @@ use std::collections::HashMap;
use crate::{
DataType,
- backends::metal::{
- FunctionConstantValuesSetValue, MTLBuffer, MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext,
- MTLError, MTLFunctionConstantValues, MTLSize, ProtocolObject, Retained,
- metal_extensions::ComputeEncoderSetValue,
+ backends::{
+ common::kernel::matmul::{
+ QuantizedMatmulArguments as GenericQuantizedMatmulArguments, QuantizedMatmulConfiguration,
+ QuantizedMatmulKernel as QuantizedMatmulKernelTrait, QuantizedMatmulType,
+ },
+ metal::{
+ FunctionConstantValuesSetValue, MTLBuffer, MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext,
+ MTLError, MTLFunctionConstantValues, MTLSize, Metal, ProtocolObject, Retained,
+ metal_extensions::ComputeEncoderSetValue,
+ },
},
config::QuantizationMode,
};
@@ -229,6 +235,58 @@ impl QuantizedMatmulKernel {
}
}
+impl QuantizedMatmulKernelTrait for QuantizedMatmulKernel {
+ type Backend = Metal;
+
+ fn new(
+ context: &MTLContext,
+ configuration: QuantizedMatmulConfiguration,
+ ) -> Result<Self, MTLError> {
+ let quantization_type = match configuration.quantization_type {
+ QuantizedMatmulType::ZeroPoint => QuantizationType::ZeroPoint,
+ QuantizedMatmulType::Mlx => QuantizationType::Mlx,
+ };
+
+ QuantizedMatmulKernel::new(
+ context,
+ configuration.data_type,
+ configuration.group_size,
+ configuration.input_dim,
+ configuration.output_dim,
+ configuration.mode,
+ quantization_type,
+ configuration.weights_transposed,
+ )
+ .map_err(|error| MTLError::Generic(format!("{:?}", error)))
+ }
+
+ fn encode(
+ &self,
+ encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
+ arguments: GenericQuantizedMatmulArguments<Metal>,
+ ) {
+ let quantization_type = match arguments.quantization_type {
+ QuantizedMatmulType::ZeroPoint => QuantizationType::ZeroPoint,
+ QuantizedMatmulType::Mlx => QuantizationType::Mlx,
+ };
+
+ let backend_arguments = QuantizedMatmulArguments {
+ a_buffer: arguments.a_buffer,
+ a_offset: arguments.a_offset as u64,
+ b_buffer: arguments.b_buffer,
+ scales_buffer: arguments.scales_buffer,
+ zero_points_or_biases_buffer: arguments.zero_points_or_biases_buffer,
+ output_buffer: arguments.output_buffer,
+ batch: arguments.batch as i32,
+ input_dim: arguments.input_dim as i32,
+ output_dim: arguments.output_dim as i32,
+ quantization_type,
+ };
+
+ QuantizedMatmulKernel::encode(self, encoder, backend_arguments).expect("Failed to encode quantized matmul");
+ }
+}
+
fn select_matrix_vector_kernel_name(
data_type: DataType,
group_size: usize,
diff --git a/crates/uzu/src/encodable_block/full_precision_embedding_readout.rs b/crates/uzu/src/encodable_block/full_precision_embedding_readout.rs
new file mode 100644
index 000000000..6e10e9855
--- /dev/null
+++ b/crates/uzu/src/encodable_block/full_precision_embedding_readout.rs
@@ -0,0 +1,139 @@
+use std::cell::RefCell;
+
+use thiserror::Error;
+
+use super::{EncodableBlock, EncodingParameters};
+use crate::{
+ DataType,
+ backends::common::{
+ Backend,
+ kernel::matmul::{FullPrecisionMatmulArguments, FullPrecisionMatmulKernel, MatmulKernels},
+ },
+ forward_pass::state::{ArrayId, ForwardPassState},
+ parameters::{ParameterLoaderError, ParameterTree},
+};
+
+#[derive(Debug, Error)]
+pub enum FullPrecisionEmbeddingReadoutError<B: Backend> {
+ #[error("Backend error: {0}")]
+ BackendError(#[source] B::Error),
+ #[error("Parameter loading error: {0}")]
+ ParameterError(ParameterLoaderError),
+ #[error("Unsupported data type: {0:?}")]
+ UnsupportedDataType(DataType),
+ #[error(
+ "Embedding readout weights shape mismatch: got {got:?}, expected [{expected_vocab_size}, {expected_model_dim}]"
+ )]
+ InvalidWeightsShape {
+ got: Box<[usize]>,
+ expected_vocab_size: usize,
+ expected_model_dim: usize,
+ },
+ #[error("Weights dtype mismatch: got {got:?}, expected {expected:?}")]
+ InvalidWeightsDataType {
+ expected: DataType,
+ got: DataType,
+ },
+}
+
+pub struct FullPrecisionEmbeddingReadout<B: Backend>
+where
+ B::Kernels: MatmulKernels,
+{
+ kernel: RefCell<<B::Kernels as MatmulKernels>::FullPrecisionMatmulKernel>,
+ weights_buffer: B::NativeBuffer,
+ vocab_size: usize,
+ model_dim: usize,
+}
+
+impl<B: Backend> FullPrecisionEmbeddingReadout<B>
+where
+ B::Kernels: MatmulKernels,
+{
+ pub fn new(
+ context: &B::Context,
+ data_type: DataType,
+ vocab_size: usize,
+ model_dim: usize,
+ parameter_tree: &ParameterTree<B::Context>,
+ ) -> Result<Self, FullPrecisionEmbeddingReadoutError<B>> {
+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
+ return Err(FullPrecisionEmbeddingReadoutError::UnsupportedDataType(data_type));
+ }
+
+ let weights = match parameter_tree.leaf("weights") {
+ Ok(weights) => weights,
+ Err(_) => {
+ parameter_tree.leaf("output_weights").map_err(FullPrecisionEmbeddingReadoutError::ParameterError)?
+ },
+ };
+
+ let weights_shape = weights.shape().to_vec();
+ if weights_shape != [vocab_size, model_dim] {
+ return Err(FullPrecisionEmbeddingReadoutError::InvalidWeightsShape {
+ got: weights_shape.into_boxed_slice(),
+ expected_vocab_size: vocab_size,
+ expected_model_dim: model_dim,
+ });
+ }
+
+ if weights.data_type() != data_type {
+ return Err(FullPrecisionEmbeddingReadoutError::InvalidWeightsDataType {
+ expected: data_type,
+ got: weights.data_type(),
+ });
+ }
+
+ let kernel = <B::Kernels as MatmulKernels>::FullPrecisionMatmulKernel::new(context, data_type)
+ .map_err(FullPrecisionEmbeddingReadoutError::BackendError)?;
+
+ Ok(Self {
+ kernel: RefCell::new(kernel),
+ weights_buffer: weights.buffer().clone(),
+ vocab_size,
+ model_dim,
+ })
+ }
+}
+
+impl<B: Backend> EncodableBlock<B> for FullPrecisionEmbeddingReadout<B>
+where
+ B::Kernels: MatmulKernels,
+{
+ fn supports_shared_encoder(&self) -> bool {
+ true
+ }
+
+ fn encode_with_shared_encoder(
+ &self,
+ state: &mut ForwardPassState<B>,
+ _parameters: &EncodingParameters<B>,
+ encoder: &B::ComputeEncoder,
+ ) {
+ let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);
+ let batch_size = state.sampling_length();
+ if batch_size == 0 {
+ return;
+ }
+
+ let sampling_start = state.sampling_start();
+ let input_array = arrays[0].borrow_mut();
+ let output_array = arrays[1].borrow_mut();
+ let element_size = input_array.data_type().size_in_bytes();
+
+ self.kernel.borrow_mut().encode(
+ state.mtl_context(),
+ encoder,
+ FullPrecisionMatmulArguments {
+ a: input_array.buffer(),
+ a_offset: sampling_start * self.model_dim * element_size,
+ b: &self.weights_buffer,
+ output: output_array.buffer(),
+ bias: None,
+ batch: batch_size,
+ input_dim: self.model_dim,
+ output_dim: self.vocab_size,
+ },
+ );
+ }
+}
diff --git a/crates/uzu/src/encodable_block/full_precision_linear.rs b/crates/uzu/src/encodable_block/full_precision_linear.rs
new file mode 100644
index 000000000..17b8e83a6
--- /dev/null
+++ b/crates/uzu/src/encodable_block/full_precision_linear.rs
@@ -0,0 +1,165 @@
+use std::cell::RefCell;
+
+use thiserror::Error;
+
+use super::{EncodableBlock, EncodingParameters};
+use crate::{
+ DataType,
+ backends::common::{
+ Backend,
+ kernel::matmul::{FullPrecisionMatmulArguments, FullPrecisionMatmulKernel, MatmulKernels},
+ },
+ forward_pass::state::{ArrayId, ForwardPassState},
+ parameters::{ParameterLoaderError, ParameterTree},
+};
+
+#[derive(Debug, Error)]
+pub enum FullPrecisionLinearError<B: Backend> {
+ #[error("Backend error: {0}")]
+ BackendError(#[source] B::Error),
+ #[error("Parameter loading error: {0}")]
+ ParameterError(ParameterLoaderError),
+ #[error("Unsupported data type for full precision linear kernel: {0:?}")]
+ UnsupportedDataType(DataType),
+ #[error("Unexpected weights shape: got {got:?}, expected [{expected_output_dim}, {expected_input_dim}]")]
+ InvalidWeightsShape {
+ got: Box<[usize]>,
+ expected_output_dim: usize,
+ expected_input_dim: usize,
+ },
+ #[error("Weights dtype mismatch: got {got:?}, expected {expected:?}")]
+ InvalidWeightsDataType {
+ expected: DataType,
+ got: DataType,
+ },
+ #[error("Bias shape mismatch: got {got:?}, expected [{expected_output_dim}]")]
+ InvalidBiasShape {
+ got: Box<[usize]>,
+ expected_output_dim: usize,
+ },
+ #[error("Bias dtype mismatch: got {got:?}, expected {expected:?}")]
+ InvalidBiasDataType {
+ expected: DataType,
+ got: DataType,
+ },
+}
+
+pub struct FullPrecisionLinear<B: Backend>
+where
+ B::Kernels: MatmulKernels,
+{
+ kernel: RefCell<<B::Kernels as MatmulKernels>::FullPrecisionMatmulKernel>,
+ bias_buffer: Option<B::NativeBuffer>,
+ weights_buffer: B::NativeBuffer,
+ input_dim: usize,
+ output_dim: usize,
+ input_array_id: ArrayId,
+ output_array_id: ArrayId,
+}
+
+impl<B: Backend> FullPrecisionLinear<B>
+where
+ B::Kernels: MatmulKernels,
+{
+ pub fn new(
+ context: &B::Context,
+ precision: DataType,
+ input_dim: usize,
+ output_dim: usize,
+ parameter_tree: &ParameterTree<B::Context>,
+ input_array_id: ArrayId,
+ output_array_id: ArrayId,
+ ) -> Result<Self, FullPrecisionLinearError<B>> {
+ if !matches!(precision, DataType::F16 | DataType::BF16 | DataType::F32) {
+ return Err(FullPrecisionLinearError::UnsupportedDataType(precision));
+ }
+
+ let weights = parameter_tree.leaf("weights").map_err(FullPrecisionLinearError::ParameterError)?;
+ let weights_shape = weights.shape().to_vec();
+ if weights_shape != [output_dim, input_dim] {
+ return Err(FullPrecisionLinearError::InvalidWeightsShape {
+ got: weights_shape.into_boxed_slice(),
+ expected_output_dim: output_dim,
+ expected_input_dim: input_dim,
+ });
+ }
+
+ if weights.data_type() != precision {
+ return Err(FullPrecisionLinearError::InvalidWeightsDataType {
+ expected: precision,
+ got: weights.data_type(),
+ });
+ }
+
+ let bias_buffer = match parameter_tree.leaf("biases") {
+ Ok(biases) => {
+ let bias_shape = biases.shape().to_vec();
+ if bias_shape != [output_dim] {
+ return Err(FullPrecisionLinearError::InvalidBiasShape {
+ got: bias_shape.into_boxed_slice(),
+ expected_output_dim: output_dim,
+ });
+ }
+
+ if biases.data_type() != precision {
+ return Err(FullPrecisionLinearError::InvalidBiasDataType {
+ expected: precision,
+ got: biases.data_type(),
+ });
+ }
+
+ Some(biases.buffer().clone())
+ },
+ Err(_) => None,
+ };
+
+ let kernel = <B::Kernels as MatmulKernels>::FullPrecisionMatmulKernel::new(context, precision)
+ .map_err(FullPrecisionLinearError::BackendError)?;
+
+ Ok(Self {
+ kernel: RefCell::new(kernel),
+ bias_buffer,
+ weights_buffer: weights.buffer().clone(),
+ input_dim,
+ output_dim,
+ input_array_id,
+ output_array_id,
+ })
+ }
+}
+
+impl<B: Backend> EncodableBlock<B> for FullPrecisionLinear<B>
+where
+ B::Kernels: MatmulKernels,
+{
+ fn supports_shared_encoder(&self) -> bool {
+ true
+ }
+
+ fn encode_with_shared_encoder(
+ &self,
+ state: &mut ForwardPassState<B>,
+ _parameters: &EncodingParameters<B>,
+ encoder: &B::ComputeEncoder,
+ ) {
+ let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);
+ let batch_size = state.active_suffix_length();
+ let input_array = arrays[0].borrow_mut();
+ let output_array = arrays[1].borrow_mut();
+
+ self.kernel.borrow_mut().encode(
+ state.mtl_context(),
+ encoder,
+ FullPrecisionMatmulArguments {
+ a: input_array.buffer(),
+ a_offset: 0,
+ b: &self.weights_buffer,
+ output: output_array.buffer(),
+ bias: self.bias_buffer.as_ref(),
+ batch: batch_size,
+ input_dim: self.input_dim,
+ output_dim: self.output_dim,
+ },
+ );
+ }
+}
diff --git a/crates/uzu/src/encodable_block/mod.rs b/crates/uzu/src/encodable_block/mod.rs
index e017b4f7a..79b75fec2 100644
--- a/crates/uzu/src/encodable_block/mod.rs
+++ b/crates/uzu/src/encodable_block/mod.rs
@@ -5,11 +5,14 @@ use crate::{
mod activation;
mod full_precision_embedding_lookup;
+mod full_precision_embedding_readout;
+mod full_precision_linear;
mod layer_norm;
mod normalization;
mod pooling;
mod qk_norm;
mod quantized_embedding_lookup;
+mod quantized_linear;
mod rms_norm;
mod rope;
mod sampling;
@@ -18,11 +21,14 @@ mod tensor_copy;
pub use activation::Activation;
pub use full_precision_embedding_lookup::{FullPrecisionEmbeddingLookup, FullPrecisionEmbeddingLookupError};
+pub use full_precision_embedding_readout::{FullPrecisionEmbeddingReadout, FullPrecisionEmbeddingReadoutError};
+pub use full_precision_linear::{FullPrecisionLinear, FullPrecisionLinearError};
pub use layer_norm::{LayerNorm, LayerNormError};
pub use normalization::{Normalization, NormalizationError};
pub use pooling::Pooling;
pub use qk_norm::{QKNorm, QKNormError};
pub use quantized_embedding_lookup::{QuantizedEmbeddingLookup, QuantizedEmbeddingLookupError};
+pub use quantized_linear::{QuantizedLinear, QuantizedLinearError};
pub use rms_norm::{RMSNorm, RMSNormError};
pub use rope::Rope;
pub use sampling::Sampling;
diff --git a/crates/uzu/src/encodable_block/quantized_linear.rs b/crates/uzu/src/encodable_block/quantized_linear.rs
new file mode 100644
index 000000000..86b4bb437
--- /dev/null
+++ b/crates/uzu/src/encodable_block/quantized_linear.rs
@@ -0,0 +1,293 @@
+use thiserror::Error;
+
+use super::{EncodableBlock, EncodingParameters};
+use crate::{
+ DataType,
+ backends::common::{
+ Backend,
+ kernel::{
+ Kernels, TensorAddBiasKernel,
+ matmul::{
+ MatmulKernels, QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulKernel,
+ QuantizedMatmulType,
+ },
+ },
+ },
+ config::QuantizationConfig,
+ forward_pass::state::{ArrayId, ForwardPassState},
+ parameters::{ParameterLoaderError, ParameterTree},
+};
+
+#[derive(Debug, Error)]
+pub enum QuantizedLinearError<B: Backend> {
+ #[error("Backend error: {0}")]
+ BackendError(#[source] B::Error),
+ #[error("Parameter loading error: {0}")]
+ ParameterError(ParameterLoaderError),
+ #[error("Unsupported data type for quantized kernel: {0:?}")]
+ UnsupportedDataType(DataType),
+ #[error("Expected weights of type {expected:?}, got {got:?}")]
+ InvalidWeightsDataType {
+ expected: DataType,
+ got: DataType,
+ },
+ #[error("Scales dtype mismatch: got {got:?}, expected {expected:?}")]
+ InvalidScalesDataType {
+ expected: DataType,
+ got: DataType,
+ },
+ #[error(
+ "Unexpected MLX shapes. weights={weights:?}, scales={scales:?}, deq_biases={deq_biases:?}; expected [N,K/{packing_divisor}],[N,K_g],[N,K_g]"
+ )]
+ InvalidMlxShapes {
+ weights: Box<[usize]>,
+ scales: Box<[usize]>,
+ deq_biases: Box<[usize]>,
+ packing_divisor: usize,
+ },
+ #[error("deq_biases dtype mismatch: got {got:?}, expected {expected:?}")]
+ InvalidDeqBiasesDataType {
+ expected: DataType,
+ got: DataType,
+ },
+ #[error(
+ "Unexpected AWQ shapes. weights={weights:?}, scales={scales:?}, zero_points={zero_points:?}; expected [N,K/{packing_divisor}],[N,K_g],[N,(K_g+{packing_minus_one})/{packing_divisor}]"
+ )]
+ InvalidAwqShapes {
+ weights: Box<[usize]>,
+ scales: Box<[usize]>,
+ zero_points: Box<[usize]>,
+ packing_divisor: usize,
+ packing_minus_one: usize,
+ },
+ #[error("Zero-points dtype mismatch: got {got:?}, expected {expected:?}")]
+ InvalidZeroPointsDataType {
+ expected: DataType,
+ got: DataType,
+ },
+ #[error("Bias shape mismatch: got {got:?}, expected [{expected_output_dim}]")]
+ InvalidBiasShape {
+ got: Box<[usize]>,
+ expected_output_dim: usize,
+ },
+ #[error("Bias dtype mismatch: got {got:?}, expected {expected:?}")]
+ InvalidBiasDataType {
+ expected: DataType,
+ got: DataType,
+ },
+}
+
+pub struct QuantizedLinear<B: Backend>
+where
+ B::Kernels: MatmulKernels,
+{
+ kernel: <B::Kernels as MatmulKernels>::QuantizedMatmulKernel,
+ bias_add_kernel: Option<<B::Kernels as Kernels>::TensorAddBiasKernel>,
+ biases_buffer: Option<B::NativeBuffer>,
+ weights_buffer: B::NativeBuffer,
+ scales_buffer: B::NativeBuffer,
+ zero_points_or_biases_buffer: B::NativeBuffer,
+ quantization_type: QuantizedMatmulType,
+ input_dim: usize,
+ output_dim: usize,
+ input_array_id: ArrayId,
+ output_array_id: ArrayId,
+}
+
+impl<B: Backend> QuantizedLinear<B>
+where
+ B::Kernels: MatmulKernels,
+{
+ pub fn new(
+ context: &B::Context,
+ config: &QuantizationConfig,
+ input_dim: usize,
+ output_dim: usize,
+ parameter_tree: &ParameterTree<B::Context>,
+ input_array_id: ArrayId,
+ output_array_id: ArrayId,
+ ) -> Result<Self, QuantizedLinearError<B>> {
+ let kernel_data_type: DataType = config.activation_precision.into();
+ if !matches!(kernel_data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
+ return Err(QuantizedLinearError::UnsupportedDataType(kernel_data_type));
+ }
+
+ let weights = parameter_tree.leaf("weights").map_err(QuantizedLinearError::ParameterError)?;
+ let packing_divisor = config.weight_quantization_mode.packing_divisor();
+ let storage_type = config.weight_quantization_mode.storage_type();
+ if weights.data_type() != storage_type {
+ return Err(QuantizedLinearError::InvalidWeightsDataType {
+ expected: storage_type,
+ got: weights.data_type(),
+ });
+ }
+
+ let scales = parameter_tree.leaf("scales").map_err(QuantizedLinearError::ParameterError)?;
+ if scales.data_type() != kernel_data_type {
+ return Err(QuantizedLinearError::InvalidScalesDataType {
+ expected: kernel_data_type,
+ got: scales.data_type(),
+ });
+ }
+
+ let k_g = (input_dim + config.group_size - 1) / config.group_size;
+ let weights_shape = weights.shape().to_vec();
+ let scales_shape = scales.shape().to_vec();
+ let weights_transposed = weights_shape[0] == output_dim;
+
+ let (quantization_type, zero_points_or_biases_buffer) = match parameter_tree.leaf("deq_biases") {
+ Ok(deq_biases) => {
+ let deq_biases_shape = deq_biases.shape().to_vec();
+ if !(weights_shape == [output_dim, input_dim / packing_divisor]
+ && scales_shape == [output_dim, k_g]
+ && deq_biases_shape == [output_dim, k_g])
+ {
+ return Err(QuantizedLinearError::InvalidMlxShapes {
+ weights: weights_shape.clone().into_boxed_slice(),
+ scales: scales_shape.clone().into_boxed_slice(),
+ deq_biases: deq_biases_shape.into_boxed_slice(),
+ packing_divisor,
+ });
+ }
+
+ if deq_biases.data_type() != kernel_data_type {
+ return Err(QuantizedLinearError::InvalidDeqBiasesDataType {
+ expected: kernel_data_type,
+ got: deq_biases.data_type(),
+ });
+ }
+
+ (QuantizedMatmulType::Mlx, deq_biases.buffer().clone())
+ },
+ Err(_) => {
+ let zero_points = parameter_tree.leaf("zero_points").map_err(QuantizedLinearError::ParameterError)?;
+ let zero_points_shape = zero_points.shape().to_vec();
+ let expected_zero_points_entries = (k_g + packing_divisor - 1) / packing_divisor;
+ if !(weights_shape == [output_dim, input_dim / packing_divisor]
+ && scales_shape == [output_dim, k_g]
+ && zero_points_shape == [output_dim, expected_zero_points_entries])
+ {
+ return Err(QuantizedLinearError::InvalidAwqShapes {
+ weights: weights_shape.clone().into_boxed_slice(),
+ scales: scales_shape.clone().into_boxed_slice(),
+ zero_points: zero_points_shape.into_boxed_slice(),
+ packing_divisor,
+ packing_minus_one: packing_divisor - 1,
+ });
+ }
+
+ if zero_points.data_type() != storage_type {
+ return Err(QuantizedLinearError::InvalidZeroPointsDataType {
+ expected: storage_type,
+ got: zero_points.data_type(),
+ });
+ }
+
+ (QuantizedMatmulType::ZeroPoint, zero_points.buffer().clone())
+ },
+ };
+
+ let (bias_add_kernel, biases_buffer) = match parameter_tree.leaf("biases") {
+ Ok(biases) => {
+ let bias_shape = biases.shape().to_vec();
+ if bias_shape != [output_dim] {
+ return Err(QuantizedLinearError::InvalidBiasShape {
+ got: bias_shape.into_boxed_slice(),
+ expected_output_dim: output_dim,
+ });
+ }
+
+ if biases.data_type() != kernel_data_type {
+ return Err(QuantizedLinearError::InvalidBiasDataType {
+ expected: kernel_data_type,
+ got: biases.data_type(),
+ });
+ }
+
+ let bias_add_kernel = <B::Kernels as Kernels>::TensorAddBiasKernel::new(context, kernel_data_type)
+ .map_err(QuantizedLinearError::BackendError)?;
+ (Some(bias_add_kernel), Some(biases.buffer().clone()))
+ },
+ Err(_) => (None, None),
+ };
+
+ let kernel = <B::Kernels as MatmulKernels>::QuantizedMatmulKernel::new(
+ context,
+ QuantizedMatmulConfiguration {
+ data_type: kernel_data_type,
+ group_size: config.group_size,
+ input_dim,
+ output_dim,
+ mode: config.weight_quantization_mode,
+ quantization_type,
+ weights_transposed,
+ },
+ )
+ .map_err(QuantizedLinearError::BackendError)?;
+
+ Ok(Self {
+ kernel,
+ bias_add_kernel,
+ biases_buffer,
+ weights_buffer: weights.buffer().clone(),
+ scales_buffer: scales.buffer().clone(),
+ zero_points_or_biases_buffer,
+ quantization_type,
+ input_dim,
+ output_dim,
+ input_array_id,
+ output_array_id,
+ })
+ }
+}
+
+impl<B: Backend> EncodableBlock<B> for QuantizedLinear<B>
+where
+ B::Kernels: MatmulKernels,
+{
+ fn supports_shared_encoder(&self) -> bool {
+ true
+ }
+
+ fn encode_with_shared_encoder(
+ &self,
+ state: &mut ForwardPassState<B>,
+ parameters: &EncodingParameters<B>,
+ encoder: &B::ComputeEncoder,
+ ) {
+ let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);
+ let batch_size = state.active_suffix_length();
+ let input_array = arrays[0].borrow_mut();
+ let output_array = arrays[1].borrow_mut();
+ let output_buffer = output_array.buffer();
+
+ self.kernel.encode(
+ encoder,
+ QuantizedMatmulArguments {
+ a_buffer: input_array.buffer(),
+ a_offset: 0,
+ b_buffer: &self.weights_buffer,
+ scales_buffer: &self.scales_buffer,
+ zero_points_or_biases_buffer: &self.zero_points_or_biases_buffer,
+ output_buffer,
+ batch: batch_size,
+ input_dim: self.input_dim,
+ output_dim: self.output_dim,
+ quantization_type: self.quantization_type,
+ },
+ );
+
+ if let (Some(bias_add_kernel), Some(biases_buffer)) = (&self.bias_add_kernel, &self.biases_buffer) {
+ let total_length = batch_size * self.output_dim;
+ bias_add_kernel.encode_if(
+ output_buffer,
+ biases_buffer,
+ output_buffer,
+ self.output_dim as u32,
+ total_length as u32,
+ encoder,
+ parameters.predicate_ref(),
+ );
+ }
+ }
+}