A high-performance inference engine for AI models
generalize linear encodables
eugene0b58815e402d
diff --git a/crates/uzu/src/backends/common/kernel/matmul/full_precision.rs b/crates/uzu/src/backends/common/kernel/matmul/full_precision.rsnew file mode 100644index 000000000..22476608f--- /dev/null+++ b/crates/uzu/src/backends/common/kernel/matmul/full_precision.rs@@ -0,0 +1,29 @@+use super::MatmulKernels;+use crate::{DataType, backends::common::Backend};++pub struct FullPrecisionMatmulArguments<'a, B: Backend> {+ pub a: &'a B::NativeBuffer,+ pub a_offset: usize,+ pub b: &'a B::NativeBuffer,+ pub output: &'a B::NativeBuffer,+ pub bias: Option<&'a B::NativeBuffer>,+ pub batch: usize,+ pub input_dim: usize,+ pub output_dim: usize,+}++pub trait FullPrecisionMatmulKernel: Sized {+ type Backend: Backend<Kernels: MatmulKernels<FullPrecisionMatmulKernel = Self>>;++ fn new(+ context: &<Self::Backend as Backend>::Context,+ data_type: DataType,+ ) -> Result<Self, <Self::Backend as Backend>::Error>;++ fn encode(+ &mut self,+ context: &<Self::Backend as Backend>::Context,+ encoder: &<Self::Backend as Backend>::ComputeEncoder,+ arguments: FullPrecisionMatmulArguments<Self::Backend>,+ );+}diff --git a/crates/uzu/src/backends/common/kernel/matmul/mod.rs b/crates/uzu/src/backends/common/kernel/matmul/mod.rsnew file mode 100644index 000000000..d3f0d7c44--- /dev/null+++ b/crates/uzu/src/backends/common/kernel/matmul/mod.rs@@ -0,0 +1,14 @@+mod full_precision;+mod quantized;++pub use full_precision::{FullPrecisionMatmulArguments, FullPrecisionMatmulKernel};+pub use quantized::{+ QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulKernel, QuantizedMatmulType,+};++use super::Kernels;++pub trait MatmulKernels: Kernels {+ type FullPrecisionMatmulKernel: FullPrecisionMatmulKernel<Backend = Self::Backend>;+ type QuantizedMatmulKernel: QuantizedMatmulKernel<Backend = Self::Backend>;+}diff --git a/crates/uzu/src/backends/common/kernel/matmul/quantized.rs b/crates/uzu/src/backends/common/kernel/matmul/quantized.rsnew file mode 100644index 000000000..88da264dd--- /dev/null+++ b/crates/uzu/src/backends/common/kernel/matmul/quantized.rs@@ -0,0 +1,47 @@+use super::MatmulKernels;+use crate::{DataType, backends::common::Backend, config::QuantizationMode};++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum QuantizedMatmulType {+ ZeroPoint,+ Mlx,+}++#[derive(Debug, Clone, Copy)]+pub struct QuantizedMatmulConfiguration {+ pub data_type: DataType,+ pub group_size: usize,+ pub input_dim: usize,+ pub output_dim: usize,+ pub mode: QuantizationMode,+ pub quantization_type: QuantizedMatmulType,+ pub weights_transposed: bool,+}++pub struct QuantizedMatmulArguments<'a, B: Backend> {+ pub a_buffer: &'a B::NativeBuffer,+ pub a_offset: usize,+ pub b_buffer: &'a B::NativeBuffer,+ pub scales_buffer: &'a B::NativeBuffer,+ pub zero_points_or_biases_buffer: &'a B::NativeBuffer,+ pub output_buffer: &'a B::NativeBuffer,+ pub batch: usize,+ pub input_dim: usize,+ pub output_dim: usize,+ pub quantization_type: QuantizedMatmulType,+}++pub trait QuantizedMatmulKernel: Sized {+ type Backend: Backend<Kernels: MatmulKernels<QuantizedMatmulKernel = Self>>;++ fn new(+ context: &<Self::Backend as Backend>::Context,+ configuration: QuantizedMatmulConfiguration,+ ) -> Result<Self, <Self::Backend as Backend>::Error>;++ fn encode(+ &self,+ encoder: &<Self::Backend as Backend>::ComputeEncoder,+ arguments: QuantizedMatmulArguments<Self::Backend>,+ );+}diff --git a/crates/uzu/src/backends/common/kernel/mod.rs b/crates/uzu/src/backends/common/kernel/mod.rsindex fb7523be0..ca042ab4e 100644--- a/crates/uzu/src/backends/common/kernel/mod.rs+++ b/crates/uzu/src/backends/common/kernel/mod.rs@@ -3,4 +3,5 @@include!(concat!(env!("OUT_DIR"), "/traits.rs"));pub mod kv_cache_update;+pub mod matmul;pub mod sampling;diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rsdeleted file mode 100644index 2a1212e09..000000000--- a/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs+++ /dev/null@@ -1,142 +0,0 @@-use std::cell::RefCell;--use super::{- super::{EncodableBlock, Metal},- EmbeddingError,-};-use crate::{- DataType,- backends::metal::{- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, MTLError, ProtocolObject,- Retained,- kernel::matmul::{MatmulArguments, MatmulKernel},- },- encodable_block::EncodingParameters,- forward_pass::state::{ArrayId, ForwardPassState},- parameters::ParameterTree,-};--pub struct FullPrecisionEmbeddingReadout {- kernel: RefCell<MatmulKernel>,- weights_buffer: Retained<ProtocolObject<dyn MTLBuffer>>,- vocab_size: usize,- model_dim: usize,-}--impl FullPrecisionEmbeddingReadout {- pub fn new(- _mtl_context: &MTLContext,- data_type: DataType,- vocab_size: usize,- model_dim: usize,- parameter_tree: &ParameterTree<MTLContext>,- ) -> Result<Self, EmbeddingError> {- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {- return Err(EmbeddingError::UnsupportedDataType(data_type));- }-- let weights = match parameter_tree.leaf("weights") {- Ok(weights) => weights,- Err(_) => parameter_tree.leaf("output_weights").map_err(|e| {- EmbeddingError::MetalError(MTLError::Generic(format!("Failed to load weights: {:?}", e)))- })?,- };-- if weights.shape() != [vocab_size, model_dim] {- return Err(EmbeddingError::MetalError(MTLError::Generic(format!(- "Embedding readout weights shape mismatch: got {:?}, expected [{}, {}]",- weights.shape(),- vocab_size,- model_dim- ))));- }-- if weights.data_type() != data_type {- return Err(EmbeddingError::MetalError(MTLError::Generic(format!(- "Weights dtype mismatch: got {:?}, expected {:?}",- weights.data_type(),- data_type- ))));- }-- let weights_buffer = weights.buffer().to_owned();-- let mut kernel = MatmulKernel::new(data_type).map_err(EmbeddingError::MetalError)?;- kernel.precompile(_mtl_context).map_err(EmbeddingError::MetalError)?;-- Ok(Self {- kernel: RefCell::new(kernel),- weights_buffer: weights_buffer.into(),- vocab_size,- model_dim,- })- }-}--impl EncodableBlock<Metal> for FullPrecisionEmbeddingReadout {- fn encode(- &self,- state: &mut ForwardPassState<Metal>,- parameters: &EncodingParameters<Metal>,- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- ) {- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");- self.encode_with_shared_encoder(state, parameters, &encoder);- encoder.end_encoding();-- if parameters.wait_until_completed {- command_buffer.commit();- command_buffer.wait_until_completed();- }- }-- fn supports_shared_encoder(&self) -> bool {- true- }-- fn encode_with_shared_encoder(- &self,- state: &mut ForwardPassState<Metal>,- _parameters: &EncodingParameters<Metal>,- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- ) {- let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);- let batch_size = state.sampling_length();- if batch_size == 0 {- return;- }- let sampling_start = state.sampling_start();- let input_array_mut = arrays[0].borrow_mut();- let output_array_mut = arrays[1].borrow_mut();-- let elem_size = input_array_mut.data_type().size_in_bytes();- let input_buffer = input_array_mut.buffer();- let output_buffer = output_array_mut.buffer();- let a_offset = (sampling_start * self.model_dim * elem_size) as u64;-- let args = MatmulArguments {- a: input_buffer,- a_offset,- b: &self.weights_buffer,- c: None,- d: output_buffer,- bias: None,- batch: batch_size as i32,- input_dim: self.model_dim as i32,- output_dim: self.vocab_size as i32,- lda: self.model_dim as i32,- ldb: self.model_dim as i32,- ldd: self.vocab_size as i32,- batch_count: 1,- alpha: 1.0,- beta: 0.0,- transpose_a: false,- transpose_b: true,- };-- self.kernel- .borrow_mut()- .encode(state.mtl_context(), encoder, args)- .expect("Failed to encode full precision embedding readout kernel");- }-}diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rsindex 836bb9c2c..f90626864 100644--- a/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rs+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/mod.rs@@ -1,18 +1,18 @@//! Embedding encodables (quantized and full-precision).mod error;-mod full_precision_readout;mod quantized_readout;pub use error::EmbeddingError;-pub use full_precision_readout::FullPrecisionEmbeddingReadout;pub use quantized_readout::QuantizedEmbeddingReadout;use super::super::Metal;use crate::encodable_block::{FullPrecisionEmbeddingLookup as GenericFullPrecisionEmbeddingLookup,+ FullPrecisionEmbeddingReadout as GenericFullPrecisionEmbeddingReadout,QuantizedEmbeddingLookup as GenericQuantizedEmbeddingLookup,};+pub type FullPrecisionEmbeddingReadout = GenericFullPrecisionEmbeddingReadout<Metal>;pub type FullPrecisionEmbeddingLookup = GenericFullPrecisionEmbeddingLookup<Metal>;pub type QuantizedEmbeddingLookup = GenericQuantizedEmbeddingLookup<Metal>;diff --git a/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs b/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rsdeleted file mode 100644index a04831507..000000000--- a/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs+++ /dev/null@@ -1,193 +0,0 @@-use std::cell::RefCell;--use crate::{- DataType,- backends::metal::{- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, MTLError, Metal,- ProtocolObject, Retained,- encodable_block::EncodableBlock,- kernel::matmul::{MatmulArguments, MatmulKernel},- },- encodable_block::EncodingParameters,- forward_pass::state::{ArrayId, ForwardPassState},- parameters::ParameterTree,-};--pub struct FullPrecisionLinear {- kernel: RefCell<MatmulKernel>,- bias_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>>,- weights_buffer: Retained<ProtocolObject<dyn MTLBuffer>>,- input_dim: usize,- output_dim: usize,- input_array_id: ArrayId,- output_array_id: ArrayId,-}--impl FullPrecisionLinear {- pub fn new(- _mtl_context: &MTLContext,- precision: DataType,- input_dim: usize,- output_dim: usize,- parameter_tree: &ParameterTree<MTLContext>,- input_array_id: ArrayId,- output_array_id: ArrayId,- ) -> Result<Self, MTLError> {- if !matches!(precision, DataType::F16 | DataType::BF16 | DataType::F32) {- return Err(MTLError::Generic(format!(- "Unsupported data type for full precision linear kernel: {:?}",- precision- )));- }-- let weights = parameter_tree- .leaf("weights")- .map_err(|e| MTLError::Generic(format!("Failed to load weights: {:?}", e)))?;-- let w_shape = weights.shape();- if w_shape != [output_dim, input_dim] {- return Err(MTLError::Generic(format!(- "Unexpected weights shape: got {:?}, expected [{}, {}]",- w_shape, output_dim, input_dim- )));- }-- if weights.data_type() != precision {- return Err(MTLError::Generic(format!(- "Weights dtype mismatch: got {:?}, expected {:?}",- weights.data_type(),- precision- )));- }-- let weights_buffer: Retained<ProtocolObject<dyn MTLBuffer>> = weights.buffer().to_owned().into();-- let bias_buffer = match parameter_tree.leaf("biases") {- Ok(biases) => {- if biases.shape() != [output_dim] {- return Err(MTLError::Generic(format!(- "Bias shape mismatch: got {:?}, expected [{:?}]",- biases.shape(),- output_dim- )));- }- if biases.data_type() != precision {- return Err(MTLError::Generic(format!(- "Bias dtype mismatch: got {:?}, expected {:?}",- biases.data_type(),- precision- )));- }- let bias_buffer: Retained<ProtocolObject<dyn MTLBuffer>> = biases.buffer().to_owned().into();- Some(bias_buffer)- },- Err(_) => None,- };-- let mut kernel = MatmulKernel::new(precision)?;- kernel.precompile(_mtl_context)?;-- Ok(Self {- kernel: RefCell::new(kernel),- bias_buffer,- weights_buffer,- input_dim,- output_dim,- input_array_id,- output_array_id,- })- }-}--impl EncodableBlock<Metal> for FullPrecisionLinear {- fn encode(- &self,- state: &mut ForwardPassState<Metal>,- parameters: &EncodingParameters<Metal>,- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- ) {- let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);- let batch_size = state.active_suffix_length();-- let input_array_mut = arrays[0].borrow_mut();- let output_array_mut = arrays[1].borrow_mut();-- let input_buffer = input_array_mut.buffer();- let output_buffer = output_array_mut.buffer();-- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");-- let args = MatmulArguments {- a: input_buffer,- a_offset: 0,- b: &self.weights_buffer,- c: None,- d: output_buffer,- bias: self.bias_buffer.as_ref(),- batch: batch_size as i32,- input_dim: self.input_dim as i32,- output_dim: self.output_dim as i32,- lda: self.input_dim as i32,- ldb: self.input_dim as i32,- ldd: self.output_dim as i32,- batch_count: 1,- alpha: 1.0,- beta: 0.0,- transpose_a: false,- transpose_b: true,- };-- let mut kernel = self.kernel.borrow_mut();- kernel.encode(state.mtl_context(), &encoder, args).expect("Failed to encode matmul kernel");-- encoder.end_encoding();-- if parameters.wait_until_completed {- command_buffer.commit();- command_buffer.wait_until_completed();- }- }-- fn supports_shared_encoder(&self) -> bool {- true- }-- fn encode_with_shared_encoder(- &self,- state: &mut ForwardPassState<Metal>,- _parameters: &EncodingParameters<Metal>,- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- ) {- let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);- let batch_size = state.active_suffix_length();-- let input_array_mut = arrays[0].borrow_mut();- let output_array_mut = arrays[1].borrow_mut();-- let input_buffer = input_array_mut.buffer();- let output_buffer = output_array_mut.buffer();-- let args = MatmulArguments {- a: input_buffer,- a_offset: 0,- b: &self.weights_buffer,- c: None,- d: output_buffer,- bias: self.bias_buffer.as_ref(),- batch: batch_size as i32,- input_dim: self.input_dim as i32,- output_dim: self.output_dim as i32,- lda: self.input_dim as i32,- ldb: self.input_dim as i32,- ldd: self.output_dim as i32,- batch_count: 1,- alpha: 1.0,- beta: 0.0,- transpose_a: false,- transpose_b: true,- };-- let mut kernel = self.kernel.borrow_mut();- kernel.encode(state.mtl_context(), encoder, args).expect("Failed to encode matmul kernel");- }-}diff --git a/crates/uzu/src/backends/metal/encodable_block/linear/mod.rs b/crates/uzu/src/backends/metal/encodable_block/linear/mod.rsindex 139ecef36..f393ef8ba 100644--- a/crates/uzu/src/backends/metal/encodable_block/linear/mod.rs+++ b/crates/uzu/src/backends/metal/encodable_block/linear/mod.rs@@ -1,5 +1,7 @@-mod full_precision;-mod quantized;+use super::super::Metal;+use crate::encodable_block::{+ FullPrecisionLinear as GenericFullPrecisionLinear, QuantizedLinear as GenericQuantizedLinear,+};-pub use full_precision::FullPrecisionLinear;-pub use quantized::QuantizedLinear;+pub type FullPrecisionLinear = GenericFullPrecisionLinear<Metal>;+pub type QuantizedLinear = GenericQuantizedLinear<Metal>;diff --git a/crates/uzu/src/backends/metal/encodable_block/linear/quantized.rs b/crates/uzu/src/backends/metal/encodable_block/linear/quantized.rsdeleted file mode 100644index 51bd932a8..000000000--- a/crates/uzu/src/backends/metal/encodable_block/linear/quantized.rs+++ /dev/null@@ -1,303 +0,0 @@-use crate::{- DataType,- backends::{- common::kernel::TensorAddBiasKernel,- metal::{- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, MTLError, Metal,- ProtocolObject, Retained,- encodable_block::EncodableBlock,- kernel::{- dsl::TensorAddBiasMetalKernel,- quant_matmul::{QuantizationType, QuantizedMatmulArguments, QuantizedMatmulKernel},- },- },- },- config::QuantizationConfig,- encodable_block::EncodingParameters,- forward_pass::state::{ArrayId, ForwardPassState},- parameters::ParameterTree,-};--pub struct QuantizedLinear {- kernel: QuantizedMatmulKernel,- bias_add_kernel: Option<TensorAddBiasMetalKernel>,- biases_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>>,- weights_buffer: Retained<ProtocolObject<dyn MTLBuffer>>,- scales_buffer: Retained<ProtocolObject<dyn MTLBuffer>>,- zero_points_or_biases_buffer: Retained<ProtocolObject<dyn MTLBuffer>>,- quantization_type: QuantizationType,- input_dim: usize,- output_dim: usize,- input_array_id: ArrayId,- output_array_id: ArrayId,-}--impl QuantizedLinear {- pub fn new(- mtl_context: &MTLContext,- config: &QuantizationConfig,- input_dim: usize,- output_dim: usize,- parameter_tree: &ParameterTree<MTLContext>,- input_array_id: ArrayId,- output_array_id: ArrayId,- ) -> Result<Self, MTLError> {- let kernel_data_type: DataType = config.activation_precision.into();-- if !matches!(kernel_data_type, DataType::F16 | DataType::BF16 | DataType::F32) {- return Err(MTLError::Generic(format!(- "Unsupported data type for quantized kernel: {:?}",- kernel_data_type- )));- }-- let weights = parameter_tree- .leaf("weights")- .map_err(|e| MTLError::Generic(format!("Failed to load weights: {:?}", e)))?;-- let packing_divisor = config.weight_quantization_mode.packing_divisor();- let storage_type = config.weight_quantization_mode.storage_type();-- if weights.data_type() != storage_type {- return Err(MTLError::Generic(format!(- "Expected weights of type {:?}, got {:?}",- storage_type,- weights.data_type()- )));- }-- let scales =- parameter_tree.leaf("scales").map_err(|e| MTLError::Generic(format!("Failed to load scales: {:?}", e)))?;- let k_g = (input_dim + config.group_size - 1) / config.group_size;-- let w_shape = weights.shape();- let s_shape = scales.shape();-- // Determine if weights are transposed by checking shape- let weights_transposed = w_shape[0] == output_dim;-- // Determine quantization style: prefer MLX (deq_biases), else AWQ (zero_points)- let (quantization_type, zero_points_or_biases_buffer, scales_buffer) = {- if scales.data_type() != kernel_data_type {- return Err(MTLError::Generic(format!(- "Scales dtype mismatch: got {:?}, expected {:?}",- scales.data_type(),- kernel_data_type- )));- }- match parameter_tree.leaf("deq_biases") {- Ok(deq_biases) => {- let db_shape = deq_biases.shape();- if !(w_shape == [output_dim, input_dim / packing_divisor]- && s_shape == [output_dim, k_g]- && db_shape == [output_dim, k_g])- {- return Err(MTLError::Generic(format!(- "Unexpected MLX shapes. weights={:?}, scales={:?}, deq_biases={:?}; expected [N,K/{}],[N,K_g],[N,K_g]",- w_shape, s_shape, db_shape, packing_divisor- )));- }- if deq_biases.data_type() != kernel_data_type {- return Err(MTLError::Generic(format!(- "deq_biases dtype mismatch: got {:?}, expected {:?}",- deq_biases.data_type(),- kernel_data_type- )));- }- let scales_buffer: Retained<ProtocolObject<dyn MTLBuffer>> = scales.buffer().clone();- let biases_buf: Retained<ProtocolObject<dyn MTLBuffer>> = deq_biases.buffer().clone();- (QuantizationType::Mlx, biases_buf, scales_buffer)- },- Err(_) => {- let zero_points = parameter_tree- .leaf("zero_points")- .map_err(|e| MTLError::Generic(format!("Failed to load zero_points: {:?}", e)))?;- let zp_shape = zero_points.shape();- let expected_zp_entries = (k_g + packing_divisor - 1) / packing_divisor;- if !(w_shape == [output_dim, input_dim / packing_divisor]- && s_shape == [output_dim, k_g]- && zp_shape == [output_dim, expected_zp_entries])- {- return Err(MTLError::Generic(format!(- "Unexpected AWQ shapes. weights={:?}, scales={:?}, zero_points={:?}; expected [N,K/{}],[N,K_g],[N,(K_g+{})/{}]",- w_shape,- s_shape,- zp_shape,- packing_divisor,- packing_divisor - 1,- packing_divisor- )));- }- if zero_points.data_type() != storage_type {- return Err(MTLError::Generic(format!(- "Zero-points dtype mismatch: got {:?}, expected {:?}",- zero_points.data_type(),- storage_type- )));- }- let scales_buffer: Retained<ProtocolObject<dyn MTLBuffer>> = scales.buffer().clone();- let zps_buf: Retained<ProtocolObject<dyn MTLBuffer>> = zero_points.buffer().clone();- (QuantizationType::ZeroPoint, zps_buf, scales_buffer)- },- }- };-- let weights_buffer: Retained<ProtocolObject<dyn MTLBuffer>> = weights.buffer().to_owned().into();-- // Optional trainable bias support- let (bias_add_kernel, biases_buffer) = match parameter_tree.leaf("biases") {- Ok(biases) => {- if biases.shape() != [output_dim] {- return Err(MTLError::Generic(format!(- "Bias shape mismatch: got {:?}, expected [{:?}]",- biases.shape(),- output_dim- )));- }- if biases.data_type() != kernel_data_type {- return Err(MTLError::Generic(format!(- "Bias dtype mismatch: got {:?}, expected {:?}",- biases.data_type(),- kernel_data_type- )));- }- let bias_add_kernel = Some(TensorAddBiasMetalKernel::new(mtl_context, kernel_data_type)?);- let biases_buffer: Retained<ProtocolObject<dyn MTLBuffer>> = biases.buffer().to_owned().into();- (bias_add_kernel, Some(biases_buffer))- },- Err(_) => (None, None),- };-- let g = config.group_size;- let mode = config.weight_quantization_mode;-- let kernel = QuantizedMatmulKernel::new(- mtl_context,- kernel_data_type,- g,- input_dim,- output_dim,- mode,- quantization_type,- weights_transposed,- )- .map_err(|e| MTLError::Generic(format!("{:?}", e)))?;-- Ok(Self {- kernel,- bias_add_kernel,- biases_buffer,- weights_buffer,- scales_buffer: scales_buffer.into(),- zero_points_or_biases_buffer: zero_points_or_biases_buffer.into(),- quantization_type,- input_dim,- output_dim,- input_array_id,- output_array_id,- })- }-}--impl EncodableBlock<Metal> for QuantizedLinear {- fn encode(- &self,- state: &mut ForwardPassState<Metal>,- parameters: &EncodingParameters<Metal>,- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- ) {- let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);- let batch_size = state.active_suffix_length();-- let input_array_mut = arrays[0].borrow_mut();- let output_array_mut = arrays[1].borrow_mut();-- let input_buffer = input_array_mut.buffer();- let output_buffer = output_array_mut.buffer();-- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");-- let args = QuantizedMatmulArguments {- a_buffer: input_buffer,- a_offset: 0,- b_buffer: &self.weights_buffer,- scales_buffer: &self.scales_buffer,- zero_points_or_biases_buffer: &self.zero_points_or_biases_buffer,- output_buffer: output_buffer,- batch: batch_size as i32,- input_dim: self.input_dim as i32,- output_dim: self.output_dim as i32,- quantization_type: self.quantization_type,- };-- self.kernel.encode(&encoder, args).expect("Failed to encode quantized matmul kernel");-- encoder.end_encoding();-- if let (Some(bias_add), Some(bias_buf)) = (&self.bias_add_kernel, &self.biases_buffer) {- let total_len = batch_size * self.output_dim;- bias_add.encode_if(- output_buffer,- bias_buf,- output_buffer,- self.output_dim as u32,- total_len as u32,- &encoder,- parameters.predicate,- );- }-- if parameters.wait_until_completed {- command_buffer.commit();- command_buffer.wait_until_completed();- }- }-- fn supports_shared_encoder(&self) -> bool {- true- }-- fn encode_with_shared_encoder(- &self,- state: &mut ForwardPassState<Metal>,- parameters: &EncodingParameters<Metal>,- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- ) {- let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);- let batch_size = state.active_suffix_length();-- let input_array_mut = arrays[0].borrow_mut();- let output_array_mut = arrays[1].borrow_mut();-- let input_buffer = input_array_mut.buffer();- let output_buffer = output_array_mut.buffer();-- let args = QuantizedMatmulArguments {- a_buffer: input_buffer,- a_offset: 0,- b_buffer: &self.weights_buffer,- scales_buffer: &self.scales_buffer,- zero_points_or_biases_buffer: &self.zero_points_or_biases_buffer,- output_buffer: output_buffer,- batch: batch_size as i32,- input_dim: self.input_dim as i32,- output_dim: self.output_dim as i32,- quantization_type: self.quantization_type,- };-- self.kernel.encode(encoder, args).expect("Failed to encode quantized matmul kernel");-- if let (Some(bias_add), Some(bias_buf)) = (&self.bias_add_kernel, &self.biases_buffer) {- let total_len = batch_size * self.output_dim;- bias_add.encode_if(- output_buffer,- bias_buf,- output_buffer,- self.output_dim as u32,- total_len as u32,- encoder,- parameters.predicate,- );- }- }-}diff --git a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rsindex 30685df8b..ac2bfa1c9 100644--- a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs+++ b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs@@ -36,7 +36,8 @@ pub fn linear_block<const N: usize>(parameter_tree,input_array_id,output_array_id,- )?;+ )+ .map_err(|error| MTLError::Generic(format!("{:?}", error)))?;Ok(Box::new(block))},LinearConfig::FullPrecision {@@ -50,7 +51,8 @@ pub fn linear_block<const N: usize>(parameter_tree,input_array_id,output_array_id,- )?;+ )+ .map_err(|error| MTLError::Generic(format!("{:?}", error)))?;Ok(Box::new(block))},LinearConfig::QLoRA {@@ -162,7 +164,8 @@ pub fn mlp_fused_block(.map_err(|error| MTLError::Generic(format!("{:?}", error)))?,ArrayId::MlpHidden,ArrayId::Main,- )?;+ )+ .map_err(|error| MTLError::Generic(format!("{:?}", error)))?;let fused_block = MlpFusedBlock::new_quantized(context,@@ -208,7 +211,8 @@ pub fn mlp_fused_block(.map_err(|error| MTLError::Generic(format!("{:?}", error)))?,ArrayId::MlpHidden,ArrayId::Main,- )?;+ )+ .map_err(|error| MTLError::Generic(format!("{:?}", error)))?;let fused_block = MlpFusedBlock::new_full_precision(context,diff --git a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/mod.rsindex fcf105d50..8c3b55834 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/mod.rs@@ -10,3 +10,56 @@ pub use dispatch_descriptor::{MatmulKernelVariant, determine_kernel_variant};pub use gemv::GemvKernel;pub use kernel::MatmulKernel;pub use split_k::SplitKGemm;++use crate::{+ DataType,+ backends::{+ common::kernel::matmul::{+ FullPrecisionMatmulArguments, FullPrecisionMatmulKernel as FullPrecisionMatmulKernelTrait,+ },+ metal::{MTLComputeCommandEncoder, MTLContext, MTLError, Metal, ProtocolObject},+ },+};++impl FullPrecisionMatmulKernelTrait for MatmulKernel {+ type Backend = Metal;++ fn new(+ context: &MTLContext,+ data_type: DataType,+ ) -> Result<Self, MTLError> {+ let mut kernel = MatmulKernel::new(data_type)?;+ kernel.precompile(context)?;+ Ok(kernel)+ }++ fn encode(+ &mut self,+ context: &MTLContext,+ encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,+ arguments: FullPrecisionMatmulArguments<Metal>,+ ) {+ let matmul_arguments = MatmulArguments {+ a: arguments.a,+ a_offset: arguments.a_offset as u64,+ b: arguments.b,+ c: None,+ d: arguments.output,+ bias: arguments.bias,+ batch: arguments.batch as i32,+ input_dim: arguments.input_dim as i32,+ output_dim: arguments.output_dim as i32,+ lda: arguments.input_dim as i32,+ ldb: arguments.input_dim as i32,+ ldd: arguments.output_dim as i32,+ batch_count: 1,+ alpha: 1.0,+ beta: 0.0,+ transpose_a: false,+ transpose_b: true,+ };++ MatmulKernel::encode(self, context, encoder, matmul_arguments)+ .expect("Failed to encode full precision matmul kernel");+ }+}diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rsindex 17f263a99..202e9b4d7 100644--- a/crates/uzu/src/backends/metal/kernel/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/mod.rs@@ -1,5 +1,6 @@pub mod attention;mod data_type;+use crate::backends::common::kernel::matmul::MatmulKernels;pub mod dsl {include!(concat!(env!("OUT_DIR"), "/dsl.rs"));}@@ -33,3 +34,8 @@ pub use short_conv::{ShortConvDecodeArguments, ShortConvKernel, ShortConvKernelEpub use ssm::{Conv1dPackArguments, Conv1dScanArguments, Conv1dScanKernel, SSDPrefillArguments, SSDPrefillMode, SSMKernelError,};++impl MatmulKernels for dsl::MetalKernels {+ type FullPrecisionMatmulKernel = MatmulKernel;+ type QuantizedMatmulKernel = quant_matmul::QuantizedMatmulKernel;+}diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rsindex 0311dd7bc..ace98f02d 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs@@ -2,10 +2,16 @@ use std::collections::HashMap;use crate::{DataType,- backends::metal::{- FunctionConstantValuesSetValue, MTLBuffer, MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext,- MTLError, MTLFunctionConstantValues, MTLSize, ProtocolObject, Retained,- metal_extensions::ComputeEncoderSetValue,+ backends::{+ common::kernel::matmul::{+ QuantizedMatmulArguments as GenericQuantizedMatmulArguments, QuantizedMatmulConfiguration,+ QuantizedMatmulKernel as QuantizedMatmulKernelTrait, QuantizedMatmulType,+ },+ metal::{+ FunctionConstantValuesSetValue, MTLBuffer, MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext,+ MTLError, MTLFunctionConstantValues, MTLSize, Metal, ProtocolObject, Retained,+ metal_extensions::ComputeEncoderSetValue,+ },},config::QuantizationMode,};@@ -229,6 +235,58 @@ impl QuantizedMatmulKernel {}}+impl QuantizedMatmulKernelTrait for QuantizedMatmulKernel {+ type Backend = Metal;++ fn new(+ context: &MTLContext,+ configuration: QuantizedMatmulConfiguration,+ ) -> Result<Self, MTLError> {+ let quantization_type = match configuration.quantization_type {+ QuantizedMatmulType::ZeroPoint => QuantizationType::ZeroPoint,+ QuantizedMatmulType::Mlx => QuantizationType::Mlx,+ };++ QuantizedMatmulKernel::new(+ context,+ configuration.data_type,+ configuration.group_size,+ configuration.input_dim,+ configuration.output_dim,+ configuration.mode,+ quantization_type,+ configuration.weights_transposed,+ )+ .map_err(|error| MTLError::Generic(format!("{:?}", error)))+ }++ fn encode(+ &self,+ encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,+ arguments: GenericQuantizedMatmulArguments<Metal>,+ ) {+ let quantization_type = match arguments.quantization_type {+ QuantizedMatmulType::ZeroPoint => QuantizationType::ZeroPoint,+ QuantizedMatmulType::Mlx => QuantizationType::Mlx,+ };++ let backend_arguments = QuantizedMatmulArguments {+ a_buffer: arguments.a_buffer,+ a_offset: arguments.a_offset as u64,+ b_buffer: arguments.b_buffer,+ scales_buffer: arguments.scales_buffer,+ zero_points_or_biases_buffer: arguments.zero_points_or_biases_buffer,+ output_buffer: arguments.output_buffer,+ batch: arguments.batch as i32,+ input_dim: arguments.input_dim as i32,+ output_dim: arguments.output_dim as i32,+ quantization_type,+ };++ QuantizedMatmulKernel::encode(self, encoder, backend_arguments).expect("Failed to encode quantized matmul");+ }+}+fn select_matrix_vector_kernel_name(data_type: DataType,group_size: usize,diff --git a/crates/uzu/src/encodable_block/full_precision_embedding_readout.rs b/crates/uzu/src/encodable_block/full_precision_embedding_readout.rsnew file mode 100644index 000000000..6e10e9855--- /dev/null+++ b/crates/uzu/src/encodable_block/full_precision_embedding_readout.rs@@ -0,0 +1,139 @@+use std::cell::RefCell;++use thiserror::Error;++use super::{EncodableBlock, EncodingParameters};+use crate::{+ DataType,+ backends::common::{+ Backend,+ kernel::matmul::{FullPrecisionMatmulArguments, FullPrecisionMatmulKernel, MatmulKernels},+ },+ forward_pass::state::{ArrayId, ForwardPassState},+ parameters::{ParameterLoaderError, ParameterTree},+};++#[derive(Debug, Error)]+pub enum FullPrecisionEmbeddingReadoutError<B: Backend> {+ #[error("Backend error: {0}")]+ BackendError(#[source] B::Error),+ #[error("Parameter loading error: {0}")]+ ParameterError(ParameterLoaderError),+ #[error("Unsupported data type: {0:?}")]+ UnsupportedDataType(DataType),+ #[error(+ "Embedding readout weights shape mismatch: got {got:?}, expected [{expected_vocab_size}, {expected_model_dim}]"+ )]+ InvalidWeightsShape {+ got: Box<[usize]>,+ expected_vocab_size: usize,+ expected_model_dim: usize,+ },+ #[error("Weights dtype mismatch: got {got:?}, expected {expected:?}")]+ InvalidWeightsDataType {+ expected: DataType,+ got: DataType,+ },+}++pub struct FullPrecisionEmbeddingReadout<B: Backend>+where+ B::Kernels: MatmulKernels,+{+ kernel: RefCell<<B::Kernels as MatmulKernels>::FullPrecisionMatmulKernel>,+ weights_buffer: B::NativeBuffer,+ vocab_size: usize,+ model_dim: usize,+}++impl<B: Backend> FullPrecisionEmbeddingReadout<B>+where+ B::Kernels: MatmulKernels,+{+ pub fn new(+ context: &B::Context,+ data_type: DataType,+ vocab_size: usize,+ model_dim: usize,+ parameter_tree: &ParameterTree<B::Context>,+ ) -> Result<Self, FullPrecisionEmbeddingReadoutError<B>> {+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {+ return Err(FullPrecisionEmbeddingReadoutError::UnsupportedDataType(data_type));+ }++ let weights = match parameter_tree.leaf("weights") {+ Ok(weights) => weights,+ Err(_) => {+ parameter_tree.leaf("output_weights").map_err(FullPrecisionEmbeddingReadoutError::ParameterError)?+ },+ };++ let weights_shape = weights.shape().to_vec();+ if weights_shape != [vocab_size, model_dim] {+ return Err(FullPrecisionEmbeddingReadoutError::InvalidWeightsShape {+ got: weights_shape.into_boxed_slice(),+ expected_vocab_size: vocab_size,+ expected_model_dim: model_dim,+ });+ }++ if weights.data_type() != data_type {+ return Err(FullPrecisionEmbeddingReadoutError::InvalidWeightsDataType {+ expected: data_type,+ got: weights.data_type(),+ });+ }++ let kernel = <B::Kernels as MatmulKernels>::FullPrecisionMatmulKernel::new(context, data_type)+ .map_err(FullPrecisionEmbeddingReadoutError::BackendError)?;++ Ok(Self {+ kernel: RefCell::new(kernel),+ weights_buffer: weights.buffer().clone(),+ vocab_size,+ model_dim,+ })+ }+}++impl<B: Backend> EncodableBlock<B> for FullPrecisionEmbeddingReadout<B>+where+ B::Kernels: MatmulKernels,+{+ fn supports_shared_encoder(&self) -> bool {+ true+ }++ fn encode_with_shared_encoder(+ &self,+ state: &mut ForwardPassState<B>,+ _parameters: &EncodingParameters<B>,+ encoder: &B::ComputeEncoder,+ ) {+ let arrays = state.arrays(&[ArrayId::Main, ArrayId::Logits]);+ let batch_size = state.sampling_length();+ if batch_size == 0 {+ return;+ }++ let sampling_start = state.sampling_start();+ let input_array = arrays[0].borrow_mut();+ let output_array = arrays[1].borrow_mut();+ let element_size = input_array.data_type().size_in_bytes();++ self.kernel.borrow_mut().encode(+ state.mtl_context(),+ encoder,+ FullPrecisionMatmulArguments {+ a: input_array.buffer(),+ a_offset: sampling_start * self.model_dim * element_size,+ b: &self.weights_buffer,+ output: output_array.buffer(),+ bias: None,+ batch: batch_size,+ input_dim: self.model_dim,+ output_dim: self.vocab_size,+ },+ );+ }+}diff --git a/crates/uzu/src/encodable_block/full_precision_linear.rs b/crates/uzu/src/encodable_block/full_precision_linear.rsnew file mode 100644index 000000000..17b8e83a6--- /dev/null+++ b/crates/uzu/src/encodable_block/full_precision_linear.rs@@ -0,0 +1,165 @@+use std::cell::RefCell;++use thiserror::Error;++use super::{EncodableBlock, EncodingParameters};+use crate::{+ DataType,+ backends::common::{+ Backend,+ kernel::matmul::{FullPrecisionMatmulArguments, FullPrecisionMatmulKernel, MatmulKernels},+ },+ forward_pass::state::{ArrayId, ForwardPassState},+ parameters::{ParameterLoaderError, ParameterTree},+};++#[derive(Debug, Error)]+pub enum FullPrecisionLinearError<B: Backend> {+ #[error("Backend error: {0}")]+ BackendError(#[source] B::Error),+ #[error("Parameter loading error: {0}")]+ ParameterError(ParameterLoaderError),+ #[error("Unsupported data type for full precision linear kernel: {0:?}")]+ UnsupportedDataType(DataType),+ #[error("Unexpected weights shape: got {got:?}, expected [{expected_output_dim}, {expected_input_dim}]")]+ InvalidWeightsShape {+ got: Box<[usize]>,+ expected_output_dim: usize,+ expected_input_dim: usize,+ },+ #[error("Weights dtype mismatch: got {got:?}, expected {expected:?}")]+ InvalidWeightsDataType {+ expected: DataType,+ got: DataType,+ },+ #[error("Bias shape mismatch: got {got:?}, expected [{expected_output_dim}]")]+ InvalidBiasShape {+ got: Box<[usize]>,+ expected_output_dim: usize,+ },+ #[error("Bias dtype mismatch: got {got:?}, expected {expected:?}")]+ InvalidBiasDataType {+ expected: DataType,+ got: DataType,+ },+}++pub struct FullPrecisionLinear<B: Backend>+where+ B::Kernels: MatmulKernels,+{+ kernel: RefCell<<B::Kernels as MatmulKernels>::FullPrecisionMatmulKernel>,+ bias_buffer: Option<B::NativeBuffer>,+ weights_buffer: B::NativeBuffer,+ input_dim: usize,+ output_dim: usize,+ input_array_id: ArrayId,+ output_array_id: ArrayId,+}++impl<B: Backend> FullPrecisionLinear<B>+where+ B::Kernels: MatmulKernels,+{+ pub fn new(+ context: &B::Context,+ precision: DataType,+ input_dim: usize,+ output_dim: usize,+ parameter_tree: &ParameterTree<B::Context>,+ input_array_id: ArrayId,+ output_array_id: ArrayId,+ ) -> Result<Self, FullPrecisionLinearError<B>> {+ if !matches!(precision, DataType::F16 | DataType::BF16 | DataType::F32) {+ return Err(FullPrecisionLinearError::UnsupportedDataType(precision));+ }++ let weights = parameter_tree.leaf("weights").map_err(FullPrecisionLinearError::ParameterError)?;+ let weights_shape = weights.shape().to_vec();+ if weights_shape != [output_dim, input_dim] {+ return Err(FullPrecisionLinearError::InvalidWeightsShape {+ got: weights_shape.into_boxed_slice(),+ expected_output_dim: output_dim,+ expected_input_dim: input_dim,+ });+ }++ if weights.data_type() != precision {+ return Err(FullPrecisionLinearError::InvalidWeightsDataType {+ expected: precision,+ got: weights.data_type(),+ });+ }++ let bias_buffer = match parameter_tree.leaf("biases") {+ Ok(biases) => {+ let bias_shape = biases.shape().to_vec();+ if bias_shape != [output_dim] {+ return Err(FullPrecisionLinearError::InvalidBiasShape {+ got: bias_shape.into_boxed_slice(),+ expected_output_dim: output_dim,+ });+ }++ if biases.data_type() != precision {+ return Err(FullPrecisionLinearError::InvalidBiasDataType {+ expected: precision,+ got: biases.data_type(),+ });+ }++ Some(biases.buffer().clone())+ },+ Err(_) => None,+ };++ let kernel = <B::Kernels as MatmulKernels>::FullPrecisionMatmulKernel::new(context, precision)+ .map_err(FullPrecisionLinearError::BackendError)?;++ Ok(Self {+ kernel: RefCell::new(kernel),+ bias_buffer,+ weights_buffer: weights.buffer().clone(),+ input_dim,+ output_dim,+ input_array_id,+ output_array_id,+ })+ }+}++impl<B: Backend> EncodableBlock<B> for FullPrecisionLinear<B>+where+ B::Kernels: MatmulKernels,+{+ fn supports_shared_encoder(&self) -> bool {+ true+ }++ fn encode_with_shared_encoder(+ &self,+ state: &mut ForwardPassState<B>,+ _parameters: &EncodingParameters<B>,+ encoder: &B::ComputeEncoder,+ ) {+ let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);+ let batch_size = state.active_suffix_length();+ let input_array = arrays[0].borrow_mut();+ let output_array = arrays[1].borrow_mut();++ self.kernel.borrow_mut().encode(+ state.mtl_context(),+ encoder,+ FullPrecisionMatmulArguments {+ a: input_array.buffer(),+ a_offset: 0,+ b: &self.weights_buffer,+ output: output_array.buffer(),+ bias: self.bias_buffer.as_ref(),+ batch: batch_size,+ input_dim: self.input_dim,+ output_dim: self.output_dim,+ },+ );+ }+}diff --git a/crates/uzu/src/encodable_block/mod.rs b/crates/uzu/src/encodable_block/mod.rsindex e017b4f7a..79b75fec2 100644--- a/crates/uzu/src/encodable_block/mod.rs+++ b/crates/uzu/src/encodable_block/mod.rs@@ -5,11 +5,14 @@ use crate::{mod activation;mod full_precision_embedding_lookup;+mod full_precision_embedding_readout;+mod full_precision_linear;mod layer_norm;mod normalization;mod pooling;mod qk_norm;mod quantized_embedding_lookup;+mod quantized_linear;mod rms_norm;mod rope;mod sampling;@@ -18,11 +21,14 @@ mod tensor_copy;pub use activation::Activation;pub use full_precision_embedding_lookup::{FullPrecisionEmbeddingLookup, FullPrecisionEmbeddingLookupError};+pub use full_precision_embedding_readout::{FullPrecisionEmbeddingReadout, FullPrecisionEmbeddingReadoutError};+pub use full_precision_linear::{FullPrecisionLinear, FullPrecisionLinearError};pub use layer_norm::{LayerNorm, LayerNormError};pub use normalization::{Normalization, NormalizationError};pub use pooling::Pooling;pub use qk_norm::{QKNorm, QKNormError};pub use quantized_embedding_lookup::{QuantizedEmbeddingLookup, QuantizedEmbeddingLookupError};+pub use quantized_linear::{QuantizedLinear, QuantizedLinearError};pub use rms_norm::{RMSNorm, RMSNormError};pub use rope::Rope;pub use sampling::Sampling;diff --git a/crates/uzu/src/encodable_block/quantized_linear.rs b/crates/uzu/src/encodable_block/quantized_linear.rsnew file mode 100644index 000000000..86b4bb437--- /dev/null+++ b/crates/uzu/src/encodable_block/quantized_linear.rs@@ -0,0 +1,293 @@+use thiserror::Error;++use super::{EncodableBlock, EncodingParameters};+use crate::{+ DataType,+ backends::common::{+ Backend,+ kernel::{+ Kernels, TensorAddBiasKernel,+ matmul::{+ MatmulKernels, QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulKernel,+ QuantizedMatmulType,+ },+ },+ },+ config::QuantizationConfig,+ forward_pass::state::{ArrayId, ForwardPassState},+ parameters::{ParameterLoaderError, ParameterTree},+};++#[derive(Debug, Error)]+pub enum QuantizedLinearError<B: Backend> {+ #[error("Backend error: {0}")]+ BackendError(#[source] B::Error),+ #[error("Parameter loading error: {0}")]+ ParameterError(ParameterLoaderError),+ #[error("Unsupported data type for quantized kernel: {0:?}")]+ UnsupportedDataType(DataType),+ #[error("Expected weights of type {expected:?}, got {got:?}")]+ InvalidWeightsDataType {+ expected: DataType,+ got: DataType,+ },+ #[error("Scales dtype mismatch: got {got:?}, expected {expected:?}")]+ InvalidScalesDataType {+ expected: DataType,+ got: DataType,+ },+ #[error(+ "Unexpected MLX shapes. weights={weights:?}, scales={scales:?}, deq_biases={deq_biases:?}; expected [N,K/{packing_divisor}],[N,K_g],[N,K_g]"+ )]+ InvalidMlxShapes {+ weights: Box<[usize]>,+ scales: Box<[usize]>,+ deq_biases: Box<[usize]>,+ packing_divisor: usize,+ },+ #[error("deq_biases dtype mismatch: got {got:?}, expected {expected:?}")]+ InvalidDeqBiasesDataType {+ expected: DataType,+ got: DataType,+ },+ #[error(+ "Unexpected AWQ shapes. weights={weights:?}, scales={scales:?}, zero_points={zero_points:?}; expected [N,K/{packing_divisor}],[N,K_g],[N,(K_g+{packing_minus_one})/{packing_divisor}]"+ )]+ InvalidAwqShapes {+ weights: Box<[usize]>,+ scales: Box<[usize]>,+ zero_points: Box<[usize]>,+ packing_divisor: usize,+ packing_minus_one: usize,+ },+ #[error("Zero-points dtype mismatch: got {got:?}, expected {expected:?}")]+ InvalidZeroPointsDataType {+ expected: DataType,+ got: DataType,+ },+ #[error("Bias shape mismatch: got {got:?}, expected [{expected_output_dim}]")]+ InvalidBiasShape {+ got: Box<[usize]>,+ expected_output_dim: usize,+ },+ #[error("Bias dtype mismatch: got {got:?}, expected {expected:?}")]+ InvalidBiasDataType {+ expected: DataType,+ got: DataType,+ },+}++pub struct QuantizedLinear<B: Backend>+where+ B::Kernels: MatmulKernels,+{+ kernel: <B::Kernels as MatmulKernels>::QuantizedMatmulKernel,+ bias_add_kernel: Option<<B::Kernels as Kernels>::TensorAddBiasKernel>,+ biases_buffer: Option<B::NativeBuffer>,+ weights_buffer: B::NativeBuffer,+ scales_buffer: B::NativeBuffer,+ zero_points_or_biases_buffer: B::NativeBuffer,+ quantization_type: QuantizedMatmulType,+ input_dim: usize,+ output_dim: usize,+ input_array_id: ArrayId,+ output_array_id: ArrayId,+}++impl<B: Backend> QuantizedLinear<B>+where+ B::Kernels: MatmulKernels,+{+ pub fn new(+ context: &B::Context,+ config: &QuantizationConfig,+ input_dim: usize,+ output_dim: usize,+ parameter_tree: &ParameterTree<B::Context>,+ input_array_id: ArrayId,+ output_array_id: ArrayId,+ ) -> Result<Self, QuantizedLinearError<B>> {+ let kernel_data_type: DataType = config.activation_precision.into();+ if !matches!(kernel_data_type, DataType::F16 | DataType::BF16 | DataType::F32) {+ return Err(QuantizedLinearError::UnsupportedDataType(kernel_data_type));+ }++ let weights = parameter_tree.leaf("weights").map_err(QuantizedLinearError::ParameterError)?;+ let packing_divisor = config.weight_quantization_mode.packing_divisor();+ let storage_type = config.weight_quantization_mode.storage_type();+ if weights.data_type() != storage_type {+ return Err(QuantizedLinearError::InvalidWeightsDataType {+ expected: storage_type,+ got: weights.data_type(),+ });+ }++ let scales = parameter_tree.leaf("scales").map_err(QuantizedLinearError::ParameterError)?;+ if scales.data_type() != kernel_data_type {+ return Err(QuantizedLinearError::InvalidScalesDataType {+ expected: kernel_data_type,+ got: scales.data_type(),+ });+ }++ let k_g = (input_dim + config.group_size - 1) / config.group_size;+ let weights_shape = weights.shape().to_vec();+ let scales_shape = scales.shape().to_vec();+ let weights_transposed = weights_shape[0] == output_dim;++ let (quantization_type, zero_points_or_biases_buffer) = match parameter_tree.leaf("deq_biases") {+ Ok(deq_biases) => {+ let deq_biases_shape = deq_biases.shape().to_vec();+ if !(weights_shape == [output_dim, input_dim / packing_divisor]+ && scales_shape == [output_dim, k_g]+ && deq_biases_shape == [output_dim, k_g])+ {+ return Err(QuantizedLinearError::InvalidMlxShapes {+ weights: weights_shape.clone().into_boxed_slice(),+ scales: scales_shape.clone().into_boxed_slice(),+ deq_biases: deq_biases_shape.into_boxed_slice(),+ packing_divisor,+ });+ }++ if deq_biases.data_type() != kernel_data_type {+ return Err(QuantizedLinearError::InvalidDeqBiasesDataType {+ expected: kernel_data_type,+ got: deq_biases.data_type(),+ });+ }++ (QuantizedMatmulType::Mlx, deq_biases.buffer().clone())+ },+ Err(_) => {+ let zero_points = parameter_tree.leaf("zero_points").map_err(QuantizedLinearError::ParameterError)?;+ let zero_points_shape = zero_points.shape().to_vec();+ let expected_zero_points_entries = (k_g + packing_divisor - 1) / packing_divisor;+ if !(weights_shape == [output_dim, input_dim / packing_divisor]+ && scales_shape == [output_dim, k_g]+ && zero_points_shape == [output_dim, expected_zero_points_entries])+ {+ return Err(QuantizedLinearError::InvalidAwqShapes {+ weights: weights_shape.clone().into_boxed_slice(),+ scales: scales_shape.clone().into_boxed_slice(),+ zero_points: zero_points_shape.into_boxed_slice(),+ packing_divisor,+ packing_minus_one: packing_divisor - 1,+ });+ }++ if zero_points.data_type() != storage_type {+ return Err(QuantizedLinearError::InvalidZeroPointsDataType {+ expected: storage_type,+ got: zero_points.data_type(),+ });+ }++ (QuantizedMatmulType::ZeroPoint, zero_points.buffer().clone())+ },+ };++ let (bias_add_kernel, biases_buffer) = match parameter_tree.leaf("biases") {+ Ok(biases) => {+ let bias_shape = biases.shape().to_vec();+ if bias_shape != [output_dim] {+ return Err(QuantizedLinearError::InvalidBiasShape {+ got: bias_shape.into_boxed_slice(),+ expected_output_dim: output_dim,+ });+ }++ if biases.data_type() != kernel_data_type {+ return Err(QuantizedLinearError::InvalidBiasDataType {+ expected: kernel_data_type,+ got: biases.data_type(),+ });+ }++ let bias_add_kernel = <B::Kernels as Kernels>::TensorAddBiasKernel::new(context, kernel_data_type)+ .map_err(QuantizedLinearError::BackendError)?;+ (Some(bias_add_kernel), Some(biases.buffer().clone()))+ },+ Err(_) => (None, None),+ };++ let kernel = <B::Kernels as MatmulKernels>::QuantizedMatmulKernel::new(+ context,+ QuantizedMatmulConfiguration {+ data_type: kernel_data_type,+ group_size: config.group_size,+ input_dim,+ output_dim,+ mode: config.weight_quantization_mode,+ quantization_type,+ weights_transposed,+ },+ )+ .map_err(QuantizedLinearError::BackendError)?;++ Ok(Self {+ kernel,+ bias_add_kernel,+ biases_buffer,+ weights_buffer: weights.buffer().clone(),+ scales_buffer: scales.buffer().clone(),+ zero_points_or_biases_buffer,+ quantization_type,+ input_dim,+ output_dim,+ input_array_id,+ output_array_id,+ })+ }+}++impl<B: Backend> EncodableBlock<B> for QuantizedLinear<B>+where+ B::Kernels: MatmulKernels,+{+ fn supports_shared_encoder(&self) -> bool {+ true+ }++ fn encode_with_shared_encoder(+ &self,+ state: &mut ForwardPassState<B>,+ parameters: &EncodingParameters<B>,+ encoder: &B::ComputeEncoder,+ ) {+ let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);+ let batch_size = state.active_suffix_length();+ let input_array = arrays[0].borrow_mut();+ let output_array = arrays[1].borrow_mut();+ let output_buffer = output_array.buffer();++ self.kernel.encode(+ encoder,+ QuantizedMatmulArguments {+ a_buffer: input_array.buffer(),+ a_offset: 0,+ b_buffer: &self.weights_buffer,+ scales_buffer: &self.scales_buffer,+ zero_points_or_biases_buffer: &self.zero_points_or_biases_buffer,+ output_buffer,+ batch: batch_size,+ input_dim: self.input_dim,+ output_dim: self.output_dim,+ quantization_type: self.quantization_type,+ },+ );++ if let (Some(bias_add_kernel), Some(biases_buffer)) = (&self.bias_add_kernel, &self.biases_buffer) {+ let total_length = batch_size * self.output_dim;+ bias_add_kernel.encode_if(+ output_buffer,+ biases_buffer,+ output_buffer,+ self.output_dim as u32,+ total_length as u32,+ encoder,+ parameters.predicate_ref(),+ );+ }+ }+}