A high-performance inference engine for AI models
Quantized Matmul Kernles (#25)
LuckyIYI12e103b34623
diff --git a/crates/uzu/src/backends/metal/executable_builder/decoder_executables.rs b/crates/uzu/src/backends/metal/executable_builder/decoder_executables.rsindex b4c8196f7..ed02cc591 100644--- a/crates/uzu/src/backends/metal/executable_builder/decoder_executables.rs+++ b/crates/uzu/src/backends/metal/executable_builder/decoder_executables.rs@@ -9,7 +9,7 @@ use crate::{KernelDataType, MTLContext, ModelShape,compilation_parameters::CompilationConfig,forward_pass::{- ArrayId, ForwardPassState, MPSGraphBlock, RopeType,+ ArrayId, ForwardPassState, RopeType,encodable_with_state::{EncodableWithState, EncodingParameters},transformer_layer::{embed_block, readout_block},},@@ -20,10 +20,10 @@ use crate::{};pub struct DecoderExecutables {- pub embed: MPSGraphBlock,+ pub embed: Box<dyn EncodableWithState>,pub layers: Box<[LayerExecutables]>,pub norm: Box<dyn EncodableWithState>,- pub readout: MPSGraphBlock,+ pub readout: Box<dyn EncodableWithState>,pub global_rope: Rc<Box<dyn EncodableWithState>>,pub local_rope: Option<Rc<Box<dyn EncodableWithState>>>,}diff --git a/crates/uzu/src/backends/metal/executable_builder/layer_executables.rs b/crates/uzu/src/backends/metal/executable_builder/layer_executables.rsindex 4bc0a74e9..0d0d17180 100644--- a/crates/uzu/src/backends/metal/executable_builder/layer_executables.rs+++ b/crates/uzu/src/backends/metal/executable_builder/layer_executables.rs@@ -9,7 +9,7 @@ use crate::{MTLContext,compilation_parameters::CompilationConfig,forward_pass::{- ArrayId, ForwardPassState, MPSGraphBlock,+ ArrayId, ForwardPassState,encodable_with_state::{EncodableWithState, EncodingParameters},transformer_layer,},@@ -26,15 +26,15 @@ pub struct LayerExecutables {pub layer_index: usize,pub copy_main_to_shortcut: Box<dyn EncodableWithState>,pub pre_attention_norm: Box<dyn EncodableWithState>,- pub qkv_projection: MPSGraphBlock,+ pub qkv_projection: Box<dyn EncodableWithState>,pub qk_norm: Option<Box<dyn EncodableWithState>>,pub rope: Rc<Box<dyn EncodableWithState>>,pub attention: Box<dyn EncodableWithState>,- pub out_projection: MPSGraphBlock,+ pub out_projection: Box<dyn EncodableWithState>,pub post_attention_norm: Option<Box<dyn EncodableWithState>>,pub main_shortcut_add_swap: Box<dyn EncodableWithState>,pub pre_mlp_norm: Box<dyn EncodableWithState>,- pub mlp: MPSGraphBlock,+ pub mlp: Box<dyn EncodableWithState>,pub post_mlp_norm: Option<Box<dyn EncodableWithState>>,}@@ -121,10 +121,10 @@ impl LayerExecutables {head_dim,) {Ok(qk_norm) => Some(Box::new(qk_norm)),- Err(e) => {- eprintln!("Failed to create QK norm kernel: {:?}", e);- None- },+ Err(e) => panic!(+ "Failed to create QK norm kernel for layer {}: {:?}",+ layer_index, e+ ),}} else {Nonediff --git a/crates/uzu/src/backends/metal/forward_pass/buffers.rs b/crates/uzu/src/backends/metal/forward_pass/buffers.rsindex eee037f34..07084db99 100644--- a/crates/uzu/src/backends/metal/forward_pass/buffers.rs+++ b/crates/uzu/src/backends/metal/forward_pass/buffers.rs@@ -23,6 +23,8 @@ pub struct ForwardPassBuffers {pub shortcut: MTLBuffer,pub qkv: MTLBuffer,pub attention_output: MTLBuffer,+ pub mlp_fused_up: MTLBuffer,+ pub mlp_hidden: MTLBuffer,// 3-Dpub rotated_queries: MTLBuffer,@@ -54,16 +56,9 @@ impl ForwardPassBuffers {let act_ty = model_shape.activation_data_type();- let total_blocks_count = 32;- let num_heads = model_shape.num_heads();- let head_dim = model_shape.head_dim();-- // Partials buffer: [num_heads, max_suffix_len, total_blocks_count, head_dim]let partials_shape =- [num_heads * max_suffix_len * total_blocks_count * head_dim];-- // Sums and maxs buffers: [num_heads, max_suffix_len, total_blocks_count]- let sums_maxs_shape = [num_heads * max_suffix_len * total_blocks_count];+ model_shape.attention_partials_shape(max_suffix_len);+ let sums_maxs_shape = model_shape.attention_sums_shape(max_suffix_len);Self {// 1-D@@ -92,6 +87,14 @@ impl ForwardPassBuffers {&model_shape.attention_output_shape(max_suffix_len),act_ty,),+ mlp_fused_up: alloc(+ &model_shape.mlp_fused_up_shape(max_suffix_len),+ act_ty,+ ),+ mlp_hidden: alloc(+ &model_shape.mlp_hidden_shape(max_suffix_len),+ act_ty,+ ),// 3-Drotated_queries: alloc(diff --git a/crates/uzu/src/backends/metal/forward_pass/model_shape.rs b/crates/uzu/src/backends/metal/forward_pass/model_shape.rsindex d18d3fc16..e0469041c 100644--- a/crates/uzu/src/backends/metal/forward_pass/model_shape.rs+++ b/crates/uzu/src/backends/metal/forward_pass/model_shape.rs@@ -7,6 +7,7 @@ pub struct ModelShape {vocabulary_size: usize,model_dim: usize,+ hidden_dim: usize,context_length: usize,num_heads: usize,@@ -30,6 +31,7 @@ impl ModelShape {kv_cache_type: activation_type,vocabulary_size: decoder_config.vocab_size,model_dim: decoder_config.model_dim,+ hidden_dim: decoder_config.hidden_dim,context_length: decoder_config.context_length,num_heads: decoder_config.num_heads,num_groups: decoder_config.num_groups,@@ -65,6 +67,20 @@ impl ModelShape {[suffix_length, self.model_dim]}+ pub fn mlp_hidden_shape(+ &self,+ suffix_length: usize,+ ) -> [usize; 2] {+ [suffix_length, self.hidden_dim]+ }++ pub fn mlp_fused_up_shape(+ &self,+ suffix_length: usize,+ ) -> [usize; 2] {+ [suffix_length, 2 * self.hidden_dim]+ }+pub fn qkv_shape(&self,suffix_length: usize,diff --git a/crates/uzu/src/backends/metal/forward_pass/state.rs b/crates/uzu/src/backends/metal/forward_pass/state.rsindex 6dc272366..56331e285 100644--- a/crates/uzu/src/backends/metal/forward_pass/state.rs+++ b/crates/uzu/src/backends/metal/forward_pass/state.rs@@ -227,6 +227,10 @@ struct AuxBuffers {qkv: ArrayCell,/// [suffix_length, num_heads * head_dim]attention_output: ArrayCell,+ /// [suffix_length, 2 * hidden_dim]+ mlp_fused_up: ArrayCell,+ /// [suffix_length, hidden_dim]+ mlp_hidden: ArrayCell,/// [num_heads, max_suffix_length, head_dim]rotated_queries: ArrayCell,/// [num_groups, max_suffix_length, head_dim]@@ -269,6 +273,16 @@ impl AuxBuffers {&model_shape.attention_output_shape(suffix_length),act_dtype,)),+ mlp_fused_up: RefCell::new(MetalArray::new(+ scratch.mlp_fused_up.clone(),+ &model_shape.mlp_fused_up_shape(suffix_length),+ act_dtype,+ )),+ mlp_hidden: RefCell::new(MetalArray::new(+ scratch.mlp_hidden.clone(),+ &model_shape.mlp_hidden_shape(suffix_length),+ act_dtype,+ )),rotated_queries: RefCell::new(MetalArray::new(scratch.rotated_queries.clone(),&model_shape.rotated_queries_shape(suffix_length),@@ -478,6 +492,8 @@ impl ForwardPassState {ArrayId::AttentionOutput => {self.aux_buffers.attention_output.clone()},+ ArrayId::MlpFusedUp => self.aux_buffers.mlp_fused_up.clone(),+ ArrayId::MlpHidden => self.aux_buffers.mlp_hidden.clone(),ArrayId::Keys(layer_index) => {self.kv_cache.borrow().data[layer_index].keys.clone()},@@ -540,7 +556,7 @@ impl ForwardPassState {.borrow().local_rope.as_ref()- .unwrap()+ .expect("Local rope requested but local_rope buffers are not initialized").cosines.clone(),},@@ -553,7 +569,7 @@ impl ForwardPassState {.borrow().local_rope.as_ref()- .unwrap()+ .expect("Local rope requested but local_rope buffers are not initialized").sines.clone(),},@@ -640,6 +656,8 @@ pub enum ArrayId {Shortcut,QKV,AttentionOutput,+ MlpFusedUp,+ MlpHidden,Keys(usize),Values(usize),diff --git a/crates/uzu/src/backends/metal/forward_pass/transformer_layer.rs b/crates/uzu/src/backends/metal/forward_pass/transformer_layer.rsindex ad33e3b1d..6ec2f5768 100644--- a/crates/uzu/src/backends/metal/forward_pass/transformer_layer.rs+++ b/crates/uzu/src/backends/metal/forward_pass/transformer_layer.rs@@ -17,11 +17,17 @@ use super::{};use crate::{DataType,- backends::metal::graph::{- embeddings_dequantize_weights_subgraph, embeddings_embed_subgraph,- embeddings_readout_subgraph,+ backends::metal::{+ graph::{+ embeddings_dequantize_weights_subgraph, embeddings_embed_subgraph,+ embeddings_readout_subgraph,+ },+ kernel::{QuantizedLinearKernelBlock, mlp::MlpGateActMulEncodable},+ },+ config::{+ DecoderConfig, EmbeddingConfig, LinearConfig, MLPConfig,+ QuantizationConfig,},- config::{DecoderConfig, EmbeddingConfig, LinearConfig, MLPConfig},parameters::ParameterTree,};@@ -41,7 +47,23 @@ pub fn linear_block<const N: usize>(input_array_id: ArrayId,output_array_id: ArrayId,compilation_descriptor: &CompilationDescriptor,-) -> MPSGraphBlock {+) -> Box<dyn super::encodable_with_state::EncodableWithState> {+ if let LinearConfig::Quantized(quant_config) = config {+ if !has_biases {+ let out_sum: usize = output_dims.iter().sum();+ return quantized_linear_block_custom(+ quant_config,+ input_dim,+ out_sum,+ context,+ parameter_tree,+ input_array_id,+ output_array_id,+ )+ .unwrap();+ }+ }+autoreleasepool(|_| {let graph = Graph::new();@@ -78,10 +100,38 @@ pub fn linear_block<const N: usize>(vec![output_array_id].into_boxed_slice(),);- MPSGraphBlock::new(executable, make_execution_descriptor(), arguments)+ Box::new(MPSGraphBlock::new(+ executable,+ make_execution_descriptor(),+ arguments,+ ))})}+pub fn quantized_linear_block_custom(+ config: &QuantizationConfig,+ input_dim: usize,+ output_dim: usize,+ context: &MTLContext,+ parameter_tree: &ParameterTree<Rc<MTLContext>>,+ input_array_id: ArrayId,+ output_array_id: ArrayId,+) -> Result<+ Box<dyn super::encodable_with_state::EncodableWithState>,+ crate::backends::metal::MTLError,+> {+ let block = crate::backends::metal::kernel::linear::QuantizedLinearKernelBlock::new(+ context,+ config,+ input_dim,+ output_dim,+ parameter_tree,+ input_array_id,+ output_array_id,+ )?;+ Ok(Box::new(block))+}+pub fn mlp_block(config: &MLPConfig,model_dim: usize,@@ -89,7 +139,52 @@ pub fn mlp_block(context: &MTLContext,parameter_tree: &ParameterTree<Rc<MTLContext>>,compilation_descriptor: &CompilationDescriptor,-) -> MPSGraphBlock {+) -> Box<dyn super::encodable_with_state::EncodableWithState> {+ if let crate::config::LinearConfig::Quantized(ref quant_config) =+ config.linear_config+ {+ // Quantized MLP path: up (2H) -> act+mul -> down+ let dtype: DataType =+ config.linear_config.activation_precision().into();++ // Up fused: Main -> MlpFusedUp+ let up = QuantizedLinearKernelBlock::new(+ context,+ quant_config,+ model_dim,+ 2 * hidden_dim,+ ¶meter_tree.subtree("up_projection").unwrap(),+ ArrayId::Main,+ ArrayId::MlpFusedUp,+ )+ .expect("Failed to build MLP up quantized block");++ // Activation+mul: MlpFusedUp -> MlpHidden+ let gate_op = MlpGateActMulEncodable::new(+ context,+ dtype,+ config.activation.clone(),+ hidden_dim,+ )+ .expect("Failed to build MLP gate activation kernel");++ // Down: MlpHidden -> Main+ let down = QuantizedLinearKernelBlock::new(+ context,+ quant_config,+ hidden_dim,+ model_dim,+ ¶meter_tree.subtree("down_projection").unwrap(),+ ArrayId::MlpHidden,+ ArrayId::Main,+ )+ .expect("Failed to build MLP down quantized block");++ let enc = crate::backends::metal::kernel::mlp::MlpBlockEncodable::new(+ up, gate_op, down,+ );+ return Box::new(enc);+ }autoreleasepool(|_| {let graph = Graph::new();@@ -126,7 +221,11 @@ pub fn mlp_block(vec![ArrayId::Main].into_boxed_slice(),);- MPSGraphBlock::new(executable, make_execution_descriptor(), arguments)+ Box::new(MPSGraphBlock::new(+ executable,+ make_execution_descriptor(),+ arguments,+ ))})}@@ -134,7 +233,7 @@ pub fn embed_block(config: &DecoderConfig,context: &MTLContext,compilation_descriptor: &CompilationDescriptor,-) -> MPSGraphBlock {+) -> Box<dyn super::encodable_with_state::EncodableWithState> {let graph = Graph::new();let input_shape = [-1 as isize];@@ -186,7 +285,7 @@ pub fn embed_block(),);- block+ Box::new(block)},EmbeddingConfig::Untied {precision,@@ -231,7 +330,7 @@ pub fn embed_block(),);- block+ Box::new(block)},EmbeddingConfig::QuantizedTied {embedding_quantization_mode,@@ -297,7 +396,7 @@ pub fn embed_block(),);- block+ Box::new(block)},}}@@ -306,7 +405,7 @@ pub fn readout_block(config: &DecoderConfig,context: &MTLContext,compilation_descriptor: &CompilationDescriptor,-) -> MPSGraphBlock {+) -> Box<dyn super::encodable_with_state::EncodableWithState> {let graph = Graph::new();let input_shape = [-1 as isize, config.model_dim as isize];@@ -360,7 +459,7 @@ pub fn readout_block(),);- block+ Box::new(block)},EmbeddingConfig::Untied {precision,@@ -404,7 +503,7 @@ pub fn readout_block(),);- block+ Box::new(block)},EmbeddingConfig::QuantizedTied {embedding_quantization_mode,@@ -469,7 +568,7 @@ pub fn readout_block(),);- block+ Box::new(block)},}}diff --git a/crates/uzu/src/backends/metal/kernel/linear/mod.rs b/crates/uzu/src/backends/metal/kernel/linear/mod.rsnew file mode 100644index 000000000..daf01628f--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/linear/mod.rs@@ -0,0 +1,213 @@+use std::rc::Rc;++use metal::Buffer as MTLBuffer;+use mpsgraph::CommandBuffer as MPSCommandBuffer;++use super::quant_matmul::{QuantizedMatmulArguments, QuantizedMatmulKernel};+use crate::{+ Array, DataType,+ backends::metal::{+ MTLContext, MTLError,+ forward_pass::{+ ArrayId, ForwardPassState,+ encodable_with_state::{EncodableWithState, EncodingParameters},+ },+ },+ config::QuantizationConfig,+ parameters::ParameterTree,+};++pub struct QuantizedLinearKernelBlock {+ kernel_mm: QuantizedMatmulKernel,+ kernel_mv: QuantizedMatmulKernel,+ weights_buffer: MTLBuffer,+ scales_buffer: MTLBuffer,+ zero_points_buffer: MTLBuffer,+ input_dim: usize,+ output_dim: usize,+ #[allow(dead_code)]+ group_size: usize,+ #[allow(dead_code)]+ kernel_data_type: DataType,+ input_array_id: ArrayId,+ output_array_id: ArrayId,+}++impl QuantizedLinearKernelBlock {+ pub fn new(+ mtl_context: &MTLContext,+ config: &QuantizationConfig,+ input_dim: usize,+ output_dim: usize,+ parameter_tree: &ParameterTree<Rc<MTLContext>>,+ input_array_id: ArrayId,+ output_array_id: ArrayId,+ ) -> Result<Self, MTLError> {+ let kernel_data_type: DataType = config.activation_precision.into();++ if !matches!(kernel_data_type, DataType::F16 | DataType::BF16) {+ return Err(MTLError::Generic(format!(+ "Unsupported data type for quantized kernel: {:?}",+ kernel_data_type+ )));+ }++ let mut weights = parameter_tree.leaf("weights").map_err(|e| {+ MTLError::Generic(format!("Failed to load weights: {:?}", e))+ })?;++ if weights.data_type() != DataType::U8 {+ return Err(MTLError::Generic(format!(+ "Expected packed U8 weights, got {:?}",+ weights.data_type()+ )));+ }++ let mut scales = parameter_tree.leaf("scales").map_err(|e| {+ MTLError::Generic(format!("Failed to load scales: {:?}", e))+ })?;++ let mut zero_points =+ parameter_tree.leaf("zero_points").map_err(|e| {+ MTLError::Generic(format!(+ "Failed to load zero_points: {:?}",+ e+ ))+ })?;++ let k_g = (input_dim + config.group_size - 1) / config.group_size;++ let w_shape = weights.shape();+ let s_shape = scales.shape();+ let zp_shape = zero_points.shape();++ if !(w_shape == [output_dim, input_dim / 2]+ && s_shape == [output_dim, k_g]+ && zp_shape == [output_dim, (k_g + 1) / 2])+ {+ return Err(MTLError::Generic(format!(+ "Unexpected shapes. weights={:?}, scales={:?}, zero_points={:?}; expected [N,K/2],[N,K_g],[N,(K_g+1)/2]",+ w_shape, s_shape, zp_shape,+ )));+ }++ let (scales_buffer, zero_points_buffer) = {+ if scales.data_type() != kernel_data_type {+ return Err(MTLError::Generic(format!(+ "Scales dtype mismatch: got {:?}, expected {:?}",+ scales.data_type(),+ kernel_data_type+ )));+ }+ let scales_buffer = unsafe { scales.mtl_buffer() }.to_owned();+ let zero_points_buffer =+ unsafe { zero_points.mtl_buffer() }.to_owned();+ (scales_buffer, zero_points_buffer)+ };++ let weights_buffer: MTLBuffer =+ unsafe { weights.mtl_buffer() }.to_owned();++ let g = config.group_size;+ let (kernel_name_mm, kernel_name_mv) = match (kernel_data_type, g) {+ (DataType::F16, 64) => {+ ("qmm_transposed_f16_g64_b4", "qmv_f16_g64_b4")+ },+ (DataType::F16, 128) => {+ ("qmm_transposed_f16_g128_b4", "qmv_f16_g128_b4")+ },+ (DataType::BF16, 64) => {+ ("qmm_transposed_bf16_g64_b4", "qmv_bf16_g64_b4")+ },+ (DataType::BF16, 128) => {+ ("qmm_transposed_bf16_g128_b4", "qmv_bf16_g128_b4")+ },+ (dtype, other) => {+ return Err(MTLError::Generic(format!(+ "Unsupported group size {} for transposed {:?} kernel",+ other, dtype+ )));+ },+ };++ let kernel_mm = QuantizedMatmulKernel::new(+ mtl_context,+ kernel_data_type,+ kernel_name_mm,+ )+ .map_err(|e| MTLError::Generic(format!("{:?}", e)))?;+ let kernel_mv = QuantizedMatmulKernel::new(+ mtl_context,+ kernel_data_type,+ kernel_name_mv,+ )+ .map_err(|e| MTLError::Generic(format!("{:?}", e)))?;++ Ok(Self {+ kernel_mm,+ kernel_mv,+ weights_buffer,+ scales_buffer,+ zero_points_buffer,+ input_dim,+ output_dim,+ group_size: config.group_size,+ kernel_data_type,+ input_array_id,+ output_array_id,+ })+ }+}++impl EncodableWithState for QuantizedLinearKernelBlock {+ fn encode(+ &self,+ state: &mut ForwardPassState,+ command_buffer: &MPSCommandBuffer,+ _parameters: &EncodingParameters,+ ) {+ let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);+ let input_array = arrays[0].borrow();++ let batch_size = input_array.shape()[0];++ drop(input_array);++ let mut input_array_mut = arrays[0].borrow_mut();+ let mut output_array_mut = arrays[1].borrow_mut();++ let input_buffer = unsafe { input_array_mut.mtl_buffer() };+ let output_buffer = unsafe { output_array_mut.mtl_buffer() };++ let root_command_buffer = command_buffer.root_command_buffer();+ let encoder = root_command_buffer.new_compute_command_encoder();++ let m = batch_size;+ let n = self.output_dim;+ let k = self.input_dim;++ let args = QuantizedMatmulArguments {+ a_buffer: input_buffer,+ b_buffer: &self.weights_buffer,+ scales_buffer: &self.scales_buffer,+ zero_points_buffer: &self.zero_points_buffer,+ output_buffer: output_buffer,+ m: m as i32,+ n: n as i32,+ k: k as i32,+ };++ let use_gemv = batch_size == 1;+ let kernel = if use_gemv {+ &self.kernel_mv+ } else {+ &self.kernel_mm+ };++ kernel+ .encode(encoder, args)+ .expect("Failed to encode quantized matmul kernel");++ encoder.end_encoding();+ }+}diff --git a/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal b/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metalnew file mode 100644index 000000000..8d08af8fc--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal@@ -0,0 +1,78 @@+#include <metal_stdlib>+#include "../definitions.metal"+using namespace metal;++enum ActivationType : ushort {+ ACT_SILU = 0,+ ACT_GELU = 1,+};++template <typename T>+inline T silu(T x) {+ float xf = float(x);+ float yf = xf / (1.0f + exp(-xf));+ return T(yf);+}++template <typename T>+inline T gelu_approx(T x) {+ constexpr float k0 = 0.044715f;+ constexpr float k1 = 0.7978845608f; // sqrt(2/pi)+ float xf = float(x);+ float yf = 0.5f * xf * (1.0f + tanh(k1 * (xf + k0 * xf * xf * xf)));+ return T(yf);+}++template <typename T>+inline T activate(T x, ushort act) {+ if (act == ACT_SILU) return silu(x);+ return gelu_approx(x);+}++template <typename T>+[[kernel]] void mlp_activation_mul(+ const device T* fused_up [[buffer(0)]],+ device T* hidden [[buffer(1)]],+ const constant int& H [[buffer(2)]],+ const constant int& M [[buffer(3)]],+ const constant ushort& act_type [[buffer(4)]],+ uint2 tid [[thread_position_in_grid]]) {+ int row = tid.y;+ int j = tid.x;+ if (row >= M || j >= H) return;+ int base = row * (2 * H);+ T up = fused_up[base + j];+ T gate = fused_up[base + H + j];+ T g = activate(gate, act_type);+ hidden[row * H + j] = up * g;+}++// Explicit instantiations with stable host names+template [[host_name("mlp_activation_mul_f16")]]+[[kernel]] void mlp_activation_mul<half>(+ const device half* fused_up [[buffer(0)]],+ device half* hidden [[buffer(1)]],+ const constant int& H [[buffer(2)]],+ const constant int& M [[buffer(3)]],+ const constant ushort& act_type [[buffer(4)]],+ uint2 tid [[thread_position_in_grid]]);++template [[host_name("mlp_activation_mul_f32")]]+[[kernel]] void mlp_activation_mul<float>(+ const device float* fused_up [[buffer(0)]],+ device float* hidden [[buffer(1)]],+ const constant int& H [[buffer(2)]],+ const constant int& M [[buffer(3)]],+ const constant ushort& act_type [[buffer(4)]],+ uint2 tid [[thread_position_in_grid]]);++template [[host_name("mlp_activation_mul_bf16")]]+[[kernel]] void mlp_activation_mul<bfloat>(+ const device bfloat* fused_up [[buffer(0)]],+ device bfloat* hidden [[buffer(1)]],+ const constant int& H [[buffer(2)]],+ const constant int& M [[buffer(3)]],+ const constant ushort& act_type [[buffer(4)]],+ uint2 tid [[thread_position_in_grid]]);++diff --git a/crates/uzu/src/backends/metal/kernel/mlp/mod.rs b/crates/uzu/src/backends/metal/kernel/mlp/mod.rsnew file mode 100644index 000000000..3d39c4f99--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/mlp/mod.rs@@ -0,0 +1,180 @@+use metal::{+ Buffer as MTLBuffer, ComputeCommandEncoderRef, ComputePipelineState,+ MTLSize,+};+use mpsgraph::CommandBuffer as MPSCommandBuffer;++use crate::{+ DataType,+ array::Array,+ backends::metal::{+ MTLContext, MTLError,+ forward_pass::{+ ArrayId, ForwardPassState,+ encodable_with_state::{EncodableWithState, EncodingParameters},+ },+ kernel::QuantizedLinearKernelBlock,+ },+ config::Activation,+};++pub struct MlpGateActMulKernel {+ pipeline: ComputePipelineState,+}++pub struct MlpGateActMulEncodable {+ kernel: MlpGateActMulKernel,+ activation: Activation,+ hidden_dim: usize,+}++impl MlpGateActMulEncodable {+ pub fn new(+ context: &MTLContext,+ data_type: DataType,+ activation: Activation,+ hidden_dim: usize,+ ) -> Result<Self, MTLError> {+ let kernel = MlpGateActMulKernel::new(context, data_type)?;+ Ok(Self {+ kernel,+ activation,+ hidden_dim,+ })+ }++ pub fn encode(+ &self,+ encoder: &ComputeCommandEncoderRef,+ fused_up: &MTLBuffer,+ hidden: &MTLBuffer,+ m: i32,+ ) -> Result<(), MTLError> {+ self.kernel.encode(+ encoder,+ &self.activation,+ fused_up,+ hidden,+ m,+ self.hidden_dim as i32,+ )+ }+}++impl MlpGateActMulKernel {+ fn kernel_name_for_type(+ data_type: DataType+ ) -> Result<&'static str, MTLError> {+ match data_type {+ DataType::F16 => Ok("mlp_activation_mul_f16"),+ DataType::F32 => Ok("mlp_activation_mul_f32"),+ DataType::BF16 => Ok("mlp_activation_mul_bf16"),+ other => Err(MTLError::Generic(format!(+ "Unsupported dtype for MLP activation: {:?}",+ other+ ))),+ }+ }++ pub fn new(+ context: &MTLContext,+ data_type: DataType,+ ) -> Result<Self, MTLError> {+ let fn_name = Self::kernel_name_for_type(data_type)?;+ let pipeline = context.compute_pipeline_state(fn_name, None)?;+ Ok(Self {+ pipeline,+ })+ }++ fn act_code(act: &Activation) -> u16 {+ match act {+ Activation::SILU => 0,+ Activation::GELU => 1,+ }+ }++ pub fn encode(+ &self,+ encoder: &ComputeCommandEncoderRef,+ activation: &Activation,+ fused_up_buffer: &MTLBuffer,+ hidden_buffer: &MTLBuffer,+ m: i32,+ h: i32,+ ) -> Result<(), MTLError> {+ let act_code = Self::act_code(activation);+ let threads_per_tg = MTLSize::new(64, 1, 1);+ encoder.set_compute_pipeline_state(&self.pipeline);+ encoder.set_buffer(0, Some(fused_up_buffer), 0);+ encoder.set_buffer(1, Some(hidden_buffer), 0);+ encoder.set_bytes(+ 2,+ std::mem::size_of::<i32>() as u64,+ &h as *const i32 as *const _,+ );+ encoder.set_bytes(+ 3,+ std::mem::size_of::<i32>() as u64,+ &m as *const i32 as *const _,+ );+ encoder.set_bytes(+ 4,+ std::mem::size_of::<u16>() as u64,+ &act_code as *const u16 as *const _,+ );++ let grid = MTLSize::new(h as u64, m as u64, 1);+ encoder.dispatch_threads(grid, threads_per_tg);+ Ok(())+ }+}++pub struct MlpBlockEncodable {+ up: QuantizedLinearKernelBlock,+ gate: MlpGateActMulEncodable,+ down: QuantizedLinearKernelBlock,+}++impl MlpBlockEncodable {+ pub fn new(+ up: QuantizedLinearKernelBlock,+ gate: MlpGateActMulEncodable,+ down: QuantizedLinearKernelBlock,+ ) -> Self {+ Self {+ up,+ gate,+ down,+ }+ }+}++impl EncodableWithState for MlpBlockEncodable {+ fn encode(+ &self,+ state: &mut ForwardPassState,+ command_buffer: &MPSCommandBuffer,+ _params: &EncodingParameters,+ ) {+ // Up+ self.up.encode(state, command_buffer, _params);+ // Gate act+mul (fused_up -> hidden)+ let arrays = state.arrays(&[ArrayId::MlpFusedUp, ArrayId::MlpHidden]);+ let mut fused = arrays[0].borrow_mut();+ let mut hidden = arrays[1].borrow_mut();+ let m = fused.shape()[0] as i32;+ let root = command_buffer.root_command_buffer();+ let encoder = root.new_compute_command_encoder();+ let fused_buf = unsafe { fused.mtl_buffer() };+ let hidden_buf = unsafe { hidden.mtl_buffer() };+ self.gate+ .encode(encoder, fused_buf, hidden_buf, m)+ .expect("Failed to encode MLP activation/mul kernel");+ encoder.end_encoding();+ drop(fused);+ drop(hidden);+ // Down+ self.down.encode(state, command_buffer, _params);+ }+}diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rsindex 0b0c376fd..7f417cbf7 100644--- a/crates/uzu/src/backends/metal/kernel/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/mod.rs@@ -1,7 +1,10 @@pub mod attention;mod data_type;pub mod kv_cache_update;+pub mod linear;pub mod media_kernels;+pub mod mlp;+pub mod quant_matmul;pub mod rms_norm;pub mod rope;pub mod sampling;@@ -21,6 +24,7 @@ pub use tensor_add_swap::TensorAddSwap;pub use tensor_copy::TensorCopy;use super::{MTLContext, metal_extensions};+pub use crate::backends::metal::kernel::linear::QuantizedLinearKernelBlock;pub mod media {pub use super::media_kernels::{diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/mma.h b/crates/uzu/src/backends/metal/kernel/quant_matmul/mma.hnew file mode 100644index 000000000..9be4dee9b--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/mma.h@@ -0,0 +1,486 @@+#pragma once++#include <metal_simdgroup>+#include <metal_simdgroup_matrix>+#include <metal_stdlib>++#define UZU_MTL_CONST static constant constexpr+#define UZU_PRAGMA_UNROLL _Pragma("clang loop unroll(full)")+#define SIMD_SIZE 32++using namespace metal;++namespace matmul_utils {++template<typename T>+struct pointer_element_t_impl {+ using type = T;+};+template<typename T>+struct pointer_element_t_impl<device T*> {+ using type = T;+};+template<typename T>+using pointer_element_t = typename pointer_element_t_impl<T>::type;++template <typename T, typename U = T>+struct TransformNone {+ METAL_FUNC static U apply(U val) {+ return val;+ }+};++template <typename T, int kFragRows_, int kFragCols_>+struct BaseMMAFrag {+ static_assert(+ kFragRows_ == 8,+ "Only 8 x 8 fragment matrices are currently supported");+ static_assert(+ kFragCols_ == 8,+ "Only 8 x 8 fragment matrices are currently supported");+};++template <typename T>+struct BaseMMAFrag<T, 8, 8> {+ UZU_MTL_CONST int kFragRows = 8;+ UZU_MTL_CONST int kFragCols = 8;+ UZU_MTL_CONST int kElemsPerFrag = (kFragRows * kFragCols) / 32;+ UZU_MTL_CONST int kElemRows = 1;+ UZU_MTL_CONST int kElemCols = 2;++ static_assert(+ kElemRows * kElemCols == kElemsPerFrag,+ "MMAFrag shape is not consistent with MMAFrag size");++ typedef metal::simdgroup_matrix<T, kFragRows, kFragCols> mat_type;+ typedef metal::vec<T, kElemsPerFrag> frag_type;++ METAL_FUNC static constexpr short2 get_coord(ushort simd_lane_id+ [[thread_index_in_simdgroup]]) {+ const short qid = simd_lane_id / 4;+ const short fm = (qid & 4) + ((simd_lane_id / 2) % 4);+ const short fn = (qid & 2) * 2 + (simd_lane_id % 2) * 2;+ return short2{fn, fm};+ }++ template <typename SrcPtrType, typename StrX, typename StrY>+ METAL_FUNC static constexpr void+ load(thread frag_type& dst, SrcPtrType src, StrX str_x, StrY str_y) {+ UZU_PRAGMA_UNROLL+ for (short i = 0; i < kElemRows; i++) {+ UZU_PRAGMA_UNROLL+ for (short j = 0; j < kElemCols; j++) {+ dst[i * kElemCols + j] = static_cast<T>(src[i * str_x + j * str_y]);+ }+ }+ }+++ template <typename DstPtrType, typename StrX, typename StrY>+ METAL_FUNC static constexpr void+ store(const thread frag_type& src, DstPtrType dst, StrX str_x, StrY str_y) {+ using U = pointer_element_t<DstPtrType>;++ UZU_PRAGMA_UNROLL+ for (short i = 0; i < kElemRows; i++) {+ UZU_PRAGMA_UNROLL+ for (short j = 0; j < kElemCols; j++) {+ dst[i * str_x + j * str_y] = static_cast<U>(src[i * kElemCols + j]);+ }+ }+ }++ template <+ typename DstPtrType,+ typename StrX,+ typename StrY,+ typename LimX,+ typename LimY,+ typename OffX,+ typename OffY>+ METAL_FUNC static constexpr void store_safe(+ const thread frag_type& src,+ DstPtrType dst,+ StrX str_x,+ StrY str_y,+ LimX lim_x,+ LimY lim_y,+ OffX off_x = 0,+ OffY off_y = 0) {+ using U = pointer_element_t<DstPtrType>;++ UZU_PRAGMA_UNROLL+ for (short i = 0; i < kElemRows; i++) {+ UZU_PRAGMA_UNROLL+ for (short j = 0; j < kElemCols; j++) {+ if ((off_x + i) < lim_x && (off_y + j) < lim_y) {+ dst[(off_x + i) * str_x + (off_y + j) * str_y] =+ static_cast<U>(src[i * kElemCols + j]);+ }+ }+ }+ }+++ METAL_FUNC static constexpr void mma(+ thread frag_type& D,+ thread frag_type& A,+ thread frag_type& B,+ thread frag_type& C) {+ mat_type D_mat;+ mat_type A_mat;+ mat_type B_mat;+ mat_type C_mat;++ reinterpret_cast<thread frag_type&>(A_mat.thread_elements()) = A;+ reinterpret_cast<thread frag_type&>(B_mat.thread_elements()) = B;+ reinterpret_cast<thread frag_type&>(C_mat.thread_elements()) = C;++ mma(D_mat, A_mat, B_mat, C_mat);++ D = reinterpret_cast<thread frag_type&>(D_mat.thread_elements());+ }++ METAL_FUNC static constexpr void mma(+ thread mat_type& D,+ thread mat_type& A,+ thread mat_type& B,+ thread mat_type& C) {+ simdgroup_multiply_accumulate(D, A, B, C);+ }+};++template <+ typename T,+ int kTileRows_,+ int kTileCols_,+ class MMAFrag_ = BaseMMAFrag<T, 8, 8>>+struct MMATile {+ using MMAFrag_t = MMAFrag_;+ using elem_type = T;+ UZU_MTL_CONST int kFragRows = MMAFrag_t::kFragRows;+ UZU_MTL_CONST int kFragCols = MMAFrag_t::kFragCols;+ UZU_MTL_CONST int kElemsPerFrag = MMAFrag_t::kElemsPerFrag;++ UZU_MTL_CONST int kTileRows = kTileRows_;+ UZU_MTL_CONST int kTileCols = kTileCols_;++ UZU_MTL_CONST int kRows = kTileRows * kFragRows;+ UZU_MTL_CONST int kCols = kTileCols * kFragCols;++ UZU_MTL_CONST int kNumFrags = kTileRows * kTileCols;+ UZU_MTL_CONST int kElemsPerTile = kNumFrags * kElemsPerFrag;++ typedef typename MMAFrag_t::mat_type mat_type;+ typedef typename MMAFrag_t::frag_type frag_type;++ frag_type val_frags[kNumFrags] = {frag_type(0)};++ METAL_FUNC MMATile() thread {}++ METAL_FUNC constexpr void clear() {+ UZU_PRAGMA_UNROLL+ for (short i = 0; i < kNumFrags; ++i) {+ val_frags[i] = frag_type(0);+ }+ }++ METAL_FUNC constexpr thread frag_type& frag_at(const short i, const short j) {+ return val_frags[i * kTileCols + j];+ }++ METAL_FUNC constexpr const thread frag_type& frag_at(+ const short i,+ const short j) const {+ return val_frags[i * kTileCols + j];+ }+++ METAL_FUNC thread elem_type* elems() {+ return reinterpret_cast<thread elem_type*>(val_frags);+ }++ METAL_FUNC const thread elem_type* elems() const {+ return reinterpret_cast<const thread elem_type*>(val_frags);+ }++ template <typename U, int w_x, int w_y, int str_x, int str_y>+ METAL_FUNC void load(const threadgroup U* src) {+ UZU_PRAGMA_UNROLL+ for (short i = 0; i < kTileRows; ++i) {+ UZU_PRAGMA_UNROLL+ for (short j = 0; j < kTileCols; ++j) {+ MMAFrag_t::load(+ frag_at(i, j),+ &(+ src[(i * kFragRows) * w_x * str_x ++ (j * kFragCols) * w_y * str_y]),+ str_x,+ str_y);+ }+ }+ }+++ template <typename U, int w_x, int w_y>+ METAL_FUNC void store(device U* dst, const int ld) const {+ UZU_PRAGMA_UNROLL+ for (short i = 0; i < kTileRows; ++i) {+ UZU_PRAGMA_UNROLL+ for (short j = 0; j < kTileCols; ++j) {+ MMAFrag_t::store(+ frag_at(i, j),+ &(dst[(i * kFragRows) * w_x * ld + (j * kFragCols) * w_y]),+ ld,+ 1);+ }+ }+ }+++ template <typename U, int w_x, int w_y>+ METAL_FUNC void+ store_safe(device U* dst, const int ld, const short2 dst_tile_dims) const {+ UZU_PRAGMA_UNROLL+ for (int i = 0; i < kTileRows; ++i) {+ UZU_PRAGMA_UNROLL+ for (int j = 0; j < kTileCols; ++j) {+ MMAFrag_t::store_safe(+ frag_at(i, j),+ dst,+ ld,+ 1,+ dst_tile_dims.y,+ dst_tile_dims.x,+ (i * kFragRows) * w_x,+ (j * kFragCols) * w_y);+ }+ }+ }++};++template <typename T, typename U, int M, int N, int K>+METAL_FUNC void tile_matmad(+ thread MMATile<T, M, N>& D,+ thread MMATile<U, M, K>& A,+ thread MMATile<U, K, N>& B,+ thread MMATile<T, M, N>& C) {+ UZU_PRAGMA_UNROLL+ for (short m = 0; m < M; ++m) {+ UZU_PRAGMA_UNROLL+ for (short n = 0; n < N; ++n) {+ short n_serp = (m % 2) ? (N - 1 - n) : n;+ UZU_PRAGMA_UNROLL+ for (short k = 0; k < K; ++k) {+ MMATile<T, M, N>::MMAFrag_t::mma(+ D.frag_at(m, n_serp),+ A.frag_at(m, k),+ B.frag_at(k, n_serp),+ C.frag_at(m, n_serp));+ }+ }+ }+}++template <+ typename T,+ typename U,+ int BM,+ int BN,+ int BK,+ int WM,+ int WN,+ bool transpose_a,+ bool transpose_b,+ short lda_tgp,+ short ldb_tgp,+ typename AccumType = float,+ typename Epilogue = TransformNone<U, AccumType>>+struct BlockMMA {+ // MMAFrag size+ UZU_MTL_CONST short kFragSize = 8;+ using MMAFrag_acc_t = BaseMMAFrag<AccumType, kFragSize, kFragSize>;++ // Warp tile simdgroup matrix strides along M+ UZU_MTL_CONST short TM_stride = kFragSize * WM;+ // Warp tile simdgroup matrix strides along M+ UZU_MTL_CONST short TN_stride = kFragSize * WN;++ // Warp tile size along M+ UZU_MTL_CONST short TM = BM / (kFragSize * WM);+ // Warp tile size along N+ UZU_MTL_CONST short TN = BN / (kFragSize * WN);++ // Threadgroup A strides+ UZU_MTL_CONST short A_str_m = transpose_a ? 1 : lda_tgp; // M+ UZU_MTL_CONST short A_str_k = transpose_a ? lda_tgp : 1; // K++ // Threadgroup B strides+ UZU_MTL_CONST short B_str_k = transpose_b ? 1 : ldb_tgp; // K+ UZU_MTL_CONST short B_str_n = transpose_b ? ldb_tgp : 1; // N++ // Threadgroup strides along K+ UZU_MTL_CONST short tile_stride_a = kFragSize * A_str_k;+ UZU_MTL_CONST short tile_stride_b = kFragSize * B_str_k;++ // Simdgroup matrices+ MMATile<AccumType, TM, 1, MMAFrag_acc_t> Atile;+ MMATile<AccumType, 1, TN, MMAFrag_acc_t> Btile;+ MMATile<AccumType, TM, TN, MMAFrag_acc_t> Ctile;++ // Offsets within threadgroup+ short sm;+ short sn;++ short As_offset;+ short Bs_offset;++ /* Constructor */+ METAL_FUNC BlockMMA(+ ushort simd_group_id [[simdgroup_index_in_threadgroup]],+ ushort simd_lane_id [[thread_index_in_simdgroup]]) {+ // Determine thread position in simdgroup matrix+ short tm = kFragSize * (simd_group_id / WN);+ short tn = kFragSize * (simd_group_id % WN);++ short2 simd_coord = MMAFrag_acc_t::get_coord(simd_lane_id);+ sm = simd_coord.y;+ sn = simd_coord.x;++ // Determine thread and simdgroup offset+ As_offset = (tm + sm) * A_str_m + (sn)*A_str_k; // M, K+ Bs_offset = (sm)*B_str_k + (tn + sn) * B_str_n; // K, N++ sm += tm;+ sn += tn;+ }++ /* (BM, BK) X (BK, BN) multiply accumulate function */+ METAL_FUNC void mma(const threadgroup T* As, const threadgroup T* Bs) {+ // Adjust for simdgroup and thread location+ As += As_offset;+ Bs += Bs_offset;++ // Iterate over BK in blocks of kFragSize+ UZU_PRAGMA_UNROLL+ for (short kk = 0; kk < BK; kk += kFragSize) {+ simdgroup_barrier(mem_flags::mem_none);++ Atile.template load<T, WM, 1, A_str_m, A_str_k>(As);++ simdgroup_barrier(mem_flags::mem_none);++ Btile.template load<T, 1, WN, B_str_k, B_str_n>(Bs);++ simdgroup_barrier(mem_flags::mem_none);++ tile_matmad(Ctile, Atile, Btile, Ctile);++ // Progress to next simdgroup tile+ As += tile_stride_a;+ Bs += tile_stride_b;+ }+ }++ /* Store results from simdgroup_matrix results into device memory */+ METAL_FUNC void store_result(device U* D, const int ldd) {+ // Apply epilogue+ UZU_PRAGMA_UNROLL+ for (short i = 0; i < decltype(Ctile)::kElemsPerTile; i++) {+ Ctile.elems()[i] = Epilogue::apply(Ctile.elems()[i]);+ }++ // Adjust for simdgroup and thread location+ D += sm * ldd + sn;++ Ctile.template store<U, WM, WN>(D, ldd);+ }+++ METAL_FUNC void+ store_result_safe(device U* D, const int ldd, short2 dst_tile_dims) {+ // Apply epilogue+ UZU_PRAGMA_UNROLL+ for (short i = 0; i < decltype(Ctile)::kElemsPerTile; i++) {+ Ctile.elems()[i] = Epilogue::apply(Ctile.elems()[i]);+ }++ // Adjust for simdgroup and thread location+ D += sm * ldd + sn;+ dst_tile_dims -= short2(sn, sm);++ if (dst_tile_dims.x <= 0 || dst_tile_dims.y <= 0)+ return;++ Ctile.template store_safe<U, WM, WN>(D, ldd, dst_tile_dims);+ }++};++template <+ typename T,+ int BM,+ int BK,+ int lda_tgp,+ int reduction_dim,+ int tgp_size,+ int align_M = 1,+ int align_K = 4>+struct BlockLoader {+ UZU_MTL_CONST short n_reads = (BM * BK < tgp_size) ? 1 : (BM * BK) / tgp_size;++ const int src_ld;+ const short thread_idx;+ const short bi;+ const short bj;++ threadgroup T* dst;+ const device T* src;++ BlockLoader(+ const device T* src_,+ const int src_ld_,+ threadgroup T* dst_,+ ushort simd_group_id [[simdgroup_index_in_threadgroup]],+ ushort simd_lane_id [[thread_index_in_simdgroup]])+ : src_ld(src_ld_),+ thread_idx(simd_group_id * 32 + simd_lane_id),+ bi(n_reads * thread_idx / BK),+ bj((n_reads * thread_idx) % BK),+ dst(dst_ + bi * lda_tgp + bj),+ src(src_ + bi * src_ld + bj) {}++ void load_unsafe() const {+ if (BM * BK < tgp_size && bi >= BM) {+ return;+ }++ for (int i = 0; i < n_reads; i++) {+ dst[i] = src[i];+ }+ }++ void load_safe(short2 src_tile_dim) const {+ if (BM * BK < tgp_size && bi >= BM) {+ return;+ }++ if (bi >= src_tile_dim.y || bj >= src_tile_dim.x) {+ for (int i = 0; i < n_reads; i++) {+ dst[i] = T(0);+ }+ return;+ }++ for (int i = 0; i < n_reads; i++) {+ dst[i] = src[i];+ }+ }++ void next() {+ src += BK; // Advance by tile size+ }+};++}\ No newline at end of filediff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rsnew file mode 100644index 000000000..192ba21b7--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs@@ -0,0 +1,202 @@+use std::mem::size_of;++use metal::{+ Buffer as MTLBuffer, ComputeCommandEncoderRef,+ ComputePipelineState as MTLComputePipelineState, MTLSize,+};++use super::super::MTLContext;+use crate::{DataType, backends::metal::MTLError};++#[derive(Debug, thiserror::Error)]+pub enum QuantizedMatmulError {+ #[error("Metal error: {0}")]+ MetalError(#[from] MTLError),+ #[error("Unsupported data type: {0:?}")]+ UnsupportedDataType(DataType),+ #[error("Unsupported group size: {0}")]+ UnsupportedGroupSize(usize),+ #[error("Invalid dimensions: M={m}, N={n}, K={k}")]+ InvalidDimensions {+ m: usize,+ n: usize,+ k: usize,+ },+}++pub struct QuantizedMatmulKernel {+ pipeline: MTLComputePipelineState,+ kind: KernelKind,+}++#[derive(Debug)]+pub struct QuantizedMatmulArguments<'a> {+ pub a_buffer: &'a MTLBuffer, // Input A (float)+ pub b_buffer: &'a MTLBuffer, // Input B (quantized)+ pub scales_buffer: &'a MTLBuffer,+ pub zero_points_buffer: &'a MTLBuffer,+ pub output_buffer: &'a MTLBuffer,+ pub m: i32,+ pub n: i32,+ pub k: i32,+}++impl QuantizedMatmulKernel {+ pub fn new(+ mtl_context: &MTLContext,+ data_type: DataType,+ kernel_name: &str,+ ) -> Result<Self, QuantizedMatmulError> {+ if !matches!(data_type, DataType::F16 | DataType::BF16) {+ return Err(QuantizedMatmulError::UnsupportedDataType(data_type));+ }++ let pipeline = mtl_context+ .compute_pipeline_state(kernel_name, None)+ .map_err(QuantizedMatmulError::MetalError)?;++ let kind = if kernel_name.starts_with("qmv") {+ KernelKind::Qmv+ } else if kernel_name.starts_with("qvm") {+ KernelKind::Qvm+ } else {+ KernelKind::Qmm+ };++ Ok(Self {+ pipeline,+ kind,+ })+ }++ #[allow(dead_code)]+ fn kernel_name_for_config(data_type: DataType) -> String {+ let type_suffix = match data_type {+ DataType::F16 => "f16",+ DataType::BF16 => "bf16",+ _ => unreachable!(),+ };++ format!("qmm_t_{}_gs_64_b_4_batch_0", type_suffix)+ }++ pub fn encode(+ &self,+ encoder: &ComputeCommandEncoderRef,+ args: QuantizedMatmulArguments,+ ) -> Result<(), QuantizedMatmulError> {+ encoder.set_compute_pipeline_state(&self.pipeline);++ // Set buffers+ encoder.set_buffer(0, Some(args.b_buffer), 0);+ encoder.set_buffer(1, Some(args.scales_buffer), 0);+ encoder.set_buffer(2, Some(args.zero_points_buffer), 0);+ encoder.set_buffer(3, Some(args.a_buffer), 0);+ encoder.set_buffer(4, Some(args.output_buffer), 0);++ // Set constants+ encoder.set_bytes(+ 5,+ size_of::<i32>() as u64,+ &args.k as *const i32 as *const _,+ );+ encoder.set_bytes(+ 6,+ size_of::<i32>() as u64,+ &args.n as *const i32 as *const _,+ );+ encoder.set_bytes(+ 7,+ size_of::<i32>() as u64,+ &args.m as *const i32 as *const _,+ );++ match self.kind {+ KernelKind::Qmv => {+ let bk = 32;+ let bn = 8;+ let n_tgp_y = (args.n + bn - 1) / bn; // ceiling division+ let threadgroups =+ MTLSize::new(args.m as u64, n_tgp_y as u64, 1);+ let threads_per_threadgroup = MTLSize::new(bk as u64, 2, 1);+ encoder.dispatch_thread_groups(+ threadgroups,+ threads_per_threadgroup,+ );+ },+ KernelKind::Qvm => {+ let bk = 32;+ let bn = 64;+ let n_tgp_y = (args.n + bn - 1) / bn; // ceiling division for columns+ let threadgroups =+ MTLSize::new(args.m as u64, n_tgp_y as u64, 1);+ let threads_per_threadgroup = MTLSize::new(bk as u64, 2, 1);+ encoder.dispatch_thread_groups(+ threadgroups,+ threads_per_threadgroup,+ );+ },+ KernelKind::Qmm => {+ let bm = 32;+ let bn = 32;+ let wm = 2;+ let wn = 2;+ let threads_per_threadgroup = MTLSize::new(32, wn, wm);+ let threadgroups = MTLSize::new(+ (args.n as u64 + bn - 1) / bn,+ (args.m as u64 + bm - 1) / bm,+ 1,+ );+ encoder.dispatch_thread_groups(+ threadgroups,+ threads_per_threadgroup,+ );+ },+ }++ Ok(())+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+enum KernelKind {+ Qmm,+ Qmv,+ Qvm,+}++pub fn encode_quantized_matmul(+ mtl_context: &MTLContext,+ encoder: &ComputeCommandEncoderRef,+ kernel_data_type: DataType,+ group_size: usize,+ transpose: bool,+ args: QuantizedMatmulArguments,+) -> Result<(), QuantizedMatmulError> {+ let type_suffix = match kernel_data_type {+ DataType::F16 => "f16",+ DataType::BF16 => "bf16",+ other => return Err(QuantizedMatmulError::UnsupportedDataType(other)),+ };++ let kernel_name = match (type_suffix, group_size, transpose) {+ ("f16", 64, false) => "qmm_f16_g64_b4".to_string(),+ ("f16", 64, true) => "qmm_transposed_f16_g64_b4".to_string(),+ ("f16", 128, false) => "qmm_f16_g128_b4".to_string(),+ ("f16", 128, true) => "qmm_transposed_f16_g128_b4".to_string(),+ ("bf16", 64, false) => "qmm_bf16_g64_b4".to_string(),+ ("bf16", 64, true) => "qmm_transposed_bf16_g64_b4".to_string(),+ ("bf16", 128, false) => "qmm_bf16_g128_b4".to_string(),+ ("bf16", 128, true) => "qmm_transposed_bf16_g128_b4".to_string(),+ _ => {+ return Err(QuantizedMatmulError::UnsupportedGroupSize(group_size));+ },+ };++ let kernel = QuantizedMatmulKernel::new(+ mtl_context,+ kernel_data_type,+ &kernel_name,+ )?;+ kernel.encode(encoder, args)+}diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metalnew file mode 100644index 000000000..6c9270588--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal@@ -0,0 +1,1179 @@+#include <metal_simdgroup>+#include <metal_stdlib>++using namespace metal;++#include "mma.h"+++template <int bits, int wsize = 8>+inline constexpr short get_pack_factor() {+ return (bits == 3 || bits == 5) ? 8 : (bits == 6 ? 4 : wsize / bits);+}++template <int bits, int wsize = 8>+inline constexpr short get_bytes_per_pack() {+ constexpr int power_of_2_bits = (bits & (bits - 1)) == 0;+ return power_of_2_bits ? (wsize / 8) : (bits == 5 ? 5 : 3);+}++template <typename T, typename U, int values_per_thread, int bits>+inline U load_vector(const device T* x, thread U* x_thread) {+ static_assert(bits == 4, "Only int4 supported");++ U sum = 0;+ for (int i = 0; i < values_per_thread; i += 4) {+ sum += x[i] + x[i + 1] + x[i + 2] + x[i + 3];+ x_thread[i] = x[i];+ x_thread[i + 1] = x[i + 1] / 16.0f;+ x_thread[i + 2] = x[i + 2] / 256.0f;+ x_thread[i + 3] = x[i + 3] / 4096.0f;+ }+ return sum;+}++template <typename T, typename U, int values_per_thread, int bits>+inline U load_vector_safe(const device T* x, thread U* x_thread, int N) {+ static_assert(bits == 4, "Only int4 supported");++ const U scale_lut[4] = {+ static_cast<U>(1.0f),+ static_cast<U>(1.0f / 16.0f),+ static_cast<U>(1.0f / 256.0f),+ static_cast<U>(1.0f / 4096.0f)+ };++ U sum = 0;+ for (int i = 0; i < values_per_thread; ++i) {+ x_thread[i] = 0;+ }+ for (int i = 0; i < N; ++i) {+ U v = x[i];+ sum += v;+ x_thread[i] = v * scale_lut[i & 3];+ }+ return sum;+}++template <typename U, int values_per_thread, int bits>+inline U qdot(+ const device uint8_t* w,+ const thread U* x_thread,+ U scale,+ U bias,+ U sum) {+ static_assert(bits == 4, "Only int4 supported");++ U accum = 0;+ const device uint16_t* ws = (const device uint16_t*)w;+ for (int i = 0; i < (values_per_thread / 4); i++) {+ accum +=+ (x_thread[4 * i] * (ws[i] & 0x000f) ++ x_thread[4 * i + 1] * (ws[i] & 0x00f0) ++ x_thread[4 * i + 2] * (ws[i] & 0x0f00) ++ x_thread[4 * i + 3] * (ws[i] & 0xf000));+ }+ return scale * accum + sum * bias;+}++template <typename U, int values_per_thread, int bits>+inline U qdot_safe(+ const device uint8_t* w,+ const thread U* x_thread,+ U scale,+ U bias,+ U sum,+ int N) {+ static_assert(bits == 4, "Only int4 supported");++ U accum = 0;+ const device uint16_t* ws = (const device uint16_t*)w;++ int full = N / 4;+ for (int i = 0; i < full; i++) {+ accum +=+ (x_thread[4 * i] * (ws[i] & 0x000f) ++ x_thread[4 * i + 1] * (ws[i] & 0x00f0) ++ x_thread[4 * i + 2] * (ws[i] & 0x0f00) ++ x_thread[4 * i + 3] * (ws[i] & 0xf000));+ }++ int rem = N & 3;+ if (rem > 0) {+ uint16_t wv = ws[full];+ int base = 4 * full;+ if (rem > 0) accum += x_thread[base] * (wv & 0x000f);+ if (rem > 1) accum += x_thread[base + 1] * (wv & 0x00f0);+ if (rem > 2) accum += x_thread[base + 2] * (wv & 0x0f00);+ }++ return scale * accum + sum * bias;+}++template <typename U, int N, int bits>+inline void dequantize(const device uint8_t* w, U scale, U bias, threadgroup U* w_local) {+ static_assert(bits == 4, "Only int4 supported");++ U s[2] = {scale, scale / static_cast<U>(16.0f)};+ for (int i = 0; i < (N / 2); i++) {+ w_local[2 * i] = s[0] * (w[i] & 0x0f) + bias;+ w_local[2 * i + 1] = s[1] * (w[i] & 0xf0) + bias;+ }+}++template <+ typename T,+ short BROWS,+ short BCOLS,+ short dst_ld,+ short reduction_dim,+ short tgp_size,+ short group_size,+ short bits>+struct QuantizedBlockLoader {+ static_assert(BCOLS <= group_size, "Group size should be larger than columns");+ static_assert(group_size % BCOLS == 0, "Group size should be divisible by columns");+ static_assert(bits == 4, "Only int4 supported");++ UZU_MTL_CONST short pack_factor = get_pack_factor<bits, 8>();+ UZU_MTL_CONST short bytes_per_pack = get_bytes_per_pack<bits>();+ UZU_MTL_CONST short BCOLS_PACKED = BCOLS / pack_factor;+ UZU_MTL_CONST short n_reads = (BCOLS_PACKED * BROWS < tgp_size) ? 1 : (BCOLS_PACKED * BROWS) / tgp_size;+ UZU_MTL_CONST short group_steps = group_size / BCOLS;++ const int src_ld;+ const int tile_stride;+ short group_step_cnt;+ int k_base;+ const int group_stride;++ const short thread_idx;+ const short bi;+ const short bj;++ threadgroup T* dst;+ const device uint8_t* src;+ const device T* scales;+ const device T* biases;++ QuantizedBlockLoader(+ const device uint8_t* src_,+ const device T* scales_,+ const device T* biases_,+ const int src_ld_,+ threadgroup T* dst_,+ ushort simd_group_id [[simdgroup_index_in_threadgroup]],+ ushort simd_lane_id [[thread_index_in_simdgroup]])+ : src_ld(src_ld_),+ tile_stride(+ reduction_dim ? BCOLS_PACKED * bytes_per_pack+ : BROWS * src_ld * bytes_per_pack / pack_factor),+ group_step_cnt(0),+ group_stride(BROWS * src_ld / group_size),+ thread_idx(simd_group_id * 32 + simd_lane_id),+ bi(n_reads * thread_idx / BCOLS_PACKED),+ bj((n_reads * thread_idx) % BCOLS_PACKED),+ dst(dst_ + bi * dst_ld + bj * pack_factor),+ src(src_ + bi * src_ld * bytes_per_pack / pack_factor + bj * bytes_per_pack),+ scales(scales_ + bi * src_ld / group_size),+ biases(biases_ + bi * src_ld / group_size) {}++ void load_unsafe() const {+ if (BCOLS_PACKED * BROWS < tgp_size && bi >= BROWS) {+ return;+ }++ T scale = *scales;+ T bias = *biases;+ for (int i = 0; i < n_reads; i++) {+ dequantize<T, pack_factor, bits>(+ src + i * bytes_per_pack, scale, bias, dst + i * pack_factor);+ }+ }++ void load_safe(short2 src_tile_dim) const {+ if (BCOLS_PACKED * BROWS < tgp_size && bi >= BROWS) {+ return;+ }++ if (reduction_dim == 1) {+ // N-tail: zero out rows beyond valid outputs+ if (bi >= src_tile_dim.x) {+ for (int i = 0; i < n_reads * pack_factor; i++) {+ dst[i] = T(0);+ }+ return;+ }++ // K-tail: strictly limit to valid packs and zero trailing nibbles+ int valid_cols = src_tile_dim.y; // 0..BK+ int valid_packs = (valid_cols + pack_factor - 1) / pack_factor;++ T scale;+ T bias;+ current_scale_bias(scale, bias);+ for (int i = 0; i < n_reads; i++) {+ int pack_idx = bj + i; // global pack index across the BK packs+ if (pack_idx < valid_packs) {+ dequantize<T, pack_factor, bits>(+ src + i * bytes_per_pack, scale, bias, dst + i * pack_factor);+ if (pack_idx == valid_packs - 1) {+ int rem = valid_cols - (valid_packs - 1) * pack_factor;+ if (rem > 0 && rem < pack_factor) {+ for (int r = rem; r < pack_factor; ++r) {+ dst[i * pack_factor + r] = T(0);+ }+ }+ }+ } else {+ for (int j = 0; j < pack_factor; ++j) {+ dst[i * pack_factor + j] = T(0);+ }+ }+ }+ return;+ }++ if (reduction_dim == 0 && bi >= src_tile_dim.y) {+ for (int i = 0; i < n_reads * pack_factor; i++) {+ dst[i] = T(0);+ }+ return;+ }++ T scale = *scales;+ T bias = *biases;+ for (int i = 0; i < n_reads; i++) {+ dequantize<T, pack_factor, bits>(+ src + i * bytes_per_pack, scale, bias, dst + i * pack_factor);+ }+ }++ void next() {+ src += tile_stride;+ if (reduction_dim == 1) {+ if (group_steps > 1) {+ group_step_cnt++;+ if (group_step_cnt == group_steps) {+ group_step_cnt = 0;+ scales++;+ biases++;+ }+ } else {+ scales++;+ biases++;+ }+ } else {+ scales += group_stride;+ biases += group_stride;+ }+ }+};++// Loader that computes bias from packed zero-points (U8) and scales on-GPU+template <+ typename T,+ short BROWS,+ short BCOLS,+ short dst_ld,+ short reduction_dim,+ short tgp_size,+ short group_size,+ short bits>+struct QuantizedBlockLoaderZp {+ static_assert(BCOLS <= group_size, "Group size should be larger than columns");+ static_assert(group_size % BCOLS == 0, "Group size should be divisible by columns");+ static_assert(bits == 4, "Only int4 supported");++ UZU_MTL_CONST short pack_factor = get_pack_factor<bits, 8>();+ UZU_MTL_CONST short bytes_per_pack = get_bytes_per_pack<bits>();+ UZU_MTL_CONST short BCOLS_PACKED = BCOLS / pack_factor;+ UZU_MTL_CONST short n_reads = (BCOLS_PACKED * BROWS < tgp_size) ? 1 : (BCOLS_PACKED * BROWS) / tgp_size;+ UZU_MTL_CONST short group_steps = group_size / BCOLS;++ const int src_ld;+ const int groups_per_row;+ const int tile_stride;+ short group_step_cnt;+ int k_base;+ const int group_stride;++ const short thread_idx;+ const short bi;+ const short bj;++ threadgroup T* dst;+ const device uint8_t* src;+ const device T* scales;+ const device T* scales_row_start;+ const device uint8_t* zps_row_start;++ QuantizedBlockLoaderZp(+ const device uint8_t* src_,+ const device T* scales_,+ const device uint8_t* zero_points_row_start_,+ const int src_ld_,+ const int groups_per_row_,+ threadgroup T* dst_,+ ushort simd_group_id [[simdgroup_index_in_threadgroup]],+ ushort simd_lane_id [[thread_index_in_simdgroup]])+ : src_ld(src_ld_),+ groups_per_row(groups_per_row_),+ tile_stride(+ reduction_dim ? BCOLS_PACKED * bytes_per_pack+ : BROWS * src_ld * bytes_per_pack / pack_factor),+ group_step_cnt(0),+ k_base(0),+ group_stride(BROWS * groups_per_row_),+ thread_idx(simd_group_id * 32 + simd_lane_id),+ bi(n_reads * thread_idx / BCOLS_PACKED),+ bj((n_reads * thread_idx) % BCOLS_PACKED),+ dst(dst_ + bi * dst_ld + bj * pack_factor),+ src(src_ + bi * src_ld * bytes_per_pack / pack_factor + bj * bytes_per_pack),+ scales(reduction_dim == 1 ? (scales_ + bi * groups_per_row_) : scales_),+ scales_row_start(reduction_dim == 1 ? (scales_ + bi * groups_per_row_) : scales_),+ zps_row_start(reduction_dim == 1 ? (zero_points_row_start_ + bi * ((groups_per_row_ + 1) / 2))+ : zero_points_row_start_) {}++ inline void current_scale_bias(thread T& out_scale, thread T& out_bias) const {+ int g = reduction_dim == 0 ? (k_base / group_size)+ : (int)(scales - scales_row_start);+ const device uint8_t* zp_ptr = zps_row_start + (g >> 1);+ uint8_t zp_b = *zp_ptr;+ uint zp_n = (g & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F);+ T scale_val = reduction_dim == 0 ? scales_row_start[g] : *scales;+ out_scale = scale_val;+ out_bias = static_cast<T>(-scale_val * static_cast<T>(zp_n));+ }++ void load_unsafe() const {+ if (BCOLS_PACKED * BROWS < tgp_size && bi >= BROWS) {+ return;+ }++ T scale;+ T bias;+ current_scale_bias(scale, bias);+ for (int i = 0; i < n_reads; i++) {+ dequantize<T, pack_factor, bits>(+ src + i * bytes_per_pack, scale, bias, dst + i * pack_factor);+ }+ }++ void load_safe(short2 src_tile_dim) const {+ if (BCOLS_PACKED * BROWS < tgp_size && bi >= BROWS) {+ return;+ }++ if (reduction_dim == 1 && bi >= src_tile_dim.x) {+ for (int i = 0; i < n_reads * pack_factor; i++) {+ dst[i] = T(0);+ }+ return;+ }++ if (reduction_dim == 0 && bi >= src_tile_dim.y) {+ for (int i = 0; i < n_reads * pack_factor; i++) {+ dst[i] = T(0);+ }+ return;+ }++ T scale;+ T bias;+ current_scale_bias(scale, bias);+ for (int i = 0; i < n_reads; i++) {+ dequantize<T, pack_factor, bits>(+ src + i * bytes_per_pack, scale, bias, dst + i * pack_factor);+ }+ }++ void next() {+ src += tile_stride;+ if (reduction_dim == 1) {+ if (group_steps > 1) {+ group_step_cnt++;+ if (group_step_cnt == group_steps) {+ group_step_cnt = 0;+ scales++;+ }+ } else {+ scales++;+ }+ } else {+ k_base += BROWS;+ }+ }+};++template <+ typename T,+ const int group_size,+ const int bits,+ const int BM = 32,+ const int BK = 32,+ const int BN = 32>+void qmm_impl(+ const device uint32_t* w,+ const device T* scales,+ const device uint8_t* zero_points,+ const device T* x,+ device T* y,+ threadgroup T* Xs,+ threadgroup T* Ws,+ const constant int& K,+ const constant int& N,+ const constant int& M,+ uint3 tid [[threadgroup_position_in_grid]],+ uint lid [[thread_index_in_threadgroup]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]) {+ static_assert(BK >= 32, "BK should be larger than SIMD_SIZE");+ static_assert(BK % 32 == 0, "BK should be divisible by SIMD_SIZE");++ (void)lid;++ constexpr int WM = 2;+ constexpr int WN = 2;+ constexpr int pack_factor = get_pack_factor<bits, 8>();+ constexpr int bytes_per_pack = get_bytes_per_pack<bits>();++ constexpr int BK_padded = (BK + 16 / sizeof(T));+ constexpr int BN_padded = (BN + 16 / sizeof(T));++ using mma_t = matmul_utils::BlockMMA<T, T, BM, BN, BK, WM, WN, false, false, BK_padded, BN_padded>;+ using loader_x_t = matmul_utils::BlockLoader<T, BM, BK, BK_padded, 1, WM * WN * 32, 1, 4>;+ using loader_w_t = QuantizedBlockLoaderZp<+ T,+ BK,+ BN,+ BN_padded,+ 0,+ WM * WN * 32,+ group_size,+ bits>;++ auto wl = (const device uint8_t*)w;++ const int y_row = tid.y * BM;+ const int y_col = tid.x * BN;+ x += y_row * static_cast<int64_t>(K);+ wl += y_col * bytes_per_pack / pack_factor;+ const int groups_per_row = (K + group_size - 1) / group_size;+ scales += y_col * groups_per_row;+ const device uint8_t* zps_row_start = zero_points + y_col * ((groups_per_row + 1) / 2);+ y += y_row * static_cast<int64_t>(N) + y_col;++ const short num_els = min(BM, M - y_row);+ const short num_outs = min(BN, N - y_col);+ loader_x_t loader_x(x, K, Xs, simd_gid, simd_lid);+ loader_w_t loader_w(wl, scales, zps_row_start, N, groups_per_row, Ws, simd_gid, simd_lid);+ mma_t mma_op(simd_gid, simd_lid);++ if (num_els < BM) {+ if ((K % BK) != 0) {+ const int k_blocks = K / BK;+ for (int k = 0; k < k_blocks; k++) {+ threadgroup_barrier(mem_flags::mem_threadgroup);+ loader_x.load_safe(short2(BK, num_els));+ loader_w.load_unsafe();+ threadgroup_barrier(mem_flags::mem_threadgroup);+ mma_op.mma(Xs, Ws);+ loader_x.next();+ loader_w.next();+ }+ const short num_k = K - k_blocks * BK;+ threadgroup_barrier(mem_flags::mem_threadgroup);+ loader_x.load_safe(short2(num_k, num_els));+ loader_w.load_safe(short2(BN, num_k));+ threadgroup_barrier(mem_flags::mem_threadgroup);+ mma_op.mma(Xs, Ws);+ } else {+ for (int k = 0; k < K; k += BK) {+ threadgroup_barrier(mem_flags::mem_threadgroup);+ loader_x.load_safe(short2(BK, num_els));+ loader_w.load_unsafe();+ threadgroup_barrier(mem_flags::mem_threadgroup);+ mma_op.mma(Xs, Ws);+ loader_x.next();+ loader_w.next();+ }+ }+ } else {+ if ((K % BK) != 0) {+ const int k_blocks = K / BK;+ for (int k = 0; k < k_blocks; k++) {+ threadgroup_barrier(mem_flags::mem_threadgroup);+ loader_x.load_unsafe();+ loader_w.load_unsafe();+ threadgroup_barrier(mem_flags::mem_threadgroup);+ mma_op.mma(Xs, Ws);+ loader_x.next();+ loader_w.next();+ }+ const short num_k = K - k_blocks * BK;+ threadgroup_barrier(mem_flags::mem_threadgroup);+ loader_x.load_safe(short2(num_k, BM));+ loader_w.load_safe(short2(BN, num_k));+ threadgroup_barrier(mem_flags::mem_threadgroup);+ mma_op.mma(Xs, Ws);+ } else {+ for (int k = 0; k < K; k += BK) {+ threadgroup_barrier(mem_flags::mem_threadgroup);+ loader_x.load_unsafe();+ loader_w.load_unsafe();+ threadgroup_barrier(mem_flags::mem_threadgroup);+ mma_op.mma(Xs, Ws);+ loader_x.next();+ loader_w.next();+ }+ }+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);+ if (num_els < BM || num_outs < BN) {+ mma_op.store_result_safe(y, N, short2(num_outs, num_els));+ } else {+ mma_op.store_result(y, N);+ }+}++template <+ typename T,+ const int group_size,+ const int bits,+ const bool aligned_N,+ const int BM = 32,+ const int BK = 32,+ const int BN = 32>+void qmm_transposed_impl(+ const device uint32_t* w,+ const device T* scales,+ const device uint8_t* zero_points,+ const device T* x,+ device T* y,+ threadgroup T* Xs,+ threadgroup T* Ws,+ const constant int& K,+ const constant int& N,+ const constant int& M,+ uint3 tid [[threadgroup_position_in_grid]],+ uint lid [[thread_index_in_threadgroup]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]) {+ static_assert(BK >= 32, "BK should be larger than SIMD_SIZE");+ static_assert(BK % 32 == 0, "BK should be divisible by SIMD_SIZE");++ (void)lid;++ constexpr int WM = 2;+ constexpr int WN = 2;+ constexpr int pack_factor = get_pack_factor<bits, 8>();+ constexpr int bytes_per_pack = get_bytes_per_pack<bits>();++ constexpr int BK_padded = (BK + 16 / sizeof(T));++ using mma_t = matmul_utils::BlockMMA<T, T, BM, BN, BK, WM, WN, false, true, BK_padded, BK_padded>;+ using loader_x_t = matmul_utils::BlockLoader<T, BM, BK, BK_padded, 1, WM * WN * 32>;+ using loader_w_t = QuantizedBlockLoaderZp<+ T,+ BN,+ BK,+ BK_padded,+ 1,+ WM * WN * 32,+ group_size,+ bits>;++ const int K_w = K * bytes_per_pack / pack_factor;+ const int K_g = (K + group_size - 1) / group_size;+ const int y_row = tid.y * BM;+ const int y_col = tid.x * BN;++ auto wl = (const device uint8_t*)w;++ x += y_row * static_cast<int64_t>(K);+ wl += y_col * K_w;+ scales += y_col * K_g;+ const device uint8_t* zps_row_start = zero_points + y_col * ((K_g + 1) / 2);+ y += y_row * static_cast<int64_t>(N) + y_col;++ const short num_els = min(BM, M - y_row);+ const short num_outs = min(BN, N - y_col);+ loader_x_t loader_x(x, K, Xs, simd_gid, simd_lid);+ loader_w_t loader_w(wl, scales, zps_row_start, K, K_g, Ws, simd_gid, simd_lid);+ mma_t mma_op(simd_gid, simd_lid);++ const int k_blocks = (K + BK - 1) / BK;+ for (int kb = 0; kb < k_blocks; kb++) {+ const short k_len = (kb < (K / BK)) ? BK : static_cast<short>(K - (K / BK) * BK);+ threadgroup_barrier(mem_flags::mem_threadgroup);+ if (num_els < BM || k_len < BK) {+ loader_x.load_safe(short2(k_len, num_els));+ } else {+ loader_x.load_unsafe();+ }+ loader_w.load_safe(short2(num_outs, k_len));+ threadgroup_barrier(mem_flags::mem_threadgroup);+ mma_op.mma(Xs, Ws);+ if (kb + 1 < k_blocks) {+ loader_x.next();+ loader_w.next();+ }+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);+ if (num_els < BM || num_outs < BN) {+ mma_op.store_result_safe(y, N, short2(num_outs, num_els));+ } else {+ mma_op.store_result(y, N);+ }+}++template <typename T, int group_size, int bits>+void qmv_impl(+ const device uint32_t* w,+ const device T* scales,+ const device uint8_t* zero_points,+ const device T* x,+ device T* y,+ const constant int& K, // in_vec_size+ const constant int& N, // out_vec_size+ uint3 tid [[threadgroup_position_in_grid]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]) {+ constexpr int num_simdgroups = 2;+ constexpr int results_per_simdgroup = 4;+ constexpr int packs_per_thread = 1;+ constexpr int pack_factor = get_pack_factor<bits, 32>();+ constexpr int bytes_per_pack = get_bytes_per_pack<bits, 32>();++ constexpr int values_per_thread = pack_factor * packs_per_thread;+ constexpr int block_size = values_per_thread * 32;+ constexpr int scale_step_per_thread = group_size / values_per_thread;++ const device uint8_t* ws = (const device uint8_t*)w;+ typedef float U;+ thread U x_thread[values_per_thread];+ thread U result[results_per_simdgroup] = {0};++ const int in_vec_size_w = K * bytes_per_pack / pack_factor;+ const int in_vec_size_g = (K + group_size - 1) / group_size; // ceil(K / group_size)+ const int out_row = tid.y * (num_simdgroups * results_per_simdgroup) + simd_gid * results_per_simdgroup;+ const int used_out_row = min(N - results_per_simdgroup, out_row);++ if (out_row >= N) {+ return;+ }++ if (N < (num_simdgroups * results_per_simdgroup)) {+ ws += out_row * in_vec_size_w + simd_lid * packs_per_thread * bytes_per_pack;+ scales += out_row * in_vec_size_g + simd_lid / scale_step_per_thread;+ const device uint8_t* zps_row_base = zero_points + out_row * ((in_vec_size_g + 1) / 2);+ x += tid.x * K + simd_lid * values_per_thread;+ y += tid.x * N + out_row;++ int k = 0;+ for (; k < K - block_size; k += block_size) {+ U sum = load_vector<T, U, values_per_thread, bits>(x, x_thread);++ for (int row = 0; out_row + row < N; row++) {+ if (row >= results_per_simdgroup) break;+ auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);+ const device T* sl = scales + row * in_vec_size_g;+ const device uint8_t* zl = zps_row_base + row * ((in_vec_size_g + 1) / 2);++ int g = (k + simd_lid * values_per_thread) / group_size;+ U s = sl[0];+ uint8_t zp_b = zl[g >> 1];+ U b = static_cast<U>(-s * static_cast<U>((g & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F)));+ result[row] += qdot<U, values_per_thread, bits>(wl, x_thread, s, b, sum);+ }++ ws += block_size * bytes_per_pack / pack_factor;+ scales += block_size / group_size;+ x += block_size;+ }+ const int remaining = clamp(+ static_cast<int>(K - k - simd_lid * values_per_thread),+ 0,+ values_per_thread);+ if (remaining > 0) {+ U sum = load_vector_safe<T, U, values_per_thread, bits>(+ x, x_thread, remaining);++ for (int row = 0; out_row + row < N; row++) {+ if (row >= results_per_simdgroup) break;+ auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);+ const device T* sl = scales + row * in_vec_size_g;+ const device uint8_t* zl = zps_row_base + row * ((in_vec_size_g + 1) / 2);++ int g = (k + simd_lid * values_per_thread) / group_size;+ U s = sl[0];+ uint8_t zp_b = zl[g >> 1];+ U b = static_cast<U>(-s * static_cast<U>((g & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F)));+ result[row] +=+ qdot<U, values_per_thread, bits>(wl, x_thread, s, b, sum);+ }+ }++ for (int row = 0; out_row + row < N; row++) {+ if (row >= results_per_simdgroup) break;+ result[row] = simd_sum(result[row]);+ if (simd_lid == 0) {+ y[row] = static_cast<T>(result[row]);+ }+ }+ }+ else {+ ws += used_out_row * in_vec_size_w + simd_lid * packs_per_thread * bytes_per_pack;+ scales += used_out_row * in_vec_size_g + simd_lid / scale_step_per_thread;+ const device uint8_t* zps_row_base = zero_points + used_out_row * ((in_vec_size_g + 1) / 2);+ x += tid.x * K + simd_lid * values_per_thread;+ y += tid.x * N + used_out_row;++ int k = 0;+ for (; k < K - block_size; k += block_size) {+ U sum = load_vector<T, U, values_per_thread, bits>(x, x_thread);++ for (int row = 0; row < results_per_simdgroup; row++) {+ auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);+ const device T* sl = scales + row * in_vec_size_g;+ const device uint8_t* zl = zps_row_base + row * ((in_vec_size_g + 1) / 2);++ int g = (k + simd_lid * values_per_thread) / group_size;+ U s = sl[0];+ uint8_t zp_b = zl[g >> 1];+ U b = static_cast<U>(-s * static_cast<U>((g & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F)));+ result[row] += qdot<U, values_per_thread, bits>(wl, x_thread, s, b, sum);+ }++ ws += block_size * bytes_per_pack / pack_factor;+ scales += block_size / group_size;+ x += block_size;+ }+ const int remaining = clamp(+ static_cast<int>(K - k - simd_lid * values_per_thread),+ 0,+ values_per_thread);++ if (remaining > 0) {+ U sum = load_vector_safe<T, U, values_per_thread, bits>(+ x, x_thread, remaining);++ for (int row = 0; row < results_per_simdgroup; row++) {+ auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);+ const device T* sl = scales + row * in_vec_size_g;+ const device uint8_t* zl = zps_row_base + row * ((in_vec_size_g + 1) / 2);++ int g = (k + simd_lid * values_per_thread) / group_size;+ U s = sl[0];+ uint8_t zp_b = zl[g >> 1];+ U b = static_cast<U>(-s * static_cast<U>((g & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F)));+ result[row] += qdot_safe<U, values_per_thread, bits>(+ wl, x_thread, s, b, sum, remaining);+ }+ }++ for (int row = 0; row < results_per_simdgroup; row++) {+ result[row] = simd_sum(result[row]);+ if (simd_lid == 0) {+ y[row] = static_cast<T>(result[row]);+ }+ }+ }+}++template <typename T, int group_size, int bits>+void qvm_impl(+ const device uint32_t* w,+ const device T* scales,+ const device uint8_t* zero_points,+ const device T* x,+ device T* y,+ const constant int& K, // in_vec_size+ const constant int& N, // out_vec_size+ uint3 tid [[threadgroup_position_in_grid]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]) {++ typedef float U;+ constexpr int BN = 64;+ constexpr int BK = 32;+ constexpr int WN = 2;+ constexpr int pack_factor = get_pack_factor<bits, 32>();+ constexpr int bytes_per_pack = get_bytes_per_pack<bits, 32>();++ const int lidy = simd_gid;+ const int lidx = simd_lid;+ const int bn = BN / WN;++ const int out_col = tid.y * BN + lidy * bn;+ const int in_row = tid.x;++ if (out_col >= N) {+ return;+ }++ const device uint8_t* w_row = ((const device uint8_t*)w) + out_col * K * bytes_per_pack / pack_factor;+ const int in_vec_size_g = (K + group_size - 1) / group_size;+ const device T* scales_row = scales + out_col * in_vec_size_g;+ const device uint8_t* zps_row = zero_points + out_col * ((in_vec_size_g + 1) / 2);+ const device T* x_row = x + in_row * K;+ device T* y_row = y + in_row * N + out_col;++ U result[BN / WN] = {0};+ thread U x_thread[pack_factor];++ for (int k = lidx * pack_factor; k < K; k += BK * pack_factor) {+ const int k_pack = k / pack_factor;+ const int remaining = min(pack_factor, K - k);++ U x_sum = load_vector_safe<T, U, pack_factor, bits>(+ x_row + k, x_thread, remaining);++ for (int n = 0; n < min(bn, N - out_col); n++) {+ const int w_offset = n * K * bytes_per_pack / pack_factor + k_pack * bytes_per_pack;+ const int scale_idx = n * ((K + group_size - 1) / group_size) + k / group_size;++ U scale = static_cast<U>(scales_row[scale_idx]);+ uint8_t zp_b = zps_row[k / group_size / 2];+ U bias = static_cast<U>(-scale * static_cast<U>(((k / group_size) & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F)));++ result[n] += qdot_safe<U, pack_factor, bits>(+ w_row + w_offset, x_thread, scale, bias, x_sum, remaining);+ }+ }++ for (int n = 0; n < min(bn, N - out_col); n++) {+ result[n] = simd_sum(result[n]);+ if (lidx == 0) {+ y_row[n] = static_cast<T>(result[n]);+ }+ }+}+++template <+ typename T,+ const int group_size,+ const int bits,+ const int BM = 32,+ const int BK = 32,+ const int BN = 32>+[[kernel]] void qmm(+ const device uint32_t* w [[buffer(0)]],+ const device T* scales [[buffer(1)]],+ const device uint8_t* zero_points [[buffer(2)]],+ const device T* x [[buffer(3)]],+ device T* y [[buffer(4)]],+ const constant int& K [[buffer(5)]],+ const constant int& N [[buffer(6)]],+ const constant int& M [[buffer(7)]],+ uint3 tid [[threadgroup_position_in_grid]],+ uint lid [[thread_index_in_threadgroup]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]) {++ (void)lid;++ constexpr int BK_padded = (BK + 16 / sizeof(T));+ constexpr int BN_padded = (BN + 16 / sizeof(T));++ threadgroup T Xs[BM * BK_padded];+ threadgroup T Ws[BK * BN_padded];++ qmm_impl<T, group_size, bits, BM, BK, BN>(+ w, scales, zero_points, x, y, Xs, Ws, K, N, M, tid, lid, simd_gid, simd_lid);+}++template <+ typename T,+ const int group_size,+ const int bits,+ const bool aligned_N,+ const int BM = 32,+ const int BK = 32,+ const int BN = 32>+[[kernel]] void qmm_transposed(+ const device uint32_t* w [[buffer(0)]],+ const device T* scales [[buffer(1)]],+ const device uint8_t* zero_points [[buffer(2)]],+ const device T* x [[buffer(3)]],+ device T* y [[buffer(4)]],+ const constant int& K [[buffer(5)]],+ const constant int& N [[buffer(6)]],+ const constant int& M [[buffer(7)]],+ uint3 tid [[threadgroup_position_in_grid]],+ uint lid [[thread_index_in_threadgroup]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]) {++ (void)lid;++ constexpr int BK_padded = (BK + 16 / sizeof(T));++ threadgroup T Xs[BM * BK_padded];+ threadgroup T Ws[BN * BK_padded];++ qmm_transposed_impl<T, group_size, bits, aligned_N, BM, BK, BN>(+ w, scales, zero_points, x, y, Xs, Ws, K, N, M, tid, lid, simd_gid, simd_lid);+}++template <typename T, int group_size, int bits>+[[kernel]] void qmv(+ const device uint32_t* w [[buffer(0)]],+ const device T* scales [[buffer(1)]],+ const device uint8_t* zero_points [[buffer(2)]],+ const device T* x [[buffer(3)]],+ device T* y [[buffer(4)]],+ const constant int& K [[buffer(5)]],+ const constant int& N [[buffer(6)]],+ uint3 tid [[threadgroup_position_in_grid]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]) {++ qmv_impl<T, group_size, bits>(w, scales, zero_points, x, y, K, N, tid, simd_gid, simd_lid);+}++template <typename T, int group_size, int bits>+[[kernel]] void qvm(+ const device uint32_t* w [[buffer(0)]],+ const device T* scales [[buffer(1)]],+ const device uint8_t* zero_points [[buffer(2)]],+ const device T* x [[buffer(3)]],+ device T* y [[buffer(4)]],+ const constant int& K [[buffer(5)]],+ const constant int& N [[buffer(6)]],+ uint3 tid [[threadgroup_position_in_grid]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]) {++ qvm_impl<T, group_size, bits>(w, scales, zero_points, x, y, K, N, tid, simd_gid, simd_lid);+}++// Group size 64 (F16)+template [[host_name("qmm_f16_g64_b4")]]+[[kernel]] void qmm<half, 64, 4>(+ const device uint32_t* w [[buffer(0)]],+ const device half* scales [[buffer(1)]],+ const device uint8_t* zero_points [[buffer(2)]],+ const device half* x [[buffer(3)]],+ device half* y [[buffer(4)]],+ const constant int& K [[buffer(5)]],+ const constant int& N [[buffer(6)]],+ const constant int& M [[buffer(7)]],+ uint3 tid [[threadgroup_position_in_grid]],+ uint lid [[thread_index_in_threadgroup]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]);++template [[host_name("qmm_transposed_f16_g64_b4")]]+[[kernel]] void qmm_transposed<half, 64, 4, true>(+ const device uint32_t* w [[buffer(0)]],+ const device half* scales [[buffer(1)]],+ const device uint8_t* zero_points [[buffer(2)]],+ const device half* x [[buffer(3)]],+ device half* y [[buffer(4)]],+ const constant int& K [[buffer(5)]],+ const constant int& N [[buffer(6)]],+ const constant int& M [[buffer(7)]],+ uint3 tid [[threadgroup_position_in_grid]],+ uint lid [[thread_index_in_threadgroup]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]);++template [[host_name("qmv_f16_g64_b4")]]+[[kernel]] void qmv<half, 64, 4>(+ const device uint32_t* w [[buffer(0)]],+ const device half* scales [[buffer(1)]],+ const device uint8_t* zero_points [[buffer(2)]],+ const device half* x [[buffer(3)]],+ device half* y [[buffer(4)]],+ const constant int& K [[buffer(5)]],+ const constant int& N [[buffer(6)]],+ uint3 tid [[threadgroup_position_in_grid]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]);++template [[host_name("qvm_f16_g64_b4")]]+[[kernel]] void qvm<half, 64, 4>(+ const device uint32_t* w [[buffer(0)]],+ const device half* scales [[buffer(1)]],+ const device uint8_t* zero_points [[buffer(2)]],+ const device half* x [[buffer(3)]],+ device half* y [[buffer(4)]],+ const constant int& K [[buffer(5)]],+ const constant int& N [[buffer(6)]],+ uint3 tid [[threadgroup_position_in_grid]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]);++template [[host_name("qmm_f16_g128_b4")]]+[[kernel]] void qmm<half, 128, 4>(+ const device uint32_t* w [[buffer(0)]],+ const device half* scales [[buffer(1)]],+ const device uint8_t* zero_points [[buffer(2)]],+ const device half* x [[buffer(3)]],+ device half* y [[buffer(4)]],+ const constant int& K [[buffer(5)]],+ const constant int& N [[buffer(6)]],+ const constant int& M [[buffer(7)]],+ uint3 tid [[threadgroup_position_in_grid]],+ uint lid [[thread_index_in_threadgroup]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]);++template [[host_name("qmm_transposed_f16_g128_b4")]]+[[kernel]] void qmm_transposed<half, 128, 4, true>(+ const device uint32_t* w [[buffer(0)]],+ const device half* scales [[buffer(1)]],+ const device uint8_t* zero_points [[buffer(2)]],+ const device half* x [[buffer(3)]],+ device half* y [[buffer(4)]],+ const constant int& K [[buffer(5)]],+ const constant int& N [[buffer(6)]],+ const constant int& M [[buffer(7)]],+ uint3 tid [[threadgroup_position_in_grid]],+ uint lid [[thread_index_in_threadgroup]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]);++template [[host_name("qmv_f16_g128_b4")]]+[[kernel]] void qmv<half, 128, 4>(+ const device uint32_t* w [[buffer(0)]],+ const device half* scales [[buffer(1)]],+ const device uint8_t* zero_points [[buffer(2)]],+ const device half* x [[buffer(3)]],+ device half* y [[buffer(4)]],+ const constant int& K [[buffer(5)]],+ const constant int& N [[buffer(6)]],+ uint3 tid [[threadgroup_position_in_grid]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]);++template [[host_name("qvm_f16_g128_b4")]]+[[kernel]] void qvm<half, 128, 4>(+ const device uint32_t* w [[buffer(0)]],+ const device half* scales [[buffer(1)]],+ const device uint8_t* zero_points [[buffer(2)]],+ const device half* x [[buffer(3)]],+ device half* y [[buffer(4)]],+ const constant int& K [[buffer(5)]],+ const constant int& N [[buffer(6)]],+ uint3 tid [[threadgroup_position_in_grid]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]);++template [[host_name("qmm_bf16_g64_b4")]]+[[kernel]] void qmm<bfloat, 64, 4>(+ const device uint32_t* w [[buffer(0)]],+ const device bfloat* scales [[buffer(1)]],+ const device uint8_t* zero_points [[buffer(2)]],+ const device bfloat* x [[buffer(3)]],+ device bfloat* y [[buffer(4)]],+ const constant int& K [[buffer(5)]],+ const constant int& N [[buffer(6)]],+ const constant int& M [[buffer(7)]],+ uint3 tid [[threadgroup_position_in_grid]],+ uint lid [[thread_index_in_threadgroup]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]);++template [[host_name("qmm_transposed_bf16_g64_b4")]]+[[kernel]] void qmm_transposed<bfloat, 64, 4, true>(+ const device uint32_t* w [[buffer(0)]],+ const device bfloat* scales [[buffer(1)]],+ const device uint8_t* zero_points [[buffer(2)]],+ const device bfloat* x [[buffer(3)]],+ device bfloat* y [[buffer(4)]],+ const constant int& K [[buffer(5)]],+ const constant int& N [[buffer(6)]],+ const constant int& M [[buffer(7)]],+ uint3 tid [[threadgroup_position_in_grid]],+ uint lid [[thread_index_in_threadgroup]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]);++template [[host_name("qmv_bf16_g64_b4")]]+[[kernel]] void qmv<bfloat, 64, 4>(+ const device uint32_t* w [[buffer(0)]],+ const device bfloat* scales [[buffer(1)]],+ const device uint8_t* zero_points [[buffer(2)]],+ const device bfloat* x [[buffer(3)]],+ device bfloat* y [[buffer(4)]],+ const constant int& K [[buffer(5)]],+ const constant int& N [[buffer(6)]],+ uint3 tid [[threadgroup_position_in_grid]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]);++template [[host_name("qvm_bf16_g64_b4")]]+[[kernel]] void qvm<bfloat, 64, 4>(+ const device uint32_t* w [[buffer(0)]],+ const device bfloat* scales [[buffer(1)]],+ const device uint8_t* zero_points [[buffer(2)]],+ const device bfloat* x [[buffer(3)]],+ device bfloat* y [[buffer(4)]],+ const constant int& K [[buffer(5)]],+ const constant int& N [[buffer(6)]],+ uint3 tid [[threadgroup_position_in_grid]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]);++// Group size 128 (BF16)+template [[host_name("qmm_bf16_g128_b4")]]+[[kernel]] void qmm<bfloat, 128, 4>(+ const device uint32_t* w [[buffer(0)]],+ const device bfloat* scales [[buffer(1)]],+ const device uint8_t* zero_points [[buffer(2)]],+ const device bfloat* x [[buffer(3)]],+ device bfloat* y [[buffer(4)]],+ const constant int& K [[buffer(5)]],+ const constant int& N [[buffer(6)]],+ const constant int& M [[buffer(7)]],+ uint3 tid [[threadgroup_position_in_grid]],+ uint lid [[thread_index_in_threadgroup]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]);++template [[host_name("qmm_transposed_bf16_g128_b4")]]+[[kernel]] void qmm_transposed<bfloat, 128, 4, true>(+ const device uint32_t* w [[buffer(0)]],+ const device bfloat* scales [[buffer(1)]],+ const device uint8_t* zero_points [[buffer(2)]],+ const device bfloat* x [[buffer(3)]],+ device bfloat* y [[buffer(4)]],+ const constant int& K [[buffer(5)]],+ const constant int& N [[buffer(6)]],+ const constant int& M [[buffer(7)]],+ uint3 tid [[threadgroup_position_in_grid]],+ uint lid [[thread_index_in_threadgroup]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]);++template [[host_name("qmv_bf16_g128_b4")]]+[[kernel]] void qmv<bfloat, 128, 4>(+ const device uint32_t* w [[buffer(0)]],+ const device bfloat* scales [[buffer(1)]],+ const device uint8_t* zero_points [[buffer(2)]],+ const device bfloat* x [[buffer(3)]],+ device bfloat* y [[buffer(4)]],+ const constant int& K [[buffer(5)]],+ const constant int& N [[buffer(6)]],+ uint3 tid [[threadgroup_position_in_grid]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]);++template [[host_name("qvm_bf16_g128_b4")]]+[[kernel]] void qvm<bfloat, 128, 4>(+ const device uint32_t* w [[buffer(0)]],+ const device bfloat* scales [[buffer(1)]],+ const device uint8_t* zero_points [[buffer(2)]],+ const device bfloat* x [[buffer(3)]],+ device bfloat* y [[buffer(4)]],+ const constant int& K [[buffer(5)]],+ const constant int& N [[buffer(6)]],+ uint3 tid [[threadgroup_position_in_grid]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]);\ No newline at end of filediff --git a/crates/uzu/tests/quant_matmul_test.rs b/crates/uzu/tests/quant_matmul_test.rsnew file mode 100644index 000000000..669bc489e--- /dev/null+++ b/crates/uzu/tests/quant_matmul_test.rs@@ -0,0 +1,596 @@+use std::time::Instant;++use half::f16;+use metal::{Device, MTLResourceOptions};+use uzu::{+ DataType,+ backends::metal::{+ MTLContext,+ kernel::quant_matmul::{+ QuantizedMatmulArguments, QuantizedMatmulKernel,+ },+ },+};++fn create_test_context() -> Option<MTLContext> {+ let device = Device::system_default()?;+ let command_queue = device.new_command_queue();+ MTLContext::new(device, command_queue).ok()+}++// Packs 4-bit weights into bytes (4 values per 16-bit word, matching qdot expectations)+fn pack_u4_weights(values: &[u8]) -> Vec<u8> {+ let mut out = Vec::with_capacity((values.len() + 3) / 4 * 2); // 4 weights per 16-bit word+ for chunk in values.chunks(4) {+ let w0 = (*chunk.get(0).unwrap_or(&0) as u16) & 0x0F; // Bits 0-3+ let w1 = ((*chunk.get(1).unwrap_or(&0) as u16) & 0x0F) << 4; // Bits 4-7+ let w2 = ((*chunk.get(2).unwrap_or(&0) as u16) & 0x0F) << 8; // Bits 8-11+ let w3 = ((*chunk.get(3).unwrap_or(&0) as u16) & 0x0F) << 12; // Bits 12-15++ let word: u16 = w0 | w1 | w2 | w3;+ out.push(word as u8); // Low byte+ out.push((word >> 8) as u8); // High byte+ }+ out+}++fn cpu_reference(+ m: usize,+ n: usize,+ k: usize,+ a: &[f32], // A matrix (M x K)+ b_quant: &[u8], // B matrix (N x K), quantized and packed, (N * K / 2) in size+ scales: &[f32], // scales for B (N x ceil(K/group_size))+ biases: &[f32], // biases for B (N x ceil(K/group_size))+ transpose_b: bool, // Whether B matrix weights are stored in transposed layout+) -> Vec<f32> {+ let group_size = 64;+ let num_groups = (k + group_size - 1) / group_size;+ let mut y = vec![0.0f32; m * n];+ for i in 0..m {+ for j in 0..n {+ let mut acc = 0.0f64;+ for g in 0..num_groups {+ let scale = scales[j * num_groups + g] as f64;+ let bias = biases[j * num_groups + g] as f64;+ let l_start = g * group_size;+ let l_end = (l_start + group_size).min(k);+ let mut group_acc = 0.0f64;+ let mut group_sum = 0.0f64;+ for l in l_start..l_end {+ // Weight indexing based on kernel expectations:+ // Non-transposed kernel expects: W[N][K] layout -> w[output_idx * k + input_idx]+ // Transposed kernel expects: W[K][N] layout -> w[input_idx * n + output_idx]+ let weight_linear_idx = if transpose_b {+ l * n + j // W[K][N]: w[k_idx * n + n_idx]+ } else {+ j * k + l // W[N][K]: w[n_idx * k + k_idx]+ };++ let word_idx = weight_linear_idx / 4; // 4 weights per 16-bit word+ let word_offset = weight_linear_idx % 4; // Which nibble in the word+ let byte_idx = word_idx * 2; // 2 bytes per word++ let word = if byte_idx + 1 < b_quant.len() {+ b_quant[byte_idx] as u16+ | ((b_quant[byte_idx + 1] as u16) << 8)+ } else {+ 0+ };++ let val_q = match word_offset {+ 0 => word & 0x000F, // Bits 0-3+ 1 => (word & 0x00F0) >> 4, // Bits 4-7+ 2 => (word & 0x0F00) >> 8, // Bits 8-11+ 3 => (word & 0xF000) >> 12, // Bits 12-15+ _ => 0,+ } as f64;+ let val_a = a[i * k + l] as f64;+ group_acc += val_a * val_q;+ group_sum += val_a;+ }+ acc += scale * group_acc + bias * group_sum;+ }+ y[i * n + j] = acc as f32;+ }+ }+ y+}++fn execute_quantized_matmul(+ ctx: &MTLContext,+ m: usize,+ n: usize,+ k: usize,+ kernel_name: &str,+ cpu_transpose: bool,+ iterations: usize,+ validate: bool,+ randomize_zp: bool,+) -> f64 {+ // Prepare deterministic weights: w(row,k) = row+1+ // Always create weights in the same layout (test_m × test_k)+ let mut weights_q4: Vec<u8> = Vec::with_capacity(m * k);+ for row in 0..m {+ for _k in 0..k {+ let v = (row as u8 + 1) & 0x0F;+ weights_q4.push(v);+ }+ }+ let weights_packed = pack_u4_weights(&weights_q4);++ let num_groups = (k + 63) / 64;+ let scales: Vec<f16> = vec![f16::from_f32(1.0); m * num_groups];+ let mut biases: Vec<f16> = vec![f16::from_f32(0.0); m * num_groups];++ // Prepare input data+ let (x_f32, x_f16) = if n == 1 {+ // GEMV case: single input vector+ let x_f32: Vec<f32> = (1..=k).map(|i| i as f32 / k as f32).collect();+ let x_f16: Vec<f16> = x_f32.iter().map(|&v| f16::from_f32(v)).collect();+ (x_f32, x_f16)+ } else {+ // GEMM case: multiple input vectors (column-major)+ let mut x_f32: Vec<f32> = Vec::with_capacity(k * n);+ for _col in 0..n {+ for i in 0..k {+ x_f32.push((i + 1) as f32 / k as f32);+ }+ }+ let x_f16: Vec<f16> = x_f32.iter().map(|&v| f16::from_f32(v)).collect();+ (x_f32, x_f16)+ };++ // Create buffers+ let w_buf = ctx.device.new_buffer_with_data(+ weights_packed.as_ptr() as *const _,+ (weights_packed.len() * std::mem::size_of::<u8>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );+ let s_buf = ctx.device.new_buffer_with_data(+ scales.as_ptr() as *const _,+ (scales.len() * std::mem::size_of::<f16>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );+ // Build zero-points buffer+ let mut zps_bytes = vec![0u8; m * ((num_groups + 1) / 2)];+ if randomize_zp {+ // Randomized U8 zero-points per group and corresponding float biases.+ // even g -> low nibble, odd g -> high nibble+ for j in 0..m {+ for g in 0..num_groups {+ let zp_val: u8 = ((j + 3 * g) as u8) & 0x0F; // deterministic 0..15+ let byte_index = j * ((num_groups + 1) / 2) + (g >> 1);+ if (g & 1) == 0 {+ zps_bytes[byte_index] =+ (zps_bytes[byte_index] & 0xF0) | (zp_val & 0x0F);+ } else {+ zps_bytes[byte_index] =+ (zps_bytes[byte_index] & 0x0F) | ((zp_val & 0x0F) << 4);+ }+ // bias = -scale * zp+ let s = scales[j * num_groups + g].to_f32();+ let b = -s * (zp_val as f32);+ biases[j * num_groups + g] = f16::from_f32(b);+ }+ }+ }+ let b_buf = ctx.device.new_buffer_with_data(+ zps_bytes.as_ptr() as *const _,+ (zps_bytes.len() * std::mem::size_of::<u8>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );+ let x_buf = ctx.device.new_buffer_with_data(+ x_f16.as_ptr() as *const _,+ (x_f16.len() * std::mem::size_of::<f16>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );+ let y_buf = ctx.device.new_buffer(+ (m * n * std::mem::size_of::<f16>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );++ // Create kernel+ let kernel =+ QuantizedMatmulKernel::new(&ctx, DataType::F16, kernel_name).unwrap();++ // Warmup (only for benchmarks)+ if iterations > 1 {+ for _ in 0..3 {+ let args = QuantizedMatmulArguments {+ a_buffer: &x_buf,+ b_buffer: &w_buf,+ scales_buffer: &s_buf,+ zero_points_buffer: &b_buf,+ output_buffer: &y_buf,+ m: n as i32,+ n: m as i32,+ k: k as i32,+ };+ let cb_ref = ctx.command_queue.new_command_buffer();+ let encoder = cb_ref.new_compute_command_encoder();+ kernel.encode(encoder, args).unwrap();+ encoder.end_encoding();+ cb_ref.commit();+ cb_ref.wait_until_completed();+ }+ }++ // Execute and time+ let start = Instant::now();+ for _ in 0..iterations {+ let args = QuantizedMatmulArguments {+ a_buffer: &x_buf,+ b_buffer: &w_buf,+ scales_buffer: &s_buf,+ zero_points_buffer: &b_buf,+ output_buffer: &y_buf,+ m: n as i32,+ n: m as i32,+ k: k as i32,+ };+ let cb_ref = ctx.command_queue.new_command_buffer();+ let encoder = cb_ref.new_compute_command_encoder();+ kernel.encode(encoder, args).unwrap();+ encoder.end_encoding();+ cb_ref.commit();+ cb_ref.wait_until_completed();+ }+ let elapsed = start.elapsed();++ // Validate if requested+ if validate {+ let gpu_m = n; // GPU sees test_n as M+ let gpu_n = m; // GPU sees test_m as N+ let gpu_k = k; // GPU sees test_k as K++ let y_expected = cpu_reference(+ gpu_m,+ gpu_n,+ gpu_k,+ &x_f32,+ &weights_packed,+ &scales.iter().map(|&v| v.to_f32()).collect::<Vec<_>>(),+ &biases.iter().map(|&v| v.to_f32()).collect::<Vec<_>>(),+ cpu_transpose,+ );++ let y_ptr = y_buf.contents() as *const f16;+ let y_out = unsafe { std::slice::from_raw_parts(y_ptr, m * n) };+ let y_out_f32: Vec<f32> = y_out.iter().map(|&v| v.to_f32()).collect();++ let tol = if randomize_zp {+ 2.5+ } else {+ 1.0+ };+ let display_size = if n == 1 {+ m+ } else {+ n+ };+ println!(+ "first {} expected: {:?}",+ display_size,+ &y_expected[..display_size]+ );+ println!(+ "first {} got: {:?}",+ display_size,+ &y_out_f32[..display_size]+ );++ for (i, (&exp, &got)) in+ y_expected.iter().zip(y_out_f32.iter()).enumerate()+ {+ let diff = (exp - got).abs();+ assert!(+ diff <= tol,+ "M={} N={} K={} idx {} diff {} exp {} got {}",+ m,+ n,+ k,+ i,+ diff,+ exp,+ got+ );+ }+ }++ elapsed.as_secs_f64()+}++fn run_gemv_test(+ ctx: &MTLContext,+ m: usize,+ k: usize,+) {+ println!("--- Testing GEMV M={}, K={} ---", m, k);+ // GEMV: weights are in W[N][K] layout, CPU uses them as-is+ execute_quantized_matmul(+ ctx,+ m,+ 1,+ k,+ "qmv_f16_g64_b4",+ false,+ 1,+ true,+ true,+ );+ println!("✅ GEMV M={}, K={} passed", m, k);+}++fn run_qvm_test(+ ctx: &MTLContext,+ n: usize,+ k: usize,+) {+ println!("--- Testing QVM N={}, K={} ---", n, k);+ // QVM: weights are in W[N][K] layout, CPU uses them as-is+ execute_quantized_matmul(+ ctx,+ 1,+ n,+ k,+ "qvm_f16_g64_b4",+ false,+ 1,+ true,+ true,+ );+ println!("✅ QVM N={}, K={} passed", n, k);+}++#[test]+fn test_quant_gemv() {+ let ctx = match create_test_context() {+ Some(c) => c,+ None => {+ println!("Metal not available — skipping GEMV test");+ return;+ },+ };++ run_gemv_test(&ctx, 3, 64);+ run_gemv_test(&ctx, 1, 128);+ run_gemv_test(&ctx, 7, 256);+ run_gemv_test(&ctx, 13, 512);+ run_gemv_test(&ctx, 13, 511);+ run_gemv_test(&ctx, 8, 64);+ run_gemv_test(&ctx, 9, 64);+ run_gemv_test(&ctx, 25, 128);+ run_gemv_test(&ctx, 31, 512);+}++#[test]+fn test_quant_qevm() {+ let ctx = match create_test_context() {+ Some(c) => c,+ None => {+ println!("Metal not available — skipping QVM test");+ return;+ },+ };++ run_qvm_test(&ctx, 3, 64);+ run_qvm_test(&ctx, 1, 128);+ run_qvm_test(&ctx, 7, 256);+ run_qvm_test(&ctx, 13, 512);+ run_qvm_test(&ctx, 13, 511);+ run_qvm_test(&ctx, 8, 64);+ run_qvm_test(&ctx, 9, 64);+ run_qvm_test(&ctx, 25, 128);+ run_qvm_test(&ctx, 31, 512);+}++fn run_gemm_test(+ ctx: &MTLContext,+ m: usize,+ n: usize,+ k: usize,+) {+ println!("--- Testing GEMM M={}, N={}, K={} ---", m, n, k);+ // GEMM: weights are in W[N][K] layout, CPU needs to transpose for correct computation+ execute_quantized_matmul(+ ctx,+ m,+ n,+ k,+ "qmm_f16_g64_b4",+ true,+ 1,+ true,+ false,+ );+ println!("✅ GEMM M={}, N={}, K={} passed", m, n, k);+}++fn run_gemm_transposed_test(+ ctx: &MTLContext,+ m: usize,+ n: usize,+ k: usize,+) {+ println!("--- Testing GEMM Transposed M={}, N={}, K={} ---", m, n, k);+ // GEMM Transposed: weights are in W[N][K] but kernel expects W[K][N], CPU uses them as-is+ execute_quantized_matmul(+ ctx,+ m,+ n,+ k,+ "qmm_transposed_f16_g64_b4",+ false,+ 1,+ true,+ true,+ );+ println!("✅ GEMM Transposed M={}, N={}, K={} passed", m, n, k);+}++#[test]+fn test_quant_gemm() {+ let ctx = match create_test_context() {+ Some(c) => c,+ None => {+ println!("Metal not available — skipping GEMM test");+ return;+ },+ };++ run_gemm_test(&ctx, 1, 1, 128);+ run_gemm_test(&ctx, 8, 16, 256);+ run_gemm_test(&ctx, 16, 8, 511);+ run_gemm_test(&ctx, 9, 9, 64);+ run_gemm_test(&ctx, 25, 17, 128);+}++#[test]+fn test_quant_gemm_transposed() {+ let ctx = match create_test_context() {+ Some(c) => c,+ None => {+ println!("Metal not available — skipping GEMM transposed test");+ return;+ },+ };++ run_gemm_transposed_test(&ctx, 1, 1, 128);+ run_gemm_transposed_test(&ctx, 8, 16, 256);+ run_gemm_transposed_test(&ctx, 16, 8, 511);+ run_gemm_transposed_test(&ctx, 9, 9, 64);+ run_gemm_transposed_test(&ctx, 25, 17, 128);+}++fn benchmark_quantized_gemv(+ ctx: &MTLContext,+ m: usize,+ k: usize,+ iterations: usize,+) -> f64 {+ execute_quantized_matmul(+ ctx,+ m,+ 1,+ k,+ "qmv_f16_g64_b4",+ false,+ iterations,+ false,+ true,+ )+}++fn benchmark_quantized_qevm(+ ctx: &MTLContext,+ n: usize,+ k: usize,+ iterations: usize,+) -> f64 {+ execute_quantized_matmul(+ ctx,+ 1,+ n,+ k,+ "qvm_f16_g64_b4",+ false,+ iterations,+ false,+ true,+ )+}++fn benchmark_quantized_gemm(+ ctx: &MTLContext,+ m: usize,+ n: usize,+ k: usize,+ iterations: usize,+) -> f64 {+ execute_quantized_matmul(+ ctx,+ m,+ n,+ k,+ "qmm_f16_g64_b4",+ true,+ iterations,+ false,+ true,+ )+}++#[ignore]+#[test]+fn test_quantized_matmul_performance() {+ let ctx = match create_test_context() {+ Some(c) => c,+ None => {+ println!("Metal not available — skipping performance test");+ return;+ },+ };++ println!("\n🚀 QUANTIZED MATMUL PERFORMANCE BENCHMARKS");+ println!("==========================================");++ // Test different sizes+ let test_configs = vec![+ // (M, N, K, iterations, description)+ (512, 1, 512, 100, "Small GEMV (512×512 × 512×1)"),+ (1024, 1, 1024, 50, "Medium GEMV (1024×1024 × 1024×1)"),+ (2048, 1, 2048, 20, "Large GEMV (2048×2048 × 2048×1)"),+ (4096, 1, 4096, 10, "XLarge GEMV (4096×4096 × 4096×1)"),+ (1, 512, 512, 100, "Small QVM (1×512 × 512×512)"),+ (1, 1024, 1024, 50, "Medium QVM (1×1024 × 1024×1024)"),+ (1, 2048, 2048, 20, "Large QVM (1×2048 × 2048×2048)"),+ (1, 4096, 4096, 10, "XLarge QVM (1×4096 × 4096×4096)"),+ (512, 16, 512, 50, "Small GEMM (512×512 × 512×16)"),+ (1024, 32, 1024, 20, "Medium GEMM (1024×1024 × 1024×32)"),+ (2048, 64, 2048, 10, "Large GEMM (2048×2048 × 2048×64)"),+ (4096, 128, 4096, 5, "XLarge GEMM (4096×4096 × 4096×128)"),+ ];++ for (m, n, k, iterations, description) in test_configs {+ let ops = 2.0 * (m as f64) * (n as f64) * (k as f64);++ if n == 1 {+ // GEMV+ let time = benchmark_quantized_gemv(&ctx, m, k, iterations);+ let avg_time = time / (iterations as f64);+ let throughput = (ops / avg_time) / 1e12;++ println!("{}", description);+ println!(+ " Time: {:.4}ms/iter, Throughput: {:.4} TFLOPS",+ avg_time * 1000.0,+ throughput+ );+ } else if m == 1 {+ // QVM+ let time = benchmark_quantized_qevm(&ctx, n, k, iterations);+ let avg_time = time / (iterations as f64);+ let throughput = (ops / avg_time) / 1e12;++ println!("{}", description);+ println!(+ " Time: {:.4}ms/iter, Throughput: {:.4} TFLOPS",+ avg_time * 1000.0,+ throughput+ );+ } else {+ // GEMM+ let time = benchmark_quantized_gemm(&ctx, m, n, k, iterations);+ let avg_time = time / (iterations as f64);+ let throughput = (ops / avg_time) / 1e12;++ println!("{}", description);+ println!(+ " Time: {:.4}ms/iter, Throughput: {:.4} TFLOPS",+ avg_time * 1000.0,+ throughput+ );+ }+ }+}