Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Quantized Matmul Kernles (#25)

LuckyIYI12e103b34623
diff --git a/crates/uzu/src/backends/metal/executable_builder/decoder_executables.rs b/crates/uzu/src/backends/metal/executable_builder/decoder_executables.rs
index b4c8196f7..ed02cc591 100644
--- a/crates/uzu/src/backends/metal/executable_builder/decoder_executables.rs
+++ b/crates/uzu/src/backends/metal/executable_builder/decoder_executables.rs
@@ -9,7 +9,7 @@ use crate::{
KernelDataType, MTLContext, ModelShape,
compilation_parameters::CompilationConfig,
forward_pass::{
- ArrayId, ForwardPassState, MPSGraphBlock, RopeType,
+ ArrayId, ForwardPassState, RopeType,
encodable_with_state::{EncodableWithState, EncodingParameters},
transformer_layer::{embed_block, readout_block},
},
@@ -20,10 +20,10 @@ use crate::{
};
pub struct DecoderExecutables {
- pub embed: MPSGraphBlock,
+ pub embed: Box<dyn EncodableWithState>,
pub layers: Box<[LayerExecutables]>,
pub norm: Box<dyn EncodableWithState>,
- pub readout: MPSGraphBlock,
+ pub readout: Box<dyn EncodableWithState>,
pub global_rope: Rc<Box<dyn EncodableWithState>>,
pub local_rope: Option<Rc<Box<dyn EncodableWithState>>>,
}
diff --git a/crates/uzu/src/backends/metal/executable_builder/layer_executables.rs b/crates/uzu/src/backends/metal/executable_builder/layer_executables.rs
index 4bc0a74e9..0d0d17180 100644
--- a/crates/uzu/src/backends/metal/executable_builder/layer_executables.rs
+++ b/crates/uzu/src/backends/metal/executable_builder/layer_executables.rs
@@ -9,7 +9,7 @@ use crate::{
MTLContext,
compilation_parameters::CompilationConfig,
forward_pass::{
- ArrayId, ForwardPassState, MPSGraphBlock,
+ ArrayId, ForwardPassState,
encodable_with_state::{EncodableWithState, EncodingParameters},
transformer_layer,
},
@@ -26,15 +26,15 @@ pub struct LayerExecutables {
pub layer_index: usize,
pub copy_main_to_shortcut: Box<dyn EncodableWithState>,
pub pre_attention_norm: Box<dyn EncodableWithState>,
- pub qkv_projection: MPSGraphBlock,
+ pub qkv_projection: Box<dyn EncodableWithState>,
pub qk_norm: Option<Box<dyn EncodableWithState>>,
pub rope: Rc<Box<dyn EncodableWithState>>,
pub attention: Box<dyn EncodableWithState>,
- pub out_projection: MPSGraphBlock,
+ pub out_projection: Box<dyn EncodableWithState>,
pub post_attention_norm: Option<Box<dyn EncodableWithState>>,
pub main_shortcut_add_swap: Box<dyn EncodableWithState>,
pub pre_mlp_norm: Box<dyn EncodableWithState>,
- pub mlp: MPSGraphBlock,
+ pub mlp: Box<dyn EncodableWithState>,
pub post_mlp_norm: Option<Box<dyn EncodableWithState>>,
}
@@ -121,10 +121,10 @@ impl LayerExecutables {
head_dim,
) {
Ok(qk_norm) => Some(Box::new(qk_norm)),
- Err(e) => {
- eprintln!("Failed to create QK norm kernel: {:?}", e);
- None
- },
+ Err(e) => panic!(
+ "Failed to create QK norm kernel for layer {}: {:?}",
+ layer_index, e
+ ),
}
} else {
None
diff --git a/crates/uzu/src/backends/metal/forward_pass/buffers.rs b/crates/uzu/src/backends/metal/forward_pass/buffers.rs
index eee037f34..07084db99 100644
--- a/crates/uzu/src/backends/metal/forward_pass/buffers.rs
+++ b/crates/uzu/src/backends/metal/forward_pass/buffers.rs
@@ -23,6 +23,8 @@ pub struct ForwardPassBuffers {
pub shortcut: MTLBuffer,
pub qkv: MTLBuffer,
pub attention_output: MTLBuffer,
+ pub mlp_fused_up: MTLBuffer,
+ pub mlp_hidden: MTLBuffer,
// 3-D
pub rotated_queries: MTLBuffer,
@@ -54,16 +56,9 @@ impl ForwardPassBuffers {
let act_ty = model_shape.activation_data_type();
- let total_blocks_count = 32;
- let num_heads = model_shape.num_heads();
- let head_dim = model_shape.head_dim();
-
- // Partials buffer: [num_heads, max_suffix_len, total_blocks_count, head_dim]
let partials_shape =
- [num_heads * max_suffix_len * total_blocks_count * head_dim];
-
- // Sums and maxs buffers: [num_heads, max_suffix_len, total_blocks_count]
- let sums_maxs_shape = [num_heads * max_suffix_len * total_blocks_count];
+ model_shape.attention_partials_shape(max_suffix_len);
+ let sums_maxs_shape = model_shape.attention_sums_shape(max_suffix_len);
Self {
// 1-D
@@ -92,6 +87,14 @@ impl ForwardPassBuffers {
&model_shape.attention_output_shape(max_suffix_len),
act_ty,
),
+ mlp_fused_up: alloc(
+ &model_shape.mlp_fused_up_shape(max_suffix_len),
+ act_ty,
+ ),
+ mlp_hidden: alloc(
+ &model_shape.mlp_hidden_shape(max_suffix_len),
+ act_ty,
+ ),
// 3-D
rotated_queries: alloc(
diff --git a/crates/uzu/src/backends/metal/forward_pass/model_shape.rs b/crates/uzu/src/backends/metal/forward_pass/model_shape.rs
index d18d3fc16..e0469041c 100644
--- a/crates/uzu/src/backends/metal/forward_pass/model_shape.rs
+++ b/crates/uzu/src/backends/metal/forward_pass/model_shape.rs
@@ -7,6 +7,7 @@ pub struct ModelShape {
vocabulary_size: usize,
model_dim: usize,
+ hidden_dim: usize,
context_length: usize,
num_heads: usize,
@@ -30,6 +31,7 @@ impl ModelShape {
kv_cache_type: activation_type,
vocabulary_size: decoder_config.vocab_size,
model_dim: decoder_config.model_dim,
+ hidden_dim: decoder_config.hidden_dim,
context_length: decoder_config.context_length,
num_heads: decoder_config.num_heads,
num_groups: decoder_config.num_groups,
@@ -65,6 +67,20 @@ impl ModelShape {
[suffix_length, self.model_dim]
}
+ pub fn mlp_hidden_shape(
+ &self,
+ suffix_length: usize,
+ ) -> [usize; 2] {
+ [suffix_length, self.hidden_dim]
+ }
+
+ pub fn mlp_fused_up_shape(
+ &self,
+ suffix_length: usize,
+ ) -> [usize; 2] {
+ [suffix_length, 2 * self.hidden_dim]
+ }
+
pub fn qkv_shape(
&self,
suffix_length: usize,
diff --git a/crates/uzu/src/backends/metal/forward_pass/state.rs b/crates/uzu/src/backends/metal/forward_pass/state.rs
index 6dc272366..56331e285 100644
--- a/crates/uzu/src/backends/metal/forward_pass/state.rs
+++ b/crates/uzu/src/backends/metal/forward_pass/state.rs
@@ -227,6 +227,10 @@ struct AuxBuffers {
qkv: ArrayCell,
/// [suffix_length, num_heads * head_dim]
attention_output: ArrayCell,
+ /// [suffix_length, 2 * hidden_dim]
+ mlp_fused_up: ArrayCell,
+ /// [suffix_length, hidden_dim]
+ mlp_hidden: ArrayCell,
/// [num_heads, max_suffix_length, head_dim]
rotated_queries: ArrayCell,
/// [num_groups, max_suffix_length, head_dim]
@@ -269,6 +273,16 @@ impl AuxBuffers {
&model_shape.attention_output_shape(suffix_length),
act_dtype,
)),
+ mlp_fused_up: RefCell::new(MetalArray::new(
+ scratch.mlp_fused_up.clone(),
+ &model_shape.mlp_fused_up_shape(suffix_length),
+ act_dtype,
+ )),
+ mlp_hidden: RefCell::new(MetalArray::new(
+ scratch.mlp_hidden.clone(),
+ &model_shape.mlp_hidden_shape(suffix_length),
+ act_dtype,
+ )),
rotated_queries: RefCell::new(MetalArray::new(
scratch.rotated_queries.clone(),
&model_shape.rotated_queries_shape(suffix_length),
@@ -478,6 +492,8 @@ impl ForwardPassState {
ArrayId::AttentionOutput => {
self.aux_buffers.attention_output.clone()
},
+ ArrayId::MlpFusedUp => self.aux_buffers.mlp_fused_up.clone(),
+ ArrayId::MlpHidden => self.aux_buffers.mlp_hidden.clone(),
ArrayId::Keys(layer_index) => {
self.kv_cache.borrow().data[layer_index].keys.clone()
},
@@ -540,7 +556,7 @@ impl ForwardPassState {
.borrow()
.local_rope
.as_ref()
- .unwrap()
+ .expect("Local rope requested but local_rope buffers are not initialized")
.cosines
.clone(),
},
@@ -553,7 +569,7 @@ impl ForwardPassState {
.borrow()
.local_rope
.as_ref()
- .unwrap()
+ .expect("Local rope requested but local_rope buffers are not initialized")
.sines
.clone(),
},
@@ -640,6 +656,8 @@ pub enum ArrayId {
Shortcut,
QKV,
AttentionOutput,
+ MlpFusedUp,
+ MlpHidden,
Keys(usize),
Values(usize),
diff --git a/crates/uzu/src/backends/metal/forward_pass/transformer_layer.rs b/crates/uzu/src/backends/metal/forward_pass/transformer_layer.rs
index ad33e3b1d..6ec2f5768 100644
--- a/crates/uzu/src/backends/metal/forward_pass/transformer_layer.rs
+++ b/crates/uzu/src/backends/metal/forward_pass/transformer_layer.rs
@@ -17,11 +17,17 @@ use super::{
};
use crate::{
DataType,
- backends::metal::graph::{
- embeddings_dequantize_weights_subgraph, embeddings_embed_subgraph,
- embeddings_readout_subgraph,
+ backends::metal::{
+ graph::{
+ embeddings_dequantize_weights_subgraph, embeddings_embed_subgraph,
+ embeddings_readout_subgraph,
+ },
+ kernel::{QuantizedLinearKernelBlock, mlp::MlpGateActMulEncodable},
+ },
+ config::{
+ DecoderConfig, EmbeddingConfig, LinearConfig, MLPConfig,
+ QuantizationConfig,
},
- config::{DecoderConfig, EmbeddingConfig, LinearConfig, MLPConfig},
parameters::ParameterTree,
};
@@ -41,7 +47,23 @@ pub fn linear_block<const N: usize>(
input_array_id: ArrayId,
output_array_id: ArrayId,
compilation_descriptor: &CompilationDescriptor,
-) -> MPSGraphBlock {
+) -> Box<dyn super::encodable_with_state::EncodableWithState> {
+ if let LinearConfig::Quantized(quant_config) = config {
+ if !has_biases {
+ let out_sum: usize = output_dims.iter().sum();
+ return quantized_linear_block_custom(
+ quant_config,
+ input_dim,
+ out_sum,
+ context,
+ parameter_tree,
+ input_array_id,
+ output_array_id,
+ )
+ .unwrap();
+ }
+ }
+
autoreleasepool(|_| {
let graph = Graph::new();
@@ -78,10 +100,38 @@ pub fn linear_block<const N: usize>(
vec![output_array_id].into_boxed_slice(),
);
- MPSGraphBlock::new(executable, make_execution_descriptor(), arguments)
+ Box::new(MPSGraphBlock::new(
+ executable,
+ make_execution_descriptor(),
+ arguments,
+ ))
})
}
+pub fn quantized_linear_block_custom(
+ config: &QuantizationConfig,
+ input_dim: usize,
+ output_dim: usize,
+ context: &MTLContext,
+ parameter_tree: &ParameterTree<Rc<MTLContext>>,
+ input_array_id: ArrayId,
+ output_array_id: ArrayId,
+) -> Result<
+ Box<dyn super::encodable_with_state::EncodableWithState>,
+ crate::backends::metal::MTLError,
+> {
+ let block = crate::backends::metal::kernel::linear::QuantizedLinearKernelBlock::new(
+ context,
+ config,
+ input_dim,
+ output_dim,
+ parameter_tree,
+ input_array_id,
+ output_array_id,
+ )?;
+ Ok(Box::new(block))
+}
+
pub fn mlp_block(
config: &MLPConfig,
model_dim: usize,
@@ -89,7 +139,52 @@ pub fn mlp_block(
context: &MTLContext,
parameter_tree: &ParameterTree<Rc<MTLContext>>,
compilation_descriptor: &CompilationDescriptor,
-) -> MPSGraphBlock {
+) -> Box<dyn super::encodable_with_state::EncodableWithState> {
+ if let crate::config::LinearConfig::Quantized(ref quant_config) =
+ config.linear_config
+ {
+ // Quantized MLP path: up (2H) -> act+mul -> down
+ let dtype: DataType =
+ config.linear_config.activation_precision().into();
+
+ // Up fused: Main -> MlpFusedUp
+ let up = QuantizedLinearKernelBlock::new(
+ context,
+ quant_config,
+ model_dim,
+ 2 * hidden_dim,
+ &parameter_tree.subtree("up_projection").unwrap(),
+ ArrayId::Main,
+ ArrayId::MlpFusedUp,
+ )
+ .expect("Failed to build MLP up quantized block");
+
+ // Activation+mul: MlpFusedUp -> MlpHidden
+ let gate_op = MlpGateActMulEncodable::new(
+ context,
+ dtype,
+ config.activation.clone(),
+ hidden_dim,
+ )
+ .expect("Failed to build MLP gate activation kernel");
+
+ // Down: MlpHidden -> Main
+ let down = QuantizedLinearKernelBlock::new(
+ context,
+ quant_config,
+ hidden_dim,
+ model_dim,
+ &parameter_tree.subtree("down_projection").unwrap(),
+ ArrayId::MlpHidden,
+ ArrayId::Main,
+ )
+ .expect("Failed to build MLP down quantized block");
+
+ let enc = crate::backends::metal::kernel::mlp::MlpBlockEncodable::new(
+ up, gate_op, down,
+ );
+ return Box::new(enc);
+ }
autoreleasepool(|_| {
let graph = Graph::new();
@@ -126,7 +221,11 @@ pub fn mlp_block(
vec![ArrayId::Main].into_boxed_slice(),
);
- MPSGraphBlock::new(executable, make_execution_descriptor(), arguments)
+ Box::new(MPSGraphBlock::new(
+ executable,
+ make_execution_descriptor(),
+ arguments,
+ ))
})
}
@@ -134,7 +233,7 @@ pub fn embed_block(
config: &DecoderConfig,
context: &MTLContext,
compilation_descriptor: &CompilationDescriptor,
-) -> MPSGraphBlock {
+) -> Box<dyn super::encodable_with_state::EncodableWithState> {
let graph = Graph::new();
let input_shape = [-1 as isize];
@@ -186,7 +285,7 @@ pub fn embed_block(
),
);
- block
+ Box::new(block)
},
EmbeddingConfig::Untied {
precision,
@@ -231,7 +330,7 @@ pub fn embed_block(
),
);
- block
+ Box::new(block)
},
EmbeddingConfig::QuantizedTied {
embedding_quantization_mode,
@@ -297,7 +396,7 @@ pub fn embed_block(
),
);
- block
+ Box::new(block)
},
}
}
@@ -306,7 +405,7 @@ pub fn readout_block(
config: &DecoderConfig,
context: &MTLContext,
compilation_descriptor: &CompilationDescriptor,
-) -> MPSGraphBlock {
+) -> Box<dyn super::encodable_with_state::EncodableWithState> {
let graph = Graph::new();
let input_shape = [-1 as isize, config.model_dim as isize];
@@ -360,7 +459,7 @@ pub fn readout_block(
),
);
- block
+ Box::new(block)
},
EmbeddingConfig::Untied {
precision,
@@ -404,7 +503,7 @@ pub fn readout_block(
),
);
- block
+ Box::new(block)
},
EmbeddingConfig::QuantizedTied {
embedding_quantization_mode,
@@ -469,7 +568,7 @@ pub fn readout_block(
),
);
- block
+ Box::new(block)
},
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/linear/mod.rs b/crates/uzu/src/backends/metal/kernel/linear/mod.rs
new file mode 100644
index 000000000..daf01628f
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/linear/mod.rs
@@ -0,0 +1,213 @@
+use std::rc::Rc;
+
+use metal::Buffer as MTLBuffer;
+use mpsgraph::CommandBuffer as MPSCommandBuffer;
+
+use super::quant_matmul::{QuantizedMatmulArguments, QuantizedMatmulKernel};
+use crate::{
+ Array, DataType,
+ backends::metal::{
+ MTLContext, MTLError,
+ forward_pass::{
+ ArrayId, ForwardPassState,
+ encodable_with_state::{EncodableWithState, EncodingParameters},
+ },
+ },
+ config::QuantizationConfig,
+ parameters::ParameterTree,
+};
+
+pub struct QuantizedLinearKernelBlock {
+ kernel_mm: QuantizedMatmulKernel,
+ kernel_mv: QuantizedMatmulKernel,
+ weights_buffer: MTLBuffer,
+ scales_buffer: MTLBuffer,
+ zero_points_buffer: MTLBuffer,
+ input_dim: usize,
+ output_dim: usize,
+ #[allow(dead_code)]
+ group_size: usize,
+ #[allow(dead_code)]
+ kernel_data_type: DataType,
+ input_array_id: ArrayId,
+ output_array_id: ArrayId,
+}
+
+impl QuantizedLinearKernelBlock {
+ pub fn new(
+ mtl_context: &MTLContext,
+ config: &QuantizationConfig,
+ input_dim: usize,
+ output_dim: usize,
+ parameter_tree: &ParameterTree<Rc<MTLContext>>,
+ input_array_id: ArrayId,
+ output_array_id: ArrayId,
+ ) -> Result<Self, MTLError> {
+ let kernel_data_type: DataType = config.activation_precision.into();
+
+ if !matches!(kernel_data_type, DataType::F16 | DataType::BF16) {
+ return Err(MTLError::Generic(format!(
+ "Unsupported data type for quantized kernel: {:?}",
+ kernel_data_type
+ )));
+ }
+
+ let mut weights = parameter_tree.leaf("weights").map_err(|e| {
+ MTLError::Generic(format!("Failed to load weights: {:?}", e))
+ })?;
+
+ if weights.data_type() != DataType::U8 {
+ return Err(MTLError::Generic(format!(
+ "Expected packed U8 weights, got {:?}",
+ weights.data_type()
+ )));
+ }
+
+ let mut scales = parameter_tree.leaf("scales").map_err(|e| {
+ MTLError::Generic(format!("Failed to load scales: {:?}", e))
+ })?;
+
+ let mut zero_points =
+ parameter_tree.leaf("zero_points").map_err(|e| {
+ MTLError::Generic(format!(
+ "Failed to load zero_points: {:?}",
+ e
+ ))
+ })?;
+
+ let k_g = (input_dim + config.group_size - 1) / config.group_size;
+
+ let w_shape = weights.shape();
+ let s_shape = scales.shape();
+ let zp_shape = zero_points.shape();
+
+ if !(w_shape == [output_dim, input_dim / 2]
+ && s_shape == [output_dim, k_g]
+ && zp_shape == [output_dim, (k_g + 1) / 2])
+ {
+ return Err(MTLError::Generic(format!(
+ "Unexpected shapes. weights={:?}, scales={:?}, zero_points={:?}; expected [N,K/2],[N,K_g],[N,(K_g+1)/2]",
+ w_shape, s_shape, zp_shape,
+ )));
+ }
+
+ let (scales_buffer, zero_points_buffer) = {
+ if scales.data_type() != kernel_data_type {
+ return Err(MTLError::Generic(format!(
+ "Scales dtype mismatch: got {:?}, expected {:?}",
+ scales.data_type(),
+ kernel_data_type
+ )));
+ }
+ let scales_buffer = unsafe { scales.mtl_buffer() }.to_owned();
+ let zero_points_buffer =
+ unsafe { zero_points.mtl_buffer() }.to_owned();
+ (scales_buffer, zero_points_buffer)
+ };
+
+ let weights_buffer: MTLBuffer =
+ unsafe { weights.mtl_buffer() }.to_owned();
+
+ let g = config.group_size;
+ let (kernel_name_mm, kernel_name_mv) = match (kernel_data_type, g) {
+ (DataType::F16, 64) => {
+ ("qmm_transposed_f16_g64_b4", "qmv_f16_g64_b4")
+ },
+ (DataType::F16, 128) => {
+ ("qmm_transposed_f16_g128_b4", "qmv_f16_g128_b4")
+ },
+ (DataType::BF16, 64) => {
+ ("qmm_transposed_bf16_g64_b4", "qmv_bf16_g64_b4")
+ },
+ (DataType::BF16, 128) => {
+ ("qmm_transposed_bf16_g128_b4", "qmv_bf16_g128_b4")
+ },
+ (dtype, other) => {
+ return Err(MTLError::Generic(format!(
+ "Unsupported group size {} for transposed {:?} kernel",
+ other, dtype
+ )));
+ },
+ };
+
+ let kernel_mm = QuantizedMatmulKernel::new(
+ mtl_context,
+ kernel_data_type,
+ kernel_name_mm,
+ )
+ .map_err(|e| MTLError::Generic(format!("{:?}", e)))?;
+ let kernel_mv = QuantizedMatmulKernel::new(
+ mtl_context,
+ kernel_data_type,
+ kernel_name_mv,
+ )
+ .map_err(|e| MTLError::Generic(format!("{:?}", e)))?;
+
+ Ok(Self {
+ kernel_mm,
+ kernel_mv,
+ weights_buffer,
+ scales_buffer,
+ zero_points_buffer,
+ input_dim,
+ output_dim,
+ group_size: config.group_size,
+ kernel_data_type,
+ input_array_id,
+ output_array_id,
+ })
+ }
+}
+
+impl EncodableWithState for QuantizedLinearKernelBlock {
+ fn encode(
+ &self,
+ state: &mut ForwardPassState,
+ command_buffer: &MPSCommandBuffer,
+ _parameters: &EncodingParameters,
+ ) {
+ let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);
+ let input_array = arrays[0].borrow();
+
+ let batch_size = input_array.shape()[0];
+
+ drop(input_array);
+
+ let mut input_array_mut = arrays[0].borrow_mut();
+ let mut output_array_mut = arrays[1].borrow_mut();
+
+ let input_buffer = unsafe { input_array_mut.mtl_buffer() };
+ let output_buffer = unsafe { output_array_mut.mtl_buffer() };
+
+ let root_command_buffer = command_buffer.root_command_buffer();
+ let encoder = root_command_buffer.new_compute_command_encoder();
+
+ let m = batch_size;
+ let n = self.output_dim;
+ let k = self.input_dim;
+
+ let args = QuantizedMatmulArguments {
+ a_buffer: input_buffer,
+ b_buffer: &self.weights_buffer,
+ scales_buffer: &self.scales_buffer,
+ zero_points_buffer: &self.zero_points_buffer,
+ output_buffer: output_buffer,
+ m: m as i32,
+ n: n as i32,
+ k: k as i32,
+ };
+
+ let use_gemv = batch_size == 1;
+ let kernel = if use_gemv {
+ &self.kernel_mv
+ } else {
+ &self.kernel_mm
+ };
+
+ kernel
+ .encode(encoder, args)
+ .expect("Failed to encode quantized matmul kernel");
+
+ encoder.end_encoding();
+ }
+}
diff --git a/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal b/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal
new file mode 100644
index 000000000..8d08af8fc
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal
@@ -0,0 +1,78 @@
+#include <metal_stdlib>
+#include "../definitions.metal"
+using namespace metal;
+
+enum ActivationType : ushort {
+ ACT_SILU = 0,
+ ACT_GELU = 1,
+};
+
+template <typename T>
+inline T silu(T x) {
+ float xf = float(x);
+ float yf = xf / (1.0f + exp(-xf));
+ return T(yf);
+}
+
+template <typename T>
+inline T gelu_approx(T x) {
+ constexpr float k0 = 0.044715f;
+ constexpr float k1 = 0.7978845608f; // sqrt(2/pi)
+ float xf = float(x);
+ float yf = 0.5f * xf * (1.0f + tanh(k1 * (xf + k0 * xf * xf * xf)));
+ return T(yf);
+}
+
+template <typename T>
+inline T activate(T x, ushort act) {
+ if (act == ACT_SILU) return silu(x);
+ return gelu_approx(x);
+}
+
+template <typename T>
+[[kernel]] void mlp_activation_mul(
+ const device T* fused_up [[buffer(0)]],
+ device T* hidden [[buffer(1)]],
+ const constant int& H [[buffer(2)]],
+ const constant int& M [[buffer(3)]],
+ const constant ushort& act_type [[buffer(4)]],
+ uint2 tid [[thread_position_in_grid]]) {
+ int row = tid.y;
+ int j = tid.x;
+ if (row >= M || j >= H) return;
+ int base = row * (2 * H);
+ T up = fused_up[base + j];
+ T gate = fused_up[base + H + j];
+ T g = activate(gate, act_type);
+ hidden[row * H + j] = up * g;
+}
+
+// Explicit instantiations with stable host names
+template [[host_name("mlp_activation_mul_f16")]]
+[[kernel]] void mlp_activation_mul<half>(
+ const device half* fused_up [[buffer(0)]],
+ device half* hidden [[buffer(1)]],
+ const constant int& H [[buffer(2)]],
+ const constant int& M [[buffer(3)]],
+ const constant ushort& act_type [[buffer(4)]],
+ uint2 tid [[thread_position_in_grid]]);
+
+template [[host_name("mlp_activation_mul_f32")]]
+[[kernel]] void mlp_activation_mul<float>(
+ const device float* fused_up [[buffer(0)]],
+ device float* hidden [[buffer(1)]],
+ const constant int& H [[buffer(2)]],
+ const constant int& M [[buffer(3)]],
+ const constant ushort& act_type [[buffer(4)]],
+ uint2 tid [[thread_position_in_grid]]);
+
+template [[host_name("mlp_activation_mul_bf16")]]
+[[kernel]] void mlp_activation_mul<bfloat>(
+ const device bfloat* fused_up [[buffer(0)]],
+ device bfloat* hidden [[buffer(1)]],
+ const constant int& H [[buffer(2)]],
+ const constant int& M [[buffer(3)]],
+ const constant ushort& act_type [[buffer(4)]],
+ uint2 tid [[thread_position_in_grid]]);
+
+
diff --git a/crates/uzu/src/backends/metal/kernel/mlp/mod.rs b/crates/uzu/src/backends/metal/kernel/mlp/mod.rs
new file mode 100644
index 000000000..3d39c4f99
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/mlp/mod.rs
@@ -0,0 +1,180 @@
+use metal::{
+ Buffer as MTLBuffer, ComputeCommandEncoderRef, ComputePipelineState,
+ MTLSize,
+};
+use mpsgraph::CommandBuffer as MPSCommandBuffer;
+
+use crate::{
+ DataType,
+ array::Array,
+ backends::metal::{
+ MTLContext, MTLError,
+ forward_pass::{
+ ArrayId, ForwardPassState,
+ encodable_with_state::{EncodableWithState, EncodingParameters},
+ },
+ kernel::QuantizedLinearKernelBlock,
+ },
+ config::Activation,
+};
+
+pub struct MlpGateActMulKernel {
+ pipeline: ComputePipelineState,
+}
+
+pub struct MlpGateActMulEncodable {
+ kernel: MlpGateActMulKernel,
+ activation: Activation,
+ hidden_dim: usize,
+}
+
+impl MlpGateActMulEncodable {
+ pub fn new(
+ context: &MTLContext,
+ data_type: DataType,
+ activation: Activation,
+ hidden_dim: usize,
+ ) -> Result<Self, MTLError> {
+ let kernel = MlpGateActMulKernel::new(context, data_type)?;
+ Ok(Self {
+ kernel,
+ activation,
+ hidden_dim,
+ })
+ }
+
+ pub fn encode(
+ &self,
+ encoder: &ComputeCommandEncoderRef,
+ fused_up: &MTLBuffer,
+ hidden: &MTLBuffer,
+ m: i32,
+ ) -> Result<(), MTLError> {
+ self.kernel.encode(
+ encoder,
+ &self.activation,
+ fused_up,
+ hidden,
+ m,
+ self.hidden_dim as i32,
+ )
+ }
+}
+
+impl MlpGateActMulKernel {
+ fn kernel_name_for_type(
+ data_type: DataType
+ ) -> Result<&'static str, MTLError> {
+ match data_type {
+ DataType::F16 => Ok("mlp_activation_mul_f16"),
+ DataType::F32 => Ok("mlp_activation_mul_f32"),
+ DataType::BF16 => Ok("mlp_activation_mul_bf16"),
+ other => Err(MTLError::Generic(format!(
+ "Unsupported dtype for MLP activation: {:?}",
+ other
+ ))),
+ }
+ }
+
+ pub fn new(
+ context: &MTLContext,
+ data_type: DataType,
+ ) -> Result<Self, MTLError> {
+ let fn_name = Self::kernel_name_for_type(data_type)?;
+ let pipeline = context.compute_pipeline_state(fn_name, None)?;
+ Ok(Self {
+ pipeline,
+ })
+ }
+
+ fn act_code(act: &Activation) -> u16 {
+ match act {
+ Activation::SILU => 0,
+ Activation::GELU => 1,
+ }
+ }
+
+ pub fn encode(
+ &self,
+ encoder: &ComputeCommandEncoderRef,
+ activation: &Activation,
+ fused_up_buffer: &MTLBuffer,
+ hidden_buffer: &MTLBuffer,
+ m: i32,
+ h: i32,
+ ) -> Result<(), MTLError> {
+ let act_code = Self::act_code(activation);
+ let threads_per_tg = MTLSize::new(64, 1, 1);
+ encoder.set_compute_pipeline_state(&self.pipeline);
+ encoder.set_buffer(0, Some(fused_up_buffer), 0);
+ encoder.set_buffer(1, Some(hidden_buffer), 0);
+ encoder.set_bytes(
+ 2,
+ std::mem::size_of::<i32>() as u64,
+ &h as *const i32 as *const _,
+ );
+ encoder.set_bytes(
+ 3,
+ std::mem::size_of::<i32>() as u64,
+ &m as *const i32 as *const _,
+ );
+ encoder.set_bytes(
+ 4,
+ std::mem::size_of::<u16>() as u64,
+ &act_code as *const u16 as *const _,
+ );
+
+ let grid = MTLSize::new(h as u64, m as u64, 1);
+ encoder.dispatch_threads(grid, threads_per_tg);
+ Ok(())
+ }
+}
+
+pub struct MlpBlockEncodable {
+ up: QuantizedLinearKernelBlock,
+ gate: MlpGateActMulEncodable,
+ down: QuantizedLinearKernelBlock,
+}
+
+impl MlpBlockEncodable {
+ pub fn new(
+ up: QuantizedLinearKernelBlock,
+ gate: MlpGateActMulEncodable,
+ down: QuantizedLinearKernelBlock,
+ ) -> Self {
+ Self {
+ up,
+ gate,
+ down,
+ }
+ }
+}
+
+impl EncodableWithState for MlpBlockEncodable {
+ fn encode(
+ &self,
+ state: &mut ForwardPassState,
+ command_buffer: &MPSCommandBuffer,
+ _params: &EncodingParameters,
+ ) {
+ // Up
+ self.up.encode(state, command_buffer, _params);
+ // Gate act+mul (fused_up -> hidden)
+ let arrays = state.arrays(&[ArrayId::MlpFusedUp, ArrayId::MlpHidden]);
+ let mut fused = arrays[0].borrow_mut();
+ let mut hidden = arrays[1].borrow_mut();
+ let m = fused.shape()[0] as i32;
+ let root = command_buffer.root_command_buffer();
+ let encoder = root.new_compute_command_encoder();
+ let fused_buf = unsafe { fused.mtl_buffer() };
+ let hidden_buf = unsafe { hidden.mtl_buffer() };
+ self.gate
+ .encode(encoder, fused_buf, hidden_buf, m)
+ .expect("Failed to encode MLP activation/mul kernel");
+ encoder.end_encoding();
+ drop(fused);
+ drop(hidden);
+ // Down
+ self.down.encode(state, command_buffer, _params);
+ }
+}
diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rs
index 0b0c376fd..7f417cbf7 100644
--- a/crates/uzu/src/backends/metal/kernel/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/mod.rs
@@ -1,7 +1,10 @@
pub mod attention;
mod data_type;
pub mod kv_cache_update;
+pub mod linear;
pub mod media_kernels;
+pub mod mlp;
+pub mod quant_matmul;
pub mod rms_norm;
pub mod rope;
pub mod sampling;
@@ -21,6 +24,7 @@ pub use tensor_add_swap::TensorAddSwap;
pub use tensor_copy::TensorCopy;
use super::{MTLContext, metal_extensions};
+pub use crate::backends::metal::kernel::linear::QuantizedLinearKernelBlock;
pub mod media {
pub use super::media_kernels::{
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/mma.h b/crates/uzu/src/backends/metal/kernel/quant_matmul/mma.h
new file mode 100644
index 000000000..9be4dee9b
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/mma.h
@@ -0,0 +1,486 @@
+#pragma once
+
+#include <metal_simdgroup>
+#include <metal_simdgroup_matrix>
+#include <metal_stdlib>
+
+#define UZU_MTL_CONST static constant constexpr
+#define UZU_PRAGMA_UNROLL _Pragma("clang loop unroll(full)")
+#define SIMD_SIZE 32
+
+using namespace metal;
+
+namespace matmul_utils {
+
+template<typename T>
+struct pointer_element_t_impl {
+ using type = T;
+};
+template<typename T>
+struct pointer_element_t_impl<device T*> {
+ using type = T;
+};
+template<typename T>
+using pointer_element_t = typename pointer_element_t_impl<T>::type;
+
+template <typename T, typename U = T>
+struct TransformNone {
+ METAL_FUNC static U apply(U val) {
+ return val;
+ }
+};
+
+template <typename T, int kFragRows_, int kFragCols_>
+struct BaseMMAFrag {
+ static_assert(
+ kFragRows_ == 8,
+ "Only 8 x 8 fragment matrices are currently supported");
+ static_assert(
+ kFragCols_ == 8,
+ "Only 8 x 8 fragment matrices are currently supported");
+};
+
+template <typename T>
+struct BaseMMAFrag<T, 8, 8> {
+ UZU_MTL_CONST int kFragRows = 8;
+ UZU_MTL_CONST int kFragCols = 8;
+ UZU_MTL_CONST int kElemsPerFrag = (kFragRows * kFragCols) / 32;
+ UZU_MTL_CONST int kElemRows = 1;
+ UZU_MTL_CONST int kElemCols = 2;
+
+ static_assert(
+ kElemRows * kElemCols == kElemsPerFrag,
+ "MMAFrag shape is not consistent with MMAFrag size");
+
+ typedef metal::simdgroup_matrix<T, kFragRows, kFragCols> mat_type;
+ typedef metal::vec<T, kElemsPerFrag> frag_type;
+
+ METAL_FUNC static constexpr short2 get_coord(ushort simd_lane_id
+ [[thread_index_in_simdgroup]]) {
+ const short qid = simd_lane_id / 4;
+ const short fm = (qid & 4) + ((simd_lane_id / 2) % 4);
+ const short fn = (qid & 2) * 2 + (simd_lane_id % 2) * 2;
+ return short2{fn, fm};
+ }
+
+ template <typename SrcPtrType, typename StrX, typename StrY>
+ METAL_FUNC static constexpr void
+ load(thread frag_type& dst, SrcPtrType src, StrX str_x, StrY str_y) {
+ UZU_PRAGMA_UNROLL
+ for (short i = 0; i < kElemRows; i++) {
+ UZU_PRAGMA_UNROLL
+ for (short j = 0; j < kElemCols; j++) {
+ dst[i * kElemCols + j] = static_cast<T>(src[i * str_x + j * str_y]);
+ }
+ }
+ }
+
+
+ template <typename DstPtrType, typename StrX, typename StrY>
+ METAL_FUNC static constexpr void
+ store(const thread frag_type& src, DstPtrType dst, StrX str_x, StrY str_y) {
+ using U = pointer_element_t<DstPtrType>;
+
+ UZU_PRAGMA_UNROLL
+ for (short i = 0; i < kElemRows; i++) {
+ UZU_PRAGMA_UNROLL
+ for (short j = 0; j < kElemCols; j++) {
+ dst[i * str_x + j * str_y] = static_cast<U>(src[i * kElemCols + j]);
+ }
+ }
+ }
+
+ template <
+ typename DstPtrType,
+ typename StrX,
+ typename StrY,
+ typename LimX,
+ typename LimY,
+ typename OffX,
+ typename OffY>
+ METAL_FUNC static constexpr void store_safe(
+ const thread frag_type& src,
+ DstPtrType dst,
+ StrX str_x,
+ StrY str_y,
+ LimX lim_x,
+ LimY lim_y,
+ OffX off_x = 0,
+ OffY off_y = 0) {
+ using U = pointer_element_t<DstPtrType>;
+
+ UZU_PRAGMA_UNROLL
+ for (short i = 0; i < kElemRows; i++) {
+ UZU_PRAGMA_UNROLL
+ for (short j = 0; j < kElemCols; j++) {
+ if ((off_x + i) < lim_x && (off_y + j) < lim_y) {
+ dst[(off_x + i) * str_x + (off_y + j) * str_y] =
+ static_cast<U>(src[i * kElemCols + j]);
+ }
+ }
+ }
+ }
+
+
+ METAL_FUNC static constexpr void mma(
+ thread frag_type& D,
+ thread frag_type& A,
+ thread frag_type& B,
+ thread frag_type& C) {
+ mat_type D_mat;
+ mat_type A_mat;
+ mat_type B_mat;
+ mat_type C_mat;
+
+ reinterpret_cast<thread frag_type&>(A_mat.thread_elements()) = A;
+ reinterpret_cast<thread frag_type&>(B_mat.thread_elements()) = B;
+ reinterpret_cast<thread frag_type&>(C_mat.thread_elements()) = C;
+
+ mma(D_mat, A_mat, B_mat, C_mat);
+
+ D = reinterpret_cast<thread frag_type&>(D_mat.thread_elements());
+ }
+
+ METAL_FUNC static constexpr void mma(
+ thread mat_type& D,
+ thread mat_type& A,
+ thread mat_type& B,
+ thread mat_type& C) {
+ simdgroup_multiply_accumulate(D, A, B, C);
+ }
+};
+
+template <
+ typename T,
+ int kTileRows_,
+ int kTileCols_,
+ class MMAFrag_ = BaseMMAFrag<T, 8, 8>>
+struct MMATile {
+ using MMAFrag_t = MMAFrag_;
+ using elem_type = T;
+ UZU_MTL_CONST int kFragRows = MMAFrag_t::kFragRows;
+ UZU_MTL_CONST int kFragCols = MMAFrag_t::kFragCols;
+ UZU_MTL_CONST int kElemsPerFrag = MMAFrag_t::kElemsPerFrag;
+
+ UZU_MTL_CONST int kTileRows = kTileRows_;
+ UZU_MTL_CONST int kTileCols = kTileCols_;
+
+ UZU_MTL_CONST int kRows = kTileRows * kFragRows;
+ UZU_MTL_CONST int kCols = kTileCols * kFragCols;
+
+ UZU_MTL_CONST int kNumFrags = kTileRows * kTileCols;
+ UZU_MTL_CONST int kElemsPerTile = kNumFrags * kElemsPerFrag;
+
+ typedef typename MMAFrag_t::mat_type mat_type;
+ typedef typename MMAFrag_t::frag_type frag_type;
+
+ frag_type val_frags[kNumFrags] = {frag_type(0)};
+
+ METAL_FUNC MMATile() thread {}
+
+ METAL_FUNC constexpr void clear() {
+ UZU_PRAGMA_UNROLL
+ for (short i = 0; i < kNumFrags; ++i) {
+ val_frags[i] = frag_type(0);
+ }
+ }
+
+ METAL_FUNC constexpr thread frag_type& frag_at(const short i, const short j) {
+ return val_frags[i * kTileCols + j];
+ }
+
+ METAL_FUNC constexpr const thread frag_type& frag_at(
+ const short i,
+ const short j) const {
+ return val_frags[i * kTileCols + j];
+ }
+
+
+ METAL_FUNC thread elem_type* elems() {
+ return reinterpret_cast<thread elem_type*>(val_frags);
+ }
+
+ METAL_FUNC const thread elem_type* elems() const {
+ return reinterpret_cast<const thread elem_type*>(val_frags);
+ }
+
+ template <typename U, int w_x, int w_y, int str_x, int str_y>
+ METAL_FUNC void load(const threadgroup U* src) {
+ UZU_PRAGMA_UNROLL
+ for (short i = 0; i < kTileRows; ++i) {
+ UZU_PRAGMA_UNROLL
+ for (short j = 0; j < kTileCols; ++j) {
+ MMAFrag_t::load(
+ frag_at(i, j),
+ &(
+ src[(i * kFragRows) * w_x * str_x +
+ (j * kFragCols) * w_y * str_y]),
+ str_x,
+ str_y);
+ }
+ }
+ }
+
+
+ template <typename U, int w_x, int w_y>
+ METAL_FUNC void store(device U* dst, const int ld) const {
+ UZU_PRAGMA_UNROLL
+ for (short i = 0; i < kTileRows; ++i) {
+ UZU_PRAGMA_UNROLL
+ for (short j = 0; j < kTileCols; ++j) {
+ MMAFrag_t::store(
+ frag_at(i, j),
+ &(dst[(i * kFragRows) * w_x * ld + (j * kFragCols) * w_y]),
+ ld,
+ 1);
+ }
+ }
+ }
+
+
+ template <typename U, int w_x, int w_y>
+ METAL_FUNC void
+ store_safe(device U* dst, const int ld, const short2 dst_tile_dims) const {
+ UZU_PRAGMA_UNROLL
+ for (int i = 0; i < kTileRows; ++i) {
+ UZU_PRAGMA_UNROLL
+ for (int j = 0; j < kTileCols; ++j) {
+ MMAFrag_t::store_safe(
+ frag_at(i, j),
+ dst,
+ ld,
+ 1,
+ dst_tile_dims.y,
+ dst_tile_dims.x,
+ (i * kFragRows) * w_x,
+ (j * kFragCols) * w_y);
+ }
+ }
+ }
+
+};
+
+template <typename T, typename U, int M, int N, int K>
+METAL_FUNC void tile_matmad(
+ thread MMATile<T, M, N>& D,
+ thread MMATile<U, M, K>& A,
+ thread MMATile<U, K, N>& B,
+ thread MMATile<T, M, N>& C) {
+ UZU_PRAGMA_UNROLL
+ for (short m = 0; m < M; ++m) {
+ UZU_PRAGMA_UNROLL
+ for (short n = 0; n < N; ++n) {
+ short n_serp = (m % 2) ? (N - 1 - n) : n;
+ UZU_PRAGMA_UNROLL
+ for (short k = 0; k < K; ++k) {
+ MMATile<T, M, N>::MMAFrag_t::mma(
+ D.frag_at(m, n_serp),
+ A.frag_at(m, k),
+ B.frag_at(k, n_serp),
+ C.frag_at(m, n_serp));
+ }
+ }
+ }
+}
+
+template <
+ typename T,
+ typename U,
+ int BM,
+ int BN,
+ int BK,
+ int WM,
+ int WN,
+ bool transpose_a,
+ bool transpose_b,
+ short lda_tgp,
+ short ldb_tgp,
+ typename AccumType = float,
+ typename Epilogue = TransformNone<U, AccumType>>
+struct BlockMMA {
+ // MMAFrag size
+ UZU_MTL_CONST short kFragSize = 8;
+ using MMAFrag_acc_t = BaseMMAFrag<AccumType, kFragSize, kFragSize>;
+
+ // Warp tile simdgroup matrix strides along M
+ UZU_MTL_CONST short TM_stride = kFragSize * WM;
+ // Warp tile simdgroup matrix strides along M
+ UZU_MTL_CONST short TN_stride = kFragSize * WN;
+
+ // Warp tile size along M
+ UZU_MTL_CONST short TM = BM / (kFragSize * WM);
+ // Warp tile size along N
+ UZU_MTL_CONST short TN = BN / (kFragSize * WN);
+
+ // Threadgroup A strides
+ UZU_MTL_CONST short A_str_m = transpose_a ? 1 : lda_tgp; // M
+ UZU_MTL_CONST short A_str_k = transpose_a ? lda_tgp : 1; // K
+
+ // Threadgroup B strides
+ UZU_MTL_CONST short B_str_k = transpose_b ? 1 : ldb_tgp; // K
+ UZU_MTL_CONST short B_str_n = transpose_b ? ldb_tgp : 1; // N
+
+ // Threadgroup strides along K
+ UZU_MTL_CONST short tile_stride_a = kFragSize * A_str_k;
+ UZU_MTL_CONST short tile_stride_b = kFragSize * B_str_k;
+
+ // Simdgroup matrices
+ MMATile<AccumType, TM, 1, MMAFrag_acc_t> Atile;
+ MMATile<AccumType, 1, TN, MMAFrag_acc_t> Btile;
+ MMATile<AccumType, TM, TN, MMAFrag_acc_t> Ctile;
+
+ // Offsets within threadgroup
+ short sm;
+ short sn;
+
+ short As_offset;
+ short Bs_offset;
+
+ /* Constructor */
+ METAL_FUNC BlockMMA(
+ ushort simd_group_id [[simdgroup_index_in_threadgroup]],
+ ushort simd_lane_id [[thread_index_in_simdgroup]]) {
+ // Determine thread position in simdgroup matrix
+ short tm = kFragSize * (simd_group_id / WN);
+ short tn = kFragSize * (simd_group_id % WN);
+
+ short2 simd_coord = MMAFrag_acc_t::get_coord(simd_lane_id);
+ sm = simd_coord.y;
+ sn = simd_coord.x;
+
+ // Determine thread and simdgroup offset
+ As_offset = (tm + sm) * A_str_m + (sn)*A_str_k; // M, K
+ Bs_offset = (sm)*B_str_k + (tn + sn) * B_str_n; // K, N
+
+ sm += tm;
+ sn += tn;
+ }
+
+ /* (BM, BK) X (BK, BN) multiply accumulate function */
+ METAL_FUNC void mma(const threadgroup T* As, const threadgroup T* Bs) {
+ // Adjust for simdgroup and thread location
+ As += As_offset;
+ Bs += Bs_offset;
+
+ // Iterate over BK in blocks of kFragSize
+ UZU_PRAGMA_UNROLL
+ for (short kk = 0; kk < BK; kk += kFragSize) {
+ simdgroup_barrier(mem_flags::mem_none);
+
+ Atile.template load<T, WM, 1, A_str_m, A_str_k>(As);
+
+ simdgroup_barrier(mem_flags::mem_none);
+
+ Btile.template load<T, 1, WN, B_str_k, B_str_n>(Bs);
+
+ simdgroup_barrier(mem_flags::mem_none);
+
+ tile_matmad(Ctile, Atile, Btile, Ctile);
+
+ // Progress to next simdgroup tile
+ As += tile_stride_a;
+ Bs += tile_stride_b;
+ }
+ }
+
+ /* Store results from simdgroup_matrix results into device memory */
+ METAL_FUNC void store_result(device U* D, const int ldd) {
+ // Apply epilogue
+ UZU_PRAGMA_UNROLL
+ for (short i = 0; i < decltype(Ctile)::kElemsPerTile; i++) {
+ Ctile.elems()[i] = Epilogue::apply(Ctile.elems()[i]);
+ }
+
+ // Adjust for simdgroup and thread location
+ D += sm * ldd + sn;
+
+ Ctile.template store<U, WM, WN>(D, ldd);
+ }
+
+
+ METAL_FUNC void
+ store_result_safe(device U* D, const int ldd, short2 dst_tile_dims) {
+ // Apply epilogue
+ UZU_PRAGMA_UNROLL
+ for (short i = 0; i < decltype(Ctile)::kElemsPerTile; i++) {
+ Ctile.elems()[i] = Epilogue::apply(Ctile.elems()[i]);
+ }
+
+ // Adjust for simdgroup and thread location
+ D += sm * ldd + sn;
+ dst_tile_dims -= short2(sn, sm);
+
+ if (dst_tile_dims.x <= 0 || dst_tile_dims.y <= 0)
+ return;
+
+ Ctile.template store_safe<U, WM, WN>(D, ldd, dst_tile_dims);
+ }
+
+};
+
+template <
+ typename T,
+ int BM,
+ int BK,
+ int lda_tgp,
+ int reduction_dim,
+ int tgp_size,
+ int align_M = 1,
+ int align_K = 4>
+struct BlockLoader {
+ UZU_MTL_CONST short n_reads = (BM * BK < tgp_size) ? 1 : (BM * BK) / tgp_size;
+
+ const int src_ld;
+ const short thread_idx;
+ const short bi;
+ const short bj;
+
+ threadgroup T* dst;
+ const device T* src;
+
+ BlockLoader(
+ const device T* src_,
+ const int src_ld_,
+ threadgroup T* dst_,
+ ushort simd_group_id [[simdgroup_index_in_threadgroup]],
+ ushort simd_lane_id [[thread_index_in_simdgroup]])
+ : src_ld(src_ld_),
+ thread_idx(simd_group_id * 32 + simd_lane_id),
+ bi(n_reads * thread_idx / BK),
+ bj((n_reads * thread_idx) % BK),
+ dst(dst_ + bi * lda_tgp + bj),
+ src(src_ + bi * src_ld + bj) {}
+
+ void load_unsafe() const {
+ if (BM * BK < tgp_size && bi >= BM) {
+ return;
+ }
+
+ for (int i = 0; i < n_reads; i++) {
+ dst[i] = src[i];
+ }
+ }
+
+ void load_safe(short2 src_tile_dim) const {
+ if (BM * BK < tgp_size && bi >= BM) {
+ return;
+ }
+
+ if (bi >= src_tile_dim.y || bj >= src_tile_dim.x) {
+ for (int i = 0; i < n_reads; i++) {
+ dst[i] = T(0);
+ }
+ return;
+ }
+
+ for (int i = 0; i < n_reads; i++) {
+ dst[i] = src[i];
+ }
+ }
+
+ void next() {
+ src += BK; // Advance by tile size
+ }
+};
+
+}
\ No newline at end of file
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs
new file mode 100644
index 000000000..192ba21b7
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs
@@ -0,0 +1,202 @@
+use std::mem::size_of;
+
+use metal::{
+ Buffer as MTLBuffer, ComputeCommandEncoderRef,
+ ComputePipelineState as MTLComputePipelineState, MTLSize,
+};
+
+use super::super::MTLContext;
+use crate::{DataType, backends::metal::MTLError};
+
+#[derive(Debug, thiserror::Error)]
+pub enum QuantizedMatmulError {
+ #[error("Metal error: {0}")]
+ MetalError(#[from] MTLError),
+ #[error("Unsupported data type: {0:?}")]
+ UnsupportedDataType(DataType),
+ #[error("Unsupported group size: {0}")]
+ UnsupportedGroupSize(usize),
+ #[error("Invalid dimensions: M={m}, N={n}, K={k}")]
+ InvalidDimensions {
+ m: usize,
+ n: usize,
+ k: usize,
+ },
+}
+
+pub struct QuantizedMatmulKernel {
+ pipeline: MTLComputePipelineState,
+ kind: KernelKind,
+}
+
+#[derive(Debug)]
+pub struct QuantizedMatmulArguments<'a> {
+ pub a_buffer: &'a MTLBuffer, // Input A (float)
+ pub b_buffer: &'a MTLBuffer, // Input B (quantized)
+ pub scales_buffer: &'a MTLBuffer,
+ pub zero_points_buffer: &'a MTLBuffer,
+ pub output_buffer: &'a MTLBuffer,
+ pub m: i32,
+ pub n: i32,
+ pub k: i32,
+}
+
+impl QuantizedMatmulKernel {
+ pub fn new(
+ mtl_context: &MTLContext,
+ data_type: DataType,
+ kernel_name: &str,
+ ) -> Result<Self, QuantizedMatmulError> {
+ if !matches!(data_type, DataType::F16 | DataType::BF16) {
+ return Err(QuantizedMatmulError::UnsupportedDataType(data_type));
+ }
+
+ let pipeline = mtl_context
+ .compute_pipeline_state(kernel_name, None)
+ .map_err(QuantizedMatmulError::MetalError)?;
+
+ let kind = if kernel_name.starts_with("qmv") {
+ KernelKind::Qmv
+ } else if kernel_name.starts_with("qvm") {
+ KernelKind::Qvm
+ } else {
+ KernelKind::Qmm
+ };
+
+ Ok(Self {
+ pipeline,
+ kind,
+ })
+ }
+
+ #[allow(dead_code)]
+ fn kernel_name_for_config(data_type: DataType) -> String {
+ let type_suffix = match data_type {
+ DataType::F16 => "f16",
+ DataType::BF16 => "bf16",
+ _ => unreachable!(),
+ };
+
+ format!("qmm_t_{}_gs_64_b_4_batch_0", type_suffix)
+ }
+
+ pub fn encode(
+ &self,
+ encoder: &ComputeCommandEncoderRef,
+ args: QuantizedMatmulArguments,
+ ) -> Result<(), QuantizedMatmulError> {
+ encoder.set_compute_pipeline_state(&self.pipeline);
+
+ // Set buffers
+ encoder.set_buffer(0, Some(args.b_buffer), 0);
+ encoder.set_buffer(1, Some(args.scales_buffer), 0);
+ encoder.set_buffer(2, Some(args.zero_points_buffer), 0);
+ encoder.set_buffer(3, Some(args.a_buffer), 0);
+ encoder.set_buffer(4, Some(args.output_buffer), 0);
+
+ // Set constants
+ encoder.set_bytes(
+ 5,
+ size_of::<i32>() as u64,
+ &args.k as *const i32 as *const _,
+ );
+ encoder.set_bytes(
+ 6,
+ size_of::<i32>() as u64,
+ &args.n as *const i32 as *const _,
+ );
+ encoder.set_bytes(
+ 7,
+ size_of::<i32>() as u64,
+ &args.m as *const i32 as *const _,
+ );
+
+ match self.kind {
+ KernelKind::Qmv => {
+ let bk = 32;
+ let bn = 8;
+ let n_tgp_y = (args.n + bn - 1) / bn; // ceiling division
+ let threadgroups =
+ MTLSize::new(args.m as u64, n_tgp_y as u64, 1);
+ let threads_per_threadgroup = MTLSize::new(bk as u64, 2, 1);
+ encoder.dispatch_thread_groups(
+ threadgroups,
+ threads_per_threadgroup,
+ );
+ },
+ KernelKind::Qvm => {
+ let bk = 32;
+ let bn = 64;
+ let n_tgp_y = (args.n + bn - 1) / bn; // ceiling division for columns
+ let threadgroups =
+ MTLSize::new(args.m as u64, n_tgp_y as u64, 1);
+ let threads_per_threadgroup = MTLSize::new(bk as u64, 2, 1);
+ encoder.dispatch_thread_groups(
+ threadgroups,
+ threads_per_threadgroup,
+ );
+ },
+ KernelKind::Qmm => {
+ let bm = 32;
+ let bn = 32;
+ let wm = 2;
+ let wn = 2;
+ let threads_per_threadgroup = MTLSize::new(32, wn, wm);
+ let threadgroups = MTLSize::new(
+ (args.n as u64 + bn - 1) / bn,
+ (args.m as u64 + bm - 1) / bm,
+ 1,
+ );
+ encoder.dispatch_thread_groups(
+ threadgroups,
+ threads_per_threadgroup,
+ );
+ },
+ }
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum KernelKind {
+ Qmm,
+ Qmv,
+ Qvm,
+}
+
+pub fn encode_quantized_matmul(
+ mtl_context: &MTLContext,
+ encoder: &ComputeCommandEncoderRef,
+ kernel_data_type: DataType,
+ group_size: usize,
+ transpose: bool,
+ args: QuantizedMatmulArguments,
+) -> Result<(), QuantizedMatmulError> {
+ let type_suffix = match kernel_data_type {
+ DataType::F16 => "f16",
+ DataType::BF16 => "bf16",
+ other => return Err(QuantizedMatmulError::UnsupportedDataType(other)),
+ };
+
+ let kernel_name = match (type_suffix, group_size, transpose) {
+ ("f16", 64, false) => "qmm_f16_g64_b4".to_string(),
+ ("f16", 64, true) => "qmm_transposed_f16_g64_b4".to_string(),
+ ("f16", 128, false) => "qmm_f16_g128_b4".to_string(),
+ ("f16", 128, true) => "qmm_transposed_f16_g128_b4".to_string(),
+ ("bf16", 64, false) => "qmm_bf16_g64_b4".to_string(),
+ ("bf16", 64, true) => "qmm_transposed_bf16_g64_b4".to_string(),
+ ("bf16", 128, false) => "qmm_bf16_g128_b4".to_string(),
+ ("bf16", 128, true) => "qmm_transposed_bf16_g128_b4".to_string(),
+ _ => {
+ return Err(QuantizedMatmulError::UnsupportedGroupSize(group_size));
+ },
+ };
+
+ let kernel = QuantizedMatmulKernel::new(
+ mtl_context,
+ kernel_data_type,
+ &kernel_name,
+ )?;
+ kernel.encode(encoder, args)
+}
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal
new file mode 100644
index 000000000..6c9270588
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal
@@ -0,0 +1,1179 @@
+#include <metal_simdgroup>
+#include <metal_stdlib>
+
+using namespace metal;
+
+#include "mma.h"
+
+
+template <int bits, int wsize = 8>
+inline constexpr short get_pack_factor() {
+ return (bits == 3 || bits == 5) ? 8 : (bits == 6 ? 4 : wsize / bits);
+}
+
+template <int bits, int wsize = 8>
+inline constexpr short get_bytes_per_pack() {
+ constexpr int power_of_2_bits = (bits & (bits - 1)) == 0;
+ return power_of_2_bits ? (wsize / 8) : (bits == 5 ? 5 : 3);
+}
+
+template <typename T, typename U, int values_per_thread, int bits>
+inline U load_vector(const device T* x, thread U* x_thread) {
+ static_assert(bits == 4, "Only int4 supported");
+
+ U sum = 0;
+ for (int i = 0; i < values_per_thread; i += 4) {
+ sum += x[i] + x[i + 1] + x[i + 2] + x[i + 3];
+ x_thread[i] = x[i];
+ x_thread[i + 1] = x[i + 1] / 16.0f;
+ x_thread[i + 2] = x[i + 2] / 256.0f;
+ x_thread[i + 3] = x[i + 3] / 4096.0f;
+ }
+ return sum;
+}
+
+template <typename T, typename U, int values_per_thread, int bits>
+inline U load_vector_safe(const device T* x, thread U* x_thread, int N) {
+ static_assert(bits == 4, "Only int4 supported");
+
+ const U scale_lut[4] = {
+ static_cast<U>(1.0f),
+ static_cast<U>(1.0f / 16.0f),
+ static_cast<U>(1.0f / 256.0f),
+ static_cast<U>(1.0f / 4096.0f)
+ };
+
+ U sum = 0;
+ for (int i = 0; i < values_per_thread; ++i) {
+ x_thread[i] = 0;
+ }
+ for (int i = 0; i < N; ++i) {
+ U v = x[i];
+ sum += v;
+ x_thread[i] = v * scale_lut[i & 3];
+ }
+ return sum;
+}
+
+template <typename U, int values_per_thread, int bits>
+inline U qdot(
+ const device uint8_t* w,
+ const thread U* x_thread,
+ U scale,
+ U bias,
+ U sum) {
+ static_assert(bits == 4, "Only int4 supported");
+
+ U accum = 0;
+ const device uint16_t* ws = (const device uint16_t*)w;
+ for (int i = 0; i < (values_per_thread / 4); i++) {
+ accum +=
+ (x_thread[4 * i] * (ws[i] & 0x000f) +
+ x_thread[4 * i + 1] * (ws[i] & 0x00f0) +
+ x_thread[4 * i + 2] * (ws[i] & 0x0f00) +
+ x_thread[4 * i + 3] * (ws[i] & 0xf000));
+ }
+ return scale * accum + sum * bias;
+}
+
+template <typename U, int values_per_thread, int bits>
+inline U qdot_safe(
+ const device uint8_t* w,
+ const thread U* x_thread,
+ U scale,
+ U bias,
+ U sum,
+ int N) {
+ static_assert(bits == 4, "Only int4 supported");
+
+ U accum = 0;
+ const device uint16_t* ws = (const device uint16_t*)w;
+
+ int full = N / 4;
+ for (int i = 0; i < full; i++) {
+ accum +=
+ (x_thread[4 * i] * (ws[i] & 0x000f) +
+ x_thread[4 * i + 1] * (ws[i] & 0x00f0) +
+ x_thread[4 * i + 2] * (ws[i] & 0x0f00) +
+ x_thread[4 * i + 3] * (ws[i] & 0xf000));
+ }
+
+ int rem = N & 3;
+ if (rem > 0) {
+ uint16_t wv = ws[full];
+ int base = 4 * full;
+ if (rem > 0) accum += x_thread[base] * (wv & 0x000f);
+ if (rem > 1) accum += x_thread[base + 1] * (wv & 0x00f0);
+ if (rem > 2) accum += x_thread[base + 2] * (wv & 0x0f00);
+ }
+
+ return scale * accum + sum * bias;
+}
+
+template <typename U, int N, int bits>
+inline void dequantize(const device uint8_t* w, U scale, U bias, threadgroup U* w_local) {
+ static_assert(bits == 4, "Only int4 supported");
+
+ U s[2] = {scale, scale / static_cast<U>(16.0f)};
+ for (int i = 0; i < (N / 2); i++) {
+ w_local[2 * i] = s[0] * (w[i] & 0x0f) + bias;
+ w_local[2 * i + 1] = s[1] * (w[i] & 0xf0) + bias;
+ }
+}
+
+template <
+ typename T,
+ short BROWS,
+ short BCOLS,
+ short dst_ld,
+ short reduction_dim,
+ short tgp_size,
+ short group_size,
+ short bits>
+struct QuantizedBlockLoader {
+ static_assert(BCOLS <= group_size, "Group size should be larger than columns");
+ static_assert(group_size % BCOLS == 0, "Group size should be divisible by columns");
+ static_assert(bits == 4, "Only int4 supported");
+
+ UZU_MTL_CONST short pack_factor = get_pack_factor<bits, 8>();
+ UZU_MTL_CONST short bytes_per_pack = get_bytes_per_pack<bits>();
+ UZU_MTL_CONST short BCOLS_PACKED = BCOLS / pack_factor;
+ UZU_MTL_CONST short n_reads = (BCOLS_PACKED * BROWS < tgp_size) ? 1 : (BCOLS_PACKED * BROWS) / tgp_size;
+ UZU_MTL_CONST short group_steps = group_size / BCOLS;
+
+ const int src_ld;
+ const int tile_stride;
+ short group_step_cnt;
+ int k_base;
+ const int group_stride;
+
+ const short thread_idx;
+ const short bi;
+ const short bj;
+
+ threadgroup T* dst;
+ const device uint8_t* src;
+ const device T* scales;
+ const device T* biases;
+
+ QuantizedBlockLoader(
+ const device uint8_t* src_,
+ const device T* scales_,
+ const device T* biases_,
+ const int src_ld_,
+ threadgroup T* dst_,
+ ushort simd_group_id [[simdgroup_index_in_threadgroup]],
+ ushort simd_lane_id [[thread_index_in_simdgroup]])
+ : src_ld(src_ld_),
+ tile_stride(
+ reduction_dim ? BCOLS_PACKED * bytes_per_pack
+ : BROWS * src_ld * bytes_per_pack / pack_factor),
+ group_step_cnt(0),
+ group_stride(BROWS * src_ld / group_size),
+ thread_idx(simd_group_id * 32 + simd_lane_id),
+ bi(n_reads * thread_idx / BCOLS_PACKED),
+ bj((n_reads * thread_idx) % BCOLS_PACKED),
+ dst(dst_ + bi * dst_ld + bj * pack_factor),
+ src(src_ + bi * src_ld * bytes_per_pack / pack_factor + bj * bytes_per_pack),
+ scales(scales_ + bi * src_ld / group_size),
+ biases(biases_ + bi * src_ld / group_size) {}
+
+ void load_unsafe() const {
+ if (BCOLS_PACKED * BROWS < tgp_size && bi >= BROWS) {
+ return;
+ }
+
+ T scale = *scales;
+ T bias = *biases;
+ for (int i = 0; i < n_reads; i++) {
+ dequantize<T, pack_factor, bits>(
+ src + i * bytes_per_pack, scale, bias, dst + i * pack_factor);
+ }
+ }
+
+ void load_safe(short2 src_tile_dim) const {
+ if (BCOLS_PACKED * BROWS < tgp_size && bi >= BROWS) {
+ return;
+ }
+
+ if (reduction_dim == 1) {
+ // N-tail: zero out rows beyond valid outputs
+ if (bi >= src_tile_dim.x) {
+ for (int i = 0; i < n_reads * pack_factor; i++) {
+ dst[i] = T(0);
+ }
+ return;
+ }
+
+ // K-tail: strictly limit to valid packs and zero trailing nibbles
+ int valid_cols = src_tile_dim.y; // 0..BK
+ int valid_packs = (valid_cols + pack_factor - 1) / pack_factor;
+
+ T scale;
+ T bias;
+ current_scale_bias(scale, bias);
+ for (int i = 0; i < n_reads; i++) {
+ int pack_idx = bj + i; // global pack index across the BK packs
+ if (pack_idx < valid_packs) {
+ dequantize<T, pack_factor, bits>(
+ src + i * bytes_per_pack, scale, bias, dst + i * pack_factor);
+ if (pack_idx == valid_packs - 1) {
+ int rem = valid_cols - (valid_packs - 1) * pack_factor;
+ if (rem > 0 && rem < pack_factor) {
+ for (int r = rem; r < pack_factor; ++r) {
+ dst[i * pack_factor + r] = T(0);
+ }
+ }
+ }
+ } else {
+ for (int j = 0; j < pack_factor; ++j) {
+ dst[i * pack_factor + j] = T(0);
+ }
+ }
+ }
+ return;
+ }
+
+ if (reduction_dim == 0 && bi >= src_tile_dim.y) {
+ for (int i = 0; i < n_reads * pack_factor; i++) {
+ dst[i] = T(0);
+ }
+ return;
+ }
+
+ T scale = *scales;
+ T bias = *biases;
+ for (int i = 0; i < n_reads; i++) {
+ dequantize<T, pack_factor, bits>(
+ src + i * bytes_per_pack, scale, bias, dst + i * pack_factor);
+ }
+ }
+
+ void next() {
+ src += tile_stride;
+ if (reduction_dim == 1) {
+ if (group_steps > 1) {
+ group_step_cnt++;
+ if (group_step_cnt == group_steps) {
+ group_step_cnt = 0;
+ scales++;
+ biases++;
+ }
+ } else {
+ scales++;
+ biases++;
+ }
+ } else {
+ scales += group_stride;
+ biases += group_stride;
+ }
+ }
+};
+
+// Loader that computes bias from packed zero-points (U8) and scales on-GPU
+template <
+ typename T,
+ short BROWS,
+ short BCOLS,
+ short dst_ld,
+ short reduction_dim,
+ short tgp_size,
+ short group_size,
+ short bits>
+struct QuantizedBlockLoaderZp {
+ static_assert(BCOLS <= group_size, "Group size should be larger than columns");
+ static_assert(group_size % BCOLS == 0, "Group size should be divisible by columns");
+ static_assert(bits == 4, "Only int4 supported");
+
+ UZU_MTL_CONST short pack_factor = get_pack_factor<bits, 8>();
+ UZU_MTL_CONST short bytes_per_pack = get_bytes_per_pack<bits>();
+ UZU_MTL_CONST short BCOLS_PACKED = BCOLS / pack_factor;
+ UZU_MTL_CONST short n_reads = (BCOLS_PACKED * BROWS < tgp_size) ? 1 : (BCOLS_PACKED * BROWS) / tgp_size;
+ UZU_MTL_CONST short group_steps = group_size / BCOLS;
+
+ const int src_ld;
+ const int groups_per_row;
+ const int tile_stride;
+ short group_step_cnt;
+ int k_base;
+ const int group_stride;
+
+ const short thread_idx;
+ const short bi;
+ const short bj;
+
+ threadgroup T* dst;
+ const device uint8_t* src;
+ const device T* scales;
+ const device T* scales_row_start;
+ const device uint8_t* zps_row_start;
+
+ QuantizedBlockLoaderZp(
+ const device uint8_t* src_,
+ const device T* scales_,
+ const device uint8_t* zero_points_row_start_,
+ const int src_ld_,
+ const int groups_per_row_,
+ threadgroup T* dst_,
+ ushort simd_group_id [[simdgroup_index_in_threadgroup]],
+ ushort simd_lane_id [[thread_index_in_simdgroup]])
+ : src_ld(src_ld_),
+ groups_per_row(groups_per_row_),
+ tile_stride(
+ reduction_dim ? BCOLS_PACKED * bytes_per_pack
+ : BROWS * src_ld * bytes_per_pack / pack_factor),
+ group_step_cnt(0),
+ k_base(0),
+ group_stride(BROWS * groups_per_row_),
+ thread_idx(simd_group_id * 32 + simd_lane_id),
+ bi(n_reads * thread_idx / BCOLS_PACKED),
+ bj((n_reads * thread_idx) % BCOLS_PACKED),
+ dst(dst_ + bi * dst_ld + bj * pack_factor),
+ src(src_ + bi * src_ld * bytes_per_pack / pack_factor + bj * bytes_per_pack),
+ scales(reduction_dim == 1 ? (scales_ + bi * groups_per_row_) : scales_),
+ scales_row_start(reduction_dim == 1 ? (scales_ + bi * groups_per_row_) : scales_),
+ zps_row_start(reduction_dim == 1 ? (zero_points_row_start_ + bi * ((groups_per_row_ + 1) / 2))
+ : zero_points_row_start_) {}
+
+ inline void current_scale_bias(thread T& out_scale, thread T& out_bias) const {
+ int g = reduction_dim == 0 ? (k_base / group_size)
+ : (int)(scales - scales_row_start);
+ const device uint8_t* zp_ptr = zps_row_start + (g >> 1);
+ uint8_t zp_b = *zp_ptr;
+ uint zp_n = (g & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F);
+ T scale_val = reduction_dim == 0 ? scales_row_start[g] : *scales;
+ out_scale = scale_val;
+ out_bias = static_cast<T>(-scale_val * static_cast<T>(zp_n));
+ }
+
+ void load_unsafe() const {
+ if (BCOLS_PACKED * BROWS < tgp_size && bi >= BROWS) {
+ return;
+ }
+
+ T scale;
+ T bias;
+ current_scale_bias(scale, bias);
+ for (int i = 0; i < n_reads; i++) {
+ dequantize<T, pack_factor, bits>(
+ src + i * bytes_per_pack, scale, bias, dst + i * pack_factor);
+ }
+ }
+
+ void load_safe(short2 src_tile_dim) const {
+ if (BCOLS_PACKED * BROWS < tgp_size && bi >= BROWS) {
+ return;
+ }
+
+ if (reduction_dim == 1 && bi >= src_tile_dim.x) {
+ for (int i = 0; i < n_reads * pack_factor; i++) {
+ dst[i] = T(0);
+ }
+ return;
+ }
+
+ if (reduction_dim == 0 && bi >= src_tile_dim.y) {
+ for (int i = 0; i < n_reads * pack_factor; i++) {
+ dst[i] = T(0);
+ }
+ return;
+ }
+
+ T scale;
+ T bias;
+ current_scale_bias(scale, bias);
+ for (int i = 0; i < n_reads; i++) {
+ dequantize<T, pack_factor, bits>(
+ src + i * bytes_per_pack, scale, bias, dst + i * pack_factor);
+ }
+ }
+
+ void next() {
+ src += tile_stride;
+ if (reduction_dim == 1) {
+ if (group_steps > 1) {
+ group_step_cnt++;
+ if (group_step_cnt == group_steps) {
+ group_step_cnt = 0;
+ scales++;
+ }
+ } else {
+ scales++;
+ }
+ } else {
+ k_base += BROWS;
+ }
+ }
+};
+
+template <
+ typename T,
+ const int group_size,
+ const int bits,
+ const int BM = 32,
+ const int BK = 32,
+ const int BN = 32>
+void qmm_impl(
+ const device uint32_t* w,
+ const device T* scales,
+ const device uint8_t* zero_points,
+ const device T* x,
+ device T* y,
+ threadgroup T* Xs,
+ threadgroup T* Ws,
+ const constant int& K,
+ const constant int& N,
+ const constant int& M,
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint lid [[thread_index_in_threadgroup]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]) {
+ static_assert(BK >= 32, "BK should be larger than SIMD_SIZE");
+ static_assert(BK % 32 == 0, "BK should be divisible by SIMD_SIZE");
+
+ (void)lid;
+
+ constexpr int WM = 2;
+ constexpr int WN = 2;
+ constexpr int pack_factor = get_pack_factor<bits, 8>();
+ constexpr int bytes_per_pack = get_bytes_per_pack<bits>();
+
+ constexpr int BK_padded = (BK + 16 / sizeof(T));
+ constexpr int BN_padded = (BN + 16 / sizeof(T));
+
+ using mma_t = matmul_utils::BlockMMA<T, T, BM, BN, BK, WM, WN, false, false, BK_padded, BN_padded>;
+ using loader_x_t = matmul_utils::BlockLoader<T, BM, BK, BK_padded, 1, WM * WN * 32, 1, 4>;
+ using loader_w_t = QuantizedBlockLoaderZp<
+ T,
+ BK,
+ BN,
+ BN_padded,
+ 0,
+ WM * WN * 32,
+ group_size,
+ bits>;
+
+ auto wl = (const device uint8_t*)w;
+
+ const int y_row = tid.y * BM;
+ const int y_col = tid.x * BN;
+ x += y_row * static_cast<int64_t>(K);
+ wl += y_col * bytes_per_pack / pack_factor;
+ const int groups_per_row = (K + group_size - 1) / group_size;
+ scales += y_col * groups_per_row;
+ const device uint8_t* zps_row_start = zero_points + y_col * ((groups_per_row + 1) / 2);
+ y += y_row * static_cast<int64_t>(N) + y_col;
+
+ const short num_els = min(BM, M - y_row);
+ const short num_outs = min(BN, N - y_col);
+ loader_x_t loader_x(x, K, Xs, simd_gid, simd_lid);
+ loader_w_t loader_w(wl, scales, zps_row_start, N, groups_per_row, Ws, simd_gid, simd_lid);
+ mma_t mma_op(simd_gid, simd_lid);
+
+ if (num_els < BM) {
+ if ((K % BK) != 0) {
+ const int k_blocks = K / BK;
+ for (int k = 0; k < k_blocks; k++) {
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ loader_x.load_safe(short2(BK, num_els));
+ loader_w.load_unsafe();
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ mma_op.mma(Xs, Ws);
+ loader_x.next();
+ loader_w.next();
+ }
+ const short num_k = K - k_blocks * BK;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ loader_x.load_safe(short2(num_k, num_els));
+ loader_w.load_safe(short2(BN, num_k));
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ mma_op.mma(Xs, Ws);
+ } else {
+ for (int k = 0; k < K; k += BK) {
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ loader_x.load_safe(short2(BK, num_els));
+ loader_w.load_unsafe();
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ mma_op.mma(Xs, Ws);
+ loader_x.next();
+ loader_w.next();
+ }
+ }
+ } else {
+ if ((K % BK) != 0) {
+ const int k_blocks = K / BK;
+ for (int k = 0; k < k_blocks; k++) {
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ loader_x.load_unsafe();
+ loader_w.load_unsafe();
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ mma_op.mma(Xs, Ws);
+ loader_x.next();
+ loader_w.next();
+ }
+ const short num_k = K - k_blocks * BK;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ loader_x.load_safe(short2(num_k, BM));
+ loader_w.load_safe(short2(BN, num_k));
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ mma_op.mma(Xs, Ws);
+ } else {
+ for (int k = 0; k < K; k += BK) {
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ loader_x.load_unsafe();
+ loader_w.load_unsafe();
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ mma_op.mma(Xs, Ws);
+ loader_x.next();
+ loader_w.next();
+ }
+ }
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (num_els < BM || num_outs < BN) {
+ mma_op.store_result_safe(y, N, short2(num_outs, num_els));
+ } else {
+ mma_op.store_result(y, N);
+ }
+}
+
+template <
+ typename T,
+ const int group_size,
+ const int bits,
+ const bool aligned_N,
+ const int BM = 32,
+ const int BK = 32,
+ const int BN = 32>
+void qmm_transposed_impl(
+ const device uint32_t* w,
+ const device T* scales,
+ const device uint8_t* zero_points,
+ const device T* x,
+ device T* y,
+ threadgroup T* Xs,
+ threadgroup T* Ws,
+ const constant int& K,
+ const constant int& N,
+ const constant int& M,
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint lid [[thread_index_in_threadgroup]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]) {
+ static_assert(BK >= 32, "BK should be larger than SIMD_SIZE");
+ static_assert(BK % 32 == 0, "BK should be divisible by SIMD_SIZE");
+
+ (void)lid;
+
+ constexpr int WM = 2;
+ constexpr int WN = 2;
+ constexpr int pack_factor = get_pack_factor<bits, 8>();
+ constexpr int bytes_per_pack = get_bytes_per_pack<bits>();
+
+ constexpr int BK_padded = (BK + 16 / sizeof(T));
+
+ using mma_t = matmul_utils::BlockMMA<T, T, BM, BN, BK, WM, WN, false, true, BK_padded, BK_padded>;
+ using loader_x_t = matmul_utils::BlockLoader<T, BM, BK, BK_padded, 1, WM * WN * 32>;
+ using loader_w_t = QuantizedBlockLoaderZp<
+ T,
+ BN,
+ BK,
+ BK_padded,
+ 1,
+ WM * WN * 32,
+ group_size,
+ bits>;
+
+ const int K_w = K * bytes_per_pack / pack_factor;
+ const int K_g = (K + group_size - 1) / group_size;
+ const int y_row = tid.y * BM;
+ const int y_col = tid.x * BN;
+
+ auto wl = (const device uint8_t*)w;
+
+ x += y_row * static_cast<int64_t>(K);
+ wl += y_col * K_w;
+ scales += y_col * K_g;
+ const device uint8_t* zps_row_start = zero_points + y_col * ((K_g + 1) / 2);
+ y += y_row * static_cast<int64_t>(N) + y_col;
+
+ const short num_els = min(BM, M - y_row);
+ const short num_outs = min(BN, N - y_col);
+ loader_x_t loader_x(x, K, Xs, simd_gid, simd_lid);
+ loader_w_t loader_w(wl, scales, zps_row_start, K, K_g, Ws, simd_gid, simd_lid);
+ mma_t mma_op(simd_gid, simd_lid);
+
+ const int k_blocks = (K + BK - 1) / BK;
+ for (int kb = 0; kb < k_blocks; kb++) {
+ const short k_len = (kb < (K / BK)) ? BK : static_cast<short>(K - (K / BK) * BK);
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (num_els < BM || k_len < BK) {
+ loader_x.load_safe(short2(k_len, num_els));
+ } else {
+ loader_x.load_unsafe();
+ }
+ loader_w.load_safe(short2(num_outs, k_len));
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ mma_op.mma(Xs, Ws);
+ if (kb + 1 < k_blocks) {
+ loader_x.next();
+ loader_w.next();
+ }
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (num_els < BM || num_outs < BN) {
+ mma_op.store_result_safe(y, N, short2(num_outs, num_els));
+ } else {
+ mma_op.store_result(y, N);
+ }
+}
+
+template <typename T, int group_size, int bits>
+void qmv_impl(
+ const device uint32_t* w,
+ const device T* scales,
+ const device uint8_t* zero_points,
+ const device T* x,
+ device T* y,
+ const constant int& K, // in_vec_size
+ const constant int& N, // out_vec_size
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]) {
+ constexpr int num_simdgroups = 2;
+ constexpr int results_per_simdgroup = 4;
+ constexpr int packs_per_thread = 1;
+ constexpr int pack_factor = get_pack_factor<bits, 32>();
+ constexpr int bytes_per_pack = get_bytes_per_pack<bits, 32>();
+
+ constexpr int values_per_thread = pack_factor * packs_per_thread;
+ constexpr int block_size = values_per_thread * 32;
+ constexpr int scale_step_per_thread = group_size / values_per_thread;
+
+ const device uint8_t* ws = (const device uint8_t*)w;
+ typedef float U;
+ thread U x_thread[values_per_thread];
+ thread U result[results_per_simdgroup] = {0};
+
+ const int in_vec_size_w = K * bytes_per_pack / pack_factor;
+ const int in_vec_size_g = (K + group_size - 1) / group_size; // ceil(K / group_size)
+ const int out_row = tid.y * (num_simdgroups * results_per_simdgroup) + simd_gid * results_per_simdgroup;
+ const int used_out_row = min(N - results_per_simdgroup, out_row);
+
+ if (out_row >= N) {
+ return;
+ }
+
+ if (N < (num_simdgroups * results_per_simdgroup)) {
+ ws += out_row * in_vec_size_w + simd_lid * packs_per_thread * bytes_per_pack;
+ scales += out_row * in_vec_size_g + simd_lid / scale_step_per_thread;
+ const device uint8_t* zps_row_base = zero_points + out_row * ((in_vec_size_g + 1) / 2);
+ x += tid.x * K + simd_lid * values_per_thread;
+ y += tid.x * N + out_row;
+
+ int k = 0;
+ for (; k < K - block_size; k += block_size) {
+ U sum = load_vector<T, U, values_per_thread, bits>(x, x_thread);
+
+ for (int row = 0; out_row + row < N; row++) {
+ if (row >= results_per_simdgroup) break;
+ auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);
+ const device T* sl = scales + row * in_vec_size_g;
+ const device uint8_t* zl = zps_row_base + row * ((in_vec_size_g + 1) / 2);
+
+ int g = (k + simd_lid * values_per_thread) / group_size;
+ U s = sl[0];
+ uint8_t zp_b = zl[g >> 1];
+ U b = static_cast<U>(-s * static_cast<U>((g & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F)));
+ result[row] += qdot<U, values_per_thread, bits>(wl, x_thread, s, b, sum);
+ }
+
+ ws += block_size * bytes_per_pack / pack_factor;
+ scales += block_size / group_size;
+ x += block_size;
+ }
+ const int remaining = clamp(
+ static_cast<int>(K - k - simd_lid * values_per_thread),
+ 0,
+ values_per_thread);
+ if (remaining > 0) {
+ U sum = load_vector_safe<T, U, values_per_thread, bits>(
+ x, x_thread, remaining);
+
+ for (int row = 0; out_row + row < N; row++) {
+ if (row >= results_per_simdgroup) break;
+ auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);
+ const device T* sl = scales + row * in_vec_size_g;
+ const device uint8_t* zl = zps_row_base + row * ((in_vec_size_g + 1) / 2);
+
+ int g = (k + simd_lid * values_per_thread) / group_size;
+ U s = sl[0];
+ uint8_t zp_b = zl[g >> 1];
+ U b = static_cast<U>(-s * static_cast<U>((g & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F)));
+ result[row] +=
+ qdot<U, values_per_thread, bits>(wl, x_thread, s, b, sum);
+ }
+ }
+
+ for (int row = 0; out_row + row < N; row++) {
+ if (row >= results_per_simdgroup) break;
+ result[row] = simd_sum(result[row]);
+ if (simd_lid == 0) {
+ y[row] = static_cast<T>(result[row]);
+ }
+ }
+ }
+ else {
+ ws += used_out_row * in_vec_size_w + simd_lid * packs_per_thread * bytes_per_pack;
+ scales += used_out_row * in_vec_size_g + simd_lid / scale_step_per_thread;
+ const device uint8_t* zps_row_base = zero_points + used_out_row * ((in_vec_size_g + 1) / 2);
+ x += tid.x * K + simd_lid * values_per_thread;
+ y += tid.x * N + used_out_row;
+
+ int k = 0;
+ for (; k < K - block_size; k += block_size) {
+ U sum = load_vector<T, U, values_per_thread, bits>(x, x_thread);
+
+ for (int row = 0; row < results_per_simdgroup; row++) {
+ auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);
+ const device T* sl = scales + row * in_vec_size_g;
+ const device uint8_t* zl = zps_row_base + row * ((in_vec_size_g + 1) / 2);
+
+ int g = (k + simd_lid * values_per_thread) / group_size;
+ U s = sl[0];
+ uint8_t zp_b = zl[g >> 1];
+ U b = static_cast<U>(-s * static_cast<U>((g & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F)));
+ result[row] += qdot<U, values_per_thread, bits>(wl, x_thread, s, b, sum);
+ }
+
+ ws += block_size * bytes_per_pack / pack_factor;
+ scales += block_size / group_size;
+ x += block_size;
+ }
+ const int remaining = clamp(
+ static_cast<int>(K - k - simd_lid * values_per_thread),
+ 0,
+ values_per_thread);
+
+ if (remaining > 0) {
+ U sum = load_vector_safe<T, U, values_per_thread, bits>(
+ x, x_thread, remaining);
+
+ for (int row = 0; row < results_per_simdgroup; row++) {
+ auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);
+ const device T* sl = scales + row * in_vec_size_g;
+ const device uint8_t* zl = zps_row_base + row * ((in_vec_size_g + 1) / 2);
+
+ int g = (k + simd_lid * values_per_thread) / group_size;
+ U s = sl[0];
+ uint8_t zp_b = zl[g >> 1];
+ U b = static_cast<U>(-s * static_cast<U>((g & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F)));
+ result[row] += qdot_safe<U, values_per_thread, bits>(
+ wl, x_thread, s, b, sum, remaining);
+ }
+ }
+
+ for (int row = 0; row < results_per_simdgroup; row++) {
+ result[row] = simd_sum(result[row]);
+ if (simd_lid == 0) {
+ y[row] = static_cast<T>(result[row]);
+ }
+ }
+ }
+}
+
+template <typename T, int group_size, int bits>
+void qvm_impl(
+ const device uint32_t* w,
+ const device T* scales,
+ const device uint8_t* zero_points,
+ const device T* x,
+ device T* y,
+ const constant int& K, // in_vec_size
+ const constant int& N, // out_vec_size
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]) {
+
+ typedef float U;
+ constexpr int BN = 64;
+ constexpr int BK = 32;
+ constexpr int WN = 2;
+ constexpr int pack_factor = get_pack_factor<bits, 32>();
+ constexpr int bytes_per_pack = get_bytes_per_pack<bits, 32>();
+
+ const int lidy = simd_gid;
+ const int lidx = simd_lid;
+ const int bn = BN / WN;
+
+ const int out_col = tid.y * BN + lidy * bn;
+ const int in_row = tid.x;
+
+ if (out_col >= N) {
+ return;
+ }
+
+ const device uint8_t* w_row = ((const device uint8_t*)w) + out_col * K * bytes_per_pack / pack_factor;
+ const int in_vec_size_g = (K + group_size - 1) / group_size;
+ const device T* scales_row = scales + out_col * in_vec_size_g;
+ const device uint8_t* zps_row = zero_points + out_col * ((in_vec_size_g + 1) / 2);
+ const device T* x_row = x + in_row * K;
+ device T* y_row = y + in_row * N + out_col;
+
+ U result[BN / WN] = {0};
+ thread U x_thread[pack_factor];
+
+ for (int k = lidx * pack_factor; k < K; k += BK * pack_factor) {
+ const int k_pack = k / pack_factor;
+ const int remaining = min(pack_factor, K - k);
+
+ U x_sum = load_vector_safe<T, U, pack_factor, bits>(
+ x_row + k, x_thread, remaining);
+
+ for (int n = 0; n < min(bn, N - out_col); n++) {
+ const int w_offset = n * K * bytes_per_pack / pack_factor + k_pack * bytes_per_pack;
+ const int scale_idx = n * ((K + group_size - 1) / group_size) + k / group_size;
+
+ U scale = static_cast<U>(scales_row[scale_idx]);
+ uint8_t zp_b = zps_row[k / group_size / 2];
+ U bias = static_cast<U>(-scale * static_cast<U>(((k / group_size) & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F)));
+
+ result[n] += qdot_safe<U, pack_factor, bits>(
+ w_row + w_offset, x_thread, scale, bias, x_sum, remaining);
+ }
+ }
+
+ for (int n = 0; n < min(bn, N - out_col); n++) {
+ result[n] = simd_sum(result[n]);
+ if (lidx == 0) {
+ y_row[n] = static_cast<T>(result[n]);
+ }
+ }
+}
+
+
+template <
+ typename T,
+ const int group_size,
+ const int bits,
+ const int BM = 32,
+ const int BK = 32,
+ const int BN = 32>
+[[kernel]] void qmm(
+ const device uint32_t* w [[buffer(0)]],
+ const device T* scales [[buffer(1)]],
+ const device uint8_t* zero_points [[buffer(2)]],
+ const device T* x [[buffer(3)]],
+ device T* y [[buffer(4)]],
+ const constant int& K [[buffer(5)]],
+ const constant int& N [[buffer(6)]],
+ const constant int& M [[buffer(7)]],
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint lid [[thread_index_in_threadgroup]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]) {
+
+ (void)lid;
+
+ constexpr int BK_padded = (BK + 16 / sizeof(T));
+ constexpr int BN_padded = (BN + 16 / sizeof(T));
+
+ threadgroup T Xs[BM * BK_padded];
+ threadgroup T Ws[BK * BN_padded];
+
+ qmm_impl<T, group_size, bits, BM, BK, BN>(
+ w, scales, zero_points, x, y, Xs, Ws, K, N, M, tid, lid, simd_gid, simd_lid);
+}
+
+template <
+ typename T,
+ const int group_size,
+ const int bits,
+ const bool aligned_N,
+ const int BM = 32,
+ const int BK = 32,
+ const int BN = 32>
+[[kernel]] void qmm_transposed(
+ const device uint32_t* w [[buffer(0)]],
+ const device T* scales [[buffer(1)]],
+ const device uint8_t* zero_points [[buffer(2)]],
+ const device T* x [[buffer(3)]],
+ device T* y [[buffer(4)]],
+ const constant int& K [[buffer(5)]],
+ const constant int& N [[buffer(6)]],
+ const constant int& M [[buffer(7)]],
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint lid [[thread_index_in_threadgroup]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]) {
+
+ (void)lid;
+
+ constexpr int BK_padded = (BK + 16 / sizeof(T));
+
+ threadgroup T Xs[BM * BK_padded];
+ threadgroup T Ws[BN * BK_padded];
+
+ qmm_transposed_impl<T, group_size, bits, aligned_N, BM, BK, BN>(
+ w, scales, zero_points, x, y, Xs, Ws, K, N, M, tid, lid, simd_gid, simd_lid);
+}
+
+template <typename T, int group_size, int bits>
+[[kernel]] void qmv(
+ const device uint32_t* w [[buffer(0)]],
+ const device T* scales [[buffer(1)]],
+ const device uint8_t* zero_points [[buffer(2)]],
+ const device T* x [[buffer(3)]],
+ device T* y [[buffer(4)]],
+ const constant int& K [[buffer(5)]],
+ const constant int& N [[buffer(6)]],
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]) {
+
+ qmv_impl<T, group_size, bits>(w, scales, zero_points, x, y, K, N, tid, simd_gid, simd_lid);
+}
+
+template <typename T, int group_size, int bits>
+[[kernel]] void qvm(
+ const device uint32_t* w [[buffer(0)]],
+ const device T* scales [[buffer(1)]],
+ const device uint8_t* zero_points [[buffer(2)]],
+ const device T* x [[buffer(3)]],
+ device T* y [[buffer(4)]],
+ const constant int& K [[buffer(5)]],
+ const constant int& N [[buffer(6)]],
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]) {
+
+ qvm_impl<T, group_size, bits>(w, scales, zero_points, x, y, K, N, tid, simd_gid, simd_lid);
+}
+
+// Group size 64 (F16)
+template [[host_name("qmm_f16_g64_b4")]]
+[[kernel]] void qmm<half, 64, 4>(
+ const device uint32_t* w [[buffer(0)]],
+ const device half* scales [[buffer(1)]],
+ const device uint8_t* zero_points [[buffer(2)]],
+ const device half* x [[buffer(3)]],
+ device half* y [[buffer(4)]],
+ const constant int& K [[buffer(5)]],
+ const constant int& N [[buffer(6)]],
+ const constant int& M [[buffer(7)]],
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint lid [[thread_index_in_threadgroup]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]);
+
+template [[host_name("qmm_transposed_f16_g64_b4")]]
+[[kernel]] void qmm_transposed<half, 64, 4, true>(
+ const device uint32_t* w [[buffer(0)]],
+ const device half* scales [[buffer(1)]],
+ const device uint8_t* zero_points [[buffer(2)]],
+ const device half* x [[buffer(3)]],
+ device half* y [[buffer(4)]],
+ const constant int& K [[buffer(5)]],
+ const constant int& N [[buffer(6)]],
+ const constant int& M [[buffer(7)]],
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint lid [[thread_index_in_threadgroup]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]);
+
+template [[host_name("qmv_f16_g64_b4")]]
+[[kernel]] void qmv<half, 64, 4>(
+ const device uint32_t* w [[buffer(0)]],
+ const device half* scales [[buffer(1)]],
+ const device uint8_t* zero_points [[buffer(2)]],
+ const device half* x [[buffer(3)]],
+ device half* y [[buffer(4)]],
+ const constant int& K [[buffer(5)]],
+ const constant int& N [[buffer(6)]],
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]);
+
+template [[host_name("qvm_f16_g64_b4")]]
+[[kernel]] void qvm<half, 64, 4>(
+ const device uint32_t* w [[buffer(0)]],
+ const device half* scales [[buffer(1)]],
+ const device uint8_t* zero_points [[buffer(2)]],
+ const device half* x [[buffer(3)]],
+ device half* y [[buffer(4)]],
+ const constant int& K [[buffer(5)]],
+ const constant int& N [[buffer(6)]],
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]);
+
+template [[host_name("qmm_f16_g128_b4")]]
+[[kernel]] void qmm<half, 128, 4>(
+ const device uint32_t* w [[buffer(0)]],
+ const device half* scales [[buffer(1)]],
+ const device uint8_t* zero_points [[buffer(2)]],
+ const device half* x [[buffer(3)]],
+ device half* y [[buffer(4)]],
+ const constant int& K [[buffer(5)]],
+ const constant int& N [[buffer(6)]],
+ const constant int& M [[buffer(7)]],
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint lid [[thread_index_in_threadgroup]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]);
+
+template [[host_name("qmm_transposed_f16_g128_b4")]]
+[[kernel]] void qmm_transposed<half, 128, 4, true>(
+ const device uint32_t* w [[buffer(0)]],
+ const device half* scales [[buffer(1)]],
+ const device uint8_t* zero_points [[buffer(2)]],
+ const device half* x [[buffer(3)]],
+ device half* y [[buffer(4)]],
+ const constant int& K [[buffer(5)]],
+ const constant int& N [[buffer(6)]],
+ const constant int& M [[buffer(7)]],
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint lid [[thread_index_in_threadgroup]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]);
+
+template [[host_name("qmv_f16_g128_b4")]]
+[[kernel]] void qmv<half, 128, 4>(
+ const device uint32_t* w [[buffer(0)]],
+ const device half* scales [[buffer(1)]],
+ const device uint8_t* zero_points [[buffer(2)]],
+ const device half* x [[buffer(3)]],
+ device half* y [[buffer(4)]],
+ const constant int& K [[buffer(5)]],
+ const constant int& N [[buffer(6)]],
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]);
+
+template [[host_name("qvm_f16_g128_b4")]]
+[[kernel]] void qvm<half, 128, 4>(
+ const device uint32_t* w [[buffer(0)]],
+ const device half* scales [[buffer(1)]],
+ const device uint8_t* zero_points [[buffer(2)]],
+ const device half* x [[buffer(3)]],
+ device half* y [[buffer(4)]],
+ const constant int& K [[buffer(5)]],
+ const constant int& N [[buffer(6)]],
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]);
+
+template [[host_name("qmm_bf16_g64_b4")]]
+[[kernel]] void qmm<bfloat, 64, 4>(
+ const device uint32_t* w [[buffer(0)]],
+ const device bfloat* scales [[buffer(1)]],
+ const device uint8_t* zero_points [[buffer(2)]],
+ const device bfloat* x [[buffer(3)]],
+ device bfloat* y [[buffer(4)]],
+ const constant int& K [[buffer(5)]],
+ const constant int& N [[buffer(6)]],
+ const constant int& M [[buffer(7)]],
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint lid [[thread_index_in_threadgroup]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]);
+
+template [[host_name("qmm_transposed_bf16_g64_b4")]]
+[[kernel]] void qmm_transposed<bfloat, 64, 4, true>(
+ const device uint32_t* w [[buffer(0)]],
+ const device bfloat* scales [[buffer(1)]],
+ const device uint8_t* zero_points [[buffer(2)]],
+ const device bfloat* x [[buffer(3)]],
+ device bfloat* y [[buffer(4)]],
+ const constant int& K [[buffer(5)]],
+ const constant int& N [[buffer(6)]],
+ const constant int& M [[buffer(7)]],
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint lid [[thread_index_in_threadgroup]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]);
+
+template [[host_name("qmv_bf16_g64_b4")]]
+[[kernel]] void qmv<bfloat, 64, 4>(
+ const device uint32_t* w [[buffer(0)]],
+ const device bfloat* scales [[buffer(1)]],
+ const device uint8_t* zero_points [[buffer(2)]],
+ const device bfloat* x [[buffer(3)]],
+ device bfloat* y [[buffer(4)]],
+ const constant int& K [[buffer(5)]],
+ const constant int& N [[buffer(6)]],
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]);
+
+template [[host_name("qvm_bf16_g64_b4")]]
+[[kernel]] void qvm<bfloat, 64, 4>(
+ const device uint32_t* w [[buffer(0)]],
+ const device bfloat* scales [[buffer(1)]],
+ const device uint8_t* zero_points [[buffer(2)]],
+ const device bfloat* x [[buffer(3)]],
+ device bfloat* y [[buffer(4)]],
+ const constant int& K [[buffer(5)]],
+ const constant int& N [[buffer(6)]],
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]);
+
+// Group size 128 (BF16)
+template [[host_name("qmm_bf16_g128_b4")]]
+[[kernel]] void qmm<bfloat, 128, 4>(
+ const device uint32_t* w [[buffer(0)]],
+ const device bfloat* scales [[buffer(1)]],
+ const device uint8_t* zero_points [[buffer(2)]],
+ const device bfloat* x [[buffer(3)]],
+ device bfloat* y [[buffer(4)]],
+ const constant int& K [[buffer(5)]],
+ const constant int& N [[buffer(6)]],
+ const constant int& M [[buffer(7)]],
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint lid [[thread_index_in_threadgroup]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]);
+
+template [[host_name("qmm_transposed_bf16_g128_b4")]]
+[[kernel]] void qmm_transposed<bfloat, 128, 4, true>(
+ const device uint32_t* w [[buffer(0)]],
+ const device bfloat* scales [[buffer(1)]],
+ const device uint8_t* zero_points [[buffer(2)]],
+ const device bfloat* x [[buffer(3)]],
+ device bfloat* y [[buffer(4)]],
+ const constant int& K [[buffer(5)]],
+ const constant int& N [[buffer(6)]],
+ const constant int& M [[buffer(7)]],
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint lid [[thread_index_in_threadgroup]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]);
+
+template [[host_name("qmv_bf16_g128_b4")]]
+[[kernel]] void qmv<bfloat, 128, 4>(
+ const device uint32_t* w [[buffer(0)]],
+ const device bfloat* scales [[buffer(1)]],
+ const device uint8_t* zero_points [[buffer(2)]],
+ const device bfloat* x [[buffer(3)]],
+ device bfloat* y [[buffer(4)]],
+ const constant int& K [[buffer(5)]],
+ const constant int& N [[buffer(6)]],
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]);
+
+template [[host_name("qvm_bf16_g128_b4")]]
+[[kernel]] void qvm<bfloat, 128, 4>(
+ const device uint32_t* w [[buffer(0)]],
+ const device bfloat* scales [[buffer(1)]],
+ const device uint8_t* zero_points [[buffer(2)]],
+ const device bfloat* x [[buffer(3)]],
+ device bfloat* y [[buffer(4)]],
+ const constant int& K [[buffer(5)]],
+ const constant int& N [[buffer(6)]],
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]);
\ No newline at end of file
diff --git a/crates/uzu/tests/quant_matmul_test.rs b/crates/uzu/tests/quant_matmul_test.rs
new file mode 100644
index 000000000..669bc489e
--- /dev/null
+++ b/crates/uzu/tests/quant_matmul_test.rs
@@ -0,0 +1,596 @@
+use std::time::Instant;
+
+use half::f16;
+use metal::{Device, MTLResourceOptions};
+use uzu::{
+ DataType,
+ backends::metal::{
+ MTLContext,
+ kernel::quant_matmul::{
+ QuantizedMatmulArguments, QuantizedMatmulKernel,
+ },
+ },
+};
+
+fn create_test_context() -> Option<MTLContext> {
+ let device = Device::system_default()?;
+ let command_queue = device.new_command_queue();
+ MTLContext::new(device, command_queue).ok()
+}
+
+// Packs 4-bit weights into bytes (4 values per 16-bit word, matching qdot expectations)
+fn pack_u4_weights(values: &[u8]) -> Vec<u8> {
+ let mut out = Vec::with_capacity((values.len() + 3) / 4 * 2); // 4 weights per 16-bit word
+ for chunk in values.chunks(4) {
+ let w0 = (*chunk.get(0).unwrap_or(&0) as u16) & 0x0F; // Bits 0-3
+ let w1 = ((*chunk.get(1).unwrap_or(&0) as u16) & 0x0F) << 4; // Bits 4-7
+ let w2 = ((*chunk.get(2).unwrap_or(&0) as u16) & 0x0F) << 8; // Bits 8-11
+ let w3 = ((*chunk.get(3).unwrap_or(&0) as u16) & 0x0F) << 12; // Bits 12-15
+
+ let word: u16 = w0 | w1 | w2 | w3;
+ out.push(word as u8); // Low byte
+ out.push((word >> 8) as u8); // High byte
+ }
+ out
+}
+
+fn cpu_reference(
+ m: usize,
+ n: usize,
+ k: usize,
+ a: &[f32], // A matrix (M x K)
+ b_quant: &[u8], // B matrix (N x K), quantized and packed, (N * K / 2) in size
+ scales: &[f32], // scales for B (N x ceil(K/group_size))
+ biases: &[f32], // biases for B (N x ceil(K/group_size))
+ transpose_b: bool, // Whether B matrix weights are stored in transposed layout
+) -> Vec<f32> {
+ let group_size = 64;
+ let num_groups = (k + group_size - 1) / group_size;
+ let mut y = vec![0.0f32; m * n];
+ for i in 0..m {
+ for j in 0..n {
+ let mut acc = 0.0f64;
+ for g in 0..num_groups {
+ let scale = scales[j * num_groups + g] as f64;
+ let bias = biases[j * num_groups + g] as f64;
+ let l_start = g * group_size;
+ let l_end = (l_start + group_size).min(k);
+ let mut group_acc = 0.0f64;
+ let mut group_sum = 0.0f64;
+ for l in l_start..l_end {
+ // Weight indexing based on kernel expectations:
+ // Non-transposed kernel expects: W[N][K] layout -> w[output_idx * k + input_idx]
+ // Transposed kernel expects: W[K][N] layout -> w[input_idx * n + output_idx]
+ let weight_linear_idx = if transpose_b {
+ l * n + j // W[K][N]: w[k_idx * n + n_idx]
+ } else {
+ j * k + l // W[N][K]: w[n_idx * k + k_idx]
+ };
+
+ let word_idx = weight_linear_idx / 4; // 4 weights per 16-bit word
+ let word_offset = weight_linear_idx % 4; // Which nibble in the word
+ let byte_idx = word_idx * 2; // 2 bytes per word
+
+ let word = if byte_idx + 1 < b_quant.len() {
+ b_quant[byte_idx] as u16
+ | ((b_quant[byte_idx + 1] as u16) << 8)
+ } else {
+ 0
+ };
+
+ let val_q = match word_offset {
+ 0 => word & 0x000F, // Bits 0-3
+ 1 => (word & 0x00F0) >> 4, // Bits 4-7
+ 2 => (word & 0x0F00) >> 8, // Bits 8-11
+ 3 => (word & 0xF000) >> 12, // Bits 12-15
+ _ => 0,
+ } as f64;
+ let val_a = a[i * k + l] as f64;
+ group_acc += val_a * val_q;
+ group_sum += val_a;
+ }
+ acc += scale * group_acc + bias * group_sum;
+ }
+ y[i * n + j] = acc as f32;
+ }
+ }
+ y
+}
+
+fn execute_quantized_matmul(
+ ctx: &MTLContext,
+ m: usize,
+ n: usize,
+ k: usize,
+ kernel_name: &str,
+ cpu_transpose: bool,
+ iterations: usize,
+ validate: bool,
+ randomize_zp: bool,
+) -> f64 {
+ // Prepare deterministic weights: w(row,k) = row+1
+ // Always create weights in the same layout (test_m × test_k)
+ let mut weights_q4: Vec<u8> = Vec::with_capacity(m * k);
+ for row in 0..m {
+ for _k in 0..k {
+ let v = (row as u8 + 1) & 0x0F;
+ weights_q4.push(v);
+ }
+ }
+ let weights_packed = pack_u4_weights(&weights_q4);
+
+ let num_groups = (k + 63) / 64;
+ let scales: Vec<f16> = vec![f16::from_f32(1.0); m * num_groups];
+ let mut biases: Vec<f16> = vec![f16::from_f32(0.0); m * num_groups];
+
+ // Prepare input data
+ let (x_f32, x_f16) = if n == 1 {
+ // GEMV case: single input vector
+ let x_f32: Vec<f32> = (1..=k).map(|i| i as f32 / k as f32).collect();
+ let x_f16: Vec<f16> = x_f32.iter().map(|&v| f16::from_f32(v)).collect();
+ (x_f32, x_f16)
+ } else {
+ // GEMM case: multiple input vectors (column-major)
+ let mut x_f32: Vec<f32> = Vec::with_capacity(k * n);
+ for _col in 0..n {
+ for i in 0..k {
+ x_f32.push((i + 1) as f32 / k as f32);
+ }
+ }
+ let x_f16: Vec<f16> = x_f32.iter().map(|&v| f16::from_f32(v)).collect();
+ (x_f32, x_f16)
+ };
+
+ // Create buffers
+ let w_buf = ctx.device.new_buffer_with_data(
+ weights_packed.as_ptr() as *const _,
+ (weights_packed.len() * std::mem::size_of::<u8>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+ let s_buf = ctx.device.new_buffer_with_data(
+ scales.as_ptr() as *const _,
+ (scales.len() * std::mem::size_of::<f16>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+ // Build zero-points buffer
+ let mut zps_bytes = vec![0u8; m * ((num_groups + 1) / 2)];
+ if randomize_zp {
+ // Randomized U8 zero-points per group and corresponding float biases.
+ // even g -> low nibble, odd g -> high nibble
+ for j in 0..m {
+ for g in 0..num_groups {
+ let zp_val: u8 = ((j + 3 * g) as u8) & 0x0F; // deterministic 0..15
+ let byte_index = j * ((num_groups + 1) / 2) + (g >> 1);
+ if (g & 1) == 0 {
+ zps_bytes[byte_index] =
+ (zps_bytes[byte_index] & 0xF0) | (zp_val & 0x0F);
+ } else {
+ zps_bytes[byte_index] =
+ (zps_bytes[byte_index] & 0x0F) | ((zp_val & 0x0F) << 4);
+ }
+ // bias = -scale * zp
+ let s = scales[j * num_groups + g].to_f32();
+ let b = -s * (zp_val as f32);
+ biases[j * num_groups + g] = f16::from_f32(b);
+ }
+ }
+ }
+ let b_buf = ctx.device.new_buffer_with_data(
+ zps_bytes.as_ptr() as *const _,
+ (zps_bytes.len() * std::mem::size_of::<u8>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+ let x_buf = ctx.device.new_buffer_with_data(
+ x_f16.as_ptr() as *const _,
+ (x_f16.len() * std::mem::size_of::<f16>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+ let y_buf = ctx.device.new_buffer(
+ (m * n * std::mem::size_of::<f16>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+
+ // Create kernel
+ let kernel =
+ QuantizedMatmulKernel::new(&ctx, DataType::F16, kernel_name).unwrap();
+
+ // Warmup (only for benchmarks)
+ if iterations > 1 {
+ for _ in 0..3 {
+ let args = QuantizedMatmulArguments {
+ a_buffer: &x_buf,
+ b_buffer: &w_buf,
+ scales_buffer: &s_buf,
+ zero_points_buffer: &b_buf,
+ output_buffer: &y_buf,
+ m: n as i32,
+ n: m as i32,
+ k: k as i32,
+ };
+ let cb_ref = ctx.command_queue.new_command_buffer();
+ let encoder = cb_ref.new_compute_command_encoder();
+ kernel.encode(encoder, args).unwrap();
+ encoder.end_encoding();
+ cb_ref.commit();
+ cb_ref.wait_until_completed();
+ }
+ }
+
+ // Execute and time
+ let start = Instant::now();
+ for _ in 0..iterations {
+ let args = QuantizedMatmulArguments {
+ a_buffer: &x_buf,
+ b_buffer: &w_buf,
+ scales_buffer: &s_buf,
+ zero_points_buffer: &b_buf,
+ output_buffer: &y_buf,
+ m: n as i32,
+ n: m as i32,
+ k: k as i32,
+ };
+ let cb_ref = ctx.command_queue.new_command_buffer();
+ let encoder = cb_ref.new_compute_command_encoder();
+ kernel.encode(encoder, args).unwrap();
+ encoder.end_encoding();
+ cb_ref.commit();
+ cb_ref.wait_until_completed();
+ }
+ let elapsed = start.elapsed();
+
+ // Validate if requested
+ if validate {
+ let gpu_m = n; // GPU sees test_n as M
+ let gpu_n = m; // GPU sees test_m as N
+ let gpu_k = k; // GPU sees test_k as K
+
+ let y_expected = cpu_reference(
+ gpu_m,
+ gpu_n,
+ gpu_k,
+ &x_f32,
+ &weights_packed,
+ &scales.iter().map(|&v| v.to_f32()).collect::<Vec<_>>(),
+ &biases.iter().map(|&v| v.to_f32()).collect::<Vec<_>>(),
+ cpu_transpose,
+ );
+
+ let y_ptr = y_buf.contents() as *const f16;
+ let y_out = unsafe { std::slice::from_raw_parts(y_ptr, m * n) };
+ let y_out_f32: Vec<f32> = y_out.iter().map(|&v| v.to_f32()).collect();
+
+ let tol = if randomize_zp {
+ 2.5
+ } else {
+ 1.0
+ };
+ let display_size = if n == 1 {
+ m
+ } else {
+ n
+ };
+ println!(
+ "first {} expected: {:?}",
+ display_size,
+ &y_expected[..display_size]
+ );
+ println!(
+ "first {} got: {:?}",
+ display_size,
+ &y_out_f32[..display_size]
+ );
+
+ for (i, (&exp, &got)) in
+ y_expected.iter().zip(y_out_f32.iter()).enumerate()
+ {
+ let diff = (exp - got).abs();
+ assert!(
+ diff <= tol,
+ "M={} N={} K={} idx {} diff {} exp {} got {}",
+ m,
+ n,
+ k,
+ i,
+ diff,
+ exp,
+ got
+ );
+ }
+ }
+
+ elapsed.as_secs_f64()
+}
+
+fn run_gemv_test(
+ ctx: &MTLContext,
+ m: usize,
+ k: usize,
+) {
+ println!("--- Testing GEMV M={}, K={} ---", m, k);
+ // GEMV: weights are in W[N][K] layout, CPU uses them as-is
+ execute_quantized_matmul(
+ ctx,
+ m,
+ 1,
+ k,
+ "qmv_f16_g64_b4",
+ false,
+ 1,
+ true,
+ true,
+ );
+ println!("✅ GEMV M={}, K={} passed", m, k);
+}
+
+fn run_qvm_test(
+ ctx: &MTLContext,
+ n: usize,
+ k: usize,
+) {
+ println!("--- Testing QVM N={}, K={} ---", n, k);
+ // QVM: weights are in W[N][K] layout, CPU uses them as-is
+ execute_quantized_matmul(
+ ctx,
+ 1,
+ n,
+ k,
+ "qvm_f16_g64_b4",
+ false,
+ 1,
+ true,
+ true,
+ );
+ println!("✅ QVM N={}, K={} passed", n, k);
+}
+
+#[test]
+fn test_quant_gemv() {
+ let ctx = match create_test_context() {
+ Some(c) => c,
+ None => {
+ println!("Metal not available — skipping GEMV test");
+ return;
+ },
+ };
+
+ run_gemv_test(&ctx, 3, 64);
+ run_gemv_test(&ctx, 1, 128);
+ run_gemv_test(&ctx, 7, 256);
+ run_gemv_test(&ctx, 13, 512);
+ run_gemv_test(&ctx, 13, 511);
+ run_gemv_test(&ctx, 8, 64);
+ run_gemv_test(&ctx, 9, 64);
+ run_gemv_test(&ctx, 25, 128);
+ run_gemv_test(&ctx, 31, 512);
+}
+
+#[test]
+fn test_quant_qevm() {
+ let ctx = match create_test_context() {
+ Some(c) => c,
+ None => {
+ println!("Metal not available — skipping QVM test");
+ return;
+ },
+ };
+
+ run_qvm_test(&ctx, 3, 64);
+ run_qvm_test(&ctx, 1, 128);
+ run_qvm_test(&ctx, 7, 256);
+ run_qvm_test(&ctx, 13, 512);
+ run_qvm_test(&ctx, 13, 511);
+ run_qvm_test(&ctx, 8, 64);
+ run_qvm_test(&ctx, 9, 64);
+ run_qvm_test(&ctx, 25, 128);
+ run_qvm_test(&ctx, 31, 512);
+}
+
+fn run_gemm_test(
+ ctx: &MTLContext,
+ m: usize,
+ n: usize,
+ k: usize,
+) {
+ println!("--- Testing GEMM M={}, N={}, K={} ---", m, n, k);
+ // GEMM: weights are in W[N][K] layout, CPU needs to transpose for correct computation
+ execute_quantized_matmul(
+ ctx,
+ m,
+ n,
+ k,
+ "qmm_f16_g64_b4",
+ true,
+ 1,
+ true,
+ false,
+ );
+ println!("✅ GEMM M={}, N={}, K={} passed", m, n, k);
+}
+
+fn run_gemm_transposed_test(
+ ctx: &MTLContext,
+ m: usize,
+ n: usize,
+ k: usize,
+) {
+ println!("--- Testing GEMM Transposed M={}, N={}, K={} ---", m, n, k);
+ // GEMM Transposed: weights are in W[N][K] but kernel expects W[K][N], CPU uses them as-is
+ execute_quantized_matmul(
+ ctx,
+ m,
+ n,
+ k,
+ "qmm_transposed_f16_g64_b4",
+ false,
+ 1,
+ true,
+ true,
+ );
+ println!("✅ GEMM Transposed M={}, N={}, K={} passed", m, n, k);
+}
+
+#[test]
+fn test_quant_gemm() {
+ let ctx = match create_test_context() {
+ Some(c) => c,
+ None => {
+ println!("Metal not available — skipping GEMM test");
+ return;
+ },
+ };
+
+ run_gemm_test(&ctx, 1, 1, 128);
+ run_gemm_test(&ctx, 8, 16, 256);
+ run_gemm_test(&ctx, 16, 8, 511);
+ run_gemm_test(&ctx, 9, 9, 64);
+ run_gemm_test(&ctx, 25, 17, 128);
+}
+
+#[test]
+fn test_quant_gemm_transposed() {
+ let ctx = match create_test_context() {
+ Some(c) => c,
+ None => {
+ println!("Metal not available — skipping GEMM transposed test");
+ return;
+ },
+ };
+
+ run_gemm_transposed_test(&ctx, 1, 1, 128);
+ run_gemm_transposed_test(&ctx, 8, 16, 256);
+ run_gemm_transposed_test(&ctx, 16, 8, 511);
+ run_gemm_transposed_test(&ctx, 9, 9, 64);
+ run_gemm_transposed_test(&ctx, 25, 17, 128);
+}
+
+fn benchmark_quantized_gemv(
+ ctx: &MTLContext,
+ m: usize,
+ k: usize,
+ iterations: usize,
+) -> f64 {
+ execute_quantized_matmul(
+ ctx,
+ m,
+ 1,
+ k,
+ "qmv_f16_g64_b4",
+ false,
+ iterations,
+ false,
+ true,
+ )
+}
+
+fn benchmark_quantized_qevm(
+ ctx: &MTLContext,
+ n: usize,
+ k: usize,
+ iterations: usize,
+) -> f64 {
+ execute_quantized_matmul(
+ ctx,
+ 1,
+ n,
+ k,
+ "qvm_f16_g64_b4",
+ false,
+ iterations,
+ false,
+ true,
+ )
+}
+
+fn benchmark_quantized_gemm(
+ ctx: &MTLContext,
+ m: usize,
+ n: usize,
+ k: usize,
+ iterations: usize,
+) -> f64 {
+ execute_quantized_matmul(
+ ctx,
+ m,
+ n,
+ k,
+ "qmm_f16_g64_b4",
+ true,
+ iterations,
+ false,
+ true,
+ )
+}
+
+#[ignore]
+#[test]
+fn test_quantized_matmul_performance() {
+ let ctx = match create_test_context() {
+ Some(c) => c,
+ None => {
+ println!("Metal not available — skipping performance test");
+ return;
+ },
+ };
+
+ println!("\n🚀 QUANTIZED MATMUL PERFORMANCE BENCHMARKS");
+ println!("==========================================");
+
+ // Test different sizes
+ let test_configs = vec![
+ // (M, N, K, iterations, description)
+ (512, 1, 512, 100, "Small GEMV (512×512 × 512×1)"),
+ (1024, 1, 1024, 50, "Medium GEMV (1024×1024 × 1024×1)"),
+ (2048, 1, 2048, 20, "Large GEMV (2048×2048 × 2048×1)"),
+ (4096, 1, 4096, 10, "XLarge GEMV (4096×4096 × 4096×1)"),
+ (1, 512, 512, 100, "Small QVM (1×512 × 512×512)"),
+ (1, 1024, 1024, 50, "Medium QVM (1×1024 × 1024×1024)"),
+ (1, 2048, 2048, 20, "Large QVM (1×2048 × 2048×2048)"),
+ (1, 4096, 4096, 10, "XLarge QVM (1×4096 × 4096×4096)"),
+ (512, 16, 512, 50, "Small GEMM (512×512 × 512×16)"),
+ (1024, 32, 1024, 20, "Medium GEMM (1024×1024 × 1024×32)"),
+ (2048, 64, 2048, 10, "Large GEMM (2048×2048 × 2048×64)"),
+ (4096, 128, 4096, 5, "XLarge GEMM (4096×4096 × 4096×128)"),
+ ];
+
+ for (m, n, k, iterations, description) in test_configs {
+ let ops = 2.0 * (m as f64) * (n as f64) * (k as f64);
+
+ if n == 1 {
+ // GEMV
+ let time = benchmark_quantized_gemv(&ctx, m, k, iterations);
+ let avg_time = time / (iterations as f64);
+ let throughput = (ops / avg_time) / 1e12;
+
+ println!("{}", description);
+ println!(
+ " Time: {:.4}ms/iter, Throughput: {:.4} TFLOPS",
+ avg_time * 1000.0,
+ throughput
+ );
+ } else if m == 1 {
+ // QVM
+ let time = benchmark_quantized_qevm(&ctx, n, k, iterations);
+ let avg_time = time / (iterations as f64);
+ let throughput = (ops / avg_time) / 1e12;
+
+ println!("{}", description);
+ println!(
+ " Time: {:.4}ms/iter, Throughput: {:.4} TFLOPS",
+ avg_time * 1000.0,
+ throughput
+ );
+ } else {
+ // GEMM
+ let time = benchmark_quantized_gemm(&ctx, m, n, k, iterations);
+ let avg_time = time / (iterations as f64);
+ let throughput = (ops / avg_time) / 1e12;
+
+ println!("{}", description);
+ println!(
+ " Time: {:.4}ms/iter, Throughput: {:.4} TFLOPS",
+ avg_time * 1000.0,
+ throughput
+ );
+ }
+ }
+}