Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Quant matmul cpu kernels (#316)

Aleksandr Golokoz6f6ddaa9787b
diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rs
index 4306fc711..5a0374c78 100644
--- a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rs
+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rs
@@ -4,33 +4,117 @@ use num_traits::Float;
use crate::ArrayElement;
+pub fn qmm_transposed<T: ArrayElement + Float>(
+ weights: *const u32,
+ scales: *const T,
+ zero_points: Option<*const u8>,
+ biases: Option<*const T>,
+ input: *const T,
+ output: *mut T,
+ in_vec_size: usize,
+ out_vec_size: usize,
+ batch_size: usize,
+ use_zero_points: bool,
+ use_mlx_quant: bool,
+ group_size: usize,
+ bits: usize,
+) {
+ let num_groups_k = in_vec_size.div_ceil(group_size);
+ let zp_stride = if bits == 4 {
+ (num_groups_k + 1) / 2
+ } else {
+ num_groups_k
+ };
+
+ unsafe {
+ for i in 0..batch_size {
+ for j in 0..out_vec_size {
+ let mut acc = 0.0f32;
+
+ for l in 0..in_vec_size {
+ let group_idx = l / group_size;
+
+ // Transposed: weight at row j, col l
+ let weight_linear_idx = j * in_vec_size + l;
+
+ let val_q = if bits == 4 {
+ let u32_idx = weight_linear_idx / 8;
+ let bit_offset = (weight_linear_idx % 8) * 4;
+ ((weights.add(u32_idx).read_unaligned() >> bit_offset) & 0xF) as f32
+ } else {
+ let u32_idx = weight_linear_idx / 4;
+ let byte_offset = (weight_linear_idx % 4) * 8;
+ ((weights.add(u32_idx).read_unaligned() >> byte_offset) & 0xFF) as f32
+ };
+
+ let val_a = (*input.add(i * in_vec_size + l)).to_f32().unwrap();
+ let scale_t = *scales.add(j * num_groups_k + group_idx);
+
+ // Dequantize in f32 to serve as a precision-neutral reference.
+ // Metal dequantizes in bf16/f16 which rounds differently across
+ // GPU hardware; computing in f32 avoids biasing toward any
+ // particular rounding pattern.
+ let w_dequant_f32 = if use_zero_points {
+ let zp = zero_points.unwrap();
+ let zp_val = if bits == 4 {
+ let byte_index = j * zp_stride + (group_idx >> 1);
+ let byte_val = *zp.add(byte_index);
+ if (group_idx & 1) == 0 {
+ (byte_val & 0x0F) as f32
+ } else {
+ ((byte_val >> 4) & 0x0F) as f32
+ }
+ } else {
+ *zp.add(j * zp_stride + group_idx) as f32
+ };
+ scale_t.to_f32().unwrap() * (val_q - zp_val)
+ } else if use_mlx_quant {
+ let bias_t = *biases.unwrap().add(j * num_groups_k + group_idx);
+ scale_t.to_f32().unwrap() * val_q + bias_t.to_f32().unwrap()
+ } else {
+ 0.0f32
+ };
+
+ acc += val_a * w_dequant_f32;
+ }
+
+ *output.add(i * out_vec_size + j) = T::from(acc).unwrap();
+ }
+ }
+ }
+}
+
#[kernel(QuantizedMatmulQmmTransposed)]
#[variants(T, f32, f16, bf16)]
#[variants(GROUP_SIZE, 32, 64, 128)]
#[variants(BITS, 4, 8)]
pub fn quantized_matmul_qmm_transposed<T: ArrayElement + Float, const GROUP_SIZE: u32, const BITS: u32>(
- #[allow(unused)] weights: *const u32,
- #[allow(unused)] scales: *const T,
- #[allow(unused)]
- #[optional(use_zero_points)]
- zero_points: Option<*const u8>,
- #[allow(unused)]
- #[optional(use_mlx_quant)]
- biases: Option<*const T>,
- #[allow(unused)] input: *const T,
- #[allow(unused)] output: *mut T,
- #[allow(unused)] in_vec_size: u32,
- #[allow(unused)] out_vec_size: u32,
- #[allow(unused)] batch_size: u32,
- #[allow(unused)]
- #[specialize]
- use_zero_points: bool,
- #[allow(unused)]
- #[specialize]
- use_mlx_quant: bool,
- #[allow(unused)]
- #[specialize]
- aligned_n: bool,
+ weights: *const u32,
+ scales: *const T,
+ #[optional(use_zero_points)] zero_points: Option<*const u8>,
+ #[optional(use_mlx_quant)] biases: Option<*const T>,
+ input: *const T,
+ output: *mut T,
+ in_vec_size: u32,
+ out_vec_size: u32,
+ batch_size: u32,
+ #[specialize] use_zero_points: bool,
+ #[specialize] use_mlx_quant: bool,
+ #[specialize] aligned_n: bool,
) {
- todo!()
+ qmm_transposed::<T>(
+ weights,
+ scales,
+ zero_points,
+ biases,
+ input,
+ output,
+ in_vec_size as usize,
+ out_vec_size as usize,
+ batch_size as usize,
+ use_zero_points,
+ use_mlx_quant,
+ GROUP_SIZE as usize,
+ BITS as usize,
+ );
}
diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_64x64.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_64x64.rs
index aee61e512..c28219b6a 100644
--- a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_64x64.rs
+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_64x64.rs
@@ -2,32 +2,38 @@ use dsl::kernel;
use half::bf16;
use num_traits::Float;
-use crate::ArrayElement;
+use crate::{ArrayElement, backends::cpu::kernel::quant_matmul::qmm_transposed::qmm_transposed};
#[kernel(QuantizedMatmulQmmTransposed64x64)]
#[variants(T, bf16)]
#[variants(GROUP_SIZE, 64, 128)]
#[variants(BITS, 4, 8)]
pub fn quantized_matmul_qmm_transposed64x64<T: ArrayElement + Float, const GROUP_SIZE: u32, const BITS: u32>(
- #[allow(unused)] weights: *const u32,
- #[allow(unused)] scales: *const T,
- #[allow(unused)]
- #[optional(use_zero_points)]
- zero_points: Option<*const u8>,
- #[allow(unused)]
- #[optional(use_mlx_quant)]
- biases: Option<*const T>,
- #[allow(unused)] input: *const T,
- #[allow(unused)] output: *mut T,
- #[allow(unused)] in_vec_size: u32,
- #[allow(unused)] out_vec_size: u32,
- #[allow(unused)] batch_size: u32,
- #[allow(unused)]
- #[specialize]
- use_zero_points: bool,
- #[allow(unused)]
- #[specialize]
- use_mlx_quant: bool,
+ weights: *const u32,
+ scales: *const T,
+ #[optional(use_zero_points)] zero_points: Option<*const u8>,
+ #[optional(use_mlx_quant)] biases: Option<*const T>,
+ input: *const T,
+ output: *mut T,
+ in_vec_size: u32,
+ out_vec_size: u32,
+ batch_size: u32,
+ #[specialize] use_zero_points: bool,
+ #[specialize] use_mlx_quant: bool,
) {
- todo!()
+ qmm_transposed::<T>(
+ weights,
+ scales,
+ zero_points,
+ biases,
+ input,
+ output,
+ in_vec_size as usize,
+ out_vec_size as usize,
+ batch_size as usize,
+ use_zero_points,
+ use_mlx_quant,
+ GROUP_SIZE as usize,
+ BITS as usize,
+ );
}
diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_wide.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_wide.rs
index aa64ed350..ad227a8d5 100644
--- a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_wide.rs
+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_wide.rs
@@ -2,32 +2,38 @@ use dsl::kernel;
use half::bf16;
use num_traits::Float;
-use crate::ArrayElement;
+use crate::{ArrayElement, backends::cpu::kernel::quant_matmul::qmm_transposed::qmm_transposed};
#[kernel(QuantizedMatmulQmmTransposedWide)]
#[variants(T, bf16)]
#[variants(GROUP_SIZE, 32, 64, 128)]
#[variants(BITS, 4, 8)]
pub fn quantized_matmul_qmm_transposed_wide<T: ArrayElement + Float, const GROUP_SIZE: u32, const BITS: u32>(
- #[allow(unused)] weights: *const u32,
- #[allow(unused)] scales: *const T,
- #[allow(unused)]
- #[optional(use_zero_points)]
- zero_points: Option<*const u8>,
- #[allow(unused)]
- #[optional(use_mlx_quant)]
- biases: Option<*const T>,
- #[allow(unused)] input: *const T,
- #[allow(unused)] output: *mut T,
- #[allow(unused)] in_vec_size: u32,
- #[allow(unused)] out_vec_size: u32,
- #[allow(unused)] batch_size: u32,
- #[allow(unused)]
- #[specialize]
- use_zero_points: bool,
- #[allow(unused)]
- #[specialize]
- use_mlx_quant: bool,
+ weights: *const u32,
+ scales: *const T,
+ #[optional(use_zero_points)] zero_points: Option<*const u8>,
+ #[optional(use_mlx_quant)] biases: Option<*const T>,
+ input: *const T,
+ output: *mut T,
+ in_vec_size: u32,
+ out_vec_size: u32,
+ batch_size: u32,
+ #[specialize] use_zero_points: bool,
+ #[specialize] use_mlx_quant: bool,
) {
- todo!()
+ qmm_transposed::<T>(
+ weights,
+ scales,
+ zero_points,
+ biases,
+ input,
+ output,
+ in_vec_size as usize,
+ out_vec_size as usize,
+ batch_size as usize,
+ use_zero_points,
+ use_mlx_quant,
+ GROUP_SIZE as usize,
+ BITS as usize,
+ );
}
diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv.rs
index 7341a22e6..0f937ff5a 100644
--- a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv.rs
+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv.rs
@@ -4,30 +4,128 @@ use num_traits::Float;
use crate::ArrayElement;
+/// CPU reference implementation of quantized matrix-vector multiply.
+///
+/// Weight layout is transposed compared to qmm:
+/// weights: [N, K], scales: [N, num_groups_k], zero_points: [N, zp_stride_k]
+/// where groups are along the K dimension.
+///
+/// Computes y[i, j] = sum_l( x[i, l] * dequant(w[j, l]) ) for each batch i and output j.
+pub fn qmv<T: ArrayElement + Float>(
+ weights: *const u32,
+ scales: *const T,
+ zero_points: Option<*const u8>,
+ biases: Option<*const T>,
+ input: *const T,
+ output: *mut T,
+ in_vec_size: usize,
+ out_vec_size: usize,
+ batch_size: usize,
+ use_zero_points: bool,
+ use_mlx_quant: bool,
+ group_size: usize,
+ bits: usize,
+) {
+ let num_groups_k = in_vec_size.div_ceil(group_size);
+ let zp_stride = if bits == 4 {
+ (num_groups_k + 1) / 2
+ } else {
+ num_groups_k
+ };
+
+ // Number of weight elements packed per u32
+ let pack_factor = if bits == 4 {
+ 8
+ } else {
+ 4
+ };
+
+ unsafe {
+ for i in 0..batch_size {
+ for j in 0..out_vec_size {
+ let mut acc = 0.0f32;
+
+ for l in 0..in_vec_size {
+ // Weights are [N, K]: row j, column l
+ let weight_linear_idx = j * in_vec_size + l;
+
+ let val_q = if bits == 4 {
+ let u32_idx = weight_linear_idx / pack_factor;
+ let bit_offset = (weight_linear_idx % pack_factor) * 4;
+ ((weights.add(u32_idx).read_unaligned() >> bit_offset) & 0xF) as f32
+ } else {
+ let u32_idx = weight_linear_idx / pack_factor;
+ let byte_offset = (weight_linear_idx % pack_factor) * 8;
+ ((weights.add(u32_idx).read_unaligned() >> byte_offset) & 0xFF) as f32
+ };
+
+ let val_a = (*input.add(i * in_vec_size + l)).to_f32().unwrap();
+
+ // Scales are [N, num_groups_k]: groups along K
+ let group_idx = l / group_size;
+ let scale = (*scales.add(j * num_groups_k + group_idx)).to_f32().unwrap();
+
+ let bias = if use_zero_points {
+ let zp = zero_points.unwrap();
+ // Zero points are [N, zp_stride_k]
+ let zp_val = if bits == 4 {
+ let byte_index = j * zp_stride + (group_idx >> 1);
+ let byte_val = *zp.add(byte_index);
+ if (group_idx & 1) == 0 {
+ (byte_val & 0x0F) as f32
+ } else {
+ ((byte_val >> 4) & 0x0F) as f32
+ }
+ } else {
+ *zp.add(j * zp_stride + group_idx) as f32
+ };
+ -scale * zp_val
+ } else if use_mlx_quant {
+ // Biases are [N, num_groups_k]
+ (*biases.unwrap().add(j * num_groups_k + group_idx)).to_f32().unwrap()
+ } else {
+ 0.0f32
+ };
+
+ acc += val_a * (scale * val_q + bias);
+ }
+
+ *output.add(i * out_vec_size + j) = T::from(acc).unwrap();
+ }
+ }
+ }
+}
+
#[kernel(QuantizedMatmulQmv)]
#[variants(T, f32, f16, bf16)]
#[variants(GROUP_SIZE, 32, 64, 128)]
#[variants(BITS, 4, 8)]
pub fn quantized_matmul_qmv<T: ArrayElement + Float, const GROUP_SIZE: u32, const BITS: u32>(
- #[allow(unused)] weights: *const u32,
- #[allow(unused)] scales: *const T,
- #[allow(unused)]
- #[optional(use_zero_points)]
- zero_points: Option<*const u8>,
- #[allow(unused)]
- #[optional(use_mlx_quant)]
- biases: Option<*const T>,
- #[allow(unused)] input: *const T,
- #[allow(unused)] output: *mut T,
- #[allow(unused)] in_vec_size: u32,
- #[allow(unused)] out_vec_size: u32,
- #[allow(unused)] batch_size: u32,
- #[allow(unused)]
- #[specialize]
- use_zero_points: bool,
- #[allow(unused)]
- #[specialize]
- use_mlx_quant: bool,
+ weights: *const u32,
+ scales: *const T,
+ #[optional(use_zero_points)] zero_points: Option<*const u8>,
+ #[optional(use_mlx_quant)] biases: Option<*const T>,
+ input: *const T,
+ output: *mut T,
+ in_vec_size: u32,
+ out_vec_size: u32,
+ batch_size: u32,
+ #[specialize] use_zero_points: bool,
+ #[specialize] use_mlx_quant: bool,
) {
- todo!()
+ qmv::<T>(
+ weights,
+ scales,
+ zero_points,
+ biases,
+ input,
+ output,
+ in_vec_size as usize,
+ out_vec_size as usize,
+ batch_size as usize,
+ use_zero_points,
+ use_mlx_quant,
+ GROUP_SIZE as usize,
+ BITS as usize,
+ );
}
diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv_fast.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv_fast.rs
index d5c53856d..aa05b7ac0 100644
--- a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv_fast.rs
+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv_fast.rs
@@ -2,32 +2,38 @@ use dsl::kernel;
use half::{bf16, f16};
use num_traits::Float;
-use crate::ArrayElement;
+use crate::{ArrayElement, backends::cpu::kernel::quant_matmul::qmv::qmv};
#[kernel(QuantizedMatmulQmvFast)]
#[variants(T, f32, f16, bf16)]
#[variants(GROUP_SIZE, 32, 64, 128)]
#[variants(BITS, 4, 8)]
pub fn quantized_matmul_qmv_fast<T: ArrayElement + Float, const GROUP_SIZE: u32, const BITS: u32>(
- #[allow(unused)] weights: *const u32,
- #[allow(unused)] scales: *const T,
- #[allow(unused)]
- #[optional(use_zero_points)]
- zero_points: Option<*const u8>,
- #[allow(unused)]
- #[optional(use_mlx_quant)]
- biases: Option<*const T>,
- #[allow(unused)] input: *const T,
- #[allow(unused)] output: *mut T,
- #[allow(unused)] in_vec_size: u32,
- #[allow(unused)] out_vec_size: u32,
- #[allow(unused)] batch_size: u32,
- #[allow(unused)]
- #[specialize]
- use_zero_points: bool,
- #[allow(unused)]
- #[specialize]
- use_mlx_quant: bool,
+ weights: *const u32,
+ scales: *const T,
+ #[optional(use_zero_points)] zero_points: Option<*const u8>,
+ #[optional(use_mlx_quant)] biases: Option<*const T>,
+ input: *const T,
+ output: *mut T,
+ in_vec_size: u32,
+ out_vec_size: u32,
+ batch_size: u32,
+ #[specialize] use_zero_points: bool,
+ #[specialize] use_mlx_quant: bool,
) {
- todo!()
+ qmv::<T>(
+ weights,
+ scales,
+ zero_points,
+ biases,
+ input,
+ output,
+ in_vec_size as usize,
+ out_vec_size as usize,
+ batch_size as usize,
+ use_zero_points,
+ use_mlx_quant,
+ GROUP_SIZE as usize,
+ BITS as usize,
+ );
}
diff --git a/crates/uzu/tests/common/helpers.rs b/crates/uzu/tests/common/helpers.rs
index 2361ab6d7..03efa27aa 100644
--- a/crates/uzu/tests/common/helpers.rs
+++ b/crates/uzu/tests/common/helpers.rs
@@ -2,6 +2,7 @@ use std::rc::Rc;
use uzu::backends::common::{Backend, Buffer, Context};
+#[allow(dead_code)]
pub fn alloc_buffer<B: Backend, T>(
context: &B::Context,
elements_count: usize,
@@ -25,6 +26,7 @@ pub fn alloc_buffer_with_data<B: Backend, T: bytemuck::NoUninit>(
buffer
}
+#[allow(dead_code)]
pub fn create_context<B: Backend>() -> Rc<<B as Backend>::Context> {
B::Context::new().expect(format!("Failed to create context for {}", std::any::type_name::<B>()).as_str())
}
diff --git a/crates/uzu/tests/common/mod.rs b/crates/uzu/tests/common/mod.rs
index 1f00f9c5b..ea1593e22 100644
--- a/crates/uzu/tests/common/mod.rs
+++ b/crates/uzu/tests/common/mod.rs
@@ -45,3 +45,21 @@ macro_rules! for_each_non_cpu_backend {
}};
}
pub(crate) use for_each_non_cpu_backend;
+
+macro_rules! for_each_float_type {
+ (|$F:ident| $body:expr) => {{
+ {
+ type $F = f32;
+ $body
+ }
+ {
+ type $F = half::f16;
+ $body
+ }
+ {
+ type $F = half::bf16;
+ $body
+ }
+ }};
+}
+pub(crate) use for_each_float_type;
diff --git a/crates/uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs b/crates/uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs
index a2a42b2a9..ebb324b74 100644
--- a/crates/uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs
+++ b/crates/uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs
@@ -1,24 +1,20 @@
-#![cfg(metal_backend)]
+#[macro_use]
+#[path = "../../../../common/mod.rs"]
+mod common;
use std::time::Instant;
-use bytemuck;
use half::{bf16, f16};
-use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};
-use objc2::{rc::Retained, runtime::ProtocolObject};
use crate::{
DataType,
- backends::{
- common::{
- Backend, Context, Encoder,
- gpu_types::QuantizationMode,
- kernel::quant_matmul::{
- QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulKernelEncodable,
- QuantizedMatmulType,
- },
+ backends::common::{
+ Backend, Buffer, Context, Encoder,
+ gpu_types::QuantizationMode,
+ kernel::quant_matmul::{
+ QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulKernelEncodable,
+ QuantizedMatmulType, tests::common::helpers::alloc_buffer_with_data,
},
- metal::Metal,
},
};
@@ -265,36 +261,29 @@ fn generate_test_quant_params(
}
}
-fn buffer_from_f32_slice(
- ctx: &<Metal as Backend>::Context,
+fn buffer_from_f32_slice<B: Backend>(
+ ctx: &B::Context,
dtype: DataType,
values: &[f32],
-) -> Retained<ProtocolObject<dyn MTLBuffer>> {
+) -> B::Buffer {
match dtype {
DataType::F16 => {
let data: Vec<f16> = values.iter().map(|&v| f16::from_f32(v)).collect();
- ctx.device
- .new_buffer_with_data(bytemuck::cast_slice(&data), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer")
+ alloc_buffer_with_data::<B, f16>(ctx, data.as_slice())
},
DataType::BF16 => {
let data: Vec<bf16> = values.iter().map(|&v| bf16::from_f32(v)).collect();
- ctx.device
- .new_buffer_with_data(bytemuck::cast_slice(&data), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer")
+ alloc_buffer_with_data::<B, bf16>(ctx, data.as_slice())
},
- DataType::F32 => ctx
- .device
- .new_buffer_with_data(bytemuck::cast_slice(values), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer"),
+ DataType::F32 => alloc_buffer_with_data::<B, f32>(ctx, values),
other => {
panic!("Unsupported dtype for buffer_from_f32_slice: {:?}", other)
},
}
}
-fn execute_quantized_matmul(
- ctx: &<Metal as Backend>::Context,
+fn execute_quantized_matmul<B: Backend>(
+ ctx: &B::Context,
batch: usize,
input_dim: usize,
output_dim: usize,
@@ -330,23 +319,17 @@ fn execute_quantized_matmul(
};
let x_quant = quantize_slice(&x_f32, data_type);
- let w_buf = ctx
- .device
- .new_buffer_with_data(bytemuck::cast_slice(&weights_packed), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
- let s_buf = buffer_from_f32_slice(ctx, data_type, &params.scales);
+ let w_buf = alloc_buffer_with_data::<B, u8>(ctx, &weights_packed);
+ let s_buf = buffer_from_f32_slice::<B>(ctx, data_type, &params.scales);
let b_buf = match quantization_type {
- QuantizedMatmulType::ZeroPoint => ctx
- .device
- .new_buffer_with_data(&params.zero_points, MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer"),
- QuantizedMatmulType::Mlx => buffer_from_f32_slice(ctx, data_type, &params.biases),
+ QuantizedMatmulType::ZeroPoint => alloc_buffer_with_data::<B, u8>(ctx, &params.zero_points),
+ QuantizedMatmulType::Mlx => buffer_from_f32_slice::<B>(ctx, data_type, &params.biases),
};
- let x_buf = buffer_from_f32_slice(ctx, data_type, &x_f32);
+ let x_buf = buffer_from_f32_slice::<B>(ctx, data_type, &x_f32);
let mut y_buf = ctx.create_buffer(batch * output_dim * data_type.size_in_bytes()).expect("Failed to create buffer");
- let kernel = QuantizedMatmulKernelEncodable::<Metal>::new(
+ let kernel = QuantizedMatmulKernelEncodable::<B>::new(
&ctx,
QuantizedMatmulConfiguration {
data_type,
@@ -416,17 +399,17 @@ fn execute_quantized_matmul(
let y_out_f32: Vec<f32> = match data_type {
DataType::F16 => {
- let y_ptr = y_buf.contents().as_ptr() as *const f16;
+ let y_ptr = y_buf.cpu_ptr().as_ptr() as *const f16;
let y_out = unsafe { std::slice::from_raw_parts(y_ptr, batch * output_dim) };
y_out.iter().map(|&v| v.to_f32()).collect()
},
DataType::BF16 => {
- let y_ptr = y_buf.contents().as_ptr() as *const bf16;
+ let y_ptr = y_buf.cpu_ptr().as_ptr() as *const bf16;
let y_out = unsafe { std::slice::from_raw_parts(y_ptr, batch * output_dim) };
y_out.iter().map(|&v| v.to_f32()).collect()
},
DataType::F32 => {
- let y_ptr = y_buf.contents().as_ptr() as *const f32;
+ let y_ptr = y_buf.cpu_ptr().as_ptr() as *const f32;
let y_out = unsafe { std::slice::from_raw_parts(y_ptr, batch * output_dim) };
y_out.to_vec()
},
@@ -488,8 +471,8 @@ struct TestConfig {
const QMV_DIMS: &[(usize, usize)] = &[(128, 512), (512, 1024), (1024, 4096)];
const QMM_DIMS: &[(usize, usize, usize)] = &[(64, 64, 64), (512, 512, 1024), (128, 128, 256)];
-fn run_kernel_test(
- ctx: &<Metal as Backend>::Context,
+fn run_kernel_test<B: Backend>(
+ ctx: &B::Context,
batch: usize,
output_dim: usize,
input_dim: usize,
@@ -499,7 +482,7 @@ fn run_kernel_test(
) -> ExecutionResult {
let randomize_zp = config.quant_type == QuantizedMatmulType::ZeroPoint;
- execute_quantized_matmul(
+ execute_quantized_matmul::<B>(
ctx,
batch,
input_dim,
@@ -514,12 +497,11 @@ fn run_kernel_test(
)
}
-#[test]
-fn test_quant_gmv() {
- let ctx = match <Metal as Backend>::Context::new() {
+fn test_quant_gmv_internal<B: Backend>() {
+ let ctx = match B::Context::new() {
Ok(c) => c,
Err(_) => {
- println!("Metal not available — skipping QMV test");
+ println!("{} backend not available — skipping QMV test", std::any::type_name::<B>());
return;
},
};
@@ -553,17 +535,16 @@ fn test_quant_gmv() {
for config in &configs {
for &(output_dim, input_dim) in QMV_DIMS {
- run_kernel_test(&ctx, 1, output_dim, input_dim, config, true, 1);
+ run_kernel_test::<B>(&ctx, 1, output_dim, input_dim, config, true, 1);
}
}
}
-#[test]
-fn test_quant_gmm_transposed() {
- let ctx = match <Metal as Backend>::Context::new() {
+fn test_quant_gmm_transposed_internal<B: Backend>() {
+ let ctx = match B::Context::new() {
Ok(c) => c,
Err(_) => {
- println!("Metal not available — skipping QMM transposed test");
+ println!("{} not available — skipping QMM transposed test", std::any::type_name::<B>());
return;
},
};
@@ -597,18 +578,16 @@ fn test_quant_gmm_transposed() {
for config in &configs {
for &(batch, output_dim, input_dim) in QMM_DIMS {
- run_kernel_test(&ctx, batch, output_dim, input_dim, config, true, 1);
+ run_kernel_test::<B>(&ctx, batch, output_dim, input_dim, config, true, 1);
}
}
}
-#[test]
-#[ignore]
-fn test_quant_matmul_perf() {
- let ctx = match <Metal as Backend>::Context::new() {
+fn test_quant_matmul_perf_internal<B: Backend>() {
+ let ctx = match B::Context::new() {
Ok(c) => c,
Err(_) => {
- println!("Metal not available — skipping Perf test");
+ println!("{} backend not available — skipping Perf test", std::any::type_name::<B>());
return;
},
};
@@ -660,10 +639,11 @@ fn test_quant_matmul_perf() {
);
println!("{}", "-".repeat(85));
+ let iterations = 20;
for config in &configs {
for &(batch, output_dim, input_dim) in &shapes {
- let result = run_kernel_test(&ctx, batch, output_dim, input_dim, config, false, 20);
- let avg = result.elapsed / 20.0 * 1000.0; // ms
+ let result = run_kernel_test::<B>(&ctx, batch, output_dim, input_dim, config, false, iterations);
+ let avg = result.elapsed / (iterations as f64) * 1000.0; // ms
println!(
"{:<20} | {:<10} | {:<6} | {:<5} | {:<5} | {:.4} ms",
@@ -677,3 +657,25 @@ fn test_quant_matmul_perf() {
}
}
}
+
+#[test]
+fn test_quant_gmv() {
+ for_each_non_cpu_backend!(|B| {
+ test_quant_gmv_internal::<B>();
+ })
+}
+
+#[test]
+fn test_quant_gmm_transposed() {
+ for_each_non_cpu_backend!(|B| {
+ test_quant_gmm_transposed_internal::<B>();
+ })
+}
+
+#[test]
+#[ignore]
+fn test_quant_matmul_perf() {
+ for_each_non_cpu_backend!(|B| {
+ test_quant_matmul_perf_internal::<B>();
+ })
+}
diff --git a/crates/uzu/tests/unit/kernel/mod.rs b/crates/uzu/tests/unit/kernel/mod.rs
index e9290f87c..dc6e2b888 100644
--- a/crates/uzu/tests/unit/kernel/mod.rs
+++ b/crates/uzu/tests/unit/kernel/mod.rs
@@ -14,6 +14,7 @@ mod matmul;
mod moe;
mod pooling;
mod qk_norm_test;
+mod quant_matmul;
mod rms_norm_test;
mod rope_test;
mod sampling;
diff --git a/crates/uzu/tests/unit/kernel/quant_matmul/mod.rs b/crates/uzu/tests/unit/kernel/quant_matmul/mod.rs
new file mode 100644
index 000000000..b97526f4f
--- /dev/null
+++ b/crates/uzu/tests/unit/kernel/quant_matmul/mod.rs
@@ -0,0 +1,89 @@
+#[macro_use]
+#[path = "../../../common/mod.rs"]
+mod common;
+
+mod qmm_transposed_64x64_test;
+mod qmm_transposed_test;
+mod qmm_transposed_wide_test;
+mod qmv_fast_test;
+mod qmv_test;
+
+use num_traits::Float;
+use uzu::ArrayElement;
+
+pub(super) struct Input<T: ArrayElement + Float> {
+ pub w_packed: Vec<u32>,
+ pub scales: Vec<T>,
+ pub zero_points: Option<Vec<u8>>,
+ pub biases: Option<Vec<T>>,
+ pub x: Vec<T>,
+ pub k: u32,
+ pub n: u32,
+ pub m: u32,
+ pub group_size: u32,
+ pub bits: u32,
+ pub use_zero_points: bool,
+ pub use_mlx_quant: bool,
+}
+
+pub(super) fn pack_weights_u32(
+ values: &[u8],
+ bits: u32,
+) -> Vec<u32> {
+ if bits == 4 {
+ values
+ .chunks(8)
+ .map(|chunk| {
+ let mut word = 0u32;
+ for (i, &v) in chunk.iter().enumerate() {
+ word |= ((v & 0xF) as u32) << (i * 4);
+ }
+ word
+ })
+ .collect()
+ } else {
+ values
+ .chunks(4)
+ .map(|chunk| {
+ let mut word = 0u32;
+ for (i, &v) in chunk.iter().enumerate() {
+ word |= (v as u32) << (i * 8);
+ }
+ word
+ })
+ .collect()
+ }
+}
+
+pub(super) fn pack_zero_points(
+ values: &[u8],
+ bits: u32,
+) -> Vec<u8> {
+ if bits == 4 {
+ values
+ .chunks(2)
+ .map(|chunk| {
+ let lo = chunk[0] & 0x0F;
+ let hi = if chunk.len() > 1 {
+ chunk[1] & 0x0F
+ } else {
+ 0
+ };
+ lo | (hi << 4)
+ })
+ .collect()
+ } else {
+ values.to_vec()
+ }
+}
+
+pub(super) fn check_tolerance(
+ expected: f32,
+ actual: f32,
+ rel_tol: f64,
+ abs_tol: f64,
+) -> bool {
+ let diff = (expected - actual).abs() as f64;
+ let tol = abs_tol.max(expected.abs() as f64 * rel_tol);
+ diff <= tol
+}
diff --git a/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_64x64_test.rs b/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_64x64_test.rs
new file mode 100644
index 000000000..62bad063e
--- /dev/null
+++ b/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_64x64_test.rs
@@ -0,0 +1,372 @@
+use std::fmt::{Debug, Display};
+
+use half::bf16;
+use num_traits::Float;
+use uzu::{
+ ArrayElement, DataType,
+ backends::{
+ common::{Backend, Buffer, Context, Encoder, Kernels, kernel::QuantizedMatmulQmmTransposed64x64Kernel},
+ cpu::Cpu,
+ },
+};
+
+use super::{Input, check_tolerance, pack_weights_u32, pack_zero_points};
+use crate::{common::helpers::alloc_buffer_with_data, uzu_test};
+
+fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
+ let context = B::Context::new().expect("Failed to create Context");
+ let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmmTransposed64x64Kernel::new(
+ &context,
+ T::data_type(),
+ input.group_size,
+ input.bits,
+ input.use_zero_points,
+ input.use_mlx_quant,
+ )
+ .expect("Failed to create QuantizedMatmulQmmTransposed64x64Kernel");
+
+ let w_buf = alloc_buffer_with_data::<B, u32>(&context, &input.w_packed);
+ let scales_buf = alloc_buffer_with_data::<B, T>(&context, &input.scales);
+
+ let zp_buf = input.zero_points.as_ref().map(|zp| alloc_buffer_with_data::<B, u8>(&context, zp));
+ let bias_buf = input.biases.as_ref().map(|b| alloc_buffer_with_data::<B, T>(&context, b));
+
+ let x_buf = alloc_buffer_with_data::<B, T>(&context, &input.x);
+ let output_size = (input.m as usize) * (input.n as usize) * T::data_type().size_in_bytes();
+ let mut y_buf = context.create_buffer(output_size).expect("Failed to create buffer");
+
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
+ kernel.encode(
+ &w_buf,
+ &scales_buf,
+ zp_buf.as_ref(),
+ bias_buf.as_ref(),
+ &x_buf,
+ &mut y_buf,
+ input.k,
+ input.n,
+ input.m,
+ &mut encoder,
+ );
+
+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
+
+ let y_ptr = y_buf.cpu_ptr().as_ptr() as *const T;
+ let y_len = (input.m as usize) * (input.n as usize);
+ unsafe { std::slice::from_raw_parts(y_ptr, y_len) }.to_vec()
+}
+
+/// Create test data for transposed 64x64 qmm: x[m×k] * w_transposed[n×k] = y[m×n]
+/// Weights are in transposed [n × k] layout.
+fn get_test_data_basic<T: ArrayElement + Float>(
+ m: usize,
+ k: usize,
+ n: usize,
+ group_size: u32,
+ bits: u32,
+ use_zero_points: bool,
+ use_mlx_quant: bool,
+) -> (Input<T>, Vec<T>) {
+ let num_groups_k = (k + group_size as usize - 1) / group_size as usize;
+
+ // Generate quantized weights [n × k]
+ let max_val = if bits == 4 {
+ 15u8
+ } else {
+ 255u8
+ };
+ let mut weights_raw: Vec<u8> = Vec::with_capacity(n * k);
+ for j in 0..n {
+ for l in 0..k {
+ weights_raw.push(((j * 3 + l * 7 + 1) % (max_val as usize + 1)) as u8);
+ }
+ }
+ let w_packed = pack_weights_u32(&weights_raw, bits);
+
+ // Scales: [n × num_groups_k]
+ let mut scales_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);
+ for j in 0..n {
+ for g in 0..num_groups_k {
+ scales_f32.push(0.5 + 0.1 * ((j + g) % 5) as f32);
+ }
+ }
+ let scales: Vec<T> = scales_f32.iter().map(|&v| T::from(v).unwrap()).collect();
+
+ // Zero points or biases
+ let zp_stride = if bits == 4 {
+ (num_groups_k + 1) / 2
+ } else {
+ num_groups_k
+ };
+
+ let (zero_points, biases) = if use_zero_points {
+ let mut zp_raw: Vec<u8> = Vec::with_capacity(n * num_groups_k);
+ for j in 0..n {
+ for g in 0..num_groups_k {
+ let zp_val = ((j * 2 + g * 3) % (max_val as usize + 1)) as u8;
+ zp_raw.push(zp_val);
+ }
+ }
+ // Pack zero points per row
+ let mut zp_packed: Vec<u8> = Vec::with_capacity(n * zp_stride);
+ for j in 0..n {
+ let row = &zp_raw[j * num_groups_k..(j + 1) * num_groups_k];
+ let packed_row = pack_zero_points(row, bits);
+ let mut padded = packed_row;
+ padded.resize(zp_stride, 0);
+ zp_packed.extend_from_slice(&padded);
+ }
+ (Some(zp_packed), None)
+ } else if use_mlx_quant {
+ let mut biases_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);
+ for j in 0..n {
+ for g in 0..num_groups_k {
+ biases_f32.push(0.01 * ((j + g * 2) % 7) as f32);
+ }
+ }
+ let biases: Vec<T> = biases_f32.iter().map(|&v| T::from(v).unwrap()).collect();
+ (None, Some(biases))
+ } else {
+ unreachable!("Must use either zero_points or mlx_quant");
+ };
+
+ // X: [m × k]
+ let mut x_f32: Vec<f32> = Vec::with_capacity(m * k);
+ for i in 0..m {
+ for l in 0..k {
+ x_f32.push(0.1 * f32::sin((i * k + l) as f32 * 0.05) + 0.5);
+ }
+ }
+ let x: Vec<T> = x_f32.iter().map(|&v| T::from(v).unwrap()).collect();
+
+ let input = Input {
+ w_packed,
+ scales,
+ zero_points,
+ biases,
+ x,
+ k: k as u32,
+ n: n as u32,
+ m: m as u32,
+ group_size,
+ bits,
+ use_zero_points,
+ use_mlx_quant,
+ };
+
+ let expected = get_output::<Cpu, T>(&input);
+ (input, expected)
+}
+
+/// Small edge-case test data with known values
+fn get_test_data_edge<T: ArrayElement + Float>(
+ group_size: u32,
+ bits: u32,
+ use_zero_points: bool,
+ use_mlx_quant: bool,
+) -> (Input<T>, Vec<T>) {
+ let m = 2usize;
+ let k = group_size as usize;
+ let n = 64usize;
+ let num_groups_k = 1;
+
+ // Simple weights [n × k]: all 1s
+ let weights_raw: Vec<u8> = vec![1u8; n * k];
+ let w_packed = pack_weights_u32(&weights_raw, bits);
+
+ // Unit scales
+ let scales: Vec<T> = vec![T::one(); n * num_groups_k];
+
+ let zp_stride = if bits == 4 {
+ (num_groups_k + 1) / 2
+ } else {
+ num_groups_k
+ };
+
+ let (zero_points, biases) = if use_zero_points {
+ // Zero points all 0 -> bias = 0
+ let zp_packed = vec![0u8; n * zp_stride];
+ (Some(zp_packed), None)
+ } else if use_mlx_quant {
+ let biases: Vec<T> = vec![T::zero(); n * num_groups_k];
+ (None, Some(biases))
+ } else {
+ unreachable!("Must use either zero_points or mlx_quant");
+ };
+
+ // X: simple increasing values
+ let x_f32: Vec<f32> = (0..m * k).map(|i| (i + 1) as f32 * 0.1).collect();
+ let x: Vec<T> = x_f32.iter().map(|&v| T::from(v).unwrap()).collect();
+
+ let input = Input {
+ w_packed,
+ scales,
+ zero_points,
+ biases,
+ x,
+ k: k as u32,
+ n: n as u32,
+ m: m as u32,
+ group_size,
+ bits,
+ use_zero_points,
+ use_mlx_quant,
+ };
+
+ let expected = get_output::<Cpu, T>(&input);
+ (input, expected)
+}
+
+fn test_internal<T: ArrayElement + Float + Debug + Display>(
+ input: &Input<T>,
+ expected: &[T],
+) {
+ let (rel_tol, abs_tol): (f64, f64) = match T::data_type() {
+ DataType::BF16 => (0.05, 0.5),
+ DataType::F16 => (0.02, 0.5),
+ _ => (0.01, 0.1),
+ };
+
+ for_each_non_cpu_backend!(|B| {
+ let output = get_output::<B, T>(input);
+ assert_eq!(expected.len(), output.len(), "Output length mismatch");
+
+ let mut errors = 0;
+ for (i, (&exp, &got)) in expected.iter().zip(output.iter()).enumerate() {
+ let exp_f32 = exp.to_f32().unwrap();
+ let got_f32 = got.to_f32().unwrap();
+ if !check_tolerance(exp_f32, got_f32, rel_tol, abs_tol) {
+ if errors < 5 {
+ eprintln!(" idx={} expected={} got={} diff={}", i, exp_f32, got_f32, (exp_f32 - got_f32).abs());
+ }
+ errors += 1;
+ }
+ }
+ assert_eq!(
+ errors,
+ 0,
+ "QMM transposed 64x64 kernel: backend={}, m={}, k={}, n={}, gs={}, bits={}, zp={}, mlx={}: {} mismatches",
+ std::any::type_name::<B>(),
+ input.m,
+ input.k,
+ input.n,
+ input.group_size,
+ input.bits,
+ input.use_zero_points,
+ input.use_mlx_quant,
+ errors,
+ );
+ });
+}
+
+fn get_test_dims(
+ group_size: u32,
+ bits: u32,
+) -> Vec<(usize, usize, usize)> {
+ let gs = group_size as usize;
+ // 64x64 kernel: BM=64, BK=64, BN=64.
+ // K must be >= group_size (transposed grouping along K).
+ // N must be a multiple of 64 (BN tile size).
+ // bf16 8-bit: keep dims small to avoid catastrophic cancellation.
+ if bits == 8 {
+ vec![(2, gs, 64), (4, gs, 64)]
+ } else {
+ vec![(2, gs * 2, 64), (4, gs * 2, 128), (8, gs, 128)]
+ }
+}
+
+fn test_basic(
+ group_size: u32,
+ bits: u32,
+ use_zero_points: bool,
+ use_mlx_quant: bool,
+) {
+ let dims = get_test_dims(group_size, bits);
+
+ for (m, k, n) in dims {
+ let (input, expected) = get_test_data_basic::<bf16>(m, k, n, group_size, bits, use_zero_points, use_mlx_quant);
+ test_internal::<bf16>(&input, &expected);
+ }
+}
+
+fn test_edge(
+ group_size: u32,
+ bits: u32,
+ use_zero_points: bool,
+ use_mlx_quant: bool,
+) {
+ let (input, expected) = get_test_data_edge::<bf16>(group_size, bits, use_zero_points, use_mlx_quant);
+ test_internal::<bf16>(&input, &expected);
+}
+
+// -- 4-bit, zero points -------------------------------------------------------
+
+#[uzu_test]
+#[ignore]
+fn test_bf16_gs64_4bit_zp() {
+ test_basic(64, 4, true, false);
+}
+
+#[uzu_test]
+#[ignore]
+fn test_bf16_gs128_4bit_zp() {
+ test_basic(128, 4, true, false);
+}
+
+// -- 8-bit, zero points -------------------------------------------------------
+
+#[uzu_test]
+fn test_bf16_gs64_8bit_zp() {
+ test_basic(64, 8, true, false);
+}
+
+#[uzu_test]
+fn test_bf16_gs128_8bit_zp() {
+ test_basic(128, 8, true, false);
+}
+
+// -- 4-bit, mlx quant ----------------------------------------------------------
+
+#[uzu_test]
+fn test_bf16_gs64_4bit_mlx() {
+ test_basic(64, 4, false, true);
+}
+
+#[uzu_test]
+fn test_bf16_gs128_4bit_mlx() {
+ test_basic(128, 4, false, true);
+}
+
+// -- 8-bit, mlx quant ----------------------------------------------------------
+
+#[uzu_test]
+fn test_bf16_gs64_8bit_mlx() {
+ test_basic(64, 8, false, true);
+}
+
+#[uzu_test]
+fn test_bf16_gs128_8bit_mlx() {
+ test_basic(128, 8, false, true);
+}
+
+// -- Edge cases ----------------------------------------------------------------
+
+#[uzu_test]
+fn test_edge_bf16_4bit_zp() {
+ test_edge(64, 4, true, false);
+}
+
+#[uzu_test]
+fn test_edge_bf16_8bit_zp() {
+ test_edge(64, 8, true, false);
+}
+
+#[uzu_test]
+fn test_edge_bf16_4bit_mlx() {
+ test_edge(64, 4, false, true);
+}
+
+#[uzu_test]
+fn test_edge_bf16_8bit_mlx() {
+ test_edge(64, 8, false, true);
+}
diff --git a/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rs b/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rs
new file mode 100644
index 000000000..db6185dd3
--- /dev/null
+++ b/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rs
@@ -0,0 +1,530 @@
+use std::fmt::{Debug, Display};
+
+use half::{bf16, f16};
+use num_traits::Float;
+use uzu::{
+ ArrayElement, DataType,
+ backends::{
+ common::{Backend, Buffer, Context, Encoder, Kernels, kernel::QuantizedMatmulQmmTransposedKernel},
+ cpu::Cpu,
+ },
+};
+
+use super::{Input, check_tolerance, pack_weights_u32, pack_zero_points};
+use crate::{common::helpers::alloc_buffer_with_data, uzu_test};
+
+fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
+ let context = B::Context::new().expect("Failed to create Context");
+ let aligned_n = (input.n % 32) == 0;
+ let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel::new(
+ &context,
+ T::data_type(),
+ input.group_size,
+ input.bits,
+ input.use_zero_points,
+ input.use_mlx_quant,
+ aligned_n,
+ )
+ .expect("Failed to create QuantizedMatmulQmmTransposedKernel");
+
+ let w_buf = alloc_buffer_with_data::<B, u32>(&context, &input.w_packed);
+ let scales_buf = alloc_buffer_with_data::<B, T>(&context, &input.scales);
+
+ let zp_buf = input.zero_points.as_ref().map(|zp| alloc_buffer_with_data::<B, u8>(&context, zp));
+ let bias_buf = input.biases.as_ref().map(|b| alloc_buffer_with_data::<B, T>(&context, b));
+
+ let x_buf = alloc_buffer_with_data::<B, T>(&context, &input.x);
+ let output_size = (input.m as usize) * (input.n as usize) * T::data_type().size_in_bytes();
+ let mut y_buf = context.create_buffer(output_size).expect("Failed to create buffer");
+
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
+ kernel.encode(
+ &w_buf,
+ &scales_buf,
+ zp_buf.as_ref(),
+ bias_buf.as_ref(),
+ &x_buf,
+ &mut y_buf,
+ input.k,
+ input.n,
+ input.m,
+ &mut encoder,
+ );
+
+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
+
+ let y_ptr = y_buf.cpu_ptr().as_ptr() as *const T;
+ let y_len = (input.m as usize) * (input.n as usize);
+ unsafe { std::slice::from_raw_parts(y_ptr, y_len) }.to_vec()
+}
+
+/// Create test data for transposed qmm: x[m×k] * w_transposed[n×k] = y[m×n]
+/// Weights are in transposed [n × k] layout.
+fn get_test_data_basic<T: ArrayElement + Float>(
+ m: usize,
+ k: usize,
+ n: usize,
+ group_size: u32,
+ bits: u32,
+ use_zero_points: bool,
+ use_mlx_quant: bool,
+) -> (Input<T>, Vec<T>) {
+ let num_groups_k = (k + group_size as usize - 1) / group_size as usize;
+
+ // Generate quantized weights [n × k]
+ let max_val = if bits == 4 {
+ 15u8
+ } else {
+ 255u8
+ };
+ let mut weights_raw: Vec<u8> = Vec::with_capacity(n * k);
+ for j in 0..n {
+ for l in 0..k {
+ weights_raw.push(((j * 3 + l * 7 + 1) % (max_val as usize + 1)) as u8);
+ }
+ }
+ let w_packed = pack_weights_u32(&weights_raw, bits);
+
+ // Scales: [n × num_groups_k]
+ let mut scales_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);
+ for j in 0..n {
+ for g in 0..num_groups_k {
+ scales_f32.push(0.5 + 0.1 * ((j + g) % 5) as f32);
+ }
+ }
+ let scales: Vec<T> = scales_f32.iter().map(|&v| T::from(v).unwrap()).collect();
+
+ // Zero points or biases
+ let zp_stride = if bits == 4 {
+ (num_groups_k + 1) / 2
+ } else {
+ num_groups_k
+ };
+
+ let (zero_points, biases) = if use_zero_points {
+ let mut zp_raw: Vec<u8> = Vec::with_capacity(n * num_groups_k);
+ for j in 0..n {
+ for g in 0..num_groups_k {
+ let zp_val = ((j * 2 + g * 3) % (max_val as usize + 1)) as u8;
+ zp_raw.push(zp_val);
+ }
+ }
+ // Pack zero points per row
+ let mut zp_packed: Vec<u8> = Vec::with_capacity(n * zp_stride);
+ for j in 0..n {
+ let row = &zp_raw[j * num_groups_k..(j + 1) * num_groups_k];
+ let packed_row = pack_zero_points(row, bits);
+ let mut padded = packed_row;
+ padded.resize(zp_stride, 0);
+ zp_packed.extend_from_slice(&padded);
+ }
+ (Some(zp_packed), None)
+ } else if use_mlx_quant {
+ let mut biases_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);
+ for j in 0..n {
+ for g in 0..num_groups_k {
+ biases_f32.push(0.01 * ((j + g * 2) % 7) as f32);
+ }
+ }
+ let biases: Vec<T> = biases_f32.iter().map(|&v| T::from(v).unwrap()).collect();
+ (None, Some(biases))
+ } else {
+ unreachable!("Must use either zero_points or mlx_quant");
+ };
+
+ // X: [m × k]
+ let mut x_f32: Vec<f32> = Vec::with_capacity(m * k);
+ for i in 0..m {
+ for l in 0..k {
+ x_f32.push(0.1 * f32::sin((i * k + l) as f32 * 0.05) + 0.5);
+ }
+ }
+ let x: Vec<T> = x_f32.iter().map(|&v| T::from(v).unwrap()).collect();
+
+ let input = Input {
+ w_packed,
+ scales,
+ zero_points,
+ biases,
+ x,
+ k: k as u32,
+ n: n as u32,
+ m: m as u32,
+ group_size,
+ bits,
+ use_zero_points,
+ use_mlx_quant,
+ };
+
+ let expected = get_output::<Cpu, T>(&input);
+ (input, expected)
+}
+
+/// Small edge-case test data with known values
+fn get_test_data_edge<T: ArrayElement + Float>(
+ group_size: u32,
+ bits: u32,
+ use_zero_points: bool,
+ use_mlx_quant: bool,
+) -> (Input<T>, Vec<T>) {
+ let m = 2usize;
+ let k = group_size as usize;
+ let n = 32usize;
+ let num_groups_k = 1;
+
+ // Simple weights [n × k]: all 1s
+ let weights_raw: Vec<u8> = vec![1u8; n * k];
+ let w_packed = pack_weights_u32(&weights_raw, bits);
+
+ // Unit scales
+ let scales: Vec<T> = vec![T::one(); n * num_groups_k];
+
+ let zp_stride = if bits == 4 {
+ (num_groups_k + 1) / 2
+ } else {
+ num_groups_k
+ };
+
+ let (zero_points, biases) = if use_zero_points {
+ // Zero points all 0 -> bias = 0
+ let zp_packed = vec![0u8; n * zp_stride];
+ (Some(zp_packed), None)
+ } else if use_mlx_quant {
+ let biases: Vec<T> = vec![T::zero(); n * num_groups_k];
+ (None, Some(biases))
+ } else {
+ unreachable!("Must use either zero_points or mlx_quant");
+ };
+
+ // X: simple increasing values
+ let x_f32: Vec<f32> = (0..m * k).map(|i| (i + 1) as f32 * 0.1).collect();
+ let x: Vec<T> = x_f32.iter().map(|&v| T::from(v).unwrap()).collect();
+
+ let input = Input {
+ w_packed,
+ scales,
+ zero_points,
+ biases,
+ x,
+ k: k as u32,
+ n: n as u32,
+ m: m as u32,
+ group_size,
+ bits,
+ use_zero_points,
+ use_mlx_quant,
+ };
+
+ let expected = get_output::<Cpu, T>(&input);
+ (input, expected)
+}
+
+fn test_internal<T: ArrayElement + Float + Debug + Display>(
+ input: &Input<T>,
+ expected: &[T],
+) {
+ // GPU MMA units use different rounding than scalar CPU code.
+ // bf16 has 7-bit mantissa (ULP ~0.8%), f16 has 10-bit (~0.1%).
+ // Tiled accumulation reorders FMA ops, compounding rounding differences.
+ // 8-bit weights (0-255) with ZP bias cause larger accumulated errors.
+ let (rel_tol, abs_tol): (f64, f64) = match T::data_type() {
+ DataType::BF16 => (0.05, 0.5),
+ DataType::F16 => (0.02, 0.5),
+ _ => (0.01, 0.1),
+ };
+
+ for_each_non_cpu_backend!(|B| {
+ let output = get_output::<B, T>(input);
+ assert_eq!(expected.len(), output.len(), "Output length mismatch");
+
+ let mut errors = 0;
+ for (i, (&exp, &got)) in expected.iter().zip(output.iter()).enumerate() {
+ let exp_f32 = exp.to_f32().unwrap();
+ let got_f32 = got.to_f32().unwrap();
+ if !check_tolerance(exp_f32, got_f32, rel_tol, abs_tol) {
+ if errors < 5 {
+ eprintln!(" idx={} expected={} got={} diff={}", i, exp_f32, got_f32, (exp_f32 - got_f32).abs());
+ }
+ errors += 1;
+ }
+ }
+ assert_eq!(
+ errors,
+ 0,
+ "QMM transposed kernel: backend={}, m={}, k={}, n={}, gs={}, bits={}, zp={}, mlx={}: {} mismatches",
+ std::any::type_name::<B>(),
+ input.m,
+ input.k,
+ input.n,
+ input.group_size,
+ input.bits,
+ input.use_zero_points,
+ input.use_mlx_quant,
+ errors,
+ );
+ });
+}
+
+fn get_test_dims(
+ group_size: u32,
+ bits: u32,
+ is_half_precision: bool,
+) -> Vec<(usize, usize, usize)> {
+ let gs = group_size as usize;
+ // K must be >= group_size for the transposed grouping along K.
+ // N must be >= 32 for the Metal tiled weight loader (BN=32).
+ // For half-precision 8-bit, keep dimensions small to avoid
+ // catastrophic cancellation from large weight values (0-255).
+ if is_half_precision && bits == 8 {
+ vec![(2, gs, gs), (4, gs, gs)]
+ } else {
+ vec![(2, gs * 2, gs), (4, gs * 2, gs * 2), (8, gs, gs * 2)]
+ }
+}
+
+fn test_basic<T: ArrayElement + Float + Debug + Display>(
+ group_size: u32,
+ bits: u32,
+ use_zero_points: bool,
+ use_mlx_quant: bool,
+) {
+ let is_half = matches!(T::data_type(), DataType::F16 | DataType::BF16);
+ let dims = get_test_dims(group_size, bits, is_half);
+
+ for (m, k, n) in dims {
+ let (input, expected) = get_test_data_basic::<T>(m, k, n, group_size, bits, use_zero_points, use_mlx_quant);
+ test_internal::<T>(&input, &expected);
+ }
+}
+
+fn test_edge<T: ArrayElement + Float + Debug + Display>(
+ group_size: u32,
+ bits: u32,
+ use_zero_points: bool,
+ use_mlx_quant: bool,
+) {
+ let (input, expected) = get_test_data_edge::<T>(group_size, bits, use_zero_points, use_mlx_quant);
+ test_internal::<T>(&input, &expected);
+}
+
+// -- 4-bit, zero points -------------------------------------------------------
+
+#[uzu_test]
+fn test_f32_gs32_4bit_zp() {
+ test_basic::<f32>(32, 4, true, false);
+}
+
+#[uzu_test]
+fn test_f32_gs64_4bit_zp() {
+ test_basic::<f32>(64, 4, true, false);
+}
+
+#[uzu_test]
+fn test_f32_gs128_4bit_zp() {
+ test_basic::<f32>(128, 4, true, false);
+}
+
+#[uzu_test]
+fn test_f16_gs32_4bit_zp() {
+ test_basic::<f16>(32, 4, true, false);
+}
+
+#[uzu_test]
+fn test_f16_gs64_4bit_zp() {
+ test_basic::<f16>(64, 4, true, false);
+}
+
+#[uzu_test]
+fn test_f16_gs128_4bit_zp() {
+ test_basic::<f16>(128, 4, true, false);
+}
+
+#[uzu_test]
+fn test_bf16_gs32_4bit_zp() {
+ test_basic::<bf16>(32, 4, true, false);
+}
+
+#[uzu_test]
+#[ignore]
+fn test_bf16_gs64_4bit_zp() {
+ test_basic::<bf16>(64, 4, true, false);
+}
+
+#[uzu_test]
+#[ignore]
+fn test_bf16_gs128_4bit_zp() {
+ test_basic::<bf16>(128, 4, true, false);
+}
+
+// -- 8-bit, zero points -------------------------------------------------------
+
+#[uzu_test]
+fn test_f32_gs32_8bit_zp() {
+ test_basic::<f32>(32, 8, true, false);
+}
+
+#[uzu_test]
+fn test_f32_gs64_8bit_zp() {
+ test_basic::<f32>(64, 8, true, false);
+}
+
+#[uzu_test]
+fn test_f32_gs128_8bit_zp() {
+ test_basic::<f32>(128, 8, true, false);
+}
+
+#[uzu_test]
+fn test_f16_gs32_8bit_zp() {
+ test_basic::<f16>(32, 8, true, false);
+}
+
+#[uzu_test]
+fn test_f16_gs64_8bit_zp() {
+ test_basic::<f16>(64, 8, true, false);
+}
+
+#[uzu_test]
+fn test_f16_gs128_8bit_zp() {
+ test_basic::<f16>(128, 8, true, false);
+}
+
+#[uzu_test]
+fn test_bf16_gs32_8bit_zp() {
+ test_basic::<bf16>(32, 8, true, false);
+}
+
+#[uzu_test]
+fn test_bf16_gs64_8bit_zp() {
+ test_basic::<bf16>(64, 8, true, false);
+}
+
+#[uzu_test]
+fn test_bf16_gs128_8bit_zp() {
+ test_basic::<bf16>(128, 8, true, false);
+}
+
+// -- 4-bit, mlx quant ----------------------------------------------------------
+
+#[uzu_test]
+fn test_f32_gs32_4bit_mlx() {
+ test_basic::<f32>(32, 4, false, true);
+}
+
+#[uzu_test]
+fn test_f32_gs64_4bit_mlx() {
+ test_basic::<f32>(64, 4, false, true);
+}
+
+#[uzu_test]
+fn test_f32_gs128_4bit_mlx() {
+ test_basic::<f32>(128, 4, false, true);
+}
+
+#[uzu_test]
+fn test_f16_gs32_4bit_mlx() {
+ test_basic::<f16>(32, 4, false, true);
+}
+
+#[uzu_test]
+fn test_f16_gs64_4bit_mlx() {
+ test_basic::<f16>(64, 4, false, true);
+}
+
+#[uzu_test]
+fn test_f16_gs128_4bit_mlx() {
+ test_basic::<f16>(128, 4, false, true);
+}
+
+#[uzu_test]
+fn test_bf16_gs32_4bit_mlx() {
+ test_basic::<bf16>(32, 4, false, true);
+}
+
+#[uzu_test]
+fn test_bf16_gs64_4bit_mlx() {
+ test_basic::<bf16>(64, 4, false, true);
+}
+
+#[uzu_test]
+fn test_bf16_gs128_4bit_mlx() {
+ test_basic::<bf16>(128, 4, false, true);
+}
+
+// -- 8-bit, mlx quant ----------------------------------------------------------
+
+#[uzu_test]
+fn test_f32_gs32_8bit_mlx() {
+ test_basic::<f32>(32, 8, false, true);
+}
+
+#[uzu_test]
+fn test_f32_gs64_8bit_mlx() {
+ test_basic::<f32>(64, 8, false, true);
+}
+
+#[uzu_test]
+fn test_f32_gs128_8bit_mlx() {
+ test_basic::<f32>(128, 8, false, true);
+}
+
+#[uzu_test]
+fn test_f16_gs32_8bit_mlx() {
+ test_basic::<f16>(32, 8, false, true);
+}
+
+#[uzu_test]
+fn test_f16_gs64_8bit_mlx() {
+ test_basic::<f16>(64, 8, false, true);
+}
+
+#[uzu_test]
+fn test_f16_gs128_8bit_mlx() {
+ test_basic::<f16>(128, 8, false, true);
+}
+
+#[uzu_test]
+fn test_bf16_gs32_8bit_mlx() {
+ test_basic::<bf16>(32, 8, false, true);
+}
+
+#[uzu_test]
+fn test_bf16_gs64_8bit_mlx() {
+ test_basic::<bf16>(64, 8, false, true);
+}
+
+#[uzu_test]
+fn test_bf16_gs128_8bit_mlx() {
+ test_basic::<bf16>(128, 8, false, true);
+}
+
+// -- Edge cases ----------------------------------------------------------------
+
+#[uzu_test]
+fn test_edge_f32_4bit_zp() {
+ test_edge::<f32>(32, 4, true, false);
+}
+
+#[uzu_test]
+fn test_edge_f32_8bit_zp() {
+ test_edge::<f32>(32, 8, true, false);
+}
+
+#[uzu_test]
+fn test_edge_f16_4bit_zp() {
+ test_edge::<f16>(32, 4, true, false);
+}
+
+#[uzu_test]
+fn test_edge_f32_4bit_mlx() {
+ test_edge::<f32>(32, 4, false, true);
+}
+
+#[uzu_test]
+fn test_edge_f32_8bit_mlx() {
+ test_edge::<f32>(32, 8, false, true);
+}
+
+#[uzu_test]
+fn test_edge_f16_4bit_mlx() {
+ test_edge::<f16>(32, 4, false, true);
+}
diff --git a/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_wide_test.rs b/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_wide_test.rs
new file mode 100644
index 000000000..59a74348b
--- /dev/null
+++ b/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_wide_test.rs
@@ -0,0 +1,392 @@
+use std::fmt::{Debug, Display};
+
+use half::bf16;
+use num_traits::Float;
+use uzu::{
+ ArrayElement, DataType,
+ backends::{
+ common::{Backend, Buffer, Context, Encoder, Kernels, kernel::QuantizedMatmulQmmTransposedWideKernel},
+ cpu::Cpu,
+ },
+};
+
+use super::{Input, check_tolerance, pack_weights_u32, pack_zero_points};
+use crate::{common::helpers::alloc_buffer_with_data, uzu_test};
+
+fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
+ let context = B::Context::new().expect("Failed to create Context");
+ let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmmTransposedWideKernel::new(
+ &context,
+ T::data_type(),
+ input.group_size,
+ input.bits,
+ input.use_zero_points,
+ input.use_mlx_quant,
+ )
+ .expect("Failed to create QuantizedMatmulQmmTransposedWideKernel");
+
+ let w_buf = alloc_buffer_with_data::<B, u32>(&context, &input.w_packed);
+ let scales_buf = alloc_buffer_with_data::<B, T>(&context, &input.scales);
+
+ let zp_buf = input.zero_points.as_ref().map(|zp| alloc_buffer_with_data::<B, u8>(&context, zp));
+ let bias_buf = input.biases.as_ref().map(|b| alloc_buffer_with_data::<B, T>(&context, b));
+
+ let x_buf = alloc_buffer_with_data::<B, T>(&context, &input.x);
+ let output_size = (input.m as usize) * (input.n as usize) * T::data_type().size_in_bytes();
+ let mut y_buf = context.create_buffer(output_size).expect("Failed to create buffer");
+
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
+ kernel.encode(
+ &w_buf,
+ &scales_buf,
+ zp_buf.as_ref(),
+ bias_buf.as_ref(),
+ &x_buf,
+ &mut y_buf,
+ input.k,
+ input.n,
+ input.m,
+ &mut encoder,
+ );
+
+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
+
+ let y_ptr = y_buf.cpu_ptr().as_ptr() as *const T;
+ let y_len = (input.m as usize) * (input.n as usize);
+ unsafe { std::slice::from_raw_parts(y_ptr, y_len) }.to_vec()
+}
+
+/// Create test data for transposed wide qmm: x[m×k] * w_transposed[n×k] = y[m×n]
+/// Weights are in transposed [n × k] layout.
+fn get_test_data_basic<T: ArrayElement + Float>(
+ m: usize,
+ k: usize,
+ n: usize,
+ group_size: u32,
+ bits: u32,
+ use_zero_points: bool,
+ use_mlx_quant: bool,
+) -> (Input<T>, Vec<T>) {
+ let num_groups_k = (k + group_size as usize - 1) / group_size as usize;
+
+ // Generate quantized weights [n × k]
+ let max_val = if bits == 4 {
+ 15u8
+ } else {
+ 255u8
+ };
+ let mut weights_raw: Vec<u8> = Vec::with_capacity(n * k);
+ for j in 0..n {
+ for l in 0..k {
+ weights_raw.push(((j * 3 + l * 7 + 1) % (max_val as usize + 1)) as u8);
+ }
+ }
+ let w_packed = pack_weights_u32(&weights_raw, bits);
+
+ // Scales: [n × num_groups_k]
+ let mut scales_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);
+ for j in 0..n {
+ for g in 0..num_groups_k {
+ scales_f32.push(0.5 + 0.1 * ((j + g) % 5) as f32);
+ }
+ }
+ let scales: Vec<T> = scales_f32.iter().map(|&v| T::from(v).unwrap()).collect();
+
+ // Zero points or biases
+ let zp_stride = if bits == 4 {
+ (num_groups_k + 1) / 2
+ } else {
+ num_groups_k
+ };
+
+ let (zero_points, biases) = if use_zero_points {
+ let mut zp_raw: Vec<u8> = Vec::with_capacity(n * num_groups_k);
+ for j in 0..n {
+ for g in 0..num_groups_k {
+ let zp_val = ((j * 2 + g * 3) % (max_val as usize + 1)) as u8;
+ zp_raw.push(zp_val);
+ }
+ }
+ // Pack zero points per row
+ let mut zp_packed: Vec<u8> = Vec::with_capacity(n * zp_stride);
+ for j in 0..n {
+ let row = &zp_raw[j * num_groups_k..(j + 1) * num_groups_k];
+ let packed_row = pack_zero_points(row, bits);
+ let mut padded = packed_row;
+ padded.resize(zp_stride, 0);
+ zp_packed.extend_from_slice(&padded);
+ }
+ (Some(zp_packed), None)
+ } else if use_mlx_quant {
+ let mut biases_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);
+ for j in 0..n {
+ for g in 0..num_groups_k {
+ biases_f32.push(0.01 * ((j + g * 2) % 7) as f32);
+ }
+ }
+ let biases: Vec<T> = biases_f32.iter().map(|&v| T::from(v).unwrap()).collect();
+ (None, Some(biases))
+ } else {
+ unreachable!("Must use either zero_points or mlx_quant");
+ };
+
+ // X: [m × k]
+ let mut x_f32: Vec<f32> = Vec::with_capacity(m * k);
+ for i in 0..m {
+ for l in 0..k {
+ x_f32.push(0.1 * f32::sin((i * k + l) as f32 * 0.05) + 0.5);
+ }
+ }
+ let x: Vec<T> = x_f32.iter().map(|&v| T::from(v).unwrap()).collect();
+
+ let input = Input {
+ w_packed,
+ scales,
+ zero_points,
+ biases,
+ x,
+ k: k as u32,
+ n: n as u32,
+ m: m as u32,
+ group_size,
+ bits,
+ use_zero_points,
+ use_mlx_quant,
+ };
+
+ let expected = get_output::<Cpu, T>(&input);
+ (input, expected)
+}
+
+/// Small edge-case test data with known values
+fn get_test_data_edge<T: ArrayElement + Float>(
+ group_size: u32,
+ bits: u32,
+ use_zero_points: bool,
+ use_mlx_quant: bool,
+) -> (Input<T>, Vec<T>) {
+ let m = 2usize;
+ let k = group_size as usize;
+ let n = 64usize;
+ let num_groups_k = 1;
+
+ // Simple weights [n × k]: all 1s
+ let weights_raw: Vec<u8> = vec![1u8; n * k];
+ let w_packed = pack_weights_u32(&weights_raw, bits);
+
+ // Unit scales
+ let scales: Vec<T> = vec![T::one(); n * num_groups_k];
+
+ let zp_stride = if bits == 4 {
+ (num_groups_k + 1) / 2
+ } else {
+ num_groups_k
+ };
+
+ let (zero_points, biases) = if use_zero_points {
+ // Zero points all 0 -> bias = 0
+ let zp_packed = vec![0u8; n * zp_stride];
+ (Some(zp_packed), None)
+ } else if use_mlx_quant {
+ let biases: Vec<T> = vec![T::zero(); n * num_groups_k];
+ (None, Some(biases))
+ } else {
+ unreachable!("Must use either zero_points or mlx_quant");
+ };
+
+ // X: simple increasing values
+ let x_f32: Vec<f32> = (0..m * k).map(|i| (i + 1) as f32 * 0.1).collect();
+ let x: Vec<T> = x_f32.iter().map(|&v| T::from(v).unwrap()).collect();
+
+ let input = Input {
+ w_packed,
+ scales,
+ zero_points,
+ biases,
+ x,
+ k: k as u32,
+ n: n as u32,
+ m: m as u32,
+ group_size,
+ bits,
+ use_zero_points,
+ use_mlx_quant,
+ };
+
+ let expected = get_output::<Cpu, T>(&input);
+ (input, expected)
+}
+
+fn test_internal<T: ArrayElement + Float + Debug + Display>(
+ input: &Input<T>,
+ expected: &[T],
+) {
+ let (rel_tol, abs_tol): (f64, f64) = match T::data_type() {
+ DataType::BF16 => (0.05, 0.5),
+ DataType::F16 => (0.02, 0.5),
+ _ => (0.01, 0.1),
+ };
+
+ for_each_non_cpu_backend!(|B| {
+ let output = get_output::<B, T>(input);
+ assert_eq!(expected.len(), output.len(), "Output length mismatch");
+
+ let mut errors = 0;
+ for (i, (&exp, &got)) in expected.iter().zip(output.iter()).enumerate() {
+ let exp_f32 = exp.to_f32().unwrap();
+ let got_f32 = got.to_f32().unwrap();
+ if !check_tolerance(exp_f32, got_f32, rel_tol, abs_tol) {
+ if errors < 5 {
+ eprintln!(" idx={} expected={} got={} diff={}", i, exp_f32, got_f32, (exp_f32 - got_f32).abs());
+ }
+ errors += 1;
+ }
+ }
+ assert_eq!(
+ errors,
+ 0,
+ "QMM transposed wide kernel: backend={}, m={}, k={}, n={}, gs={}, bits={}, zp={}, mlx={}: {} mismatches",
+ std::any::type_name::<B>(),
+ input.m,
+ input.k,
+ input.n,
+ input.group_size,
+ input.bits,
+ input.use_zero_points,
+ input.use_mlx_quant,
+ errors,
+ );
+ });
+}
+
+fn get_test_dims(
+ group_size: u32,
+ bits: u32,
+) -> Vec<(usize, usize, usize)> {
+ let gs = group_size as usize;
+ // Wide kernel: BM=64, BK=32, BN=64.
+ // K must be >= group_size (transposed grouping along K).
+ // N must be a multiple of 64 (BN tile size).
+ // bf16 8-bit: keep dims small to avoid catastrophic cancellation.
+ if bits == 8 {
+ vec![(2, gs, 64), (4, gs, 64)]
+ } else {
+ vec![(2, gs * 2, 64), (4, gs * 2, 128), (8, gs, 128)]
+ }
+}
+
+fn test_basic(
+ group_size: u32,
+ bits: u32,
+ use_zero_points: bool,
+ use_mlx_quant: bool,
+) {
+ let dims = get_test_dims(group_size, bits);
+
+ for (m, k, n) in dims {
+ let (input, expected) = get_test_data_basic::<bf16>(m, k, n, group_size, bits, use_zero_points, use_mlx_quant);
+ test_internal::<bf16>(&input, &expected);
+ }
+}
+
+fn test_edge(
+ group_size: u32,
+ bits: u32,
+ use_zero_points: bool,
+ use_mlx_quant: bool,
+) {
+ let (input, expected) = get_test_data_edge::<bf16>(group_size, bits, use_zero_points, use_mlx_quant);
+ test_internal::<bf16>(&input, &expected);
+}
+
+// -- 4-bit, zero points -------------------------------------------------------
+
+#[uzu_test]
+fn test_bf16_gs32_4bit_zp() {
+ test_basic(32, 4, true, false);
+}
+
+#[uzu_test]
+#[ignore]
+fn test_bf16_gs64_4bit_zp() {
+ test_basic(64, 4, true, false);
+}
+
+#[uzu_test]
+#[ignore]
+fn test_bf16_gs128_4bit_zp() {
+ test_basic(128, 4, true, false);
+}
+
+// -- 8-bit, zero points -------------------------------------------------------
+
+#[uzu_test]
+fn test_bf16_gs32_8bit_zp() {
+ test_basic(32, 8, true, false);
+}
+
+#[uzu_test]
+fn test_bf16_gs64_8bit_zp() {
+ test_basic(64, 8, true, false);
+}
+
+#[uzu_test]
+fn test_bf16_gs128_8bit_zp() {
+ test_basic(128, 8, true, false);
+}
+
+// -- 4-bit, mlx quant ----------------------------------------------------------
+
+#[uzu_test]
+fn test_bf16_gs32_4bit_mlx() {
+ test_basic(32, 4, false, true);
+}
+
+#[uzu_test]
+fn test_bf16_gs64_4bit_mlx() {
+ test_basic(64, 4, false, true);
+}
+
+#[uzu_test]
+fn test_bf16_gs128_4bit_mlx() {
+ test_basic(128, 4, false, true);
+}
+
+// -- 8-bit, mlx quant ----------------------------------------------------------
+
+#[uzu_test]
+fn test_bf16_gs32_8bit_mlx() {
+ test_basic(32, 8, false, true);
+}
+
+#[uzu_test]
+fn test_bf16_gs64_8bit_mlx() {
+ test_basic(64, 8, false, true);
+}
+
+#[uzu_test]
+fn test_bf16_gs128_8bit_mlx() {
+ test_basic(128, 8, false, true);
+}
+
+// -- Edge cases ----------------------------------------------------------------
+
+#[uzu_test]
+fn test_edge_bf16_4bit_zp() {
+ test_edge(32, 4, true, false);
+}
+
+#[uzu_test]
+fn test_edge_bf16_8bit_zp() {
+ test_edge(32, 8, true, false);
+}
+
+#[uzu_test]
+fn test_edge_bf16_4bit_mlx() {
+ test_edge(32, 4, false, true);
+}
+
+#[uzu_test]
+fn test_edge_bf16_8bit_mlx() {
+ test_edge(32, 8, false, true);
+}
diff --git a/crates/uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs b/crates/uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs
new file mode 100644
index 000000000..56aefbdbe
--- /dev/null
+++ b/crates/uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs
@@ -0,0 +1,366 @@
+use std::fmt::{Debug, Display};
+
+use num_traits::Float;
+use uzu::{
+ ArrayElement, DataType,
+ backends::{
+ common::{
+ Backend, Buffer, Context, Encoder, Kernels,
+ kernel::{QuantizedMatmulQmvFastKernel, QuantizedMatmulQmvKernel},
+ },
+ cpu::Cpu,
+ },
+};
+
+use super::{Input, check_tolerance, pack_weights_u32, pack_zero_points};
+use crate::{common::helpers::alloc_buffer_with_data, uzu_test};
+
+fn get_expected<T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
+ let context = <Cpu as Backend>::Context::new().expect("Failed to create Context");
+
+ let w_buf = alloc_buffer_with_data::<Cpu, u32>(&context, &input.w_packed);
+ let scales_buf = alloc_buffer_with_data::<Cpu, T>(&context, &input.scales);
+ let zp_buf = input.zero_points.as_ref().map(|zp| alloc_buffer_with_data::<Cpu, u8>(&context, zp));
+ let bias_buf = input.biases.as_ref().map(|b| alloc_buffer_with_data::<Cpu, T>(&context, b));
+ let x_buf = alloc_buffer_with_data::<Cpu, T>(&context, &input.x);
+ let output_size = (input.m as usize) * (input.n as usize) * T::data_type().size_in_bytes();
+ let mut y_buf = context.create_buffer(output_size).expect("Failed to create buffer");
+
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
+
+ let kernel = <<Cpu as Backend>::Kernels as Kernels>::QuantizedMatmulQmvKernel::new(
+ &context,
+ T::data_type(),
+ input.group_size,
+ input.bits,
+ input.use_zero_points,
+ input.use_mlx_quant,
+ )
+ .expect("Failed to create QuantizedMatmulQmvKernel");
+ kernel.encode(
+ &w_buf,
+ &scales_buf,
+ zp_buf.as_ref(),
+ bias_buf.as_ref(),
+ &x_buf,
+ &mut y_buf,
+ input.k,
+ input.n,
+ input.m,
+ &mut encoder,
+ );
+
+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
+
+ let y_ptr = y_buf.cpu_ptr().as_ptr() as *const T;
+ let y_len = (input.m as usize) * (input.n as usize);
+ unsafe { std::slice::from_raw_parts(y_ptr, y_len) }.to_vec()
+}
+
+fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
+ let context = B::Context::new().expect("Failed to create Context");
+
+ let w_buf = alloc_buffer_with_data::<B, u32>(&context, &input.w_packed);
+ let scales_buf = alloc_buffer_with_data::<B, T>(&context, &input.scales);
+ let zp_buf = input.zero_points.as_ref().map(|zp| alloc_buffer_with_data::<B, u8>(&context, zp));
+ let bias_buf = input.biases.as_ref().map(|b| alloc_buffer_with_data::<B, T>(&context, b));
+ let x_buf = alloc_buffer_with_data::<B, T>(&context, &input.x);
+ let output_size = (input.m as usize) * (input.n as usize) * T::data_type().size_in_bytes();
+ let mut y_buf = context.create_buffer(output_size).expect("Failed to create buffer");
+
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
+
+ let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmvFastKernel::new(
+ &context,
+ T::data_type(),
+ input.group_size,
+ input.bits,
+ input.use_zero_points,
+ input.use_mlx_quant,
+ )
+ .expect("Failed to create QuantizedMatmulQmvFastKernel");
+ kernel.encode(
+ &w_buf,
+ &scales_buf,
+ zp_buf.as_ref(),
+ bias_buf.as_ref(),
+ &x_buf,
+ &mut y_buf,
+ input.k,
+ input.n,
+ input.m,
+ &mut encoder,
+ );
+
+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
+
+ let y_ptr = y_buf.cpu_ptr().as_ptr() as *const T;
+ let y_len = (input.m as usize) * (input.n as usize);
+ unsafe { std::slice::from_raw_parts(y_ptr, y_len) }.to_vec()
+}
+
+fn get_test_data<T: ArrayElement + Float>(
+ m: usize,
+ k: usize,
+ n: usize,
+ group_size: u32,
+ bits: u32,
+ use_zero_points: bool,
+ use_mlx_quant: bool,
+) -> (Input<T>, Vec<T>) {
+ let num_groups_k = (k + group_size as usize - 1) / group_size as usize;
+
+ let mut weights_raw: Vec<u8> = Vec::with_capacity(n * k);
+ let max_val = if bits == 4 {
+ 15u8
+ } else {
+ 255u8
+ };
+ for j in 0..n {
+ for l in 0..k {
+ weights_raw.push(((j * 3 + l * 7 + 1) % (max_val as usize + 1)) as u8);
+ }
+ }
+ let w_packed = pack_weights_u32(&weights_raw, bits);
+
+ let mut scales_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);
+ for j in 0..n {
+ for g in 0..num_groups_k {
+ scales_f32.push(0.5 + 0.1 * ((j + g) % 5) as f32);
+ }
+ }
+ let scales: Vec<T> = scales_f32.iter().map(|&v| T::from(v).unwrap()).collect();
+
+ let zp_stride = if bits == 4 {
+ (num_groups_k + 1) / 2
+ } else {
+ num_groups_k
+ };
+
+ let (zero_points, biases) = if use_zero_points {
+ let mut zp_raw: Vec<u8> = Vec::with_capacity(n * num_groups_k);
+ for j in 0..n {
+ for g in 0..num_groups_k {
+ let zp_val = ((j * 2 + g * 3) % (max_val as usize + 1)) as u8;
+ zp_raw.push(zp_val);
+ }
+ }
+ let mut zp_packed: Vec<u8> = Vec::with_capacity(n * zp_stride);
+ for j in 0..n {
+ let row = &zp_raw[j * num_groups_k..(j + 1) * num_groups_k];
+ let packed_row = pack_zero_points(row, bits);
+ let mut padded = packed_row;
+ padded.resize(zp_stride, 0);
+ zp_packed.extend_from_slice(&padded);
+ }
+ (Some(zp_packed), None)
+ } else if use_mlx_quant {
+ let mut biases_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);
+ for j in 0..n {
+ for g in 0..num_groups_k {
+ biases_f32.push(0.01 * ((j + g * 2) % 7) as f32);
+ }
+ }
+ let biases: Vec<T> = biases_f32.iter().map(|&v| T::from(v).unwrap()).collect();
+ (None, Some(biases))
+ } else {
+ unreachable!("Must use either zero_points or mlx_quant");
+ };
+
+ let mut x_f32: Vec<f32> = Vec::with_capacity(m * k);
+ for i in 0..m {
+ for l in 0..k {
+ x_f32.push(0.1 * f32::sin((i * k + l) as f32 * 0.05) + 0.5);
+ }
+ }
+ let x: Vec<T> = x_f32.iter().map(|&v| T::from(v).unwrap()).collect();
+
+ let input = Input {
+ w_packed,
+ scales,
+ zero_points,
+ biases,
+ x,
+ k: k as u32,
+ n: n as u32,
+ m: m as u32,
+ group_size,
+ bits,
+ use_zero_points,
+ use_mlx_quant,
+ };
+
+ let expected = get_expected::<T>(&input);
+ (input, expected)
+}
+
+fn test_internal<T: ArrayElement + Float + Debug + Display>(
+ input: &Input<T>,
+ expected: &[T],
+) {
+ let (rel_tol, abs_tol): (f64, f64) = match T::data_type() {
+ DataType::BF16 => (0.05, 0.5),
+ DataType::F16 => (0.02, 0.5),
+ _ => (0.01, 0.1),
+ };
+
+ for_each_non_cpu_backend!(|B| {
+ let output = get_output::<B, T>(input);
+ assert_eq!(expected.len(), output.len(), "Output length mismatch");
+
+ let mut errors = 0;
+ for (i, (&exp, &got)) in expected.iter().zip(output.iter()).enumerate() {
+ let exp_f32 = exp.to_f32().unwrap();
+ let got_f32 = got.to_f32().unwrap();
+ if !check_tolerance(exp_f32, got_f32, rel_tol, abs_tol) {
+ if errors < 5 {
+ eprintln!(" idx={} expected={} got={} diff={}", i, exp_f32, got_f32, (exp_f32 - got_f32).abs());
+ }
+ errors += 1;
+ }
+ }
+ assert_eq!(
+ errors,
+ 0,
+ "QMV fast kernel: backend={}, m={}, k={}, n={}, gs={}, bits={}, zp={}, mlx={}: {} mismatches",
+ std::any::type_name::<B>(),
+ input.m,
+ input.k,
+ input.n,
+ input.group_size,
+ input.bits,
+ input.use_zero_points,
+ input.use_mlx_quant,
+ errors,
+ );
+ });
+}
+
+/// The fast kernel requires in_vec_size to be a multiple of block_size
+/// (512 for 4-bit, 256 for 8-bit) and out_vec_size to be a multiple of 32.
+fn get_test_dims(
+ group_size: u32,
+ bits: u32,
+ is_half_precision: bool,
+) -> Vec<(usize, usize, usize)> {
+ let gs = group_size as usize;
+ let block_size = if bits == 4 {
+ 512
+ } else {
+ 256
+ };
+ let k = block_size;
+ let n = gs.max(32);
+ if is_half_precision && bits == 8 {
+ vec![(1, k, n), (2, k, n)]
+ } else {
+ vec![(1, k, n), (2, k, n * 2), (1, k, n * 2)]
+ }
+}
+
+fn test_basic<T: ArrayElement + Float + Debug + Display>(
+ group_size: u32,
+ bits: u32,
+ use_zero_points: bool,
+ use_mlx_quant: bool,
+) {
+ let is_half = matches!(T::data_type(), DataType::F16 | DataType::BF16);
+ let dims = get_test_dims(group_size, bits, is_half);
+
+ for (m, k, n) in dims {
+ let (input, expected) = get_test_data::<T>(m, k, n, group_size, bits, use_zero_points, use_mlx_quant);
+ test_internal::<T>(&input, &expected);
+ }
+}
+
+// ── 4-bit, zero points ─────────────────────────────────────────────────────
+
+#[uzu_test]
+fn test_gs32_4bit_zp() {
+ for_each_float_type!(|F| {
+ test_basic::<F>(32, 4, true, false);
+ })
+}
+
+#[uzu_test]
+fn test_gs64_4bit_zp() {
+ for_each_float_type!(|F| {
+ test_basic::<F>(64, 4, true, false);
+ })
+}
+
+#[uzu_test]
+fn test_gs128_4bit_zp() {
+ for_each_float_type!(|F| {
+ test_basic::<F>(128, 4, true, false);
+ })
+}
+
+// ── 8-bit, zero points ─────────────────────────────────────────────────────
+
+#[uzu_test]
+fn test_gs32_8bit_zp() {
+ for_each_float_type!(|F| {
+ test_basic::<F>(32, 8, true, false);
+ })
+}
+
+#[uzu_test]
+fn test_gs64_8bit_zp() {
+ for_each_float_type!(|F| {
+ test_basic::<F>(64, 8, true, false);
+ })
+}
+
+#[uzu_test]
+fn test_gs128_8bit_zp() {
+ for_each_float_type!(|F| {
+ test_basic::<F>(128, 8, true, false);
+ })
+}
+
+// ── 4-bit, mlx quant ───────────────────────────────────────────────────────
+
+#[uzu_test]
+fn test_gs32_4bit_mlx() {
+ for_each_float_type!(|F| {
+ test_basic::<F>(32, 4, false, true);
+ })
+}
+
+#[uzu_test]
+fn test_gs64_4bit_mlx() {
+ for_each_float_type!(|F| {
+ test_basic::<F>(64, 4, false, true);
+ })
+}
+
+#[uzu_test]
+fn test_gs128_4bit_mlx() {
+ for_each_float_type!(|F| {
+ test_basic::<F>(128, 4, false, true);
+ })
+}
+
+// ── 8-bit, mlx quant ───────────────────────────────────────────────────────
+
+#[uzu_test]
+fn test_gs32_8bit_mlx() {
+ for_each_float_type!(|F| {
+ test_basic::<F>(32, 8, false, true);
+ })
+}
+
+#[uzu_test]
+fn test_gs64_8bit_mlx() {
+ for_each_float_type!(|F| {
+ test_basic::<F>(64, 8, false, true);
+ })
+}
+
+#[uzu_test]
+fn test_gs128_8bit_mlx() {
+ for_each_float_type!(|F| {
+ test_basic::<F>(128, 8, false, true);
+ })
+}
diff --git a/crates/uzu/tests/unit/kernel/quant_matmul/qmv_test.rs b/crates/uzu/tests/unit/kernel/quant_matmul/qmv_test.rs
new file mode 100644
index 000000000..8a52d7bc6
--- /dev/null
+++ b/crates/uzu/tests/unit/kernel/quant_matmul/qmv_test.rs
@@ -0,0 +1,425 @@
+use std::fmt::{Debug, Display};
+
+use num_traits::Float;
+use uzu::{
+ ArrayElement, DataType,
+ backends::{
+ common::{Backend, Buffer, Context, Encoder, Kernels, kernel::QuantizedMatmulQmvKernel},
+ cpu::Cpu,
+ },
+};
+
+use super::{Input, check_tolerance, pack_weights_u32, pack_zero_points};
+use crate::{common::helpers::alloc_buffer_with_data, uzu_test};
+
+fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
+ let context = B::Context::new().expect("Failed to create Context");
+
+ let w_buf = alloc_buffer_with_data::<B, u32>(&context, &input.w_packed);
+ let scales_buf = alloc_buffer_with_data::<B, T>(&context, &input.scales);
+
+ let zp_buf = input.zero_points.as_ref().map(|zp| alloc_buffer_with_data::<B, u8>(&context, zp));
+ let bias_buf = input.biases.as_ref().map(|b| alloc_buffer_with_data::<B, T>(&context, b));
+
+ let x_buf = alloc_buffer_with_data::<B, T>(&context, &input.x);
+ let output_size = (input.m as usize) * (input.n as usize) * T::data_type().size_in_bytes();
+ let mut y_buf = context.create_buffer(output_size).expect("Failed to create buffer");
+
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
+
+ let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmvKernel::new(
+ &context,
+ T::data_type(),
+ input.group_size,
+ input.bits,
+ input.use_zero_points,
+ input.use_mlx_quant,
+ )
+ .expect("Failed to create QuantizedMatmulQmvKernel");
+ kernel.encode(
+ &w_buf,
+ &scales_buf,
+ zp_buf.as_ref(),
+ bias_buf.as_ref(),
+ &x_buf,
+ &mut y_buf,
+ input.k,
+ input.n,
+ input.m,
+ &mut encoder,
+ );
+
+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
+
+ let y_ptr = y_buf.cpu_ptr().as_ptr() as *const T;
+ let y_len = (input.m as usize) * (input.n as usize);
+ unsafe { std::slice::from_raw_parts(y_ptr, y_len) }.to_vec()
+}
+
+/// Create test data for a qmv scenario: m×k * k×n = m×n
+/// Weights are in transposed [N × K] layout (each output row stores K input weights).
+/// Scales: [N × num_groups_k], zero_points: [N × zp_stride_k], biases: [N × num_groups_k].
+/// Groups are along the K dimension.
+fn get_test_data_basic<T: ArrayElement + Float>(
+ m: usize,
+ k: usize,
+ n: usize,
+ group_size: u32,
+ bits: u32,
+ use_zero_points: bool,
+ use_mlx_quant: bool,
+) -> (Input<T>, Vec<T>) {
+ let num_groups_k = (k + group_size as usize - 1) / group_size as usize;
+
+ // Generate quantized weights [N × K]
+ let mut weights_raw: Vec<u8> = Vec::with_capacity(n * k);
+ let max_val = if bits == 4 {
+ 15u8
+ } else {
+ 255u8
+ };
+ for j in 0..n {
+ for l in 0..k {
+ weights_raw.push(((j * 3 + l * 7 + 1) % (max_val as usize + 1)) as u8);
+ }
+ }
+ let w_packed = pack_weights_u32(&weights_raw, bits);
+
+ // Scales: [N × num_groups_k]
+ let mut scales_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);
+ for j in 0..n {
+ for g in 0..num_groups_k {
+ scales_f32.push(0.5 + 0.1 * ((j + g) % 5) as f32);
+ }
+ }
+ let scales: Vec<T> = scales_f32.iter().map(|&v| T::from(v).unwrap()).collect();
+
+ // Zero points or biases
+ let zp_stride = if bits == 4 {
+ (num_groups_k + 1) / 2
+ } else {
+ num_groups_k
+ };
+
+ let (zero_points, biases) = if use_zero_points {
+ let mut zp_raw: Vec<u8> = Vec::with_capacity(n * num_groups_k);
+ for j in 0..n {
+ for g in 0..num_groups_k {
+ let zp_val = ((j * 2 + g * 3) % (max_val as usize + 1)) as u8;
+ zp_raw.push(zp_val);
+ }
+ }
+ // Pack zero points per row (row = output neuron)
+ let mut zp_packed: Vec<u8> = Vec::with_capacity(n * zp_stride);
+ for j in 0..n {
+ let row = &zp_raw[j * num_groups_k..(j + 1) * num_groups_k];
+ let packed_row = pack_zero_points(row, bits);
+ let mut padded = packed_row;
+ padded.resize(zp_stride, 0);
+ zp_packed.extend_from_slice(&padded);
+ }
+ (Some(zp_packed), None)
+ } else if use_mlx_quant {
+ let mut biases_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);
+ for j in 0..n {
+ for g in 0..num_groups_k {
+ biases_f32.push(0.01 * ((j + g * 2) % 7) as f32);
+ }
+ }
+ let biases: Vec<T> = biases_f32.iter().map(|&v| T::from(v).unwrap()).collect();
+ (None, Some(biases))
+ } else {
+ unreachable!("Must use either zero_points or mlx_quant");
+ };
+
+ // X: [m × k]
+ let mut x_f32: Vec<f32> = Vec::with_capacity(m * k);
+ for i in 0..m {
+ for l in 0..k {
+ x_f32.push(0.1 * f32::sin((i * k + l) as f32 * 0.05) + 0.5);
+ }
+ }
+ let x: Vec<T> = x_f32.iter().map(|&v| T::from(v).unwrap()).collect();
+
+ let input = Input {
+ w_packed,
+ scales,
+ zero_points,
+ biases,
+ x,
+ k: k as u32,
+ n: n as u32,
+ m: m as u32,
+ group_size,
+ bits,
+ use_zero_points,
+ use_mlx_quant,
+ };
+
+ let expected = get_output::<Cpu, T>(&input);
+ (input, expected)
+}
+
+/// Small edge-case test data with known values
+fn get_test_data_edge<T: ArrayElement + Float>(
+ group_size: u32,
+ bits: u32,
+ use_zero_points: bool,
+ use_mlx_quant: bool,
+) -> (Input<T>, Vec<T>) {
+ let m = 1usize;
+ let k = 4usize;
+ let n = 4usize;
+ let num_groups_k = (k + group_size as usize - 1) / group_size as usize;
+
+ // Simple weights [N × K]: all 1s
+ let weights_raw: Vec<u8> = vec![1u8; n * k];
+ let w_packed = pack_weights_u32(&weights_raw, bits);
+
+ // Unit scales [N × num_groups_k]
+ let scales: Vec<T> = vec![T::one(); n * num_groups_k];
+
+ let zp_stride = if bits == 4 {
+ (num_groups_k + 1) / 2
+ } else {
+ num_groups_k
+ };
+
+ let (zero_points, biases) = if use_zero_points {
+ // Zero points all 0 → bias = 0
+ let zp_packed = vec![0u8; n * zp_stride];
+ (Some(zp_packed), None)
+ } else if use_mlx_quant {
+ let biases: Vec<T> = vec![T::zero(); n * num_groups_k];
+ (None, Some(biases))
+ } else {
+ unreachable!("Must use either zero_points or mlx_quant");
+ };
+
+ // X: simple increasing values
+ let x_f32: Vec<f32> = (0..m * k).map(|i| (i + 1) as f32 * 0.1).collect();
+ let x: Vec<T> = x_f32.iter().map(|&v| T::from(v).unwrap()).collect();
+
+ let input = Input {
+ w_packed,
+ scales,
+ zero_points,
+ biases,
+ x,
+ k: k as u32,
+ n: n as u32,
+ m: m as u32,
+ group_size,
+ bits,
+ use_zero_points,
+ use_mlx_quant,
+ };
+
+ let expected = get_output::<Cpu, T>(&input);
+ (input, expected)
+}
+
+fn test_internal<T: ArrayElement + Float + Debug + Display>(
+ input: &Input<T>,
+ expected: &[T],
+) {
+ let (rel_tol, abs_tol): (f64, f64) = match T::data_type() {
+ DataType::BF16 => (0.05, 0.5),
+ DataType::F16 => (0.02, 0.5),
+ _ => (0.01, 0.1),
+ };
+
+ for_each_non_cpu_backend!(|B| {
+ let output = get_output::<B, T>(input);
+ assert_eq!(expected.len(), output.len(), "Output length mismatch");
+
+ let mut errors = 0;
+ for (i, (&exp, &got)) in expected.iter().zip(output.iter()).enumerate() {
+ let exp_f32 = exp.to_f32().unwrap();
+ let got_f32 = got.to_f32().unwrap();
+ if !check_tolerance(exp_f32, got_f32, rel_tol, abs_tol) {
+ if errors < 5 {
+ eprintln!(" idx={} expected={} got={} diff={}", i, exp_f32, got_f32, (exp_f32 - got_f32).abs());
+ }
+ errors += 1;
+ }
+ }
+ assert_eq!(
+ errors,
+ 0,
+ "QMV kernel: backend={}, m={}, k={}, n={}, gs={}, bits={}, zp={}, mlx={}: {} mismatches",
+ std::any::type_name::<B>(),
+ input.m,
+ input.k,
+ input.n,
+ input.group_size,
+ input.bits,
+ input.use_zero_points,
+ input.use_mlx_quant,
+ errors,
+ );
+ });
+}
+
+fn get_test_dims(
+ group_size: u32,
+ bits: u32,
+ is_half_precision: bool,
+) -> Vec<(usize, usize, usize)> {
+ let gs = group_size as usize;
+ // N must be >= group_size for the Metal tiled loader's stride calculations.
+ // For half-precision 8-bit, limit k to avoid catastrophic cancellation
+ // from large weight values (0-255) and ZP bias subtraction.
+ if is_half_precision && bits == 8 {
+ let k = gs.min(64);
+ vec![(1, k, gs), (2, k, gs)]
+ } else {
+ vec![(1, gs * 2, gs), (2, gs * 2, gs * 2), (1, gs, gs * 2)]
+ }
+}
+
+fn test_basic<T: ArrayElement + Float + Debug + Display>(
+ group_size: u32,
+ bits: u32,
+ use_zero_points: bool,
+ use_mlx_quant: bool,
+) {
+ let is_half = matches!(T::data_type(), DataType::F16 | DataType::BF16);
+ let dims = get_test_dims(group_size, bits, is_half);
+
+ for (m, k, n) in dims {
+ let (input, expected) = get_test_data_basic::<T>(m, k, n, group_size, bits, use_zero_points, use_mlx_quant);
+ test_internal::<T>(&input, &expected);
+ }
+}
+
+fn test_edge<T: ArrayElement + Float + Debug + Display>(
+ group_size: u32,
+ bits: u32,
+ use_zero_points: bool,
+ use_mlx_quant: bool,
+) {
+ let (input, expected) = get_test_data_edge::<T>(group_size, bits, use_zero_points, use_mlx_quant);
+ test_internal::<T>(&input, &expected);
+}
+
+// ── 4-bit, zero points ─────────────────────────────────────────────────────
+#[uzu_test]
+fn test_gs32_4bit_zp() {
+ for_each_float_type!(|F| {
+ test_basic::<F>(32, 4, true, false);
+ })
+}
+
+#[uzu_test]
+fn test_gs64_4bit_zp() {
+ for_each_float_type!(|F| {
+ test_basic::<F>(64, 4, true, false);
+ })
+}
+
+#[uzu_test]
+fn test_gs128_4bit_zp() {
+ for_each_float_type!(|F| {
+ test_basic::<F>(128, 4, true, false);
+ })
+}
+
+// ── 8-bit, zero points ─────────────────────────────────────────────────────
+
+#[uzu_test]
+fn test_gs32_8bit_zp() {
+ for_each_float_type!(|F| {
+ test_basic::<F>(32, 8, true, false);
+ })
+}
+
+#[uzu_test]
+fn test_gs64_8bit_zp() {
+ for_each_float_type!(|F| {
+ test_basic::<F>(64, 8, true, false);
+ })
+}
+
+#[uzu_test]
+fn test_gs128_8bit_zp() {
+ for_each_float_type!(|F| {
+ test_basic::<F>(128, 8, true, false);
+ })
+}
+
+// ── 4-bit, mlx quant ───────────────────────────────────────────────────────
+
+#[uzu_test]
+fn test_gs32_4bit_mlx() {
+ for_each_float_type!(|F| {
+ test_basic::<F>(32, 4, false, true);
+ })
+}
+
+#[uzu_test]
+fn test_gs64_4bit_mlx() {
+ for_each_float_type!(|F| {
+ test_basic::<F>(64, 4, false, true);
+ })
+}
+
+#[uzu_test]
+fn test_gs128_4bit_mlx() {
+ for_each_float_type!(|F| {
+ test_basic::<F>(128, 4, false, true);
+ })
+}
+
+// ── 8-bit, mlx quant ───────────────────────────────────────────────────────
+
+#[uzu_test]
+fn test_gs32_8bit_mlx() {
+ for_each_float_type!(|F| {
+ test_basic::<F>(32, 8, false, true);
+ })
+}
+
+#[uzu_test]
+fn test_gs64_8bit_mlx() {
+ for_each_float_type!(|F| {
+ test_basic::<F>(64, 8, false, true);
+ })
+}
+
+#[uzu_test]
+fn test_gs128_8bit_mlx() {
+ for_each_float_type!(|F| {
+ test_basic::<F>(128, 8, false, true);
+ })
+}
+
+// ── Edge cases ──────────────────────────────────────────────────────────────
+
+#[uzu_test]
+fn test_edge_4bit_zp() {
+ for_each_float_type!(|F| {
+ test_edge::<F>(32, 4, true, false);
+ })
+}
+
+#[uzu_test]
+fn test_edge_8bit_zp() {
+ for_each_float_type!(|F| {
+ test_edge::<F>(32, 8, true, false);
+ })
+}
+
+#[uzu_test]
+fn test_edge_4bit_mlx() {
+ for_each_float_type!(|F| {
+ test_edge::<F>(32, 4, false, true);
+ })
+}
+
+#[uzu_test]
+fn test_edge_8bit_mlx() {
+ for_each_float_type!(|F| {
+ test_edge::<F>(32, 8, false, true);
+ })
+}