A high-performance inference engine for AI models
Quant matmul cpu kernels (#316)
Aleksandr Golokoz6f6ddaa9787b
diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rsindex 4306fc711..5a0374c78 100644--- a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rs+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rs@@ -4,33 +4,117 @@ use num_traits::Float;use crate::ArrayElement;+pub fn qmm_transposed<T: ArrayElement + Float>(+ weights: *const u32,+ scales: *const T,+ zero_points: Option<*const u8>,+ biases: Option<*const T>,+ input: *const T,+ output: *mut T,+ in_vec_size: usize,+ out_vec_size: usize,+ batch_size: usize,+ use_zero_points: bool,+ use_mlx_quant: bool,+ group_size: usize,+ bits: usize,+) {+ let num_groups_k = in_vec_size.div_ceil(group_size);+ let zp_stride = if bits == 4 {+ (num_groups_k + 1) / 2+ } else {+ num_groups_k+ };++ unsafe {+ for i in 0..batch_size {+ for j in 0..out_vec_size {+ let mut acc = 0.0f32;++ for l in 0..in_vec_size {+ let group_idx = l / group_size;++ // Transposed: weight at row j, col l+ let weight_linear_idx = j * in_vec_size + l;++ let val_q = if bits == 4 {+ let u32_idx = weight_linear_idx / 8;+ let bit_offset = (weight_linear_idx % 8) * 4;+ ((weights.add(u32_idx).read_unaligned() >> bit_offset) & 0xF) as f32+ } else {+ let u32_idx = weight_linear_idx / 4;+ let byte_offset = (weight_linear_idx % 4) * 8;+ ((weights.add(u32_idx).read_unaligned() >> byte_offset) & 0xFF) as f32+ };++ let val_a = (*input.add(i * in_vec_size + l)).to_f32().unwrap();+ let scale_t = *scales.add(j * num_groups_k + group_idx);++ // Dequantize in f32 to serve as a precision-neutral reference.+ // Metal dequantizes in bf16/f16 which rounds differently across+ // GPU hardware; computing in f32 avoids biasing toward any+ // particular rounding pattern.+ let w_dequant_f32 = if use_zero_points {+ let zp = zero_points.unwrap();+ let zp_val = if bits == 4 {+ let byte_index = j * zp_stride + (group_idx >> 1);+ let byte_val = *zp.add(byte_index);+ if (group_idx & 1) == 0 {+ (byte_val & 0x0F) as f32+ } else {+ ((byte_val >> 4) & 0x0F) as f32+ }+ } else {+ *zp.add(j * zp_stride + group_idx) as f32+ };+ scale_t.to_f32().unwrap() * (val_q - zp_val)+ } else if use_mlx_quant {+ let bias_t = *biases.unwrap().add(j * num_groups_k + group_idx);+ scale_t.to_f32().unwrap() * val_q + bias_t.to_f32().unwrap()+ } else {+ 0.0f32+ };++ acc += val_a * w_dequant_f32;+ }++ *output.add(i * out_vec_size + j) = T::from(acc).unwrap();+ }+ }+ }+}+#[kernel(QuantizedMatmulQmmTransposed)]#[variants(T, f32, f16, bf16)]#[variants(GROUP_SIZE, 32, 64, 128)]#[variants(BITS, 4, 8)]pub fn quantized_matmul_qmm_transposed<T: ArrayElement + Float, const GROUP_SIZE: u32, const BITS: u32>(- #[allow(unused)] weights: *const u32,- #[allow(unused)] scales: *const T,- #[allow(unused)]- #[optional(use_zero_points)]- zero_points: Option<*const u8>,- #[allow(unused)]- #[optional(use_mlx_quant)]- biases: Option<*const T>,- #[allow(unused)] input: *const T,- #[allow(unused)] output: *mut T,- #[allow(unused)] in_vec_size: u32,- #[allow(unused)] out_vec_size: u32,- #[allow(unused)] batch_size: u32,- #[allow(unused)]- #[specialize]- use_zero_points: bool,- #[allow(unused)]- #[specialize]- use_mlx_quant: bool,- #[allow(unused)]- #[specialize]- aligned_n: bool,+ weights: *const u32,+ scales: *const T,+ #[optional(use_zero_points)] zero_points: Option<*const u8>,+ #[optional(use_mlx_quant)] biases: Option<*const T>,+ input: *const T,+ output: *mut T,+ in_vec_size: u32,+ out_vec_size: u32,+ batch_size: u32,+ #[specialize] use_zero_points: bool,+ #[specialize] use_mlx_quant: bool,+ #[specialize] aligned_n: bool,) {- todo!()+ qmm_transposed::<T>(+ weights,+ scales,+ zero_points,+ biases,+ input,+ output,+ in_vec_size as usize,+ out_vec_size as usize,+ batch_size as usize,+ use_zero_points,+ use_mlx_quant,+ GROUP_SIZE as usize,+ BITS as usize,+ );}diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_64x64.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_64x64.rsindex aee61e512..c28219b6a 100644--- a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_64x64.rs+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_64x64.rs@@ -2,32 +2,38 @@ use dsl::kernel;use half::bf16;use num_traits::Float;-use crate::ArrayElement;+use crate::{ArrayElement, backends::cpu::kernel::quant_matmul::qmm_transposed::qmm_transposed};#[kernel(QuantizedMatmulQmmTransposed64x64)]#[variants(T, bf16)]#[variants(GROUP_SIZE, 64, 128)]#[variants(BITS, 4, 8)]pub fn quantized_matmul_qmm_transposed64x64<T: ArrayElement + Float, const GROUP_SIZE: u32, const BITS: u32>(- #[allow(unused)] weights: *const u32,- #[allow(unused)] scales: *const T,- #[allow(unused)]- #[optional(use_zero_points)]- zero_points: Option<*const u8>,- #[allow(unused)]- #[optional(use_mlx_quant)]- biases: Option<*const T>,- #[allow(unused)] input: *const T,- #[allow(unused)] output: *mut T,- #[allow(unused)] in_vec_size: u32,- #[allow(unused)] out_vec_size: u32,- #[allow(unused)] batch_size: u32,- #[allow(unused)]- #[specialize]- use_zero_points: bool,- #[allow(unused)]- #[specialize]- use_mlx_quant: bool,+ weights: *const u32,+ scales: *const T,+ #[optional(use_zero_points)] zero_points: Option<*const u8>,+ #[optional(use_mlx_quant)] biases: Option<*const T>,+ input: *const T,+ output: *mut T,+ in_vec_size: u32,+ out_vec_size: u32,+ batch_size: u32,+ #[specialize] use_zero_points: bool,+ #[specialize] use_mlx_quant: bool,) {- todo!()+ qmm_transposed::<T>(+ weights,+ scales,+ zero_points,+ biases,+ input,+ output,+ in_vec_size as usize,+ out_vec_size as usize,+ batch_size as usize,+ use_zero_points,+ use_mlx_quant,+ GROUP_SIZE as usize,+ BITS as usize,+ );}diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_wide.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_wide.rsindex aa64ed350..ad227a8d5 100644--- a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_wide.rs+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_wide.rs@@ -2,32 +2,38 @@ use dsl::kernel;use half::bf16;use num_traits::Float;-use crate::ArrayElement;+use crate::{ArrayElement, backends::cpu::kernel::quant_matmul::qmm_transposed::qmm_transposed};#[kernel(QuantizedMatmulQmmTransposedWide)]#[variants(T, bf16)]#[variants(GROUP_SIZE, 32, 64, 128)]#[variants(BITS, 4, 8)]pub fn quantized_matmul_qmm_transposed_wide<T: ArrayElement + Float, const GROUP_SIZE: u32, const BITS: u32>(- #[allow(unused)] weights: *const u32,- #[allow(unused)] scales: *const T,- #[allow(unused)]- #[optional(use_zero_points)]- zero_points: Option<*const u8>,- #[allow(unused)]- #[optional(use_mlx_quant)]- biases: Option<*const T>,- #[allow(unused)] input: *const T,- #[allow(unused)] output: *mut T,- #[allow(unused)] in_vec_size: u32,- #[allow(unused)] out_vec_size: u32,- #[allow(unused)] batch_size: u32,- #[allow(unused)]- #[specialize]- use_zero_points: bool,- #[allow(unused)]- #[specialize]- use_mlx_quant: bool,+ weights: *const u32,+ scales: *const T,+ #[optional(use_zero_points)] zero_points: Option<*const u8>,+ #[optional(use_mlx_quant)] biases: Option<*const T>,+ input: *const T,+ output: *mut T,+ in_vec_size: u32,+ out_vec_size: u32,+ batch_size: u32,+ #[specialize] use_zero_points: bool,+ #[specialize] use_mlx_quant: bool,) {- todo!()+ qmm_transposed::<T>(+ weights,+ scales,+ zero_points,+ biases,+ input,+ output,+ in_vec_size as usize,+ out_vec_size as usize,+ batch_size as usize,+ use_zero_points,+ use_mlx_quant,+ GROUP_SIZE as usize,+ BITS as usize,+ );}diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv.rsindex 7341a22e6..0f937ff5a 100644--- a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv.rs+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv.rs@@ -4,30 +4,128 @@ use num_traits::Float;use crate::ArrayElement;+/// CPU reference implementation of quantized matrix-vector multiply.+///+/// Weight layout is transposed compared to qmm:+/// weights: [N, K], scales: [N, num_groups_k], zero_points: [N, zp_stride_k]+/// where groups are along the K dimension.+///+/// Computes y[i, j] = sum_l( x[i, l] * dequant(w[j, l]) ) for each batch i and output j.+pub fn qmv<T: ArrayElement + Float>(+ weights: *const u32,+ scales: *const T,+ zero_points: Option<*const u8>,+ biases: Option<*const T>,+ input: *const T,+ output: *mut T,+ in_vec_size: usize,+ out_vec_size: usize,+ batch_size: usize,+ use_zero_points: bool,+ use_mlx_quant: bool,+ group_size: usize,+ bits: usize,+) {+ let num_groups_k = in_vec_size.div_ceil(group_size);+ let zp_stride = if bits == 4 {+ (num_groups_k + 1) / 2+ } else {+ num_groups_k+ };++ // Number of weight elements packed per u32+ let pack_factor = if bits == 4 {+ 8+ } else {+ 4+ };++ unsafe {+ for i in 0..batch_size {+ for j in 0..out_vec_size {+ let mut acc = 0.0f32;++ for l in 0..in_vec_size {+ // Weights are [N, K]: row j, column l+ let weight_linear_idx = j * in_vec_size + l;++ let val_q = if bits == 4 {+ let u32_idx = weight_linear_idx / pack_factor;+ let bit_offset = (weight_linear_idx % pack_factor) * 4;+ ((weights.add(u32_idx).read_unaligned() >> bit_offset) & 0xF) as f32+ } else {+ let u32_idx = weight_linear_idx / pack_factor;+ let byte_offset = (weight_linear_idx % pack_factor) * 8;+ ((weights.add(u32_idx).read_unaligned() >> byte_offset) & 0xFF) as f32+ };++ let val_a = (*input.add(i * in_vec_size + l)).to_f32().unwrap();++ // Scales are [N, num_groups_k]: groups along K+ let group_idx = l / group_size;+ let scale = (*scales.add(j * num_groups_k + group_idx)).to_f32().unwrap();++ let bias = if use_zero_points {+ let zp = zero_points.unwrap();+ // Zero points are [N, zp_stride_k]+ let zp_val = if bits == 4 {+ let byte_index = j * zp_stride + (group_idx >> 1);+ let byte_val = *zp.add(byte_index);+ if (group_idx & 1) == 0 {+ (byte_val & 0x0F) as f32+ } else {+ ((byte_val >> 4) & 0x0F) as f32+ }+ } else {+ *zp.add(j * zp_stride + group_idx) as f32+ };+ -scale * zp_val+ } else if use_mlx_quant {+ // Biases are [N, num_groups_k]+ (*biases.unwrap().add(j * num_groups_k + group_idx)).to_f32().unwrap()+ } else {+ 0.0f32+ };++ acc += val_a * (scale * val_q + bias);+ }++ *output.add(i * out_vec_size + j) = T::from(acc).unwrap();+ }+ }+ }+}+#[kernel(QuantizedMatmulQmv)]#[variants(T, f32, f16, bf16)]#[variants(GROUP_SIZE, 32, 64, 128)]#[variants(BITS, 4, 8)]pub fn quantized_matmul_qmv<T: ArrayElement + Float, const GROUP_SIZE: u32, const BITS: u32>(- #[allow(unused)] weights: *const u32,- #[allow(unused)] scales: *const T,- #[allow(unused)]- #[optional(use_zero_points)]- zero_points: Option<*const u8>,- #[allow(unused)]- #[optional(use_mlx_quant)]- biases: Option<*const T>,- #[allow(unused)] input: *const T,- #[allow(unused)] output: *mut T,- #[allow(unused)] in_vec_size: u32,- #[allow(unused)] out_vec_size: u32,- #[allow(unused)] batch_size: u32,- #[allow(unused)]- #[specialize]- use_zero_points: bool,- #[allow(unused)]- #[specialize]- use_mlx_quant: bool,+ weights: *const u32,+ scales: *const T,+ #[optional(use_zero_points)] zero_points: Option<*const u8>,+ #[optional(use_mlx_quant)] biases: Option<*const T>,+ input: *const T,+ output: *mut T,+ in_vec_size: u32,+ out_vec_size: u32,+ batch_size: u32,+ #[specialize] use_zero_points: bool,+ #[specialize] use_mlx_quant: bool,) {- todo!()+ qmv::<T>(+ weights,+ scales,+ zero_points,+ biases,+ input,+ output,+ in_vec_size as usize,+ out_vec_size as usize,+ batch_size as usize,+ use_zero_points,+ use_mlx_quant,+ GROUP_SIZE as usize,+ BITS as usize,+ );}diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv_fast.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv_fast.rsindex d5c53856d..aa05b7ac0 100644--- a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv_fast.rs+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmv_fast.rs@@ -2,32 +2,38 @@ use dsl::kernel;use half::{bf16, f16};use num_traits::Float;-use crate::ArrayElement;+use crate::{ArrayElement, backends::cpu::kernel::quant_matmul::qmv::qmv};#[kernel(QuantizedMatmulQmvFast)]#[variants(T, f32, f16, bf16)]#[variants(GROUP_SIZE, 32, 64, 128)]#[variants(BITS, 4, 8)]pub fn quantized_matmul_qmv_fast<T: ArrayElement + Float, const GROUP_SIZE: u32, const BITS: u32>(- #[allow(unused)] weights: *const u32,- #[allow(unused)] scales: *const T,- #[allow(unused)]- #[optional(use_zero_points)]- zero_points: Option<*const u8>,- #[allow(unused)]- #[optional(use_mlx_quant)]- biases: Option<*const T>,- #[allow(unused)] input: *const T,- #[allow(unused)] output: *mut T,- #[allow(unused)] in_vec_size: u32,- #[allow(unused)] out_vec_size: u32,- #[allow(unused)] batch_size: u32,- #[allow(unused)]- #[specialize]- use_zero_points: bool,- #[allow(unused)]- #[specialize]- use_mlx_quant: bool,+ weights: *const u32,+ scales: *const T,+ #[optional(use_zero_points)] zero_points: Option<*const u8>,+ #[optional(use_mlx_quant)] biases: Option<*const T>,+ input: *const T,+ output: *mut T,+ in_vec_size: u32,+ out_vec_size: u32,+ batch_size: u32,+ #[specialize] use_zero_points: bool,+ #[specialize] use_mlx_quant: bool,) {- todo!()+ qmv::<T>(+ weights,+ scales,+ zero_points,+ biases,+ input,+ output,+ in_vec_size as usize,+ out_vec_size as usize,+ batch_size as usize,+ use_zero_points,+ use_mlx_quant,+ GROUP_SIZE as usize,+ BITS as usize,+ );}diff --git a/crates/uzu/tests/common/helpers.rs b/crates/uzu/tests/common/helpers.rsindex 2361ab6d7..03efa27aa 100644--- a/crates/uzu/tests/common/helpers.rs+++ b/crates/uzu/tests/common/helpers.rs@@ -2,6 +2,7 @@ use std::rc::Rc;use uzu::backends::common::{Backend, Buffer, Context};+#[allow(dead_code)]pub fn alloc_buffer<B: Backend, T>(context: &B::Context,elements_count: usize,@@ -25,6 +26,7 @@ pub fn alloc_buffer_with_data<B: Backend, T: bytemuck::NoUninit>(buffer}+#[allow(dead_code)]pub fn create_context<B: Backend>() -> Rc<<B as Backend>::Context> {B::Context::new().expect(format!("Failed to create context for {}", std::any::type_name::<B>()).as_str())}diff --git a/crates/uzu/tests/common/mod.rs b/crates/uzu/tests/common/mod.rsindex 1f00f9c5b..ea1593e22 100644--- a/crates/uzu/tests/common/mod.rs+++ b/crates/uzu/tests/common/mod.rs@@ -45,3 +45,21 @@ macro_rules! for_each_non_cpu_backend {}};}pub(crate) use for_each_non_cpu_backend;++macro_rules! for_each_float_type {+ (|$F:ident| $body:expr) => {{+ {+ type $F = f32;+ $body+ }+ {+ type $F = half::f16;+ $body+ }+ {+ type $F = half::bf16;+ $body+ }+ }};+}+pub(crate) use for_each_float_type;diff --git a/crates/uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs b/crates/uzu/tests/unit/backends/common/kernel/quant_matmul_test.rsindex a2a42b2a9..ebb324b74 100644--- a/crates/uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs+++ b/crates/uzu/tests/unit/backends/common/kernel/quant_matmul_test.rs@@ -1,24 +1,20 @@-#![cfg(metal_backend)]+#[macro_use]+#[path = "../../../../common/mod.rs"]+mod common;use std::time::Instant;-use bytemuck;use half::{bf16, f16};-use metal::{MTLBuffer, MTLDeviceExt, MTLResourceOptions};-use objc2::{rc::Retained, runtime::ProtocolObject};use crate::{DataType,- backends::{- common::{- Backend, Context, Encoder,- gpu_types::QuantizationMode,- kernel::quant_matmul::{- QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulKernelEncodable,- QuantizedMatmulType,- },+ backends::common::{+ Backend, Buffer, Context, Encoder,+ gpu_types::QuantizationMode,+ kernel::quant_matmul::{+ QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulKernelEncodable,+ QuantizedMatmulType, tests::common::helpers::alloc_buffer_with_data,},- metal::Metal,},};@@ -265,36 +261,29 @@ fn generate_test_quant_params(}}-fn buffer_from_f32_slice(- ctx: &<Metal as Backend>::Context,+fn buffer_from_f32_slice<B: Backend>(+ ctx: &B::Context,dtype: DataType,values: &[f32],-) -> Retained<ProtocolObject<dyn MTLBuffer>> {+) -> B::Buffer {match dtype {DataType::F16 => {let data: Vec<f16> = values.iter().map(|&v| f16::from_f32(v)).collect();- ctx.device- .new_buffer_with_data(bytemuck::cast_slice(&data), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer")+ alloc_buffer_with_data::<B, f16>(ctx, data.as_slice())},DataType::BF16 => {let data: Vec<bf16> = values.iter().map(|&v| bf16::from_f32(v)).collect();- ctx.device- .new_buffer_with_data(bytemuck::cast_slice(&data), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer")+ alloc_buffer_with_data::<B, bf16>(ctx, data.as_slice())},- DataType::F32 => ctx- .device- .new_buffer_with_data(bytemuck::cast_slice(values), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer"),+ DataType::F32 => alloc_buffer_with_data::<B, f32>(ctx, values),other => {panic!("Unsupported dtype for buffer_from_f32_slice: {:?}", other)},}}-fn execute_quantized_matmul(- ctx: &<Metal as Backend>::Context,+fn execute_quantized_matmul<B: Backend>(+ ctx: &B::Context,batch: usize,input_dim: usize,output_dim: usize,@@ -330,23 +319,17 @@ fn execute_quantized_matmul(};let x_quant = quantize_slice(&x_f32, data_type);- let w_buf = ctx- .device- .new_buffer_with_data(bytemuck::cast_slice(&weights_packed), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");- let s_buf = buffer_from_f32_slice(ctx, data_type, ¶ms.scales);+ let w_buf = alloc_buffer_with_data::<B, u8>(ctx, &weights_packed);+ let s_buf = buffer_from_f32_slice::<B>(ctx, data_type, ¶ms.scales);let b_buf = match quantization_type {- QuantizedMatmulType::ZeroPoint => ctx- .device- .new_buffer_with_data(¶ms.zero_points, MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer"),- QuantizedMatmulType::Mlx => buffer_from_f32_slice(ctx, data_type, ¶ms.biases),+ QuantizedMatmulType::ZeroPoint => alloc_buffer_with_data::<B, u8>(ctx, ¶ms.zero_points),+ QuantizedMatmulType::Mlx => buffer_from_f32_slice::<B>(ctx, data_type, ¶ms.biases),};- let x_buf = buffer_from_f32_slice(ctx, data_type, &x_f32);+ let x_buf = buffer_from_f32_slice::<B>(ctx, data_type, &x_f32);let mut y_buf = ctx.create_buffer(batch * output_dim * data_type.size_in_bytes()).expect("Failed to create buffer");- let kernel = QuantizedMatmulKernelEncodable::<Metal>::new(+ let kernel = QuantizedMatmulKernelEncodable::<B>::new(&ctx,QuantizedMatmulConfiguration {data_type,@@ -416,17 +399,17 @@ fn execute_quantized_matmul(let y_out_f32: Vec<f32> = match data_type {DataType::F16 => {- let y_ptr = y_buf.contents().as_ptr() as *const f16;+ let y_ptr = y_buf.cpu_ptr().as_ptr() as *const f16;let y_out = unsafe { std::slice::from_raw_parts(y_ptr, batch * output_dim) };y_out.iter().map(|&v| v.to_f32()).collect()},DataType::BF16 => {- let y_ptr = y_buf.contents().as_ptr() as *const bf16;+ let y_ptr = y_buf.cpu_ptr().as_ptr() as *const bf16;let y_out = unsafe { std::slice::from_raw_parts(y_ptr, batch * output_dim) };y_out.iter().map(|&v| v.to_f32()).collect()},DataType::F32 => {- let y_ptr = y_buf.contents().as_ptr() as *const f32;+ let y_ptr = y_buf.cpu_ptr().as_ptr() as *const f32;let y_out = unsafe { std::slice::from_raw_parts(y_ptr, batch * output_dim) };y_out.to_vec()},@@ -488,8 +471,8 @@ struct TestConfig {const QMV_DIMS: &[(usize, usize)] = &[(128, 512), (512, 1024), (1024, 4096)];const QMM_DIMS: &[(usize, usize, usize)] = &[(64, 64, 64), (512, 512, 1024), (128, 128, 256)];-fn run_kernel_test(- ctx: &<Metal as Backend>::Context,+fn run_kernel_test<B: Backend>(+ ctx: &B::Context,batch: usize,output_dim: usize,input_dim: usize,@@ -499,7 +482,7 @@ fn run_kernel_test() -> ExecutionResult {let randomize_zp = config.quant_type == QuantizedMatmulType::ZeroPoint;- execute_quantized_matmul(+ execute_quantized_matmul::<B>(ctx,batch,input_dim,@@ -514,12 +497,11 @@ fn run_kernel_test()}-#[test]-fn test_quant_gmv() {- let ctx = match <Metal as Backend>::Context::new() {+fn test_quant_gmv_internal<B: Backend>() {+ let ctx = match B::Context::new() {Ok(c) => c,Err(_) => {- println!("Metal not available — skipping QMV test");+ println!("{} backend not available — skipping QMV test", std::any::type_name::<B>());return;},};@@ -553,17 +535,16 @@ fn test_quant_gmv() {for config in &configs {for &(output_dim, input_dim) in QMV_DIMS {- run_kernel_test(&ctx, 1, output_dim, input_dim, config, true, 1);+ run_kernel_test::<B>(&ctx, 1, output_dim, input_dim, config, true, 1);}}}-#[test]-fn test_quant_gmm_transposed() {- let ctx = match <Metal as Backend>::Context::new() {+fn test_quant_gmm_transposed_internal<B: Backend>() {+ let ctx = match B::Context::new() {Ok(c) => c,Err(_) => {- println!("Metal not available — skipping QMM transposed test");+ println!("{} not available — skipping QMM transposed test", std::any::type_name::<B>());return;},};@@ -597,18 +578,16 @@ fn test_quant_gmm_transposed() {for config in &configs {for &(batch, output_dim, input_dim) in QMM_DIMS {- run_kernel_test(&ctx, batch, output_dim, input_dim, config, true, 1);+ run_kernel_test::<B>(&ctx, batch, output_dim, input_dim, config, true, 1);}}}-#[test]-#[ignore]-fn test_quant_matmul_perf() {- let ctx = match <Metal as Backend>::Context::new() {+fn test_quant_matmul_perf_internal<B: Backend>() {+ let ctx = match B::Context::new() {Ok(c) => c,Err(_) => {- println!("Metal not available — skipping Perf test");+ println!("{} backend not available — skipping Perf test", std::any::type_name::<B>());return;},};@@ -660,10 +639,11 @@ fn test_quant_matmul_perf() {);println!("{}", "-".repeat(85));+ let iterations = 20;for config in &configs {for &(batch, output_dim, input_dim) in &shapes {- let result = run_kernel_test(&ctx, batch, output_dim, input_dim, config, false, 20);- let avg = result.elapsed / 20.0 * 1000.0; // ms+ let result = run_kernel_test::<B>(&ctx, batch, output_dim, input_dim, config, false, iterations);+ let avg = result.elapsed / (iterations as f64) * 1000.0; // msprintln!("{:<20} | {:<10} | {:<6} | {:<5} | {:<5} | {:.4} ms",@@ -677,3 +657,25 @@ fn test_quant_matmul_perf() {}}}++#[test]+fn test_quant_gmv() {+ for_each_non_cpu_backend!(|B| {+ test_quant_gmv_internal::<B>();+ })+}++#[test]+fn test_quant_gmm_transposed() {+ for_each_non_cpu_backend!(|B| {+ test_quant_gmm_transposed_internal::<B>();+ })+}++#[test]+#[ignore]+fn test_quant_matmul_perf() {+ for_each_non_cpu_backend!(|B| {+ test_quant_matmul_perf_internal::<B>();+ })+}diff --git a/crates/uzu/tests/unit/kernel/mod.rs b/crates/uzu/tests/unit/kernel/mod.rsindex e9290f87c..dc6e2b888 100644--- a/crates/uzu/tests/unit/kernel/mod.rs+++ b/crates/uzu/tests/unit/kernel/mod.rs@@ -14,6 +14,7 @@ mod matmul;mod moe;mod pooling;mod qk_norm_test;+mod quant_matmul;mod rms_norm_test;mod rope_test;mod sampling;diff --git a/crates/uzu/tests/unit/kernel/quant_matmul/mod.rs b/crates/uzu/tests/unit/kernel/quant_matmul/mod.rsnew file mode 100644index 000000000..b97526f4f--- /dev/null+++ b/crates/uzu/tests/unit/kernel/quant_matmul/mod.rs@@ -0,0 +1,89 @@+#[macro_use]+#[path = "../../../common/mod.rs"]+mod common;++mod qmm_transposed_64x64_test;+mod qmm_transposed_test;+mod qmm_transposed_wide_test;+mod qmv_fast_test;+mod qmv_test;++use num_traits::Float;+use uzu::ArrayElement;++pub(super) struct Input<T: ArrayElement + Float> {+ pub w_packed: Vec<u32>,+ pub scales: Vec<T>,+ pub zero_points: Option<Vec<u8>>,+ pub biases: Option<Vec<T>>,+ pub x: Vec<T>,+ pub k: u32,+ pub n: u32,+ pub m: u32,+ pub group_size: u32,+ pub bits: u32,+ pub use_zero_points: bool,+ pub use_mlx_quant: bool,+}++pub(super) fn pack_weights_u32(+ values: &[u8],+ bits: u32,+) -> Vec<u32> {+ if bits == 4 {+ values+ .chunks(8)+ .map(|chunk| {+ let mut word = 0u32;+ for (i, &v) in chunk.iter().enumerate() {+ word |= ((v & 0xF) as u32) << (i * 4);+ }+ word+ })+ .collect()+ } else {+ values+ .chunks(4)+ .map(|chunk| {+ let mut word = 0u32;+ for (i, &v) in chunk.iter().enumerate() {+ word |= (v as u32) << (i * 8);+ }+ word+ })+ .collect()+ }+}++pub(super) fn pack_zero_points(+ values: &[u8],+ bits: u32,+) -> Vec<u8> {+ if bits == 4 {+ values+ .chunks(2)+ .map(|chunk| {+ let lo = chunk[0] & 0x0F;+ let hi = if chunk.len() > 1 {+ chunk[1] & 0x0F+ } else {+ 0+ };+ lo | (hi << 4)+ })+ .collect()+ } else {+ values.to_vec()+ }+}++pub(super) fn check_tolerance(+ expected: f32,+ actual: f32,+ rel_tol: f64,+ abs_tol: f64,+) -> bool {+ let diff = (expected - actual).abs() as f64;+ let tol = abs_tol.max(expected.abs() as f64 * rel_tol);+ diff <= tol+}diff --git a/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_64x64_test.rs b/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_64x64_test.rsnew file mode 100644index 000000000..62bad063e--- /dev/null+++ b/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_64x64_test.rs@@ -0,0 +1,372 @@+use std::fmt::{Debug, Display};++use half::bf16;+use num_traits::Float;+use uzu::{+ ArrayElement, DataType,+ backends::{+ common::{Backend, Buffer, Context, Encoder, Kernels, kernel::QuantizedMatmulQmmTransposed64x64Kernel},+ cpu::Cpu,+ },+};++use super::{Input, check_tolerance, pack_weights_u32, pack_zero_points};+use crate::{common::helpers::alloc_buffer_with_data, uzu_test};++fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {+ let context = B::Context::new().expect("Failed to create Context");+ let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmmTransposed64x64Kernel::new(+ &context,+ T::data_type(),+ input.group_size,+ input.bits,+ input.use_zero_points,+ input.use_mlx_quant,+ )+ .expect("Failed to create QuantizedMatmulQmmTransposed64x64Kernel");++ let w_buf = alloc_buffer_with_data::<B, u32>(&context, &input.w_packed);+ let scales_buf = alloc_buffer_with_data::<B, T>(&context, &input.scales);++ let zp_buf = input.zero_points.as_ref().map(|zp| alloc_buffer_with_data::<B, u8>(&context, zp));+ let bias_buf = input.biases.as_ref().map(|b| alloc_buffer_with_data::<B, T>(&context, b));++ let x_buf = alloc_buffer_with_data::<B, T>(&context, &input.x);+ let output_size = (input.m as usize) * (input.n as usize) * T::data_type().size_in_bytes();+ let mut y_buf = context.create_buffer(output_size).expect("Failed to create buffer");++ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");+ kernel.encode(+ &w_buf,+ &scales_buf,+ zp_buf.as_ref(),+ bias_buf.as_ref(),+ &x_buf,+ &mut y_buf,+ input.k,+ input.n,+ input.m,+ &mut encoder,+ );++ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");++ let y_ptr = y_buf.cpu_ptr().as_ptr() as *const T;+ let y_len = (input.m as usize) * (input.n as usize);+ unsafe { std::slice::from_raw_parts(y_ptr, y_len) }.to_vec()+}++/// Create test data for transposed 64x64 qmm: x[m×k] * w_transposed[n×k] = y[m×n]+/// Weights are in transposed [n × k] layout.+fn get_test_data_basic<T: ArrayElement + Float>(+ m: usize,+ k: usize,+ n: usize,+ group_size: u32,+ bits: u32,+ use_zero_points: bool,+ use_mlx_quant: bool,+) -> (Input<T>, Vec<T>) {+ let num_groups_k = (k + group_size as usize - 1) / group_size as usize;++ // Generate quantized weights [n × k]+ let max_val = if bits == 4 {+ 15u8+ } else {+ 255u8+ };+ let mut weights_raw: Vec<u8> = Vec::with_capacity(n * k);+ for j in 0..n {+ for l in 0..k {+ weights_raw.push(((j * 3 + l * 7 + 1) % (max_val as usize + 1)) as u8);+ }+ }+ let w_packed = pack_weights_u32(&weights_raw, bits);++ // Scales: [n × num_groups_k]+ let mut scales_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);+ for j in 0..n {+ for g in 0..num_groups_k {+ scales_f32.push(0.5 + 0.1 * ((j + g) % 5) as f32);+ }+ }+ let scales: Vec<T> = scales_f32.iter().map(|&v| T::from(v).unwrap()).collect();++ // Zero points or biases+ let zp_stride = if bits == 4 {+ (num_groups_k + 1) / 2+ } else {+ num_groups_k+ };++ let (zero_points, biases) = if use_zero_points {+ let mut zp_raw: Vec<u8> = Vec::with_capacity(n * num_groups_k);+ for j in 0..n {+ for g in 0..num_groups_k {+ let zp_val = ((j * 2 + g * 3) % (max_val as usize + 1)) as u8;+ zp_raw.push(zp_val);+ }+ }+ // Pack zero points per row+ let mut zp_packed: Vec<u8> = Vec::with_capacity(n * zp_stride);+ for j in 0..n {+ let row = &zp_raw[j * num_groups_k..(j + 1) * num_groups_k];+ let packed_row = pack_zero_points(row, bits);+ let mut padded = packed_row;+ padded.resize(zp_stride, 0);+ zp_packed.extend_from_slice(&padded);+ }+ (Some(zp_packed), None)+ } else if use_mlx_quant {+ let mut biases_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);+ for j in 0..n {+ for g in 0..num_groups_k {+ biases_f32.push(0.01 * ((j + g * 2) % 7) as f32);+ }+ }+ let biases: Vec<T> = biases_f32.iter().map(|&v| T::from(v).unwrap()).collect();+ (None, Some(biases))+ } else {+ unreachable!("Must use either zero_points or mlx_quant");+ };++ // X: [m × k]+ let mut x_f32: Vec<f32> = Vec::with_capacity(m * k);+ for i in 0..m {+ for l in 0..k {+ x_f32.push(0.1 * f32::sin((i * k + l) as f32 * 0.05) + 0.5);+ }+ }+ let x: Vec<T> = x_f32.iter().map(|&v| T::from(v).unwrap()).collect();++ let input = Input {+ w_packed,+ scales,+ zero_points,+ biases,+ x,+ k: k as u32,+ n: n as u32,+ m: m as u32,+ group_size,+ bits,+ use_zero_points,+ use_mlx_quant,+ };++ let expected = get_output::<Cpu, T>(&input);+ (input, expected)+}++/// Small edge-case test data with known values+fn get_test_data_edge<T: ArrayElement + Float>(+ group_size: u32,+ bits: u32,+ use_zero_points: bool,+ use_mlx_quant: bool,+) -> (Input<T>, Vec<T>) {+ let m = 2usize;+ let k = group_size as usize;+ let n = 64usize;+ let num_groups_k = 1;++ // Simple weights [n × k]: all 1s+ let weights_raw: Vec<u8> = vec![1u8; n * k];+ let w_packed = pack_weights_u32(&weights_raw, bits);++ // Unit scales+ let scales: Vec<T> = vec![T::one(); n * num_groups_k];++ let zp_stride = if bits == 4 {+ (num_groups_k + 1) / 2+ } else {+ num_groups_k+ };++ let (zero_points, biases) = if use_zero_points {+ // Zero points all 0 -> bias = 0+ let zp_packed = vec![0u8; n * zp_stride];+ (Some(zp_packed), None)+ } else if use_mlx_quant {+ let biases: Vec<T> = vec![T::zero(); n * num_groups_k];+ (None, Some(biases))+ } else {+ unreachable!("Must use either zero_points or mlx_quant");+ };++ // X: simple increasing values+ let x_f32: Vec<f32> = (0..m * k).map(|i| (i + 1) as f32 * 0.1).collect();+ let x: Vec<T> = x_f32.iter().map(|&v| T::from(v).unwrap()).collect();++ let input = Input {+ w_packed,+ scales,+ zero_points,+ biases,+ x,+ k: k as u32,+ n: n as u32,+ m: m as u32,+ group_size,+ bits,+ use_zero_points,+ use_mlx_quant,+ };++ let expected = get_output::<Cpu, T>(&input);+ (input, expected)+}++fn test_internal<T: ArrayElement + Float + Debug + Display>(+ input: &Input<T>,+ expected: &[T],+) {+ let (rel_tol, abs_tol): (f64, f64) = match T::data_type() {+ DataType::BF16 => (0.05, 0.5),+ DataType::F16 => (0.02, 0.5),+ _ => (0.01, 0.1),+ };++ for_each_non_cpu_backend!(|B| {+ let output = get_output::<B, T>(input);+ assert_eq!(expected.len(), output.len(), "Output length mismatch");++ let mut errors = 0;+ for (i, (&exp, &got)) in expected.iter().zip(output.iter()).enumerate() {+ let exp_f32 = exp.to_f32().unwrap();+ let got_f32 = got.to_f32().unwrap();+ if !check_tolerance(exp_f32, got_f32, rel_tol, abs_tol) {+ if errors < 5 {+ eprintln!(" idx={} expected={} got={} diff={}", i, exp_f32, got_f32, (exp_f32 - got_f32).abs());+ }+ errors += 1;+ }+ }+ assert_eq!(+ errors,+ 0,+ "QMM transposed 64x64 kernel: backend={}, m={}, k={}, n={}, gs={}, bits={}, zp={}, mlx={}: {} mismatches",+ std::any::type_name::<B>(),+ input.m,+ input.k,+ input.n,+ input.group_size,+ input.bits,+ input.use_zero_points,+ input.use_mlx_quant,+ errors,+ );+ });+}++fn get_test_dims(+ group_size: u32,+ bits: u32,+) -> Vec<(usize, usize, usize)> {+ let gs = group_size as usize;+ // 64x64 kernel: BM=64, BK=64, BN=64.+ // K must be >= group_size (transposed grouping along K).+ // N must be a multiple of 64 (BN tile size).+ // bf16 8-bit: keep dims small to avoid catastrophic cancellation.+ if bits == 8 {+ vec![(2, gs, 64), (4, gs, 64)]+ } else {+ vec![(2, gs * 2, 64), (4, gs * 2, 128), (8, gs, 128)]+ }+}++fn test_basic(+ group_size: u32,+ bits: u32,+ use_zero_points: bool,+ use_mlx_quant: bool,+) {+ let dims = get_test_dims(group_size, bits);++ for (m, k, n) in dims {+ let (input, expected) = get_test_data_basic::<bf16>(m, k, n, group_size, bits, use_zero_points, use_mlx_quant);+ test_internal::<bf16>(&input, &expected);+ }+}++fn test_edge(+ group_size: u32,+ bits: u32,+ use_zero_points: bool,+ use_mlx_quant: bool,+) {+ let (input, expected) = get_test_data_edge::<bf16>(group_size, bits, use_zero_points, use_mlx_quant);+ test_internal::<bf16>(&input, &expected);+}++// -- 4-bit, zero points -------------------------------------------------------++#[uzu_test]+#[ignore]+fn test_bf16_gs64_4bit_zp() {+ test_basic(64, 4, true, false);+}++#[uzu_test]+#[ignore]+fn test_bf16_gs128_4bit_zp() {+ test_basic(128, 4, true, false);+}++// -- 8-bit, zero points -------------------------------------------------------++#[uzu_test]+fn test_bf16_gs64_8bit_zp() {+ test_basic(64, 8, true, false);+}++#[uzu_test]+fn test_bf16_gs128_8bit_zp() {+ test_basic(128, 8, true, false);+}++// -- 4-bit, mlx quant ----------------------------------------------------------++#[uzu_test]+fn test_bf16_gs64_4bit_mlx() {+ test_basic(64, 4, false, true);+}++#[uzu_test]+fn test_bf16_gs128_4bit_mlx() {+ test_basic(128, 4, false, true);+}++// -- 8-bit, mlx quant ----------------------------------------------------------++#[uzu_test]+fn test_bf16_gs64_8bit_mlx() {+ test_basic(64, 8, false, true);+}++#[uzu_test]+fn test_bf16_gs128_8bit_mlx() {+ test_basic(128, 8, false, true);+}++// -- Edge cases ----------------------------------------------------------------++#[uzu_test]+fn test_edge_bf16_4bit_zp() {+ test_edge(64, 4, true, false);+}++#[uzu_test]+fn test_edge_bf16_8bit_zp() {+ test_edge(64, 8, true, false);+}++#[uzu_test]+fn test_edge_bf16_4bit_mlx() {+ test_edge(64, 4, false, true);+}++#[uzu_test]+fn test_edge_bf16_8bit_mlx() {+ test_edge(64, 8, false, true);+}diff --git a/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rs b/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rsnew file mode 100644index 000000000..db6185dd3--- /dev/null+++ b/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rs@@ -0,0 +1,530 @@+use std::fmt::{Debug, Display};++use half::{bf16, f16};+use num_traits::Float;+use uzu::{+ ArrayElement, DataType,+ backends::{+ common::{Backend, Buffer, Context, Encoder, Kernels, kernel::QuantizedMatmulQmmTransposedKernel},+ cpu::Cpu,+ },+};++use super::{Input, check_tolerance, pack_weights_u32, pack_zero_points};+use crate::{common::helpers::alloc_buffer_with_data, uzu_test};++fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {+ let context = B::Context::new().expect("Failed to create Context");+ let aligned_n = (input.n % 32) == 0;+ let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel::new(+ &context,+ T::data_type(),+ input.group_size,+ input.bits,+ input.use_zero_points,+ input.use_mlx_quant,+ aligned_n,+ )+ .expect("Failed to create QuantizedMatmulQmmTransposedKernel");++ let w_buf = alloc_buffer_with_data::<B, u32>(&context, &input.w_packed);+ let scales_buf = alloc_buffer_with_data::<B, T>(&context, &input.scales);++ let zp_buf = input.zero_points.as_ref().map(|zp| alloc_buffer_with_data::<B, u8>(&context, zp));+ let bias_buf = input.biases.as_ref().map(|b| alloc_buffer_with_data::<B, T>(&context, b));++ let x_buf = alloc_buffer_with_data::<B, T>(&context, &input.x);+ let output_size = (input.m as usize) * (input.n as usize) * T::data_type().size_in_bytes();+ let mut y_buf = context.create_buffer(output_size).expect("Failed to create buffer");++ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");+ kernel.encode(+ &w_buf,+ &scales_buf,+ zp_buf.as_ref(),+ bias_buf.as_ref(),+ &x_buf,+ &mut y_buf,+ input.k,+ input.n,+ input.m,+ &mut encoder,+ );++ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");++ let y_ptr = y_buf.cpu_ptr().as_ptr() as *const T;+ let y_len = (input.m as usize) * (input.n as usize);+ unsafe { std::slice::from_raw_parts(y_ptr, y_len) }.to_vec()+}++/// Create test data for transposed qmm: x[m×k] * w_transposed[n×k] = y[m×n]+/// Weights are in transposed [n × k] layout.+fn get_test_data_basic<T: ArrayElement + Float>(+ m: usize,+ k: usize,+ n: usize,+ group_size: u32,+ bits: u32,+ use_zero_points: bool,+ use_mlx_quant: bool,+) -> (Input<T>, Vec<T>) {+ let num_groups_k = (k + group_size as usize - 1) / group_size as usize;++ // Generate quantized weights [n × k]+ let max_val = if bits == 4 {+ 15u8+ } else {+ 255u8+ };+ let mut weights_raw: Vec<u8> = Vec::with_capacity(n * k);+ for j in 0..n {+ for l in 0..k {+ weights_raw.push(((j * 3 + l * 7 + 1) % (max_val as usize + 1)) as u8);+ }+ }+ let w_packed = pack_weights_u32(&weights_raw, bits);++ // Scales: [n × num_groups_k]+ let mut scales_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);+ for j in 0..n {+ for g in 0..num_groups_k {+ scales_f32.push(0.5 + 0.1 * ((j + g) % 5) as f32);+ }+ }+ let scales: Vec<T> = scales_f32.iter().map(|&v| T::from(v).unwrap()).collect();++ // Zero points or biases+ let zp_stride = if bits == 4 {+ (num_groups_k + 1) / 2+ } else {+ num_groups_k+ };++ let (zero_points, biases) = if use_zero_points {+ let mut zp_raw: Vec<u8> = Vec::with_capacity(n * num_groups_k);+ for j in 0..n {+ for g in 0..num_groups_k {+ let zp_val = ((j * 2 + g * 3) % (max_val as usize + 1)) as u8;+ zp_raw.push(zp_val);+ }+ }+ // Pack zero points per row+ let mut zp_packed: Vec<u8> = Vec::with_capacity(n * zp_stride);+ for j in 0..n {+ let row = &zp_raw[j * num_groups_k..(j + 1) * num_groups_k];+ let packed_row = pack_zero_points(row, bits);+ let mut padded = packed_row;+ padded.resize(zp_stride, 0);+ zp_packed.extend_from_slice(&padded);+ }+ (Some(zp_packed), None)+ } else if use_mlx_quant {+ let mut biases_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);+ for j in 0..n {+ for g in 0..num_groups_k {+ biases_f32.push(0.01 * ((j + g * 2) % 7) as f32);+ }+ }+ let biases: Vec<T> = biases_f32.iter().map(|&v| T::from(v).unwrap()).collect();+ (None, Some(biases))+ } else {+ unreachable!("Must use either zero_points or mlx_quant");+ };++ // X: [m × k]+ let mut x_f32: Vec<f32> = Vec::with_capacity(m * k);+ for i in 0..m {+ for l in 0..k {+ x_f32.push(0.1 * f32::sin((i * k + l) as f32 * 0.05) + 0.5);+ }+ }+ let x: Vec<T> = x_f32.iter().map(|&v| T::from(v).unwrap()).collect();++ let input = Input {+ w_packed,+ scales,+ zero_points,+ biases,+ x,+ k: k as u32,+ n: n as u32,+ m: m as u32,+ group_size,+ bits,+ use_zero_points,+ use_mlx_quant,+ };++ let expected = get_output::<Cpu, T>(&input);+ (input, expected)+}++/// Small edge-case test data with known values+fn get_test_data_edge<T: ArrayElement + Float>(+ group_size: u32,+ bits: u32,+ use_zero_points: bool,+ use_mlx_quant: bool,+) -> (Input<T>, Vec<T>) {+ let m = 2usize;+ let k = group_size as usize;+ let n = 32usize;+ let num_groups_k = 1;++ // Simple weights [n × k]: all 1s+ let weights_raw: Vec<u8> = vec![1u8; n * k];+ let w_packed = pack_weights_u32(&weights_raw, bits);++ // Unit scales+ let scales: Vec<T> = vec![T::one(); n * num_groups_k];++ let zp_stride = if bits == 4 {+ (num_groups_k + 1) / 2+ } else {+ num_groups_k+ };++ let (zero_points, biases) = if use_zero_points {+ // Zero points all 0 -> bias = 0+ let zp_packed = vec![0u8; n * zp_stride];+ (Some(zp_packed), None)+ } else if use_mlx_quant {+ let biases: Vec<T> = vec![T::zero(); n * num_groups_k];+ (None, Some(biases))+ } else {+ unreachable!("Must use either zero_points or mlx_quant");+ };++ // X: simple increasing values+ let x_f32: Vec<f32> = (0..m * k).map(|i| (i + 1) as f32 * 0.1).collect();+ let x: Vec<T> = x_f32.iter().map(|&v| T::from(v).unwrap()).collect();++ let input = Input {+ w_packed,+ scales,+ zero_points,+ biases,+ x,+ k: k as u32,+ n: n as u32,+ m: m as u32,+ group_size,+ bits,+ use_zero_points,+ use_mlx_quant,+ };++ let expected = get_output::<Cpu, T>(&input);+ (input, expected)+}++fn test_internal<T: ArrayElement + Float + Debug + Display>(+ input: &Input<T>,+ expected: &[T],+) {+ // GPU MMA units use different rounding than scalar CPU code.+ // bf16 has 7-bit mantissa (ULP ~0.8%), f16 has 10-bit (~0.1%).+ // Tiled accumulation reorders FMA ops, compounding rounding differences.+ // 8-bit weights (0-255) with ZP bias cause larger accumulated errors.+ let (rel_tol, abs_tol): (f64, f64) = match T::data_type() {+ DataType::BF16 => (0.05, 0.5),+ DataType::F16 => (0.02, 0.5),+ _ => (0.01, 0.1),+ };++ for_each_non_cpu_backend!(|B| {+ let output = get_output::<B, T>(input);+ assert_eq!(expected.len(), output.len(), "Output length mismatch");++ let mut errors = 0;+ for (i, (&exp, &got)) in expected.iter().zip(output.iter()).enumerate() {+ let exp_f32 = exp.to_f32().unwrap();+ let got_f32 = got.to_f32().unwrap();+ if !check_tolerance(exp_f32, got_f32, rel_tol, abs_tol) {+ if errors < 5 {+ eprintln!(" idx={} expected={} got={} diff={}", i, exp_f32, got_f32, (exp_f32 - got_f32).abs());+ }+ errors += 1;+ }+ }+ assert_eq!(+ errors,+ 0,+ "QMM transposed kernel: backend={}, m={}, k={}, n={}, gs={}, bits={}, zp={}, mlx={}: {} mismatches",+ std::any::type_name::<B>(),+ input.m,+ input.k,+ input.n,+ input.group_size,+ input.bits,+ input.use_zero_points,+ input.use_mlx_quant,+ errors,+ );+ });+}++fn get_test_dims(+ group_size: u32,+ bits: u32,+ is_half_precision: bool,+) -> Vec<(usize, usize, usize)> {+ let gs = group_size as usize;+ // K must be >= group_size for the transposed grouping along K.+ // N must be >= 32 for the Metal tiled weight loader (BN=32).+ // For half-precision 8-bit, keep dimensions small to avoid+ // catastrophic cancellation from large weight values (0-255).+ if is_half_precision && bits == 8 {+ vec![(2, gs, gs), (4, gs, gs)]+ } else {+ vec![(2, gs * 2, gs), (4, gs * 2, gs * 2), (8, gs, gs * 2)]+ }+}++fn test_basic<T: ArrayElement + Float + Debug + Display>(+ group_size: u32,+ bits: u32,+ use_zero_points: bool,+ use_mlx_quant: bool,+) {+ let is_half = matches!(T::data_type(), DataType::F16 | DataType::BF16);+ let dims = get_test_dims(group_size, bits, is_half);++ for (m, k, n) in dims {+ let (input, expected) = get_test_data_basic::<T>(m, k, n, group_size, bits, use_zero_points, use_mlx_quant);+ test_internal::<T>(&input, &expected);+ }+}++fn test_edge<T: ArrayElement + Float + Debug + Display>(+ group_size: u32,+ bits: u32,+ use_zero_points: bool,+ use_mlx_quant: bool,+) {+ let (input, expected) = get_test_data_edge::<T>(group_size, bits, use_zero_points, use_mlx_quant);+ test_internal::<T>(&input, &expected);+}++// -- 4-bit, zero points -------------------------------------------------------++#[uzu_test]+fn test_f32_gs32_4bit_zp() {+ test_basic::<f32>(32, 4, true, false);+}++#[uzu_test]+fn test_f32_gs64_4bit_zp() {+ test_basic::<f32>(64, 4, true, false);+}++#[uzu_test]+fn test_f32_gs128_4bit_zp() {+ test_basic::<f32>(128, 4, true, false);+}++#[uzu_test]+fn test_f16_gs32_4bit_zp() {+ test_basic::<f16>(32, 4, true, false);+}++#[uzu_test]+fn test_f16_gs64_4bit_zp() {+ test_basic::<f16>(64, 4, true, false);+}++#[uzu_test]+fn test_f16_gs128_4bit_zp() {+ test_basic::<f16>(128, 4, true, false);+}++#[uzu_test]+fn test_bf16_gs32_4bit_zp() {+ test_basic::<bf16>(32, 4, true, false);+}++#[uzu_test]+#[ignore]+fn test_bf16_gs64_4bit_zp() {+ test_basic::<bf16>(64, 4, true, false);+}++#[uzu_test]+#[ignore]+fn test_bf16_gs128_4bit_zp() {+ test_basic::<bf16>(128, 4, true, false);+}++// -- 8-bit, zero points -------------------------------------------------------++#[uzu_test]+fn test_f32_gs32_8bit_zp() {+ test_basic::<f32>(32, 8, true, false);+}++#[uzu_test]+fn test_f32_gs64_8bit_zp() {+ test_basic::<f32>(64, 8, true, false);+}++#[uzu_test]+fn test_f32_gs128_8bit_zp() {+ test_basic::<f32>(128, 8, true, false);+}++#[uzu_test]+fn test_f16_gs32_8bit_zp() {+ test_basic::<f16>(32, 8, true, false);+}++#[uzu_test]+fn test_f16_gs64_8bit_zp() {+ test_basic::<f16>(64, 8, true, false);+}++#[uzu_test]+fn test_f16_gs128_8bit_zp() {+ test_basic::<f16>(128, 8, true, false);+}++#[uzu_test]+fn test_bf16_gs32_8bit_zp() {+ test_basic::<bf16>(32, 8, true, false);+}++#[uzu_test]+fn test_bf16_gs64_8bit_zp() {+ test_basic::<bf16>(64, 8, true, false);+}++#[uzu_test]+fn test_bf16_gs128_8bit_zp() {+ test_basic::<bf16>(128, 8, true, false);+}++// -- 4-bit, mlx quant ----------------------------------------------------------++#[uzu_test]+fn test_f32_gs32_4bit_mlx() {+ test_basic::<f32>(32, 4, false, true);+}++#[uzu_test]+fn test_f32_gs64_4bit_mlx() {+ test_basic::<f32>(64, 4, false, true);+}++#[uzu_test]+fn test_f32_gs128_4bit_mlx() {+ test_basic::<f32>(128, 4, false, true);+}++#[uzu_test]+fn test_f16_gs32_4bit_mlx() {+ test_basic::<f16>(32, 4, false, true);+}++#[uzu_test]+fn test_f16_gs64_4bit_mlx() {+ test_basic::<f16>(64, 4, false, true);+}++#[uzu_test]+fn test_f16_gs128_4bit_mlx() {+ test_basic::<f16>(128, 4, false, true);+}++#[uzu_test]+fn test_bf16_gs32_4bit_mlx() {+ test_basic::<bf16>(32, 4, false, true);+}++#[uzu_test]+fn test_bf16_gs64_4bit_mlx() {+ test_basic::<bf16>(64, 4, false, true);+}++#[uzu_test]+fn test_bf16_gs128_4bit_mlx() {+ test_basic::<bf16>(128, 4, false, true);+}++// -- 8-bit, mlx quant ----------------------------------------------------------++#[uzu_test]+fn test_f32_gs32_8bit_mlx() {+ test_basic::<f32>(32, 8, false, true);+}++#[uzu_test]+fn test_f32_gs64_8bit_mlx() {+ test_basic::<f32>(64, 8, false, true);+}++#[uzu_test]+fn test_f32_gs128_8bit_mlx() {+ test_basic::<f32>(128, 8, false, true);+}++#[uzu_test]+fn test_f16_gs32_8bit_mlx() {+ test_basic::<f16>(32, 8, false, true);+}++#[uzu_test]+fn test_f16_gs64_8bit_mlx() {+ test_basic::<f16>(64, 8, false, true);+}++#[uzu_test]+fn test_f16_gs128_8bit_mlx() {+ test_basic::<f16>(128, 8, false, true);+}++#[uzu_test]+fn test_bf16_gs32_8bit_mlx() {+ test_basic::<bf16>(32, 8, false, true);+}++#[uzu_test]+fn test_bf16_gs64_8bit_mlx() {+ test_basic::<bf16>(64, 8, false, true);+}++#[uzu_test]+fn test_bf16_gs128_8bit_mlx() {+ test_basic::<bf16>(128, 8, false, true);+}++// -- Edge cases ----------------------------------------------------------------++#[uzu_test]+fn test_edge_f32_4bit_zp() {+ test_edge::<f32>(32, 4, true, false);+}++#[uzu_test]+fn test_edge_f32_8bit_zp() {+ test_edge::<f32>(32, 8, true, false);+}++#[uzu_test]+fn test_edge_f16_4bit_zp() {+ test_edge::<f16>(32, 4, true, false);+}++#[uzu_test]+fn test_edge_f32_4bit_mlx() {+ test_edge::<f32>(32, 4, false, true);+}++#[uzu_test]+fn test_edge_f32_8bit_mlx() {+ test_edge::<f32>(32, 8, false, true);+}++#[uzu_test]+fn test_edge_f16_4bit_mlx() {+ test_edge::<f16>(32, 4, false, true);+}diff --git a/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_wide_test.rs b/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_wide_test.rsnew file mode 100644index 000000000..59a74348b--- /dev/null+++ b/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_wide_test.rs@@ -0,0 +1,392 @@+use std::fmt::{Debug, Display};++use half::bf16;+use num_traits::Float;+use uzu::{+ ArrayElement, DataType,+ backends::{+ common::{Backend, Buffer, Context, Encoder, Kernels, kernel::QuantizedMatmulQmmTransposedWideKernel},+ cpu::Cpu,+ },+};++use super::{Input, check_tolerance, pack_weights_u32, pack_zero_points};+use crate::{common::helpers::alloc_buffer_with_data, uzu_test};++fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {+ let context = B::Context::new().expect("Failed to create Context");+ let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmmTransposedWideKernel::new(+ &context,+ T::data_type(),+ input.group_size,+ input.bits,+ input.use_zero_points,+ input.use_mlx_quant,+ )+ .expect("Failed to create QuantizedMatmulQmmTransposedWideKernel");++ let w_buf = alloc_buffer_with_data::<B, u32>(&context, &input.w_packed);+ let scales_buf = alloc_buffer_with_data::<B, T>(&context, &input.scales);++ let zp_buf = input.zero_points.as_ref().map(|zp| alloc_buffer_with_data::<B, u8>(&context, zp));+ let bias_buf = input.biases.as_ref().map(|b| alloc_buffer_with_data::<B, T>(&context, b));++ let x_buf = alloc_buffer_with_data::<B, T>(&context, &input.x);+ let output_size = (input.m as usize) * (input.n as usize) * T::data_type().size_in_bytes();+ let mut y_buf = context.create_buffer(output_size).expect("Failed to create buffer");++ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");+ kernel.encode(+ &w_buf,+ &scales_buf,+ zp_buf.as_ref(),+ bias_buf.as_ref(),+ &x_buf,+ &mut y_buf,+ input.k,+ input.n,+ input.m,+ &mut encoder,+ );++ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");++ let y_ptr = y_buf.cpu_ptr().as_ptr() as *const T;+ let y_len = (input.m as usize) * (input.n as usize);+ unsafe { std::slice::from_raw_parts(y_ptr, y_len) }.to_vec()+}++/// Create test data for transposed wide qmm: x[m×k] * w_transposed[n×k] = y[m×n]+/// Weights are in transposed [n × k] layout.+fn get_test_data_basic<T: ArrayElement + Float>(+ m: usize,+ k: usize,+ n: usize,+ group_size: u32,+ bits: u32,+ use_zero_points: bool,+ use_mlx_quant: bool,+) -> (Input<T>, Vec<T>) {+ let num_groups_k = (k + group_size as usize - 1) / group_size as usize;++ // Generate quantized weights [n × k]+ let max_val = if bits == 4 {+ 15u8+ } else {+ 255u8+ };+ let mut weights_raw: Vec<u8> = Vec::with_capacity(n * k);+ for j in 0..n {+ for l in 0..k {+ weights_raw.push(((j * 3 + l * 7 + 1) % (max_val as usize + 1)) as u8);+ }+ }+ let w_packed = pack_weights_u32(&weights_raw, bits);++ // Scales: [n × num_groups_k]+ let mut scales_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);+ for j in 0..n {+ for g in 0..num_groups_k {+ scales_f32.push(0.5 + 0.1 * ((j + g) % 5) as f32);+ }+ }+ let scales: Vec<T> = scales_f32.iter().map(|&v| T::from(v).unwrap()).collect();++ // Zero points or biases+ let zp_stride = if bits == 4 {+ (num_groups_k + 1) / 2+ } else {+ num_groups_k+ };++ let (zero_points, biases) = if use_zero_points {+ let mut zp_raw: Vec<u8> = Vec::with_capacity(n * num_groups_k);+ for j in 0..n {+ for g in 0..num_groups_k {+ let zp_val = ((j * 2 + g * 3) % (max_val as usize + 1)) as u8;+ zp_raw.push(zp_val);+ }+ }+ // Pack zero points per row+ let mut zp_packed: Vec<u8> = Vec::with_capacity(n * zp_stride);+ for j in 0..n {+ let row = &zp_raw[j * num_groups_k..(j + 1) * num_groups_k];+ let packed_row = pack_zero_points(row, bits);+ let mut padded = packed_row;+ padded.resize(zp_stride, 0);+ zp_packed.extend_from_slice(&padded);+ }+ (Some(zp_packed), None)+ } else if use_mlx_quant {+ let mut biases_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);+ for j in 0..n {+ for g in 0..num_groups_k {+ biases_f32.push(0.01 * ((j + g * 2) % 7) as f32);+ }+ }+ let biases: Vec<T> = biases_f32.iter().map(|&v| T::from(v).unwrap()).collect();+ (None, Some(biases))+ } else {+ unreachable!("Must use either zero_points or mlx_quant");+ };++ // X: [m × k]+ let mut x_f32: Vec<f32> = Vec::with_capacity(m * k);+ for i in 0..m {+ for l in 0..k {+ x_f32.push(0.1 * f32::sin((i * k + l) as f32 * 0.05) + 0.5);+ }+ }+ let x: Vec<T> = x_f32.iter().map(|&v| T::from(v).unwrap()).collect();++ let input = Input {+ w_packed,+ scales,+ zero_points,+ biases,+ x,+ k: k as u32,+ n: n as u32,+ m: m as u32,+ group_size,+ bits,+ use_zero_points,+ use_mlx_quant,+ };++ let expected = get_output::<Cpu, T>(&input);+ (input, expected)+}++/// Small edge-case test data with known values+fn get_test_data_edge<T: ArrayElement + Float>(+ group_size: u32,+ bits: u32,+ use_zero_points: bool,+ use_mlx_quant: bool,+) -> (Input<T>, Vec<T>) {+ let m = 2usize;+ let k = group_size as usize;+ let n = 64usize;+ let num_groups_k = 1;++ // Simple weights [n × k]: all 1s+ let weights_raw: Vec<u8> = vec![1u8; n * k];+ let w_packed = pack_weights_u32(&weights_raw, bits);++ // Unit scales+ let scales: Vec<T> = vec![T::one(); n * num_groups_k];++ let zp_stride = if bits == 4 {+ (num_groups_k + 1) / 2+ } else {+ num_groups_k+ };++ let (zero_points, biases) = if use_zero_points {+ // Zero points all 0 -> bias = 0+ let zp_packed = vec![0u8; n * zp_stride];+ (Some(zp_packed), None)+ } else if use_mlx_quant {+ let biases: Vec<T> = vec![T::zero(); n * num_groups_k];+ (None, Some(biases))+ } else {+ unreachable!("Must use either zero_points or mlx_quant");+ };++ // X: simple increasing values+ let x_f32: Vec<f32> = (0..m * k).map(|i| (i + 1) as f32 * 0.1).collect();+ let x: Vec<T> = x_f32.iter().map(|&v| T::from(v).unwrap()).collect();++ let input = Input {+ w_packed,+ scales,+ zero_points,+ biases,+ x,+ k: k as u32,+ n: n as u32,+ m: m as u32,+ group_size,+ bits,+ use_zero_points,+ use_mlx_quant,+ };++ let expected = get_output::<Cpu, T>(&input);+ (input, expected)+}++fn test_internal<T: ArrayElement + Float + Debug + Display>(+ input: &Input<T>,+ expected: &[T],+) {+ let (rel_tol, abs_tol): (f64, f64) = match T::data_type() {+ DataType::BF16 => (0.05, 0.5),+ DataType::F16 => (0.02, 0.5),+ _ => (0.01, 0.1),+ };++ for_each_non_cpu_backend!(|B| {+ let output = get_output::<B, T>(input);+ assert_eq!(expected.len(), output.len(), "Output length mismatch");++ let mut errors = 0;+ for (i, (&exp, &got)) in expected.iter().zip(output.iter()).enumerate() {+ let exp_f32 = exp.to_f32().unwrap();+ let got_f32 = got.to_f32().unwrap();+ if !check_tolerance(exp_f32, got_f32, rel_tol, abs_tol) {+ if errors < 5 {+ eprintln!(" idx={} expected={} got={} diff={}", i, exp_f32, got_f32, (exp_f32 - got_f32).abs());+ }+ errors += 1;+ }+ }+ assert_eq!(+ errors,+ 0,+ "QMM transposed wide kernel: backend={}, m={}, k={}, n={}, gs={}, bits={}, zp={}, mlx={}: {} mismatches",+ std::any::type_name::<B>(),+ input.m,+ input.k,+ input.n,+ input.group_size,+ input.bits,+ input.use_zero_points,+ input.use_mlx_quant,+ errors,+ );+ });+}++fn get_test_dims(+ group_size: u32,+ bits: u32,+) -> Vec<(usize, usize, usize)> {+ let gs = group_size as usize;+ // Wide kernel: BM=64, BK=32, BN=64.+ // K must be >= group_size (transposed grouping along K).+ // N must be a multiple of 64 (BN tile size).+ // bf16 8-bit: keep dims small to avoid catastrophic cancellation.+ if bits == 8 {+ vec![(2, gs, 64), (4, gs, 64)]+ } else {+ vec![(2, gs * 2, 64), (4, gs * 2, 128), (8, gs, 128)]+ }+}++fn test_basic(+ group_size: u32,+ bits: u32,+ use_zero_points: bool,+ use_mlx_quant: bool,+) {+ let dims = get_test_dims(group_size, bits);++ for (m, k, n) in dims {+ let (input, expected) = get_test_data_basic::<bf16>(m, k, n, group_size, bits, use_zero_points, use_mlx_quant);+ test_internal::<bf16>(&input, &expected);+ }+}++fn test_edge(+ group_size: u32,+ bits: u32,+ use_zero_points: bool,+ use_mlx_quant: bool,+) {+ let (input, expected) = get_test_data_edge::<bf16>(group_size, bits, use_zero_points, use_mlx_quant);+ test_internal::<bf16>(&input, &expected);+}++// -- 4-bit, zero points -------------------------------------------------------++#[uzu_test]+fn test_bf16_gs32_4bit_zp() {+ test_basic(32, 4, true, false);+}++#[uzu_test]+#[ignore]+fn test_bf16_gs64_4bit_zp() {+ test_basic(64, 4, true, false);+}++#[uzu_test]+#[ignore]+fn test_bf16_gs128_4bit_zp() {+ test_basic(128, 4, true, false);+}++// -- 8-bit, zero points -------------------------------------------------------++#[uzu_test]+fn test_bf16_gs32_8bit_zp() {+ test_basic(32, 8, true, false);+}++#[uzu_test]+fn test_bf16_gs64_8bit_zp() {+ test_basic(64, 8, true, false);+}++#[uzu_test]+fn test_bf16_gs128_8bit_zp() {+ test_basic(128, 8, true, false);+}++// -- 4-bit, mlx quant ----------------------------------------------------------++#[uzu_test]+fn test_bf16_gs32_4bit_mlx() {+ test_basic(32, 4, false, true);+}++#[uzu_test]+fn test_bf16_gs64_4bit_mlx() {+ test_basic(64, 4, false, true);+}++#[uzu_test]+fn test_bf16_gs128_4bit_mlx() {+ test_basic(128, 4, false, true);+}++// -- 8-bit, mlx quant ----------------------------------------------------------++#[uzu_test]+fn test_bf16_gs32_8bit_mlx() {+ test_basic(32, 8, false, true);+}++#[uzu_test]+fn test_bf16_gs64_8bit_mlx() {+ test_basic(64, 8, false, true);+}++#[uzu_test]+fn test_bf16_gs128_8bit_mlx() {+ test_basic(128, 8, false, true);+}++// -- Edge cases ----------------------------------------------------------------++#[uzu_test]+fn test_edge_bf16_4bit_zp() {+ test_edge(32, 4, true, false);+}++#[uzu_test]+fn test_edge_bf16_8bit_zp() {+ test_edge(32, 8, true, false);+}++#[uzu_test]+fn test_edge_bf16_4bit_mlx() {+ test_edge(32, 4, false, true);+}++#[uzu_test]+fn test_edge_bf16_8bit_mlx() {+ test_edge(32, 8, false, true);+}diff --git a/crates/uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs b/crates/uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rsnew file mode 100644index 000000000..56aefbdbe--- /dev/null+++ b/crates/uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs@@ -0,0 +1,366 @@+use std::fmt::{Debug, Display};++use num_traits::Float;+use uzu::{+ ArrayElement, DataType,+ backends::{+ common::{+ Backend, Buffer, Context, Encoder, Kernels,+ kernel::{QuantizedMatmulQmvFastKernel, QuantizedMatmulQmvKernel},+ },+ cpu::Cpu,+ },+};++use super::{Input, check_tolerance, pack_weights_u32, pack_zero_points};+use crate::{common::helpers::alloc_buffer_with_data, uzu_test};++fn get_expected<T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {+ let context = <Cpu as Backend>::Context::new().expect("Failed to create Context");++ let w_buf = alloc_buffer_with_data::<Cpu, u32>(&context, &input.w_packed);+ let scales_buf = alloc_buffer_with_data::<Cpu, T>(&context, &input.scales);+ let zp_buf = input.zero_points.as_ref().map(|zp| alloc_buffer_with_data::<Cpu, u8>(&context, zp));+ let bias_buf = input.biases.as_ref().map(|b| alloc_buffer_with_data::<Cpu, T>(&context, b));+ let x_buf = alloc_buffer_with_data::<Cpu, T>(&context, &input.x);+ let output_size = (input.m as usize) * (input.n as usize) * T::data_type().size_in_bytes();+ let mut y_buf = context.create_buffer(output_size).expect("Failed to create buffer");++ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");++ let kernel = <<Cpu as Backend>::Kernels as Kernels>::QuantizedMatmulQmvKernel::new(+ &context,+ T::data_type(),+ input.group_size,+ input.bits,+ input.use_zero_points,+ input.use_mlx_quant,+ )+ .expect("Failed to create QuantizedMatmulQmvKernel");+ kernel.encode(+ &w_buf,+ &scales_buf,+ zp_buf.as_ref(),+ bias_buf.as_ref(),+ &x_buf,+ &mut y_buf,+ input.k,+ input.n,+ input.m,+ &mut encoder,+ );++ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");++ let y_ptr = y_buf.cpu_ptr().as_ptr() as *const T;+ let y_len = (input.m as usize) * (input.n as usize);+ unsafe { std::slice::from_raw_parts(y_ptr, y_len) }.to_vec()+}++fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {+ let context = B::Context::new().expect("Failed to create Context");++ let w_buf = alloc_buffer_with_data::<B, u32>(&context, &input.w_packed);+ let scales_buf = alloc_buffer_with_data::<B, T>(&context, &input.scales);+ let zp_buf = input.zero_points.as_ref().map(|zp| alloc_buffer_with_data::<B, u8>(&context, zp));+ let bias_buf = input.biases.as_ref().map(|b| alloc_buffer_with_data::<B, T>(&context, b));+ let x_buf = alloc_buffer_with_data::<B, T>(&context, &input.x);+ let output_size = (input.m as usize) * (input.n as usize) * T::data_type().size_in_bytes();+ let mut y_buf = context.create_buffer(output_size).expect("Failed to create buffer");++ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");++ let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmvFastKernel::new(+ &context,+ T::data_type(),+ input.group_size,+ input.bits,+ input.use_zero_points,+ input.use_mlx_quant,+ )+ .expect("Failed to create QuantizedMatmulQmvFastKernel");+ kernel.encode(+ &w_buf,+ &scales_buf,+ zp_buf.as_ref(),+ bias_buf.as_ref(),+ &x_buf,+ &mut y_buf,+ input.k,+ input.n,+ input.m,+ &mut encoder,+ );++ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");++ let y_ptr = y_buf.cpu_ptr().as_ptr() as *const T;+ let y_len = (input.m as usize) * (input.n as usize);+ unsafe { std::slice::from_raw_parts(y_ptr, y_len) }.to_vec()+}++fn get_test_data<T: ArrayElement + Float>(+ m: usize,+ k: usize,+ n: usize,+ group_size: u32,+ bits: u32,+ use_zero_points: bool,+ use_mlx_quant: bool,+) -> (Input<T>, Vec<T>) {+ let num_groups_k = (k + group_size as usize - 1) / group_size as usize;++ let mut weights_raw: Vec<u8> = Vec::with_capacity(n * k);+ let max_val = if bits == 4 {+ 15u8+ } else {+ 255u8+ };+ for j in 0..n {+ for l in 0..k {+ weights_raw.push(((j * 3 + l * 7 + 1) % (max_val as usize + 1)) as u8);+ }+ }+ let w_packed = pack_weights_u32(&weights_raw, bits);++ let mut scales_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);+ for j in 0..n {+ for g in 0..num_groups_k {+ scales_f32.push(0.5 + 0.1 * ((j + g) % 5) as f32);+ }+ }+ let scales: Vec<T> = scales_f32.iter().map(|&v| T::from(v).unwrap()).collect();++ let zp_stride = if bits == 4 {+ (num_groups_k + 1) / 2+ } else {+ num_groups_k+ };++ let (zero_points, biases) = if use_zero_points {+ let mut zp_raw: Vec<u8> = Vec::with_capacity(n * num_groups_k);+ for j in 0..n {+ for g in 0..num_groups_k {+ let zp_val = ((j * 2 + g * 3) % (max_val as usize + 1)) as u8;+ zp_raw.push(zp_val);+ }+ }+ let mut zp_packed: Vec<u8> = Vec::with_capacity(n * zp_stride);+ for j in 0..n {+ let row = &zp_raw[j * num_groups_k..(j + 1) * num_groups_k];+ let packed_row = pack_zero_points(row, bits);+ let mut padded = packed_row;+ padded.resize(zp_stride, 0);+ zp_packed.extend_from_slice(&padded);+ }+ (Some(zp_packed), None)+ } else if use_mlx_quant {+ let mut biases_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);+ for j in 0..n {+ for g in 0..num_groups_k {+ biases_f32.push(0.01 * ((j + g * 2) % 7) as f32);+ }+ }+ let biases: Vec<T> = biases_f32.iter().map(|&v| T::from(v).unwrap()).collect();+ (None, Some(biases))+ } else {+ unreachable!("Must use either zero_points or mlx_quant");+ };++ let mut x_f32: Vec<f32> = Vec::with_capacity(m * k);+ for i in 0..m {+ for l in 0..k {+ x_f32.push(0.1 * f32::sin((i * k + l) as f32 * 0.05) + 0.5);+ }+ }+ let x: Vec<T> = x_f32.iter().map(|&v| T::from(v).unwrap()).collect();++ let input = Input {+ w_packed,+ scales,+ zero_points,+ biases,+ x,+ k: k as u32,+ n: n as u32,+ m: m as u32,+ group_size,+ bits,+ use_zero_points,+ use_mlx_quant,+ };++ let expected = get_expected::<T>(&input);+ (input, expected)+}++fn test_internal<T: ArrayElement + Float + Debug + Display>(+ input: &Input<T>,+ expected: &[T],+) {+ let (rel_tol, abs_tol): (f64, f64) = match T::data_type() {+ DataType::BF16 => (0.05, 0.5),+ DataType::F16 => (0.02, 0.5),+ _ => (0.01, 0.1),+ };++ for_each_non_cpu_backend!(|B| {+ let output = get_output::<B, T>(input);+ assert_eq!(expected.len(), output.len(), "Output length mismatch");++ let mut errors = 0;+ for (i, (&exp, &got)) in expected.iter().zip(output.iter()).enumerate() {+ let exp_f32 = exp.to_f32().unwrap();+ let got_f32 = got.to_f32().unwrap();+ if !check_tolerance(exp_f32, got_f32, rel_tol, abs_tol) {+ if errors < 5 {+ eprintln!(" idx={} expected={} got={} diff={}", i, exp_f32, got_f32, (exp_f32 - got_f32).abs());+ }+ errors += 1;+ }+ }+ assert_eq!(+ errors,+ 0,+ "QMV fast kernel: backend={}, m={}, k={}, n={}, gs={}, bits={}, zp={}, mlx={}: {} mismatches",+ std::any::type_name::<B>(),+ input.m,+ input.k,+ input.n,+ input.group_size,+ input.bits,+ input.use_zero_points,+ input.use_mlx_quant,+ errors,+ );+ });+}++/// The fast kernel requires in_vec_size to be a multiple of block_size+/// (512 for 4-bit, 256 for 8-bit) and out_vec_size to be a multiple of 32.+fn get_test_dims(+ group_size: u32,+ bits: u32,+ is_half_precision: bool,+) -> Vec<(usize, usize, usize)> {+ let gs = group_size as usize;+ let block_size = if bits == 4 {+ 512+ } else {+ 256+ };+ let k = block_size;+ let n = gs.max(32);+ if is_half_precision && bits == 8 {+ vec![(1, k, n), (2, k, n)]+ } else {+ vec![(1, k, n), (2, k, n * 2), (1, k, n * 2)]+ }+}++fn test_basic<T: ArrayElement + Float + Debug + Display>(+ group_size: u32,+ bits: u32,+ use_zero_points: bool,+ use_mlx_quant: bool,+) {+ let is_half = matches!(T::data_type(), DataType::F16 | DataType::BF16);+ let dims = get_test_dims(group_size, bits, is_half);++ for (m, k, n) in dims {+ let (input, expected) = get_test_data::<T>(m, k, n, group_size, bits, use_zero_points, use_mlx_quant);+ test_internal::<T>(&input, &expected);+ }+}++// ── 4-bit, zero points ─────────────────────────────────────────────────────++#[uzu_test]+fn test_gs32_4bit_zp() {+ for_each_float_type!(|F| {+ test_basic::<F>(32, 4, true, false);+ })+}++#[uzu_test]+fn test_gs64_4bit_zp() {+ for_each_float_type!(|F| {+ test_basic::<F>(64, 4, true, false);+ })+}++#[uzu_test]+fn test_gs128_4bit_zp() {+ for_each_float_type!(|F| {+ test_basic::<F>(128, 4, true, false);+ })+}++// ── 8-bit, zero points ─────────────────────────────────────────────────────++#[uzu_test]+fn test_gs32_8bit_zp() {+ for_each_float_type!(|F| {+ test_basic::<F>(32, 8, true, false);+ })+}++#[uzu_test]+fn test_gs64_8bit_zp() {+ for_each_float_type!(|F| {+ test_basic::<F>(64, 8, true, false);+ })+}++#[uzu_test]+fn test_gs128_8bit_zp() {+ for_each_float_type!(|F| {+ test_basic::<F>(128, 8, true, false);+ })+}++// ── 4-bit, mlx quant ───────────────────────────────────────────────────────++#[uzu_test]+fn test_gs32_4bit_mlx() {+ for_each_float_type!(|F| {+ test_basic::<F>(32, 4, false, true);+ })+}++#[uzu_test]+fn test_gs64_4bit_mlx() {+ for_each_float_type!(|F| {+ test_basic::<F>(64, 4, false, true);+ })+}++#[uzu_test]+fn test_gs128_4bit_mlx() {+ for_each_float_type!(|F| {+ test_basic::<F>(128, 4, false, true);+ })+}++// ── 8-bit, mlx quant ───────────────────────────────────────────────────────++#[uzu_test]+fn test_gs32_8bit_mlx() {+ for_each_float_type!(|F| {+ test_basic::<F>(32, 8, false, true);+ })+}++#[uzu_test]+fn test_gs64_8bit_mlx() {+ for_each_float_type!(|F| {+ test_basic::<F>(64, 8, false, true);+ })+}++#[uzu_test]+fn test_gs128_8bit_mlx() {+ for_each_float_type!(|F| {+ test_basic::<F>(128, 8, false, true);+ })+}diff --git a/crates/uzu/tests/unit/kernel/quant_matmul/qmv_test.rs b/crates/uzu/tests/unit/kernel/quant_matmul/qmv_test.rsnew file mode 100644index 000000000..8a52d7bc6--- /dev/null+++ b/crates/uzu/tests/unit/kernel/quant_matmul/qmv_test.rs@@ -0,0 +1,425 @@+use std::fmt::{Debug, Display};++use num_traits::Float;+use uzu::{+ ArrayElement, DataType,+ backends::{+ common::{Backend, Buffer, Context, Encoder, Kernels, kernel::QuantizedMatmulQmvKernel},+ cpu::Cpu,+ },+};++use super::{Input, check_tolerance, pack_weights_u32, pack_zero_points};+use crate::{common::helpers::alloc_buffer_with_data, uzu_test};++fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {+ let context = B::Context::new().expect("Failed to create Context");++ let w_buf = alloc_buffer_with_data::<B, u32>(&context, &input.w_packed);+ let scales_buf = alloc_buffer_with_data::<B, T>(&context, &input.scales);++ let zp_buf = input.zero_points.as_ref().map(|zp| alloc_buffer_with_data::<B, u8>(&context, zp));+ let bias_buf = input.biases.as_ref().map(|b| alloc_buffer_with_data::<B, T>(&context, b));++ let x_buf = alloc_buffer_with_data::<B, T>(&context, &input.x);+ let output_size = (input.m as usize) * (input.n as usize) * T::data_type().size_in_bytes();+ let mut y_buf = context.create_buffer(output_size).expect("Failed to create buffer");++ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");++ let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmvKernel::new(+ &context,+ T::data_type(),+ input.group_size,+ input.bits,+ input.use_zero_points,+ input.use_mlx_quant,+ )+ .expect("Failed to create QuantizedMatmulQmvKernel");+ kernel.encode(+ &w_buf,+ &scales_buf,+ zp_buf.as_ref(),+ bias_buf.as_ref(),+ &x_buf,+ &mut y_buf,+ input.k,+ input.n,+ input.m,+ &mut encoder,+ );++ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");++ let y_ptr = y_buf.cpu_ptr().as_ptr() as *const T;+ let y_len = (input.m as usize) * (input.n as usize);+ unsafe { std::slice::from_raw_parts(y_ptr, y_len) }.to_vec()+}++/// Create test data for a qmv scenario: m×k * k×n = m×n+/// Weights are in transposed [N × K] layout (each output row stores K input weights).+/// Scales: [N × num_groups_k], zero_points: [N × zp_stride_k], biases: [N × num_groups_k].+/// Groups are along the K dimension.+fn get_test_data_basic<T: ArrayElement + Float>(+ m: usize,+ k: usize,+ n: usize,+ group_size: u32,+ bits: u32,+ use_zero_points: bool,+ use_mlx_quant: bool,+) -> (Input<T>, Vec<T>) {+ let num_groups_k = (k + group_size as usize - 1) / group_size as usize;++ // Generate quantized weights [N × K]+ let mut weights_raw: Vec<u8> = Vec::with_capacity(n * k);+ let max_val = if bits == 4 {+ 15u8+ } else {+ 255u8+ };+ for j in 0..n {+ for l in 0..k {+ weights_raw.push(((j * 3 + l * 7 + 1) % (max_val as usize + 1)) as u8);+ }+ }+ let w_packed = pack_weights_u32(&weights_raw, bits);++ // Scales: [N × num_groups_k]+ let mut scales_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);+ for j in 0..n {+ for g in 0..num_groups_k {+ scales_f32.push(0.5 + 0.1 * ((j + g) % 5) as f32);+ }+ }+ let scales: Vec<T> = scales_f32.iter().map(|&v| T::from(v).unwrap()).collect();++ // Zero points or biases+ let zp_stride = if bits == 4 {+ (num_groups_k + 1) / 2+ } else {+ num_groups_k+ };++ let (zero_points, biases) = if use_zero_points {+ let mut zp_raw: Vec<u8> = Vec::with_capacity(n * num_groups_k);+ for j in 0..n {+ for g in 0..num_groups_k {+ let zp_val = ((j * 2 + g * 3) % (max_val as usize + 1)) as u8;+ zp_raw.push(zp_val);+ }+ }+ // Pack zero points per row (row = output neuron)+ let mut zp_packed: Vec<u8> = Vec::with_capacity(n * zp_stride);+ for j in 0..n {+ let row = &zp_raw[j * num_groups_k..(j + 1) * num_groups_k];+ let packed_row = pack_zero_points(row, bits);+ let mut padded = packed_row;+ padded.resize(zp_stride, 0);+ zp_packed.extend_from_slice(&padded);+ }+ (Some(zp_packed), None)+ } else if use_mlx_quant {+ let mut biases_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);+ for j in 0..n {+ for g in 0..num_groups_k {+ biases_f32.push(0.01 * ((j + g * 2) % 7) as f32);+ }+ }+ let biases: Vec<T> = biases_f32.iter().map(|&v| T::from(v).unwrap()).collect();+ (None, Some(biases))+ } else {+ unreachable!("Must use either zero_points or mlx_quant");+ };++ // X: [m × k]+ let mut x_f32: Vec<f32> = Vec::with_capacity(m * k);+ for i in 0..m {+ for l in 0..k {+ x_f32.push(0.1 * f32::sin((i * k + l) as f32 * 0.05) + 0.5);+ }+ }+ let x: Vec<T> = x_f32.iter().map(|&v| T::from(v).unwrap()).collect();++ let input = Input {+ w_packed,+ scales,+ zero_points,+ biases,+ x,+ k: k as u32,+ n: n as u32,+ m: m as u32,+ group_size,+ bits,+ use_zero_points,+ use_mlx_quant,+ };++ let expected = get_output::<Cpu, T>(&input);+ (input, expected)+}++/// Small edge-case test data with known values+fn get_test_data_edge<T: ArrayElement + Float>(+ group_size: u32,+ bits: u32,+ use_zero_points: bool,+ use_mlx_quant: bool,+) -> (Input<T>, Vec<T>) {+ let m = 1usize;+ let k = 4usize;+ let n = 4usize;+ let num_groups_k = (k + group_size as usize - 1) / group_size as usize;++ // Simple weights [N × K]: all 1s+ let weights_raw: Vec<u8> = vec![1u8; n * k];+ let w_packed = pack_weights_u32(&weights_raw, bits);++ // Unit scales [N × num_groups_k]+ let scales: Vec<T> = vec![T::one(); n * num_groups_k];++ let zp_stride = if bits == 4 {+ (num_groups_k + 1) / 2+ } else {+ num_groups_k+ };++ let (zero_points, biases) = if use_zero_points {+ // Zero points all 0 → bias = 0+ let zp_packed = vec![0u8; n * zp_stride];+ (Some(zp_packed), None)+ } else if use_mlx_quant {+ let biases: Vec<T> = vec![T::zero(); n * num_groups_k];+ (None, Some(biases))+ } else {+ unreachable!("Must use either zero_points or mlx_quant");+ };++ // X: simple increasing values+ let x_f32: Vec<f32> = (0..m * k).map(|i| (i + 1) as f32 * 0.1).collect();+ let x: Vec<T> = x_f32.iter().map(|&v| T::from(v).unwrap()).collect();++ let input = Input {+ w_packed,+ scales,+ zero_points,+ biases,+ x,+ k: k as u32,+ n: n as u32,+ m: m as u32,+ group_size,+ bits,+ use_zero_points,+ use_mlx_quant,+ };++ let expected = get_output::<Cpu, T>(&input);+ (input, expected)+}++fn test_internal<T: ArrayElement + Float + Debug + Display>(+ input: &Input<T>,+ expected: &[T],+) {+ let (rel_tol, abs_tol): (f64, f64) = match T::data_type() {+ DataType::BF16 => (0.05, 0.5),+ DataType::F16 => (0.02, 0.5),+ _ => (0.01, 0.1),+ };++ for_each_non_cpu_backend!(|B| {+ let output = get_output::<B, T>(input);+ assert_eq!(expected.len(), output.len(), "Output length mismatch");++ let mut errors = 0;+ for (i, (&exp, &got)) in expected.iter().zip(output.iter()).enumerate() {+ let exp_f32 = exp.to_f32().unwrap();+ let got_f32 = got.to_f32().unwrap();+ if !check_tolerance(exp_f32, got_f32, rel_tol, abs_tol) {+ if errors < 5 {+ eprintln!(" idx={} expected={} got={} diff={}", i, exp_f32, got_f32, (exp_f32 - got_f32).abs());+ }+ errors += 1;+ }+ }+ assert_eq!(+ errors,+ 0,+ "QMV kernel: backend={}, m={}, k={}, n={}, gs={}, bits={}, zp={}, mlx={}: {} mismatches",+ std::any::type_name::<B>(),+ input.m,+ input.k,+ input.n,+ input.group_size,+ input.bits,+ input.use_zero_points,+ input.use_mlx_quant,+ errors,+ );+ });+}++fn get_test_dims(+ group_size: u32,+ bits: u32,+ is_half_precision: bool,+) -> Vec<(usize, usize, usize)> {+ let gs = group_size as usize;+ // N must be >= group_size for the Metal tiled loader's stride calculations.+ // For half-precision 8-bit, limit k to avoid catastrophic cancellation+ // from large weight values (0-255) and ZP bias subtraction.+ if is_half_precision && bits == 8 {+ let k = gs.min(64);+ vec![(1, k, gs), (2, k, gs)]+ } else {+ vec![(1, gs * 2, gs), (2, gs * 2, gs * 2), (1, gs, gs * 2)]+ }+}++fn test_basic<T: ArrayElement + Float + Debug + Display>(+ group_size: u32,+ bits: u32,+ use_zero_points: bool,+ use_mlx_quant: bool,+) {+ let is_half = matches!(T::data_type(), DataType::F16 | DataType::BF16);+ let dims = get_test_dims(group_size, bits, is_half);++ for (m, k, n) in dims {+ let (input, expected) = get_test_data_basic::<T>(m, k, n, group_size, bits, use_zero_points, use_mlx_quant);+ test_internal::<T>(&input, &expected);+ }+}++fn test_edge<T: ArrayElement + Float + Debug + Display>(+ group_size: u32,+ bits: u32,+ use_zero_points: bool,+ use_mlx_quant: bool,+) {+ let (input, expected) = get_test_data_edge::<T>(group_size, bits, use_zero_points, use_mlx_quant);+ test_internal::<T>(&input, &expected);+}++// ── 4-bit, zero points ─────────────────────────────────────────────────────+#[uzu_test]+fn test_gs32_4bit_zp() {+ for_each_float_type!(|F| {+ test_basic::<F>(32, 4, true, false);+ })+}++#[uzu_test]+fn test_gs64_4bit_zp() {+ for_each_float_type!(|F| {+ test_basic::<F>(64, 4, true, false);+ })+}++#[uzu_test]+fn test_gs128_4bit_zp() {+ for_each_float_type!(|F| {+ test_basic::<F>(128, 4, true, false);+ })+}++// ── 8-bit, zero points ─────────────────────────────────────────────────────++#[uzu_test]+fn test_gs32_8bit_zp() {+ for_each_float_type!(|F| {+ test_basic::<F>(32, 8, true, false);+ })+}++#[uzu_test]+fn test_gs64_8bit_zp() {+ for_each_float_type!(|F| {+ test_basic::<F>(64, 8, true, false);+ })+}++#[uzu_test]+fn test_gs128_8bit_zp() {+ for_each_float_type!(|F| {+ test_basic::<F>(128, 8, true, false);+ })+}++// ── 4-bit, mlx quant ───────────────────────────────────────────────────────++#[uzu_test]+fn test_gs32_4bit_mlx() {+ for_each_float_type!(|F| {+ test_basic::<F>(32, 4, false, true);+ })+}++#[uzu_test]+fn test_gs64_4bit_mlx() {+ for_each_float_type!(|F| {+ test_basic::<F>(64, 4, false, true);+ })+}++#[uzu_test]+fn test_gs128_4bit_mlx() {+ for_each_float_type!(|F| {+ test_basic::<F>(128, 4, false, true);+ })+}++// ── 8-bit, mlx quant ───────────────────────────────────────────────────────++#[uzu_test]+fn test_gs32_8bit_mlx() {+ for_each_float_type!(|F| {+ test_basic::<F>(32, 8, false, true);+ })+}++#[uzu_test]+fn test_gs64_8bit_mlx() {+ for_each_float_type!(|F| {+ test_basic::<F>(64, 8, false, true);+ })+}++#[uzu_test]+fn test_gs128_8bit_mlx() {+ for_each_float_type!(|F| {+ test_basic::<F>(128, 8, false, true);+ })+}++// ── Edge cases ──────────────────────────────────────────────────────────────++#[uzu_test]+fn test_edge_4bit_zp() {+ for_each_float_type!(|F| {+ test_edge::<F>(32, 4, true, false);+ })+}++#[uzu_test]+fn test_edge_8bit_zp() {+ for_each_float_type!(|F| {+ test_edge::<F>(32, 8, true, false);+ })+}++#[uzu_test]+fn test_edge_4bit_mlx() {+ for_each_float_type!(|F| {+ test_edge::<F>(32, 4, false, true);+ })+}++#[uzu_test]+fn test_edge_8bit_mlx() {+ for_each_float_type!(|F| {+ test_edge::<F>(32, 8, false, true);+ })+}