A high-performance inference engine for AI models
`VARIANTS`-aware `THREADS` and `GROUPS` (#339)
CCYeh9d5c787f7afb
diff --git a/Cargo.toml b/Cargo.tomlindex 6ce4a9469..20f06a618 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -56,7 +56,7 @@ tokio = { version = "1.49", features = ["full"] }blake3 = "1.8"quote = "1.0"proc-macro2 = "1.0"-syn = "2.0"+syn = { version = "2.0", features = ["full", "fold"] }prettyplease = "0.2"rhai = "1.24.0"mach2 = "0.6"diff --git a/crates/dsl/src/lib.rs b/crates/dsl/src/lib.rsindex 9c6a90b38..73678c534 100644--- a/crates/dsl/src/lib.rs+++ b/crates/dsl/src/lib.rs@@ -11,7 +11,7 @@ pub fn kernel(let mut func = parse_macro_input!(input as ItemFn);- func.attrs.retain(|attr| !attr.path().is_ident("variants"));+ func.attrs.retain(|attr| !attr.path().is_ident("variants") && !attr.path().is_ident("constraint"));for arg in &mut func.sig.inputs {if let FnArg::Typed(pat) = arg {diff --git a/crates/uzu/Cargo.toml b/crates/uzu/Cargo.tomlindex 90ddea471..d12802f3b 100644--- a/crates/uzu/Cargo.toml+++ b/crates/uzu/Cargo.toml@@ -69,18 +69,17 @@ syn.workspace = trueprettyplease.workspace = trueitertools.workspace = truecbindgen.workspace = true+rhai.workspace = true# metal on macos requires clang-ast for dsl build[target.'cfg(any(target_os = "macos", target_os = "ios", target_os = "tvos", target_os = "visionos"))'.build-dependencies]clang-ast = { workspace = true, optional = true }-rhai = { workspace = true, optional = true }[features]default = ["metal", "grammar"]metal = ["dep:clang-ast",- "dep:rhai","dep:objc2","dep:objc2-foundation","dep:bytesize",diff --git a/crates/uzu/build/common/constraints.rs b/crates/uzu/build/common/constraints.rsnew file mode 100644index 000000000..bc9c83298--- /dev/null+++ b/crates/uzu/build/common/constraints.rs@@ -0,0 +1,26 @@+use rhai::{Dynamic, Engine, Scope};++/// Returns true if all constraints pass for the given name/value bindings.+/// Panics on eval failure (programming error in kernel definition).+pub fn satisfied<N: AsRef<str>, V: AsRef<str>>(+ engine: &Engine,+ bindings: &[(N, V)],+ constraints: &[impl AsRef<str>],+) -> bool {+ if constraints.is_empty() {+ return true;+ }+ let mut scope = Scope::with_capacity(bindings.len());+ for (name, val) in bindings {+ let val = val.as_ref();+ scope.push(+ name.as_ref().to_owned(),+ engine.eval_expression::<Dynamic>(val).unwrap_or_else(|_| val.to_owned().into()),+ );+ }+ constraints.iter().all(|c| {+ engine+ .eval_expression_with_scope::<bool>(&mut scope, c.as_ref())+ .unwrap_or_else(|e| panic!("constraint `{}` failed to evaluate: {e}", c.as_ref()))+ })+}diff --git a/crates/uzu/build/common/mod.rs b/crates/uzu/build/common/mod.rsindex 237c705e1..b4300ff07 100644--- a/crates/uzu/build/common/mod.rs+++ b/crates/uzu/build/common/mod.rs@@ -1,6 +1,7 @@pub mod caching;pub mod codegen;pub mod compiler;+pub mod constraints;pub mod envs;pub mod gpu_types;pub mod kernel;diff --git a/crates/uzu/build/cpu/compiler.rs b/crates/uzu/build/cpu/compiler.rsindex a9c5535e0..9181ba8b8 100644--- a/crates/uzu/build/cpu/compiler.rs+++ b/crates/uzu/build/cpu/compiler.rs@@ -154,6 +154,7 @@ impl CpuCompiler {) -> anyhow::Result<Kernel> {let mut kernel_ident = None;let mut function_variants = Vec::new();+ let mut function_constraints: Vec<Expr> = Vec::new();for attr in ifn.attrs {match attr.path().get_ident().map(|i| i.to_string()).as_ref().map(|s| s.as_ref()) {@@ -176,6 +177,10 @@ impl CpuCompiler {args.collect::<Box<[_]>>(),));},+ Some("constraint") => {+ let expr = attr.parse_args::<Expr>().context("cannot parse constraint attribute")?;+ function_constraints.push(expr);+ },_ => bail!("Unexpected attr {attr:?}"),}}@@ -462,6 +467,10 @@ impl CpuCompiler {parameter_idents = quote! { (#parameter_idents) };}+ let rhai_engine = (!function_constraints.is_empty()).then(rhai::Engine::new);+ let constraint_strs: Vec<String> =+ function_constraints.iter().map(|c| c.to_token_stream().to_string()).collect();+let match_arms = function_parameters.iter().zip(function_variants.iter())@@ -481,6 +490,17 @@ impl CpuCompiler {})}).multi_cartesian_product()+ .filter(|variants| {+ let Some(engine) = &rhai_engine else {+ return true;+ };+ let bindings: Vec<(String, String)> = function_parameters+ .iter()+ .enumerate()+ .map(|(i, p)| (p.name.to_string(), variants[i].1.to_string()))+ .collect();+ crate::common::constraints::satisfied(engine, &bindings, &constraint_strs)+ }).map(|variants| {let (match_variants, generic_variants): (Vec<TokenStream>, Vec<TokenStream>) =variants.into_iter().unzip();diff --git a/crates/uzu/build/metal/bindgen.rs b/crates/uzu/build/metal/bindgen.rsindex ecf017cfa..b63e9ac1b 100644--- a/crates/uzu/build/metal/bindgen.rs+++ b/crates/uzu/build/metal/bindgen.rs@@ -1,10 +1,10 @@-use std::iter::repeat_n;+use std::{collections::BTreeSet, iter::repeat_n};use anyhow::Context;use itertools::Itertools;use proc_macro2::{Span, TokenStream};use quote::{format_ident, quote};-use syn::{Ident, Lifetime, LitInt, Type};+use syn::{Expr, Ident, Lifetime, LitInt, Type, fold::Fold};use super::{ast::{MetalArgumentType, MetalConstantType, MetalKernelInfo},@@ -15,6 +15,48 @@ use crate::{metal::ast::{MetalBufferAccess, MetalGroupsType, MetalTemplateParameterType},};+fn variant_field_ident(name: &str) -> Ident {+ format_ident!("{}", name.to_ascii_lowercase())+}++fn fold_variant_idents(+ expr: Expr,+ variant_names: &[&str],+ referenced: &mut BTreeSet<String>,+) -> Expr {+ struct VariantFolder<'a> {+ names: &'a [&'a str],+ referenced: &'a mut BTreeSet<String>,+ }+ impl syn::fold::Fold for VariantFolder<'_> {+ fn fold_expr(+ &mut self,+ e: Expr,+ ) -> Expr {+ if let Expr::Path(ref ep) = e {+ if ep.qself.is_none()+ && ep.path.leading_colon.is_none()+ && ep.path.segments.len() == 1+ && matches!(ep.path.segments[0].arguments, syn::PathArguments::None)+ {+ let name = ep.path.segments[0].ident.to_string();+ if self.names.contains(&name.as_str()) {+ self.referenced.insert(name.clone());+ let field = variant_field_ident(&name);+ return syn::parse_quote! { self.#field };+ }+ }+ }+ syn::fold::fold_expr(self, e)+ }+ }+ VariantFolder {+ names: variant_names,+ referenced,+ }+ .fold_expr(expr)+}+pub fn bindgen(kernel: &MetalKernelInfo,specialize_indices: &SpecializeBaseIndices,@@ -23,32 +65,61 @@ pub fn bindgen(let trait_name = format_ident!("{kernel_name}Kernel");let struct_name = format_ident!("{kernel_name}MetalKernel");+ let variant_param_names: Vec<&str> =+ kernel.variants.as_ref().map(|v| v.iter().map(|p| p.name.as_ref()).collect()).unwrap_or_default();+let parse_expr = |expr: &str| -> anyhow::Result<TokenStream> {syn::parse_str(expr.as_ref()).with_context(|| format!("cannot parse rust expression `{}` in kernel `{}`", expr, kernel_name))};- let (variants_extra_arguments, variants_kernel_format) = if let Some(variants) = &kernel.variants {- variants- .iter()- .map(|variant| {- let name = Ident::new(variant.name.as_ref(), Span::call_site());-- match &variant.ty {- MetalTemplateParameterType::Type => {- Ok((quote! { #[allow(non_snake_case)] #name: crate::DataType }, quote! { #name.metal_type() }))- },- MetalTemplateParameterType::Value(ty) => {- let ty: Type = syn::parse_str(ty.as_ref())?;- Ok((quote! { #[allow(non_snake_case)] #name: #ty }, quote! { #name.to_string() }))- },- }- })- .collect::<anyhow::Result<_>>()?- } else {- (Vec::new(), Vec::new())+ let mut referenced_variants: BTreeSet<String> = BTreeSet::new();+ let mut parse_expr_rewritten = |expr: &str| -> anyhow::Result<TokenStream> {+ let e: Expr = syn::parse_str(expr)+ .with_context(|| format!("cannot parse rust expression `{}` in kernel `{}`", expr, kernel_name))?;+ let folded = fold_variant_idents(e, &variant_param_names, &mut referenced_variants);+ Ok(quote! { #folded })};+ // Precomputed per-variant state shared across constructor args, mangling,+ // and the post-dispatch struct-field pass; avoids re-zipping parallel arrays.+ struct VariantBind {+ param_ident: Ident,+ field_ident: Ident,+ parsed_ty: Option<Type>,+ }++ let variant_binds: Vec<VariantBind> = kernel+ .variants+ .as_deref()+ .unwrap_or(&[])+ .iter()+ .map(|v| {+ let param_ident = Ident::new(v.name.as_ref(), Span::call_site());+ let field_ident = variant_field_ident(v.name.as_ref());+ let parsed_ty = match &v.ty {+ MetalTemplateParameterType::Type => None,+ MetalTemplateParameterType::Value(ty) => Some(syn::parse_str::<Type>(ty.as_ref())?),+ };+ Ok(VariantBind {+ param_ident,+ field_ident,+ parsed_ty,+ })+ })+ .collect::<anyhow::Result<_>>()?;++ let (variants_extra_arguments, variants_kernel_format): (Vec<TokenStream>, Vec<TokenStream>) = variant_binds+ .iter()+ .map(|v| {+ let name = &v.param_ident;+ match &v.parsed_ty {+ None => (quote! { #[allow(non_snake_case)] #name: crate::DataType }, quote! { #name.metal_type() }),+ Some(ty) => (quote! { #[allow(non_snake_case)] #name: #ty }, quote! { #name.to_string() }),+ }+ })+ .unzip();+let entry_name = dynamic_mangle(kernel.name.as_ref(), variants_kernel_format);let base_index = specialize_indices.get(&kernel.name).copied();@@ -271,8 +342,8 @@ pub fn bindgen(_ => None,}).map(|(threads_rexprs, threads_per_group_rexprs)| {- let threads = parse_expr(&threads_rexprs)?;- let threads_per_group = parse_expr(&threads_per_group_rexprs)?;+ let threads = parse_expr_rewritten(&threads_rexprs)?;+ let threads_per_group = parse_expr_rewritten(&threads_per_group_rexprs)?;Ok((threads, threads_per_group))}).collect::<anyhow::Result<Vec<(TokenStream, TokenStream)>>>()?;@@ -295,7 +366,7 @@ pub fn bindgen(.iter().filter_map(|a| {if let Ok(MetalArgumentType::Threads(rexprs)) = a.argument_type() {- Some(parse_expr(&rexprs))+ Some(parse_expr_rewritten(&rexprs))} else {None}@@ -324,7 +395,7 @@ pub fn bindgen(.iter().filter_map(|a| {if let Ok(MetalArgumentType::Groups(MetalGroupsType::Direct(rexprs))) = a.argument_type() {- Some(parse_expr(&rexprs))+ Some(parse_expr_rewritten(&rexprs))} else {None}@@ -385,10 +456,25 @@ pub fn bindgen(quote! { pub(crate) }};+ // Emit struct fields only for variants actually referenced in THREADS/GROUPS expressions.+ let (variant_fields, variant_inits): (Vec<TokenStream>, Vec<TokenStream>) = variant_binds+ .iter()+ .filter_map(|v| {+ let ty = v.parsed_ty.as_ref()?;+ if !referenced_variants.contains(&v.param_ident.to_string()) {+ return None;+ }+ let field = &v.field_ident;+ let param = &v.param_ident;+ Some((quote! { #field: #ty }, quote! { #field: #param }))+ })+ .unzip();+let kernel = quote! {pub struct #struct_name {pipeline: Retained<ProtocolObject<dyn MTLComputePipelineState>>,#(#conditional_buffer_fields,)*+ #(#variant_fields,)*}impl #maybe_trait_impl #struct_name {@@ -398,7 +484,7 @@ pub fn bindgen(let entry_name = #entry_name;#function_constants_initlet pipeline = context.compute_pipeline_state(#cache_key, &entry_name, #function_constants_arg)?;- Ok(Self { pipeline #(, #conditional_buffer_sets)* })+ Ok(Self { pipeline #(, #conditional_buffer_sets)* #(, #variant_inits)* })}#method_visibility fn encode<#(#encode_generics, )* 'encoder>(&self, #(#encode_args_defs, )* encoder: &'encoder mut crate::backends::common::Encoder<crate::backends::metal::Metal>) {diff --git a/crates/uzu/build/metal/wrapper.rs b/crates/uzu/build/metal/wrapper.rsindex d31d3a580..82f145aa4 100644--- a/crates/uzu/build/metal/wrapper.rs+++ b/crates/uzu/build/metal/wrapper.rs@@ -101,21 +101,8 @@ fn kernel_wrappers(} else {vec![None]} {- if let Some(type_variant) = &type_variant {- let mut scope = rhai::Scope::with_capacity(type_variant.len());- for (type_variant_name, type_variant_value) in type_variant {- scope.push(- type_variant_name.clone(),- engine- .eval_expression::<rhai::Dynamic>(type_variant_value.as_ref())- .unwrap_or_else(|_| type_variant_value.clone().into()),- );- }- if !kernel- .constraints- .iter()- .all(|constraint_expr| engine.eval_expression_with_scope(&mut scope, constraint_expr.as_ref()).unwrap())- {+ if let Some(ref tv) = type_variant {+ if !crate::common::constraints::satisfied(&engine, tv, &kernel.constraints) {continue;}}diff --git a/crates/uzu/src/backends/common/kernel/quant_matmul.rs b/crates/uzu/src/backends/common/kernel/quant_matmul.rsindex 56c7c6895..207f645db 100644--- a/crates/uzu/src/backends/common/kernel/quant_matmul.rs+++ b/crates/uzu/src/backends/common/kernel/quant_matmul.rs@@ -3,10 +3,7 @@ use crate::{backends::common::{Backend, Encoder, Kernels,gpu_types::QuantizationMode,- kernel::{- QuantizedMatmulQmmTransposed64x64Kernel, QuantizedMatmulQmmTransposedKernel,- QuantizedMatmulQmmTransposedWideKernel, QuantizedMatmulQmvFastKernel, QuantizedMatmulQmvKernel,- },+ kernel::{QuantizedMatmulQmmTransposedKernel, QuantizedMatmulQmvFastKernel, QuantizedMatmulQmvKernel},},};@@ -52,7 +49,7 @@ pub struct QuantizedMatmulArguments<'a, B: Backend> {pub struct QuantizedMatmulKernelEncodable<B: Backend> {matrix_vector: MatrixVectorKernel<B>,- matrix_matrix: MatrixMatrixKernel<B>,+ matrix_matrix: <B::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel,input_dim: usize,output_dim: usize,quantization_type: QuantizedMatmulType,@@ -63,12 +60,6 @@ enum MatrixVectorKernel<B: Backend> {QmvFast(<B::Kernels as Kernels>::QuantizedMatmulQmvFastKernel),}-enum MatrixMatrixKernel<B: Backend> {- QmmTransposed(<B::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel),- QmmTransposed64x64(<B::Kernels as Kernels>::QuantizedMatmulQmmTransposed64x64Kernel),- QmmTransposedWide(<B::Kernels as Kernels>::QuantizedMatmulQmmTransposedWideKernel),-}-impl<B: Backend> QuantizedMatmulKernelEncodable<B> {pub fn new(context: &B::Context,@@ -122,51 +113,34 @@ impl<B: Backend> QuantizedMatmulKernelEncodable<B> {)};- // Matrix-matrix+ // 64-tile (BM=BN=64): bf16 + output_dim%64==0; BK=64 if group_size≥64, BK=32 otherwise.+ // 32-tile fallback otherwise.let aligned_n_64 = configuration.output_dim % 64 == 0;let is_bf16 = configuration.data_type == DataType::BF16;+ let can_use_64_tile = aligned_n_64 && is_bf16;- let matrix_matrix =- if !configuration.use_hadamard && aligned_n_64 && is_bf16 && matches!(configuration.group_size, 64 | 128) {- MatrixMatrixKernel::QmmTransposed64x64(- <B::Kernels as Kernels>::QuantizedMatmulQmmTransposed64x64Kernel::new(- context,- configuration.data_type,- group_size,- bits,- use_zero_points,- use_mlx_quant,- )- .map_err(QuantizedMatmulError::BackendError)?,- )- } else if !configuration.use_hadamard && aligned_n_64 && is_bf16 {- MatrixMatrixKernel::QmmTransposedWide(- <B::Kernels as Kernels>::QuantizedMatmulQmmTransposedWideKernel::new(- context,- configuration.data_type,- group_size,- bits,- use_zero_points,- use_mlx_quant,- )- .map_err(QuantizedMatmulError::BackendError)?,- )- } else {- let aligned_n = configuration.output_dim % 32 == 0;- MatrixMatrixKernel::QmmTransposed(- <B::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel::new(- context,- configuration.data_type,- group_size,- bits,- use_zero_points,- use_mlx_quant,- configuration.use_hadamard,- aligned_n,- )- .map_err(QuantizedMatmulError::BackendError)?,- )- };+ let (bm, bk, bn, aligned_n) = if can_use_64_tile && matches!(configuration.group_size, 64 | 128) {+ (64u32, 64u32, 64u32, true)+ } else if can_use_64_tile {+ (64u32, 32u32, 64u32, true)+ } else {+ (32u32, 32u32, 32u32, configuration.output_dim % 32 == 0)+ };++ let matrix_matrix = <B::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel::new(+ context,+ configuration.data_type,+ group_size,+ bits,+ bm,+ bk,+ bn,+ use_zero_points,+ use_mlx_quant,+ configuration.use_hadamard,+ aligned_n,+ )+ .map_err(QuantizedMatmulError::BackendError)?;Ok(Self {matrix_vector,@@ -211,11 +185,7 @@ impl<B: Backend> QuantizedMatmulKernelEncodable<B> {MatrixVectorKernel::QmvFast(k) => encode_kernel!(k, arguments.hadamard_factors),}} else {- match &self.matrix_matrix {- MatrixMatrixKernel::QmmTransposed(k) => encode_kernel!(k, arguments.hadamard_factors),- MatrixMatrixKernel::QmmTransposed64x64(k) => encode_kernel!(k,),- MatrixMatrixKernel::QmmTransposedWide(k) => encode_kernel!(k,),- }+ encode_kernel!(&self.matrix_matrix, arguments.hadamard_factors);}Ok(())diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/mod.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/mod.rsindex 84dcec8b9..c8b61dac4 100644--- a/crates/uzu/src/backends/cpu/kernel/quant_matmul/mod.rs+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/mod.rs@@ -1,5 +1,3 @@pub mod qmm_transposed;-pub mod qmm_transposed_64x64;-pub mod qmm_transposed_wide;pub mod qmv;pub mod qmv_fast;diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rsindex c04bcb55f..a77a447cc 100644--- a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rs+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rs@@ -88,7 +88,21 @@ pub fn qmm_transposed<T: ArrayElement + Float>(#[variants(T, f32, f16, bf16)]#[variants(GROUP_SIZE, 32, 64, 128)]#[variants(BITS, 4, 8)]-pub fn quantized_matmul_qmm_transposed<T: ArrayElement + Float, const GROUP_SIZE: u32, const BITS: u32>(+#[variants(BM, 32, 64)]+#[variants(BK, 32, 64)]+#[variants(BN, 32, 64)]+#[constraint(BK <= BM)]+#[constraint(BK <= GROUP_SIZE)]+#[constraint(BM == BN)]+#[constraint(T != "f32" || BK < 64)]+pub fn quantized_matmul_qmm_transposed<+ T: ArrayElement + Float,+ const GROUP_SIZE: u32,+ const BITS: u32,+ const BM: u32,+ const BK: u32,+ const BN: u32,+>(weights: *const u32,scales: *const T,#[optional(use_zero_points)] zero_points: Option<*const u8>,@@ -104,6 +118,7 @@ pub fn quantized_matmul_qmm_transposed<T: ArrayElement + Float, const GROUP_SIZE#[specialize] use_hadamard: bool,#[specialize] aligned_n: bool,) {+ let _ = (BM, BK, BN); // tile params unused in tile-agnostic CPU referenceif use_hadamard {unimplemented!("not supported yet");}diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_64x64.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_64x64.rsdeleted file mode 100644index c28219b6a..000000000--- a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_64x64.rs+++ /dev/null@@ -1,39 +0,0 @@-use dsl::kernel;-use half::bf16;-use num_traits::Float;--use crate::{ArrayElement, backends::cpu::kernel::quant_matmul::qmm_transposed::qmm_transposed};--#[kernel(QuantizedMatmulQmmTransposed64x64)]-#[variants(T, bf16)]-#[variants(GROUP_SIZE, 64, 128)]-#[variants(BITS, 4, 8)]-pub fn quantized_matmul_qmm_transposed64x64<T: ArrayElement + Float, const GROUP_SIZE: u32, const BITS: u32>(- weights: *const u32,- scales: *const T,- #[optional(use_zero_points)] zero_points: Option<*const u8>,- #[optional(use_mlx_quant)] biases: Option<*const T>,- input: *const T,- output: *mut T,- in_vec_size: u32,- out_vec_size: u32,- batch_size: u32,- #[specialize] use_zero_points: bool,- #[specialize] use_mlx_quant: bool,-) {- qmm_transposed::<T>(- weights,- scales,- zero_points,- biases,- input,- output,- in_vec_size as usize,- out_vec_size as usize,- batch_size as usize,- use_zero_points,- use_mlx_quant,- GROUP_SIZE as usize,- BITS as usize,- );-}diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_wide.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_wide.rsdeleted file mode 100644index ad227a8d5..000000000--- a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_wide.rs+++ /dev/null@@ -1,39 +0,0 @@-use dsl::kernel;-use half::bf16;-use num_traits::Float;--use crate::{ArrayElement, backends::cpu::kernel::quant_matmul::qmm_transposed::qmm_transposed};--#[kernel(QuantizedMatmulQmmTransposedWide)]-#[variants(T, bf16)]-#[variants(GROUP_SIZE, 32, 64, 128)]-#[variants(BITS, 4, 8)]-pub fn quantized_matmul_qmm_transposed_wide<T: ArrayElement + Float, const GROUP_SIZE: u32, const BITS: u32>(- weights: *const u32,- scales: *const T,- #[optional(use_zero_points)] zero_points: Option<*const u8>,- #[optional(use_mlx_quant)] biases: Option<*const T>,- input: *const T,- output: *mut T,- in_vec_size: u32,- out_vec_size: u32,- batch_size: u32,- #[specialize] use_zero_points: bool,- #[specialize] use_mlx_quant: bool,-) {- qmm_transposed::<T>(- weights,- scales,- zero_points,- biases,- input,- output,- in_vec_size as usize,- out_vec_size as usize,- batch_size as usize,- use_zero_points,- use_mlx_quant,- GROUP_SIZE as usize,- BITS as usize,- );-}diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metalindex ab1ee36c0..2b00dac2d 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal@@ -3,10 +3,19 @@#include "../hadamard_transform/hadamard_transform.h"#include "quant_matmul.h"-template <typename T, uint GROUP_SIZE, uint BITS>+// Transposed QMM kernel. BM = batch tile rows, BK = K-axis tile, BN = output+// tile cols.+template <typename T, uint GROUP_SIZE, uint BITS, uint BM, uint BK, uint BN>VARIANTS(T, float, half, bfloat)VARIANTS(GROUP_SIZE, 32, 64, 128)VARIANTS(BITS, 4, 8)+VARIANTS(BM, 32, 64)+VARIANTS(BK, 32, 64)+VARIANTS(BN, 32, 64)+CONSTRAINT(BK <= BM)+CONSTRAINT(BK <= GROUP_SIZE)+CONSTRAINT(BM == BN)+CONSTRAINT(T != "float" || BK < 64)PUBLIC KERNEL(QuantizedMatmulQmmTransposed)(const device uint32_t* weights,const device T* scales,@@ -18,20 +27,20 @@ PUBLIC KERNEL(QuantizedMatmulQmmTransposed)(const constant uint& in_vec_size,const constant uint& out_vec_size,const constant uint& batch_size,- threadgroup T Xs[32 * (32 + 16 / sizeof(T))],- threadgroup T Ws[32 * (32 + 16 / sizeof(T))],+ threadgroup T Xs[BM * (BK + 16 / sizeof(T))],+ threadgroup T Ws[BN * (BK + 16 / sizeof(T))],const bool use_zero_points SPECIALIZE,const bool use_mlx_quant SPECIALIZE,const bool use_hadamard SPECIALIZE,const bool aligned_n SPECIALIZE,- const uint out_block_idx GROUPS(out_vec_size.div_ceil(32)),- const uint batch_block_idx GROUPS(batch_size.div_ceil(32)),+ const uint out_block_idx GROUPS(out_vec_size.div_ceil(BN)),+ const uint batch_block_idx GROUPS(batch_size.div_ceil(BM)),const uint simd_lane THREADS(32),const uint simd_group THREADS(4)) {if (use_mlx_quant) {if (aligned_n) {- qmm_transposed_impl<T, GROUP_SIZE, BITS, true, 32, 32, 32, true>(+ qmm_transposed_impl<T, GROUP_SIZE, BITS, true, BM, BK, BN, true>(weights,scales,zero_points,@@ -49,7 +58,7 @@ PUBLIC KERNEL(QuantizedMatmulQmmTransposed)(simd_lane);} else {- qmm_transposed_impl<T, GROUP_SIZE, BITS, false, 32, 32, 32, true>(+ qmm_transposed_impl<T, GROUP_SIZE, BITS, false, BM, BK, BN, true>(weights,scales,zero_points,@@ -69,7 +78,7 @@ PUBLIC KERNEL(QuantizedMatmulQmmTransposed)(}} else {if (aligned_n) {- qmm_transposed_impl<T, GROUP_SIZE, BITS, true, 32, 32, 32, false>(+ qmm_transposed_impl<T, GROUP_SIZE, BITS, true, BM, BK, BN, false>(weights,scales,zero_points,@@ -87,7 +96,7 @@ PUBLIC KERNEL(QuantizedMatmulQmmTransposed)(simd_lane);} else {- qmm_transposed_impl<T, GROUP_SIZE, BITS, false, 32, 32, 32, false>(+ qmm_transposed_impl<T, GROUP_SIZE, BITS, false, BM, BK, BN, false>(weights,scales,zero_points,@@ -110,26 +119,28 @@ PUBLIC KERNEL(QuantizedMatmulQmmTransposed)(if (use_hadamard) {threadgroup_barrier(mem_flags::mem_device);- constexpr uint BLOCK_SIZE = 32;+ // Each lane covers BN/32 output columns (1 at BN=32, 2 at BN=64).constexpr uint NUM_SIMD_GROUPS = 4;-- constexpr uint ROWS_PER_SIMD_GROUP = BLOCK_SIZE / NUM_SIMD_GROUPS;-- const uint out_col = out_block_idx * BLOCK_SIZE + simd_lane;- const int32_t factor = hadamard_factors[out_col];+ constexpr uint ROWS_PER_SIMD_GROUP = BM / NUM_SIMD_GROUPS;+ constexpr uint COLS_PER_LANE = BN / 32;const uint row_start =- batch_block_idx * BLOCK_SIZE + simd_group * ROWS_PER_SIMD_GROUP;+ batch_block_idx * BM + simd_group * ROWS_PER_SIMD_GROUP;const uint row_end = min(row_start + ROWS_PER_SIMD_GROUP, batch_size);- for (uint row = row_start; row < row_end; row++) {- uint output_index = row * out_vec_size + out_col;+ for (uint c = 0; c < COLS_PER_LANE; c++) {+ const uint out_col = out_block_idx * BN + c * 32 + simd_lane;+ const int32_t factor = hadamard_factors[out_col];- output[output_index] = simdgroup_random_hadamard_transform(- static_cast<ushort>(simd_lane),- output[output_index],- factor- );+ for (uint row = row_start; row < row_end; row++) {+ uint output_index = row * out_vec_size + out_col;++ output[output_index] = simdgroup_random_hadamard_transform(+ static_cast<ushort>(simd_lane),+ output[output_index],+ factor+ );+ }}}}diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metaldeleted file mode 100644index a11b6c023..000000000--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metal+++ /dev/null@@ -1,65 +0,0 @@-#include <metal_stdlib>-#include "../common/dsl.h"-#include "quant_matmul.h"--template <typename T, uint GROUP_SIZE, uint BITS>-VARIANTS(T, bfloat)-VARIANTS(GROUP_SIZE, 64, 128)-VARIANTS(BITS, 4, 8)-PUBLIC KERNEL(QuantizedMatmulQmmTransposed64x64)(- const device uint32_t* weights,- const device T* scales,- const device uint8_t* zero_points OPTIONAL(use_zero_points),- const device T* biases OPTIONAL(use_mlx_quant),- const device T* input,- device T* output,- const constant uint& in_vec_size,- const constant uint& out_vec_size,- const constant uint& batch_size,- threadgroup T Xs[64 * (64 + 16 / sizeof(T))],- threadgroup T Ws[64 * (64 + 16 / sizeof(T))],- const bool use_zero_points SPECIALIZE,- const bool use_mlx_quant SPECIALIZE,- const uint out_block_idx GROUPS(out_vec_size.div_ceil(64)),- const uint batch_block_idx GROUPS(batch_size.div_ceil(64)),- const uint simd_lane THREADS(32),- const uint simd_group THREADS(4)-) {- if (use_mlx_quant) {- qmm_transposed_impl<T, GROUP_SIZE, BITS, true, 64, 64, 64, true>(- weights,- scales,- zero_points,- biases,- input,- output,- Xs,- Ws,- in_vec_size,- out_vec_size,- batch_size,- out_block_idx,- batch_block_idx,- simd_group,- simd_lane- );- } else {- qmm_transposed_impl<T, GROUP_SIZE, BITS, true, 64, 64, 64, false>(- weights,- scales,- zero_points,- biases,- input,- output,- Xs,- Ws,- in_vec_size,- out_vec_size,- batch_size,- out_block_idx,- batch_block_idx,- simd_group,- simd_lane- );- }-}diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_wide.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_wide.metaldeleted file mode 100644index 1c6464ba9..000000000--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_wide.metal+++ /dev/null@@ -1,69 +0,0 @@-#include <metal_stdlib>-#include "../common/dsl.h"-#include "quant_matmul.h"--// Wide QMM tile: BM=64, BK=32, BN=64.-// Processes 4x more output elements per threadgroup than the 32x32 variant.-// BK=32 satisfies the group_size >= BK constraint for all group sizes (32, 64,-// 128).-template <typename T, uint GROUP_SIZE, uint BITS>-VARIANTS(T, bfloat)-VARIANTS(GROUP_SIZE, 32, 64, 128)-VARIANTS(BITS, 4, 8)-PUBLIC KERNEL(QuantizedMatmulQmmTransposedWide)(- const device uint32_t* weights,- const device T* scales,- const device uint8_t* zero_points OPTIONAL(use_zero_points),- const device T* biases OPTIONAL(use_mlx_quant),- const device T* input,- device T* output,- const constant uint& in_vec_size,- const constant uint& out_vec_size,- const constant uint& batch_size,- threadgroup T Xs[64 * (32 + 16 / sizeof(T))],- threadgroup T Ws[64 * (32 + 16 / sizeof(T))],- const bool use_zero_points SPECIALIZE,- const bool use_mlx_quant SPECIALIZE,- const uint out_block_idx GROUPS(out_vec_size.div_ceil(64)),- const uint batch_block_idx GROUPS(batch_size.div_ceil(64)),- const uint simd_lane THREADS(32),- const uint simd_group THREADS(4)-) {- if (use_mlx_quant) {- qmm_transposed_impl<T, GROUP_SIZE, BITS, true, 64, 32, 64, true>(- weights,- scales,- zero_points,- biases,- input,- output,- Xs,- Ws,- in_vec_size,- out_vec_size,- batch_size,- out_block_idx,- batch_block_idx,- simd_group,- simd_lane- );- } else {- qmm_transposed_impl<T, GROUP_SIZE, BITS, true, 64, 32, 64, false>(- weights,- scales,- zero_points,- biases,- input,- output,- Xs,- Ws,- in_vec_size,- out_vec_size,- batch_size,- out_block_idx,- batch_block_idx,- simd_group,- simd_lane- );- }-}diff --git a/crates/uzu/tests/unit/kernel/quant_matmul/mod.rs b/crates/uzu/tests/unit/kernel/quant_matmul/mod.rsindex b97526f4f..bef2b6ea4 100644--- a/crates/uzu/tests/unit/kernel/quant_matmul/mod.rs+++ b/crates/uzu/tests/unit/kernel/quant_matmul/mod.rs@@ -2,9 +2,7 @@#[path = "../../../common/mod.rs"]mod common;-mod qmm_transposed_64x64_test;mod qmm_transposed_test;-mod qmm_transposed_wide_test;mod qmv_fast_test;mod qmv_test;diff --git a/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_64x64_test.rs b/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_64x64_test.rsdeleted file mode 100644index 62bad063e..000000000--- a/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_64x64_test.rs+++ /dev/null@@ -1,372 +0,0 @@-use std::fmt::{Debug, Display};--use half::bf16;-use num_traits::Float;-use uzu::{- ArrayElement, DataType,- backends::{- common::{Backend, Buffer, Context, Encoder, Kernels, kernel::QuantizedMatmulQmmTransposed64x64Kernel},- cpu::Cpu,- },-};--use super::{Input, check_tolerance, pack_weights_u32, pack_zero_points};-use crate::{common::helpers::alloc_buffer_with_data, uzu_test};--fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {- let context = B::Context::new().expect("Failed to create Context");- let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmmTransposed64x64Kernel::new(- &context,- T::data_type(),- input.group_size,- input.bits,- input.use_zero_points,- input.use_mlx_quant,- )- .expect("Failed to create QuantizedMatmulQmmTransposed64x64Kernel");-- let w_buf = alloc_buffer_with_data::<B, u32>(&context, &input.w_packed);- let scales_buf = alloc_buffer_with_data::<B, T>(&context, &input.scales);-- let zp_buf = input.zero_points.as_ref().map(|zp| alloc_buffer_with_data::<B, u8>(&context, zp));- let bias_buf = input.biases.as_ref().map(|b| alloc_buffer_with_data::<B, T>(&context, b));-- let x_buf = alloc_buffer_with_data::<B, T>(&context, &input.x);- let output_size = (input.m as usize) * (input.n as usize) * T::data_type().size_in_bytes();- let mut y_buf = context.create_buffer(output_size).expect("Failed to create buffer");-- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");- kernel.encode(- &w_buf,- &scales_buf,- zp_buf.as_ref(),- bias_buf.as_ref(),- &x_buf,- &mut y_buf,- input.k,- input.n,- input.m,- &mut encoder,- );-- encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");-- let y_ptr = y_buf.cpu_ptr().as_ptr() as *const T;- let y_len = (input.m as usize) * (input.n as usize);- unsafe { std::slice::from_raw_parts(y_ptr, y_len) }.to_vec()-}--/// Create test data for transposed 64x64 qmm: x[m×k] * w_transposed[n×k] = y[m×n]-/// Weights are in transposed [n × k] layout.-fn get_test_data_basic<T: ArrayElement + Float>(- m: usize,- k: usize,- n: usize,- group_size: u32,- bits: u32,- use_zero_points: bool,- use_mlx_quant: bool,-) -> (Input<T>, Vec<T>) {- let num_groups_k = (k + group_size as usize - 1) / group_size as usize;-- // Generate quantized weights [n × k]- let max_val = if bits == 4 {- 15u8- } else {- 255u8- };- let mut weights_raw: Vec<u8> = Vec::with_capacity(n * k);- for j in 0..n {- for l in 0..k {- weights_raw.push(((j * 3 + l * 7 + 1) % (max_val as usize + 1)) as u8);- }- }- let w_packed = pack_weights_u32(&weights_raw, bits);-- // Scales: [n × num_groups_k]- let mut scales_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);- for j in 0..n {- for g in 0..num_groups_k {- scales_f32.push(0.5 + 0.1 * ((j + g) % 5) as f32);- }- }- let scales: Vec<T> = scales_f32.iter().map(|&v| T::from(v).unwrap()).collect();-- // Zero points or biases- let zp_stride = if bits == 4 {- (num_groups_k + 1) / 2- } else {- num_groups_k- };-- let (zero_points, biases) = if use_zero_points {- let mut zp_raw: Vec<u8> = Vec::with_capacity(n * num_groups_k);- for j in 0..n {- for g in 0..num_groups_k {- let zp_val = ((j * 2 + g * 3) % (max_val as usize + 1)) as u8;- zp_raw.push(zp_val);- }- }- // Pack zero points per row- let mut zp_packed: Vec<u8> = Vec::with_capacity(n * zp_stride);- for j in 0..n {- let row = &zp_raw[j * num_groups_k..(j + 1) * num_groups_k];- let packed_row = pack_zero_points(row, bits);- let mut padded = packed_row;- padded.resize(zp_stride, 0);- zp_packed.extend_from_slice(&padded);- }- (Some(zp_packed), None)- } else if use_mlx_quant {- let mut biases_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);- for j in 0..n {- for g in 0..num_groups_k {- biases_f32.push(0.01 * ((j + g * 2) % 7) as f32);- }- }- let biases: Vec<T> = biases_f32.iter().map(|&v| T::from(v).unwrap()).collect();- (None, Some(biases))- } else {- unreachable!("Must use either zero_points or mlx_quant");- };-- // X: [m × k]- let mut x_f32: Vec<f32> = Vec::with_capacity(m * k);- for i in 0..m {- for l in 0..k {- x_f32.push(0.1 * f32::sin((i * k + l) as f32 * 0.05) + 0.5);- }- }- let x: Vec<T> = x_f32.iter().map(|&v| T::from(v).unwrap()).collect();-- let input = Input {- w_packed,- scales,- zero_points,- biases,- x,- k: k as u32,- n: n as u32,- m: m as u32,- group_size,- bits,- use_zero_points,- use_mlx_quant,- };-- let expected = get_output::<Cpu, T>(&input);- (input, expected)-}--/// Small edge-case test data with known values-fn get_test_data_edge<T: ArrayElement + Float>(- group_size: u32,- bits: u32,- use_zero_points: bool,- use_mlx_quant: bool,-) -> (Input<T>, Vec<T>) {- let m = 2usize;- let k = group_size as usize;- let n = 64usize;- let num_groups_k = 1;-- // Simple weights [n × k]: all 1s- let weights_raw: Vec<u8> = vec![1u8; n * k];- let w_packed = pack_weights_u32(&weights_raw, bits);-- // Unit scales- let scales: Vec<T> = vec![T::one(); n * num_groups_k];-- let zp_stride = if bits == 4 {- (num_groups_k + 1) / 2- } else {- num_groups_k- };-- let (zero_points, biases) = if use_zero_points {- // Zero points all 0 -> bias = 0- let zp_packed = vec![0u8; n * zp_stride];- (Some(zp_packed), None)- } else if use_mlx_quant {- let biases: Vec<T> = vec![T::zero(); n * num_groups_k];- (None, Some(biases))- } else {- unreachable!("Must use either zero_points or mlx_quant");- };-- // X: simple increasing values- let x_f32: Vec<f32> = (0..m * k).map(|i| (i + 1) as f32 * 0.1).collect();- let x: Vec<T> = x_f32.iter().map(|&v| T::from(v).unwrap()).collect();-- let input = Input {- w_packed,- scales,- zero_points,- biases,- x,- k: k as u32,- n: n as u32,- m: m as u32,- group_size,- bits,- use_zero_points,- use_mlx_quant,- };-- let expected = get_output::<Cpu, T>(&input);- (input, expected)-}--fn test_internal<T: ArrayElement + Float + Debug + Display>(- input: &Input<T>,- expected: &[T],-) {- let (rel_tol, abs_tol): (f64, f64) = match T::data_type() {- DataType::BF16 => (0.05, 0.5),- DataType::F16 => (0.02, 0.5),- _ => (0.01, 0.1),- };-- for_each_non_cpu_backend!(|B| {- let output = get_output::<B, T>(input);- assert_eq!(expected.len(), output.len(), "Output length mismatch");-- let mut errors = 0;- for (i, (&exp, &got)) in expected.iter().zip(output.iter()).enumerate() {- let exp_f32 = exp.to_f32().unwrap();- let got_f32 = got.to_f32().unwrap();- if !check_tolerance(exp_f32, got_f32, rel_tol, abs_tol) {- if errors < 5 {- eprintln!(" idx={} expected={} got={} diff={}", i, exp_f32, got_f32, (exp_f32 - got_f32).abs());- }- errors += 1;- }- }- assert_eq!(- errors,- 0,- "QMM transposed 64x64 kernel: backend={}, m={}, k={}, n={}, gs={}, bits={}, zp={}, mlx={}: {} mismatches",- std::any::type_name::<B>(),- input.m,- input.k,- input.n,- input.group_size,- input.bits,- input.use_zero_points,- input.use_mlx_quant,- errors,- );- });-}--fn get_test_dims(- group_size: u32,- bits: u32,-) -> Vec<(usize, usize, usize)> {- let gs = group_size as usize;- // 64x64 kernel: BM=64, BK=64, BN=64.- // K must be >= group_size (transposed grouping along K).- // N must be a multiple of 64 (BN tile size).- // bf16 8-bit: keep dims small to avoid catastrophic cancellation.- if bits == 8 {- vec![(2, gs, 64), (4, gs, 64)]- } else {- vec![(2, gs * 2, 64), (4, gs * 2, 128), (8, gs, 128)]- }-}--fn test_basic(- group_size: u32,- bits: u32,- use_zero_points: bool,- use_mlx_quant: bool,-) {- let dims = get_test_dims(group_size, bits);-- for (m, k, n) in dims {- let (input, expected) = get_test_data_basic::<bf16>(m, k, n, group_size, bits, use_zero_points, use_mlx_quant);- test_internal::<bf16>(&input, &expected);- }-}--fn test_edge(- group_size: u32,- bits: u32,- use_zero_points: bool,- use_mlx_quant: bool,-) {- let (input, expected) = get_test_data_edge::<bf16>(group_size, bits, use_zero_points, use_mlx_quant);- test_internal::<bf16>(&input, &expected);-}--// -- 4-bit, zero points ---------------------------------------------------------#[uzu_test]-#[ignore]-fn test_bf16_gs64_4bit_zp() {- test_basic(64, 4, true, false);-}--#[uzu_test]-#[ignore]-fn test_bf16_gs128_4bit_zp() {- test_basic(128, 4, true, false);-}--// -- 8-bit, zero points ---------------------------------------------------------#[uzu_test]-fn test_bf16_gs64_8bit_zp() {- test_basic(64, 8, true, false);-}--#[uzu_test]-fn test_bf16_gs128_8bit_zp() {- test_basic(128, 8, true, false);-}--// -- 4-bit, mlx quant ------------------------------------------------------------#[uzu_test]-fn test_bf16_gs64_4bit_mlx() {- test_basic(64, 4, false, true);-}--#[uzu_test]-fn test_bf16_gs128_4bit_mlx() {- test_basic(128, 4, false, true);-}--// -- 8-bit, mlx quant ------------------------------------------------------------#[uzu_test]-fn test_bf16_gs64_8bit_mlx() {- test_basic(64, 8, false, true);-}--#[uzu_test]-fn test_bf16_gs128_8bit_mlx() {- test_basic(128, 8, false, true);-}--// -- Edge cases ------------------------------------------------------------------#[uzu_test]-fn test_edge_bf16_4bit_zp() {- test_edge(64, 4, true, false);-}--#[uzu_test]-fn test_edge_bf16_8bit_zp() {- test_edge(64, 8, true, false);-}--#[uzu_test]-fn test_edge_bf16_4bit_mlx() {- test_edge(64, 4, false, true);-}--#[uzu_test]-fn test_edge_bf16_8bit_mlx() {- test_edge(64, 8, false, true);-}diff --git a/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rs b/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rsindex fcbb00756..cf42d4161 100644--- a/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rs+++ b/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rs@@ -13,56 +13,7 @@ use uzu::{use super::{Input, check_tolerance, pack_weights_u32, pack_zero_points};use crate::{common::helpers::alloc_buffer_with_data, uzu_test};-fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {- let context = B::Context::new().expect("Failed to create Context");- let aligned_n = (input.n % 32) == 0;- let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel::new(- &context,- T::data_type(),- input.group_size,- input.bits,- input.use_zero_points,- input.use_mlx_quant,- false,- aligned_n,- )- .expect("Failed to create QuantizedMatmulQmmTransposedKernel");-- let w_buf = alloc_buffer_with_data::<B, u32>(&context, &input.w_packed);- let scales_buf = alloc_buffer_with_data::<B, T>(&context, &input.scales);-- let zp_buf = input.zero_points.as_ref().map(|zp| alloc_buffer_with_data::<B, u8>(&context, zp));- let bias_buf = input.biases.as_ref().map(|b| alloc_buffer_with_data::<B, T>(&context, b));-- let x_buf = alloc_buffer_with_data::<B, T>(&context, &input.x);- let output_size = (input.m as usize) * (input.n as usize) * T::data_type().size_in_bytes();- let mut y_buf = context.create_buffer(output_size).expect("Failed to create buffer");-- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");- kernel.encode(- &w_buf,- &scales_buf,- zp_buf.as_ref(),- bias_buf.as_ref(),- &x_buf,- &mut y_buf,- None::<&B::Buffer>,- input.k,- input.n,- input.m,- &mut encoder,- );-- encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");-- let y_ptr = y_buf.cpu_ptr().as_ptr() as *const T;- let y_len = (input.m as usize) * (input.n as usize);- unsafe { std::slice::from_raw_parts(y_ptr, y_len) }.to_vec()-}--/// Create test data for transposed qmm: x[m×k] * w_transposed[n×k] = y[m×n]-/// Weights are in transposed [n × k] layout.-fn get_test_data_basic<T: ArrayElement + Float>(+fn make_input_basic<T: ArrayElement + Float>(m: usize,k: usize,n: usize,@@ -70,10 +21,9 @@ fn get_test_data_basic<T: ArrayElement + Float>(bits: u32,use_zero_points: bool,use_mlx_quant: bool,-) -> (Input<T>, Vec<T>) {+) -> Input<T> {let num_groups_k = (k + group_size as usize - 1) / group_size as usize;- // Generate quantized weights [n × k]let max_val = if bits == 4 {15u8} else {@@ -87,7 +37,6 @@ fn get_test_data_basic<T: ArrayElement + Float>(}let w_packed = pack_weights_u32(&weights_raw, bits);- // Scales: [n × num_groups_k]let mut scales_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);for j in 0..n {for g in 0..num_groups_k {@@ -96,7 +45,6 @@ fn get_test_data_basic<T: ArrayElement + Float>(}let scales: Vec<T> = scales_f32.iter().map(|&v| T::from(v).unwrap()).collect();- // Zero points or biaseslet zp_stride = if bits == 4 {(num_groups_k + 1) / 2} else {@@ -111,7 +59,6 @@ fn get_test_data_basic<T: ArrayElement + Float>(zp_raw.push(zp_val);}}- // Pack zero points per rowlet mut zp_packed: Vec<u8> = Vec::with_capacity(n * zp_stride);for j in 0..n {let row = &zp_raw[j * num_groups_k..(j + 1) * num_groups_k];@@ -134,7 +81,6 @@ fn get_test_data_basic<T: ArrayElement + Float>(unreachable!("Must use either zero_points or mlx_quant");};- // X: [m × k]let mut x_f32: Vec<f32> = Vec::with_capacity(m * k);for i in 0..m {for l in 0..k {@@ -143,7 +89,7 @@ fn get_test_data_basic<T: ArrayElement + Float>(}let x: Vec<T> = x_f32.iter().map(|&v| T::from(v).unwrap()).collect();- let input = Input {+ Input {w_packed,scales,zero_points,@@ -156,29 +102,23 @@ fn get_test_data_basic<T: ArrayElement + Float>(bits,use_zero_points,use_mlx_quant,- };-- let expected = get_output::<Cpu, T>(&input);- (input, expected)+ }}-/// Small edge-case test data with known values-fn get_test_data_edge<T: ArrayElement + Float>(+fn make_input_edge<T: ArrayElement + Float>(+ n: usize,group_size: u32,bits: u32,use_zero_points: bool,use_mlx_quant: bool,-) -> (Input<T>, Vec<T>) {+) -> Input<T> {let m = 2usize;let k = group_size as usize;- let n = 32usize;let num_groups_k = 1;- // Simple weights [n × k]: all 1slet weights_raw: Vec<u8> = vec![1u8; n * k];let w_packed = pack_weights_u32(&weights_raw, bits);- // Unit scaleslet scales: Vec<T> = vec![T::one(); n * num_groups_k];let zp_stride = if bits == 4 {@@ -188,7 +128,6 @@ fn get_test_data_edge<T: ArrayElement + Float>(};let (zero_points, biases) = if use_zero_points {- // Zero points all 0 -> bias = 0let zp_packed = vec![0u8; n * zp_stride];(Some(zp_packed), None)} else if use_mlx_quant {@@ -198,11 +137,10 @@ fn get_test_data_edge<T: ArrayElement + Float>(unreachable!("Must use either zero_points or mlx_quant");};- // X: simple increasing valueslet x_f32: Vec<f32> = (0..m * k).map(|i| (i + 1) as f32 * 0.1).collect();let x: Vec<T> = x_f32.iter().map(|&v| T::from(v).unwrap()).collect();- let input = Input {+ Input {w_packed,scales,zero_points,@@ -215,20 +153,76 @@ fn get_test_data_edge<T: ArrayElement + Float>(bits,use_zero_points,use_mlx_quant,- };+ }+}++fn get_output<B: Backend, T: ArrayElement + Float>(+ input: &Input<T>,+ bm: u32,+ bk: u32,+ bn: u32,+ aligned_n: bool,+ hadamard_factors: Option<&[i32]>,+) -> Vec<T> {+ let context = B::Context::new().expect("Failed to create Context");+ let use_hadamard = hadamard_factors.is_some();+ let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel::new(+ &context,+ T::data_type(),+ input.group_size,+ input.bits,+ bm,+ bk,+ bn,+ input.use_zero_points,+ input.use_mlx_quant,+ use_hadamard,+ aligned_n,+ )+ .expect("Failed to create QuantizedMatmulQmmTransposedKernel");++ let w_buf = alloc_buffer_with_data::<B, u32>(&context, &input.w_packed);+ let scales_buf = alloc_buffer_with_data::<B, T>(&context, &input.scales);++ let zp_buf = input.zero_points.as_ref().map(|zp| alloc_buffer_with_data::<B, u8>(&context, zp));+ let bias_buf = input.biases.as_ref().map(|b| alloc_buffer_with_data::<B, T>(&context, b));+ let had_buf = hadamard_factors.map(|f| alloc_buffer_with_data::<B, i32>(&context, f));++ let x_buf = alloc_buffer_with_data::<B, T>(&context, &input.x);+ let output_size = (input.m as usize) * (input.n as usize) * T::data_type().size_in_bytes();+ let mut y_buf = context.create_buffer(output_size).expect("Failed to create buffer");- let expected = get_output::<Cpu, T>(&input);- (input, expected)+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");+ kernel.encode(+ &w_buf,+ &scales_buf,+ zp_buf.as_ref(),+ bias_buf.as_ref(),+ &x_buf,+ &mut y_buf,+ had_buf.as_ref(),+ input.k,+ input.n,+ input.m,+ &mut encoder,+ );++ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");++ let y_ptr = y_buf.cpu_ptr().as_ptr() as *const T;+ let y_len = (input.m as usize) * (input.n as usize);+ unsafe { std::slice::from_raw_parts(y_ptr, y_len) }.to_vec()}-fn test_internal<T: ArrayElement + Float + Debug + Display>(+fn check_against_expected<T: ArrayElement + Float + Debug + Display>(input: &Input<T>,expected: &[T],+ bm: u32,+ bk: u32,+ bn: u32,+ aligned_n: bool,) {- // GPU MMA units use different rounding than scalar CPU code.- // bf16 has 7-bit mantissa (ULP ~0.8%), f16 has 10-bit (~0.1%).- // Tiled accumulation reorders FMA ops, compounding rounding differences.- // 8-bit weights (0-255) with ZP bias cause larger accumulated errors.+ // GPU tiled accumulation rounds differently from CPU scalar reference.let (rel_tol, abs_tol): (f64, f64) = match T::data_type() {DataType::BF16 => (0.05, 0.5),DataType::F16 => (0.02, 0.5),@@ -236,7 +230,7 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(};for_each_non_cpu_backend!(|B| {- let output = get_output::<B, T>(input);+ let output = get_output::<B, T>(input, bm, bk, bn, aligned_n, None);assert_eq!(expected.len(), output.len(), "Output length mismatch");let mut errors = 0;@@ -253,8 +247,11 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(assert_eq!(errors,0,- "QMM transposed kernel: backend={}, m={}, k={}, n={}, gs={}, bits={}, zp={}, mlx={}: {} mismatches",+ "QMM transposed kernel: backend={}, shape=(BM={},BK={},BN={}), m={}, k={}, n={}, gs={}, bits={}, zp={}, mlx={}: {} mismatches",std::any::type_name::<B>(),+ bm,+ bk,+ bn,input.m,input.k,input.n,@@ -267,266 +264,328 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(});}-fn get_test_dims(- group_size: u32,- bits: u32,- is_half_precision: bool,-) -> Vec<(usize, usize, usize)> {- let gs = group_size as usize;- // K must be >= group_size for the transposed grouping along K.- // N must be >= 32 for the Metal tiled weight loader (BN=32).- // For half-precision 8-bit, keep dimensions small to avoid- // catastrophic cancellation from large weight values (0-255).- if is_half_precision && bits == 8 {- vec![(2, gs, gs), (4, gs, gs)]+struct TileCfg {+ bm: u32,+ bk: u32,+ bn: u32,+}++impl TileCfg {+ const fn aligned_n_default(&self) -> bool {+ // BN=64 tiles require output_dim % 64 == 0; BN=32 checks actual N at test time.+ self.bn == 64+ }+}++const CFG_32X32: TileCfg = TileCfg { bm: 32, bk: 32, bn: 32 };+const CFG_WIDE: TileCfg = TileCfg { bm: 64, bk: 32, bn: 64 };+const CFG_64X64: TileCfg = TileCfg { bm: 64, bk: 64, bn: 64 };++fn pick_aligned_n(+ cfg: &TileCfg,+ n: usize,+) -> bool {+ if cfg.bn == 32 {+ (n % 32) == 0} else {- vec![(2, gs * 2, gs), (4, gs * 2, gs * 2), (8, gs, gs * 2)]+ cfg.aligned_n_default()}}fn test_basic<T: ArrayElement + Float + Debug + Display>(+ cfg: &TileCfg,group_size: u32,bits: u32,use_zero_points: bool,use_mlx_quant: bool,) {let is_half = matches!(T::data_type(), DataType::F16 | DataType::BF16);- let dims = get_test_dims(group_size, bits, is_half);+ let gs = group_size as usize;+ let bn = cfg.bn as usize;+ // half+8bit: smaller dims to cap accumulated rounding error.+ let dims: Vec<(usize, usize, usize)> = if is_half && bits == 8 {+ vec![(2, gs, bn), (4, gs, bn)]+ } else {+ vec![(2, gs * 2, bn), (4, gs * 2, bn * 2), (8, gs, bn * 2)]+ };for (m, k, n) in dims {- let (input, expected) = get_test_data_basic::<T>(m, k, n, group_size, bits, use_zero_points, use_mlx_quant);- test_internal::<T>(&input, &expected);+ let input = make_input_basic::<T>(m, k, n, group_size, bits, use_zero_points, use_mlx_quant);+ let aligned_n = pick_aligned_n(cfg, n);+ let expected = get_output::<Cpu, T>(&input, cfg.bm, cfg.bk, cfg.bn, aligned_n, None);+ check_against_expected::<T>(&input, &expected, cfg.bm, cfg.bk, cfg.bn, aligned_n);}}fn test_edge<T: ArrayElement + Float + Debug + Display>(+ cfg: &TileCfg,group_size: u32,bits: u32,use_zero_points: bool,use_mlx_quant: bool,) {- let (input, expected) = get_test_data_edge::<T>(group_size, bits, use_zero_points, use_mlx_quant);- test_internal::<T>(&input, &expected);-}--// -- 4-bit, zero points ---------------------------------------------------------#[uzu_test]-fn test_f32_gs32_4bit_zp() {- test_basic::<f32>(32, 4, true, false);-}--#[uzu_test]-fn test_f32_gs64_4bit_zp() {- test_basic::<f32>(64, 4, true, false);-}--#[uzu_test]-fn test_f32_gs128_4bit_zp() {- test_basic::<f32>(128, 4, true, false);-}--#[uzu_test]-fn test_f16_gs32_4bit_zp() {- test_basic::<f16>(32, 4, true, false);-}--#[uzu_test]-fn test_f16_gs64_4bit_zp() {- test_basic::<f16>(64, 4, true, false);-}--#[uzu_test]-fn test_f16_gs128_4bit_zp() {- test_basic::<f16>(128, 4, true, false);-}--#[uzu_test]-fn test_bf16_gs32_4bit_zp() {- test_basic::<bf16>(32, 4, true, false);-}--#[uzu_test]-#[ignore]-fn test_bf16_gs64_4bit_zp() {- test_basic::<bf16>(64, 4, true, false);-}--#[uzu_test]-#[ignore]-fn test_bf16_gs128_4bit_zp() {- test_basic::<bf16>(128, 4, true, false);-}--// -- 8-bit, zero points ---------------------------------------------------------#[uzu_test]-fn test_f32_gs32_8bit_zp() {- test_basic::<f32>(32, 8, true, false);-}--#[uzu_test]-fn test_f32_gs64_8bit_zp() {- test_basic::<f32>(64, 8, true, false);-}--#[uzu_test]-fn test_f32_gs128_8bit_zp() {- test_basic::<f32>(128, 8, true, false);-}--#[uzu_test]-fn test_f16_gs32_8bit_zp() {- test_basic::<f16>(32, 8, true, false);-}--#[uzu_test]-fn test_f16_gs64_8bit_zp() {- test_basic::<f16>(64, 8, true, false);-}--#[uzu_test]-fn test_f16_gs128_8bit_zp() {- test_basic::<f16>(128, 8, true, false);-}--#[uzu_test]-fn test_bf16_gs32_8bit_zp() {- test_basic::<bf16>(32, 8, true, false);-}--#[uzu_test]-fn test_bf16_gs64_8bit_zp() {- test_basic::<bf16>(64, 8, true, false);-}--#[uzu_test]-fn test_bf16_gs128_8bit_zp() {- test_basic::<bf16>(128, 8, true, false);-}--// -- 4-bit, mlx quant ------------------------------------------------------------#[uzu_test]-fn test_f32_gs32_4bit_mlx() {- test_basic::<f32>(32, 4, false, true);-}--#[uzu_test]-fn test_f32_gs64_4bit_mlx() {- test_basic::<f32>(64, 4, false, true);-}--#[uzu_test]-fn test_f32_gs128_4bit_mlx() {- test_basic::<f32>(128, 4, false, true);-}--#[uzu_test]-fn test_f16_gs32_4bit_mlx() {- test_basic::<f16>(32, 4, false, true);-}--#[uzu_test]-fn test_f16_gs64_4bit_mlx() {- test_basic::<f16>(64, 4, false, true);-}--#[uzu_test]-fn test_f16_gs128_4bit_mlx() {- test_basic::<f16>(128, 4, false, true);-}--#[uzu_test]-fn test_bf16_gs32_4bit_mlx() {- test_basic::<bf16>(32, 4, false, true);-}--#[uzu_test]-fn test_bf16_gs64_4bit_mlx() {- test_basic::<bf16>(64, 4, false, true);+ // Edge N is one BN tile wide — smallest legal N for the shape.+ let n = cfg.bn as usize;+ let input = make_input_edge::<T>(n, group_size, bits, use_zero_points, use_mlx_quant);+ let aligned_n = pick_aligned_n(cfg, n);+ let expected = get_output::<Cpu, T>(&input, cfg.bm, cfg.bk, cfg.bn, aligned_n, None);+ check_against_expected::<T>(&input, &expected, cfg.bm, cfg.bk, cfg.bn, aligned_n);}-#[uzu_test]-fn test_bf16_gs128_4bit_mlx() {- test_basic::<bf16>(128, 4, false, true);-}+// No CPU Hadamard reference — verify all three tile shapes agree instead.+fn test_hadamard_consistency<T: ArrayElement + Float + Debug + Display>(+ group_size: u32,+ bits: u32,+ use_zero_points: bool,+ use_mlx_quant: bool,+) {+ // m=n=64 satisfies both BN=32 and BN=64 alignment; group_size >= 64 for BK=64.+ let m = 64usize;+ let k = group_size as usize * 2;+ let n = 64usize;-// -- 8-bit, mlx quant ----------------------------------------------------------+ let input = make_input_basic::<T>(m, k, n, group_size, bits, use_zero_points, use_mlx_quant);-#[uzu_test]-fn test_f32_gs32_8bit_mlx() {- test_basic::<f32>(32, 8, false, true);-}+ // ±1 factors: non-trivial Hadamard, bounded output across all dtypes.+ let factors: Vec<i32> = (0..n).map(|i| if (i * 13 + 7) % 2 == 0 { 1 } else { -1 }).collect();-#[uzu_test]-fn test_f32_gs64_8bit_mlx() {- test_basic::<f32>(64, 8, false, true);-}+ // Hadamard amplifies by sqrt(32) ≈ 5.65; tolerance widened accordingly.+ let (rel_tol, abs_tol): (f64, f64) = match T::data_type() {+ DataType::BF16 => (0.1, 1.0),+ DataType::F16 => (0.05, 1.0),+ _ => (0.02, 0.5),+ };-#[uzu_test]-fn test_f32_gs128_8bit_mlx() {- test_basic::<f32>(128, 8, false, true);+ for_each_non_cpu_backend!(|B| {+ let shapes: [(u32, u32, u32, &str); 3] = [+ (32, 32, 32, "32x32"),+ (64, 32, 64, "wide"),+ (64, 64, 64, "64x64"),+ ];++ let outputs: Vec<(&str, Vec<T>)> = shapes+ .iter()+ .map(|&(bm, bk, bn, name)| {+ let out = get_output::<B, T>(&input, bm, bk, bn, true, Some(&factors));+ (name, out)+ })+ .collect();++ // 32x32 is the reference — its Hadamard path is byte-identical to the original.+ let (_, ref reference) = outputs[0];++ for (name, output) in outputs.iter().skip(1) {+ let mut errors = 0;+ for (i, (&r, &o)) in reference.iter().zip(output.iter()).enumerate() {+ let r_f32 = r.to_f32().unwrap();+ let o_f32 = o.to_f32().unwrap();+ if !check_tolerance(r_f32, o_f32, rel_tol, abs_tol) {+ if errors < 5 {+ eprintln!(+ " idx={} 32x32={} {}={} diff={}",+ i,+ r_f32,+ name,+ o_f32,+ (r_f32 - o_f32).abs()+ );+ }+ errors += 1;+ }+ }+ assert_eq!(+ errors,+ 0,+ "Hadamard cross-shape mismatch: backend={}, {} vs 32x32 reference, gs={}, bits={}, zp={}, mlx={}: {} mismatches",+ std::any::type_name::<B>(),+ name,+ group_size,+ bits,+ use_zero_points,+ use_mlx_quant,+ errors,+ );+ }+ });}-#[uzu_test]-fn test_f16_gs32_8bit_mlx() {- test_basic::<f16>(32, 8, false, true);-}+// ---- Test-function macros ------------------------------------------------#[uzu_test]-fn test_f16_gs64_8bit_mlx() {- test_basic::<f16>(64, 8, false, true);+macro_rules! basic_tests {+ ($cfg:expr; $(+ $(#[$attr:meta])*+ fn $name:ident : $t:ty, $gs:literal, $bits:literal, $zp:literal, $mlx:literal;+ )*) => {+ $(+ $(#[$attr])*+ #[uzu_test]+ fn $name() {+ test_basic::<$t>($cfg, $gs, $bits, $zp, $mlx);+ }+ )*+ };}-#[uzu_test]-fn test_f16_gs128_8bit_mlx() {- test_basic::<f16>(128, 8, false, true);+macro_rules! edge_tests {+ ($cfg:expr; $(+ $(#[$attr:meta])*+ fn $name:ident : $t:ty, $gs:literal, $bits:literal, $zp:literal, $mlx:literal;+ )*) => {+ $(+ $(#[$attr])*+ #[uzu_test]+ fn $name() {+ test_edge::<$t>($cfg, $gs, $bits, $zp, $mlx);+ }+ )*+ };}-#[uzu_test]-fn test_bf16_gs32_8bit_mlx() {- test_basic::<bf16>(32, 8, false, true);+macro_rules! hadamard_tests {+ ($(+ $(#[$attr:meta])*+ fn $name:ident : $t:ty, $gs:literal, $bits:literal, $zp:literal, $mlx:literal;+ )*) => {+ $(+ $(#[$attr])*+ #[uzu_test]+ fn $name() {+ test_hadamard_consistency::<$t>($gs, $bits, $zp, $mlx);+ }+ )*+ };}-#[uzu_test]-fn test_bf16_gs64_8bit_mlx() {- test_basic::<bf16>(64, 8, false, true);-}+// ---- tile_32x32 — the original base shape, full dtype matrix -------------++mod tile_32x32 {+ use super::*;++ basic_tests!(&CFG_32X32;+ // 4-bit, zero points+ fn test_f32_gs32_4bit_zp: f32, 32, 4, true, false;+ fn test_f32_gs64_4bit_zp: f32, 64, 4, true, false;+ fn test_f32_gs128_4bit_zp: f32, 128, 4, true, false;+ fn test_f16_gs32_4bit_zp: f16, 32, 4, true, false;+ fn test_f16_gs64_4bit_zp: f16, 64, 4, true, false;+ fn test_f16_gs128_4bit_zp: f16, 128, 4, true, false;+ fn test_bf16_gs32_4bit_zp: bf16, 32, 4, true, false;+ #[ignore] fn test_bf16_gs64_4bit_zp: bf16, 64, 4, true, false;+ #[ignore] fn test_bf16_gs128_4bit_zp: bf16, 128, 4, true, false;+ // 8-bit, zero points+ fn test_f32_gs32_8bit_zp: f32, 32, 8, true, false;+ fn test_f32_gs64_8bit_zp: f32, 64, 8, true, false;+ fn test_f32_gs128_8bit_zp: f32, 128, 8, true, false;+ fn test_f16_gs32_8bit_zp: f16, 32, 8, true, false;+ fn test_f16_gs64_8bit_zp: f16, 64, 8, true, false;+ fn test_f16_gs128_8bit_zp: f16, 128, 8, true, false;+ fn test_bf16_gs32_8bit_zp: bf16, 32, 8, true, false;+ fn test_bf16_gs64_8bit_zp: bf16, 64, 8, true, false;+ fn test_bf16_gs128_8bit_zp: bf16,128, 8, true, false;+ // 4-bit, mlx quant+ fn test_f32_gs32_4bit_mlx: f32, 32, 4, false, true;+ fn test_f32_gs64_4bit_mlx: f32, 64, 4, false, true;+ fn test_f32_gs128_4bit_mlx: f32, 128, 4, false, true;+ fn test_f16_gs32_4bit_mlx: f16, 32, 4, false, true;+ fn test_f16_gs64_4bit_mlx: f16, 64, 4, false, true;+ fn test_f16_gs128_4bit_mlx: f16, 128, 4, false, true;+ fn test_bf16_gs32_4bit_mlx: bf16, 32, 4, false, true;+ fn test_bf16_gs64_4bit_mlx: bf16, 64, 4, false, true;+ fn test_bf16_gs128_4bit_mlx: bf16,128, 4, false, true;+ // 8-bit, mlx quant+ fn test_f32_gs32_8bit_mlx: f32, 32, 8, false, true;+ fn test_f32_gs64_8bit_mlx: f32, 64, 8, false, true;+ fn test_f32_gs128_8bit_mlx: f32, 128, 8, false, true;+ fn test_f16_gs32_8bit_mlx: f16, 32, 8, false, true;+ fn test_f16_gs64_8bit_mlx: f16, 64, 8, false, true;+ fn test_f16_gs128_8bit_mlx: f16, 128, 8, false, true;+ fn test_bf16_gs32_8bit_mlx: bf16, 32, 8, false, true;+ fn test_bf16_gs64_8bit_mlx: bf16, 64, 8, false, true;+ fn test_bf16_gs128_8bit_mlx: bf16,128, 8, false, true;+ );-#[uzu_test]-fn test_bf16_gs128_8bit_mlx() {- test_basic::<bf16>(128, 8, false, true);+ edge_tests!(&CFG_32X32;+ fn test_edge_f32_4bit_zp: f32, 32, 4, true, false;+ fn test_edge_f32_8bit_zp: f32, 32, 8, true, false;+ fn test_edge_f16_4bit_zp: f16, 32, 4, true, false;+ fn test_edge_f32_4bit_mlx: f32, 32, 4, false, true;+ fn test_edge_f32_8bit_mlx: f32, 32, 8, false, true;+ fn test_edge_f16_4bit_mlx: f16, 32, 4, false, true;+ );}-// -- Edge cases ----------------------------------------------------------------+// ---- tile_wide — BM=64, BK=32, BN=64, bf16 only --------------------------++mod tile_wide {+ use super::*;++ basic_tests!(&CFG_WIDE;+ // 4-bit, zero points+ fn test_bf16_gs32_4bit_zp: bf16, 32, 4, true, false;+ #[ignore] fn test_bf16_gs64_4bit_zp: bf16, 64, 4, true, false;+ #[ignore] fn test_bf16_gs128_4bit_zp: bf16, 128, 4, true, false;+ // 8-bit, zero points+ fn test_bf16_gs32_8bit_zp: bf16, 32, 8, true, false;+ fn test_bf16_gs64_8bit_zp: bf16, 64, 8, true, false;+ fn test_bf16_gs128_8bit_zp: bf16, 128, 8, true, false;+ // 4-bit, mlx quant+ fn test_bf16_gs32_4bit_mlx: bf16, 32, 4, false, true;+ fn test_bf16_gs64_4bit_mlx: bf16, 64, 4, false, true;+ fn test_bf16_gs128_4bit_mlx: bf16, 128, 4, false, true;+ // 8-bit, mlx quant+ fn test_bf16_gs32_8bit_mlx: bf16, 32, 8, false, true;+ fn test_bf16_gs64_8bit_mlx: bf16, 64, 8, false, true;+ fn test_bf16_gs128_8bit_mlx: bf16, 128, 8, false, true;+ );-#[uzu_test]-fn test_edge_f32_4bit_zp() {- test_edge::<f32>(32, 4, true, false);+ edge_tests!(&CFG_WIDE;+ fn test_edge_bf16_4bit_zp: bf16, 32, 4, true, false;+ fn test_edge_bf16_8bit_zp: bf16, 32, 8, true, false;+ fn test_edge_bf16_4bit_mlx: bf16, 32, 4, false, true;+ fn test_edge_bf16_8bit_mlx: bf16, 32, 8, false, true;+ );}-#[uzu_test]-fn test_edge_f32_8bit_zp() {- test_edge::<f32>(32, 8, true, false);-}+// ---- tile_64x64 — BM=BK=BN=64, bf16 only, group_size >= 64 --------------++mod tile_64x64 {+ use super::*;++ basic_tests!(&CFG_64X64;+ // 4-bit, zero points+ #[ignore] fn test_bf16_gs64_4bit_zp: bf16, 64, 4, true, false;+ #[ignore] fn test_bf16_gs128_4bit_zp: bf16, 128, 4, true, false;+ // 8-bit, zero points+ fn test_bf16_gs64_8bit_zp: bf16, 64, 8, true, false;+ fn test_bf16_gs128_8bit_zp: bf16, 128, 8, true, false;+ // 4-bit, mlx quant+ fn test_bf16_gs64_4bit_mlx: bf16, 64, 4, false, true;+ fn test_bf16_gs128_4bit_mlx: bf16, 128, 4, false, true;+ // 8-bit, mlx quant+ fn test_bf16_gs64_8bit_mlx: bf16, 64, 8, false, true;+ fn test_bf16_gs128_8bit_mlx: bf16, 128, 8, false, true;+ );-#[uzu_test]-fn test_edge_f16_4bit_zp() {- test_edge::<f16>(32, 4, true, false);+ edge_tests!(&CFG_64X64;+ fn test_edge_bf16_4bit_zp: bf16, 64, 4, true, false;+ fn test_edge_bf16_8bit_zp: bf16, 64, 8, true, false;+ fn test_edge_bf16_4bit_mlx: bf16, 64, 4, false, true;+ fn test_edge_bf16_8bit_mlx: bf16, 64, 8, false, true;+ );}-#[uzu_test]-fn test_edge_f32_4bit_mlx() {- test_edge::<f32>(32, 4, false, true);-}+// ---- hadamard — cross-shape consistency of the generalized HT loop --------#[uzu_test]-fn test_edge_f32_8bit_mlx() {- test_edge::<f32>(32, 8, false, true);-}+mod hadamard {+ use super::*;-#[uzu_test]-fn test_edge_f16_4bit_mlx() {- test_edge::<f16>(32, 4, false, true);+ // f32 at (64,64,64) overflows threadgroup memory (34816 > 32768 bytes).+ hadamard_tests!(+ fn test_bf16_gs64_4bit_mlx: bf16, 64, 4, false, true;+ fn test_bf16_gs128_4bit_mlx: bf16, 128, 4, false, true;+ fn test_bf16_gs64_8bit_mlx: bf16, 64, 8, false, true;+ fn test_bf16_gs128_8bit_mlx: bf16, 128, 8, false, true;+ fn test_f16_gs64_4bit_mlx: f16, 64, 4, false, true;+ fn test_f16_gs128_4bit_mlx: f16, 128, 4, false, true;+ );}diff --git a/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_wide_test.rs b/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_wide_test.rsdeleted file mode 100644index 59a74348b..000000000--- a/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_wide_test.rs+++ /dev/null@@ -1,392 +0,0 @@-use std::fmt::{Debug, Display};--use half::bf16;-use num_traits::Float;-use uzu::{- ArrayElement, DataType,- backends::{- common::{Backend, Buffer, Context, Encoder, Kernels, kernel::QuantizedMatmulQmmTransposedWideKernel},- cpu::Cpu,- },-};--use super::{Input, check_tolerance, pack_weights_u32, pack_zero_points};-use crate::{common::helpers::alloc_buffer_with_data, uzu_test};--fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {- let context = B::Context::new().expect("Failed to create Context");- let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmmTransposedWideKernel::new(- &context,- T::data_type(),- input.group_size,- input.bits,- input.use_zero_points,- input.use_mlx_quant,- )- .expect("Failed to create QuantizedMatmulQmmTransposedWideKernel");-- let w_buf = alloc_buffer_with_data::<B, u32>(&context, &input.w_packed);- let scales_buf = alloc_buffer_with_data::<B, T>(&context, &input.scales);-- let zp_buf = input.zero_points.as_ref().map(|zp| alloc_buffer_with_data::<B, u8>(&context, zp));- let bias_buf = input.biases.as_ref().map(|b| alloc_buffer_with_data::<B, T>(&context, b));-- let x_buf = alloc_buffer_with_data::<B, T>(&context, &input.x);- let output_size = (input.m as usize) * (input.n as usize) * T::data_type().size_in_bytes();- let mut y_buf = context.create_buffer(output_size).expect("Failed to create buffer");-- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");- kernel.encode(- &w_buf,- &scales_buf,- zp_buf.as_ref(),- bias_buf.as_ref(),- &x_buf,- &mut y_buf,- input.k,- input.n,- input.m,- &mut encoder,- );-- encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");-- let y_ptr = y_buf.cpu_ptr().as_ptr() as *const T;- let y_len = (input.m as usize) * (input.n as usize);- unsafe { std::slice::from_raw_parts(y_ptr, y_len) }.to_vec()-}--/// Create test data for transposed wide qmm: x[m×k] * w_transposed[n×k] = y[m×n]-/// Weights are in transposed [n × k] layout.-fn get_test_data_basic<T: ArrayElement + Float>(- m: usize,- k: usize,- n: usize,- group_size: u32,- bits: u32,- use_zero_points: bool,- use_mlx_quant: bool,-) -> (Input<T>, Vec<T>) {- let num_groups_k = (k + group_size as usize - 1) / group_size as usize;-- // Generate quantized weights [n × k]- let max_val = if bits == 4 {- 15u8- } else {- 255u8- };- let mut weights_raw: Vec<u8> = Vec::with_capacity(n * k);- for j in 0..n {- for l in 0..k {- weights_raw.push(((j * 3 + l * 7 + 1) % (max_val as usize + 1)) as u8);- }- }- let w_packed = pack_weights_u32(&weights_raw, bits);-- // Scales: [n × num_groups_k]- let mut scales_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);- for j in 0..n {- for g in 0..num_groups_k {- scales_f32.push(0.5 + 0.1 * ((j + g) % 5) as f32);- }- }- let scales: Vec<T> = scales_f32.iter().map(|&v| T::from(v).unwrap()).collect();-- // Zero points or biases- let zp_stride = if bits == 4 {- (num_groups_k + 1) / 2- } else {- num_groups_k- };-- let (zero_points, biases) = if use_zero_points {- let mut zp_raw: Vec<u8> = Vec::with_capacity(n * num_groups_k);- for j in 0..n {- for g in 0..num_groups_k {- let zp_val = ((j * 2 + g * 3) % (max_val as usize + 1)) as u8;- zp_raw.push(zp_val);- }- }- // Pack zero points per row- let mut zp_packed: Vec<u8> = Vec::with_capacity(n * zp_stride);- for j in 0..n {- let row = &zp_raw[j * num_groups_k..(j + 1) * num_groups_k];- let packed_row = pack_zero_points(row, bits);- let mut padded = packed_row;- padded.resize(zp_stride, 0);- zp_packed.extend_from_slice(&padded);- }- (Some(zp_packed), None)- } else if use_mlx_quant {- let mut biases_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);- for j in 0..n {- for g in 0..num_groups_k {- biases_f32.push(0.01 * ((j + g * 2) % 7) as f32);- }- }- let biases: Vec<T> = biases_f32.iter().map(|&v| T::from(v).unwrap()).collect();- (None, Some(biases))- } else {- unreachable!("Must use either zero_points or mlx_quant");- };-- // X: [m × k]- let mut x_f32: Vec<f32> = Vec::with_capacity(m * k);- for i in 0..m {- for l in 0..k {- x_f32.push(0.1 * f32::sin((i * k + l) as f32 * 0.05) + 0.5);- }- }- let x: Vec<T> = x_f32.iter().map(|&v| T::from(v).unwrap()).collect();-- let input = Input {- w_packed,- scales,- zero_points,- biases,- x,- k: k as u32,- n: n as u32,- m: m as u32,- group_size,- bits,- use_zero_points,- use_mlx_quant,- };-- let expected = get_output::<Cpu, T>(&input);- (input, expected)-}--/// Small edge-case test data with known values-fn get_test_data_edge<T: ArrayElement + Float>(- group_size: u32,- bits: u32,- use_zero_points: bool,- use_mlx_quant: bool,-) -> (Input<T>, Vec<T>) {- let m = 2usize;- let k = group_size as usize;- let n = 64usize;- let num_groups_k = 1;-- // Simple weights [n × k]: all 1s- let weights_raw: Vec<u8> = vec![1u8; n * k];- let w_packed = pack_weights_u32(&weights_raw, bits);-- // Unit scales- let scales: Vec<T> = vec![T::one(); n * num_groups_k];-- let zp_stride = if bits == 4 {- (num_groups_k + 1) / 2- } else {- num_groups_k- };-- let (zero_points, biases) = if use_zero_points {- // Zero points all 0 -> bias = 0- let zp_packed = vec![0u8; n * zp_stride];- (Some(zp_packed), None)- } else if use_mlx_quant {- let biases: Vec<T> = vec![T::zero(); n * num_groups_k];- (None, Some(biases))- } else {- unreachable!("Must use either zero_points or mlx_quant");- };-- // X: simple increasing values- let x_f32: Vec<f32> = (0..m * k).map(|i| (i + 1) as f32 * 0.1).collect();- let x: Vec<T> = x_f32.iter().map(|&v| T::from(v).unwrap()).collect();-- let input = Input {- w_packed,- scales,- zero_points,- biases,- x,- k: k as u32,- n: n as u32,- m: m as u32,- group_size,- bits,- use_zero_points,- use_mlx_quant,- };-- let expected = get_output::<Cpu, T>(&input);- (input, expected)-}--fn test_internal<T: ArrayElement + Float + Debug + Display>(- input: &Input<T>,- expected: &[T],-) {- let (rel_tol, abs_tol): (f64, f64) = match T::data_type() {- DataType::BF16 => (0.05, 0.5),- DataType::F16 => (0.02, 0.5),- _ => (0.01, 0.1),- };-- for_each_non_cpu_backend!(|B| {- let output = get_output::<B, T>(input);- assert_eq!(expected.len(), output.len(), "Output length mismatch");-- let mut errors = 0;- for (i, (&exp, &got)) in expected.iter().zip(output.iter()).enumerate() {- let exp_f32 = exp.to_f32().unwrap();- let got_f32 = got.to_f32().unwrap();- if !check_tolerance(exp_f32, got_f32, rel_tol, abs_tol) {- if errors < 5 {- eprintln!(" idx={} expected={} got={} diff={}", i, exp_f32, got_f32, (exp_f32 - got_f32).abs());- }- errors += 1;- }- }- assert_eq!(- errors,- 0,- "QMM transposed wide kernel: backend={}, m={}, k={}, n={}, gs={}, bits={}, zp={}, mlx={}: {} mismatches",- std::any::type_name::<B>(),- input.m,- input.k,- input.n,- input.group_size,- input.bits,- input.use_zero_points,- input.use_mlx_quant,- errors,- );- });-}--fn get_test_dims(- group_size: u32,- bits: u32,-) -> Vec<(usize, usize, usize)> {- let gs = group_size as usize;- // Wide kernel: BM=64, BK=32, BN=64.- // K must be >= group_size (transposed grouping along K).- // N must be a multiple of 64 (BN tile size).- // bf16 8-bit: keep dims small to avoid catastrophic cancellation.- if bits == 8 {- vec![(2, gs, 64), (4, gs, 64)]- } else {- vec![(2, gs * 2, 64), (4, gs * 2, 128), (8, gs, 128)]- }-}--fn test_basic(- group_size: u32,- bits: u32,- use_zero_points: bool,- use_mlx_quant: bool,-) {- let dims = get_test_dims(group_size, bits);-- for (m, k, n) in dims {- let (input, expected) = get_test_data_basic::<bf16>(m, k, n, group_size, bits, use_zero_points, use_mlx_quant);- test_internal::<bf16>(&input, &expected);- }-}--fn test_edge(- group_size: u32,- bits: u32,- use_zero_points: bool,- use_mlx_quant: bool,-) {- let (input, expected) = get_test_data_edge::<bf16>(group_size, bits, use_zero_points, use_mlx_quant);- test_internal::<bf16>(&input, &expected);-}--// -- 4-bit, zero points ---------------------------------------------------------#[uzu_test]-fn test_bf16_gs32_4bit_zp() {- test_basic(32, 4, true, false);-}--#[uzu_test]-#[ignore]-fn test_bf16_gs64_4bit_zp() {- test_basic(64, 4, true, false);-}--#[uzu_test]-#[ignore]-fn test_bf16_gs128_4bit_zp() {- test_basic(128, 4, true, false);-}--// -- 8-bit, zero points ---------------------------------------------------------#[uzu_test]-fn test_bf16_gs32_8bit_zp() {- test_basic(32, 8, true, false);-}--#[uzu_test]-fn test_bf16_gs64_8bit_zp() {- test_basic(64, 8, true, false);-}--#[uzu_test]-fn test_bf16_gs128_8bit_zp() {- test_basic(128, 8, true, false);-}--// -- 4-bit, mlx quant ------------------------------------------------------------#[uzu_test]-fn test_bf16_gs32_4bit_mlx() {- test_basic(32, 4, false, true);-}--#[uzu_test]-fn test_bf16_gs64_4bit_mlx() {- test_basic(64, 4, false, true);-}--#[uzu_test]-fn test_bf16_gs128_4bit_mlx() {- test_basic(128, 4, false, true);-}--// -- 8-bit, mlx quant ------------------------------------------------------------#[uzu_test]-fn test_bf16_gs32_8bit_mlx() {- test_basic(32, 8, false, true);-}--#[uzu_test]-fn test_bf16_gs64_8bit_mlx() {- test_basic(64, 8, false, true);-}--#[uzu_test]-fn test_bf16_gs128_8bit_mlx() {- test_basic(128, 8, false, true);-}--// -- Edge cases ------------------------------------------------------------------#[uzu_test]-fn test_edge_bf16_4bit_zp() {- test_edge(32, 4, true, false);-}--#[uzu_test]-fn test_edge_bf16_8bit_zp() {- test_edge(32, 8, true, false);-}--#[uzu_test]-fn test_edge_bf16_4bit_mlx() {- test_edge(32, 4, false, true);-}--#[uzu_test]-fn test_edge_bf16_8bit_mlx() {- test_edge(32, 8, false, true);-}