Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

`VARIANTS`-aware `THREADS` and `GROUPS` (#339)

CCYeh9d5c787f7afb
diff --git a/Cargo.toml b/Cargo.toml
index 6ce4a9469..20f06a618 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -56,7 +56,7 @@ tokio = { version = "1.49", features = ["full"] }
blake3 = "1.8"
quote = "1.0"
proc-macro2 = "1.0"
-syn = "2.0"
+syn = { version = "2.0", features = ["full", "fold"] }
prettyplease = "0.2"
rhai = "1.24.0"
mach2 = "0.6"
diff --git a/crates/dsl/src/lib.rs b/crates/dsl/src/lib.rs
index 9c6a90b38..73678c534 100644
--- a/crates/dsl/src/lib.rs
+++ b/crates/dsl/src/lib.rs
@@ -11,7 +11,7 @@ pub fn kernel(
let mut func = parse_macro_input!(input as ItemFn);
- func.attrs.retain(|attr| !attr.path().is_ident("variants"));
+ func.attrs.retain(|attr| !attr.path().is_ident("variants") && !attr.path().is_ident("constraint"));
for arg in &mut func.sig.inputs {
if let FnArg::Typed(pat) = arg {
diff --git a/crates/uzu/Cargo.toml b/crates/uzu/Cargo.toml
index 90ddea471..d12802f3b 100644
--- a/crates/uzu/Cargo.toml
+++ b/crates/uzu/Cargo.toml
@@ -69,18 +69,17 @@ syn.workspace = true
prettyplease.workspace = true
itertools.workspace = true
cbindgen.workspace = true
+rhai.workspace = true
# metal on macos requires clang-ast for dsl build
[target.'cfg(any(target_os = "macos", target_os = "ios", target_os = "tvos", target_os = "visionos"))'.build-dependencies]
clang-ast = { workspace = true, optional = true }
-rhai = { workspace = true, optional = true }
[features]
default = ["metal", "grammar"]
metal = [
"dep:clang-ast",
- "dep:rhai",
"dep:objc2",
"dep:objc2-foundation",
"dep:bytesize",
diff --git a/crates/uzu/build/common/constraints.rs b/crates/uzu/build/common/constraints.rs
new file mode 100644
index 000000000..bc9c83298
--- /dev/null
+++ b/crates/uzu/build/common/constraints.rs
@@ -0,0 +1,26 @@
+use rhai::{Dynamic, Engine, Scope};
+
+/// Returns true if all constraints pass for the given name/value bindings.
+/// Panics on eval failure (programming error in kernel definition).
+pub fn satisfied<N: AsRef<str>, V: AsRef<str>>(
+ engine: &Engine,
+ bindings: &[(N, V)],
+ constraints: &[impl AsRef<str>],
+) -> bool {
+ if constraints.is_empty() {
+ return true;
+ }
+ let mut scope = Scope::with_capacity(bindings.len());
+ for (name, val) in bindings {
+ let val = val.as_ref();
+ scope.push(
+ name.as_ref().to_owned(),
+ engine.eval_expression::<Dynamic>(val).unwrap_or_else(|_| val.to_owned().into()),
+ );
+ }
+ constraints.iter().all(|c| {
+ engine
+ .eval_expression_with_scope::<bool>(&mut scope, c.as_ref())
+ .unwrap_or_else(|e| panic!("constraint `{}` failed to evaluate: {e}", c.as_ref()))
+ })
+}
diff --git a/crates/uzu/build/common/mod.rs b/crates/uzu/build/common/mod.rs
index 237c705e1..b4300ff07 100644
--- a/crates/uzu/build/common/mod.rs
+++ b/crates/uzu/build/common/mod.rs
@@ -1,6 +1,7 @@
pub mod caching;
pub mod codegen;
pub mod compiler;
+pub mod constraints;
pub mod envs;
pub mod gpu_types;
pub mod kernel;
diff --git a/crates/uzu/build/cpu/compiler.rs b/crates/uzu/build/cpu/compiler.rs
index a9c5535e0..9181ba8b8 100644
--- a/crates/uzu/build/cpu/compiler.rs
+++ b/crates/uzu/build/cpu/compiler.rs
@@ -154,6 +154,7 @@ impl CpuCompiler {
) -> anyhow::Result<Kernel> {
let mut kernel_ident = None;
let mut function_variants = Vec::new();
+ let mut function_constraints: Vec<Expr> = Vec::new();
for attr in ifn.attrs {
match attr.path().get_ident().map(|i| i.to_string()).as_ref().map(|s| s.as_ref()) {
@@ -176,6 +177,10 @@ impl CpuCompiler {
args.collect::<Box<[_]>>(),
));
},
+ Some("constraint") => {
+ let expr = attr.parse_args::<Expr>().context("cannot parse constraint attribute")?;
+ function_constraints.push(expr);
+ },
_ => bail!("Unexpected attr {attr:?}"),
}
}
@@ -462,6 +467,10 @@ impl CpuCompiler {
parameter_idents = quote! { (#parameter_idents) };
}
+ let rhai_engine = (!function_constraints.is_empty()).then(rhai::Engine::new);
+ let constraint_strs: Vec<String> =
+ function_constraints.iter().map(|c| c.to_token_stream().to_string()).collect();
+
let match_arms = function_parameters
.iter()
.zip(function_variants.iter())
@@ -481,6 +490,17 @@ impl CpuCompiler {
})
})
.multi_cartesian_product()
+ .filter(|variants| {
+ let Some(engine) = &rhai_engine else {
+ return true;
+ };
+ let bindings: Vec<(String, String)> = function_parameters
+ .iter()
+ .enumerate()
+ .map(|(i, p)| (p.name.to_string(), variants[i].1.to_string()))
+ .collect();
+ crate::common::constraints::satisfied(engine, &bindings, &constraint_strs)
+ })
.map(|variants| {
let (match_variants, generic_variants): (Vec<TokenStream>, Vec<TokenStream>) =
variants.into_iter().unzip();
diff --git a/crates/uzu/build/metal/bindgen.rs b/crates/uzu/build/metal/bindgen.rs
index ecf017cfa..b63e9ac1b 100644
--- a/crates/uzu/build/metal/bindgen.rs
+++ b/crates/uzu/build/metal/bindgen.rs
@@ -1,10 +1,10 @@
-use std::iter::repeat_n;
+use std::{collections::BTreeSet, iter::repeat_n};
use anyhow::Context;
use itertools::Itertools;
use proc_macro2::{Span, TokenStream};
use quote::{format_ident, quote};
-use syn::{Ident, Lifetime, LitInt, Type};
+use syn::{Expr, Ident, Lifetime, LitInt, Type, fold::Fold};
use super::{
ast::{MetalArgumentType, MetalConstantType, MetalKernelInfo},
@@ -15,6 +15,48 @@ use crate::{
metal::ast::{MetalBufferAccess, MetalGroupsType, MetalTemplateParameterType},
};
+fn variant_field_ident(name: &str) -> Ident {
+ format_ident!("{}", name.to_ascii_lowercase())
+}
+
+fn fold_variant_idents(
+ expr: Expr,
+ variant_names: &[&str],
+ referenced: &mut BTreeSet<String>,
+) -> Expr {
+ struct VariantFolder<'a> {
+ names: &'a [&'a str],
+ referenced: &'a mut BTreeSet<String>,
+ }
+ impl syn::fold::Fold for VariantFolder<'_> {
+ fn fold_expr(
+ &mut self,
+ e: Expr,
+ ) -> Expr {
+ if let Expr::Path(ref ep) = e {
+ if ep.qself.is_none()
+ && ep.path.leading_colon.is_none()
+ && ep.path.segments.len() == 1
+ && matches!(ep.path.segments[0].arguments, syn::PathArguments::None)
+ {
+ let name = ep.path.segments[0].ident.to_string();
+ if self.names.contains(&name.as_str()) {
+ self.referenced.insert(name.clone());
+ let field = variant_field_ident(&name);
+ return syn::parse_quote! { self.#field };
+ }
+ }
+ }
+ syn::fold::fold_expr(self, e)
+ }
+ }
+ VariantFolder {
+ names: variant_names,
+ referenced,
+ }
+ .fold_expr(expr)
+}
+
pub fn bindgen(
kernel: &MetalKernelInfo,
specialize_indices: &SpecializeBaseIndices,
@@ -23,32 +65,61 @@ pub fn bindgen(
let trait_name = format_ident!("{kernel_name}Kernel");
let struct_name = format_ident!("{kernel_name}MetalKernel");
+ let variant_param_names: Vec<&str> =
+ kernel.variants.as_ref().map(|v| v.iter().map(|p| p.name.as_ref()).collect()).unwrap_or_default();
+
let parse_expr = |expr: &str| -> anyhow::Result<TokenStream> {
syn::parse_str(expr.as_ref())
.with_context(|| format!("cannot parse rust expression `{}` in kernel `{}`", expr, kernel_name))
};
- let (variants_extra_arguments, variants_kernel_format) = if let Some(variants) = &kernel.variants {
- variants
- .iter()
- .map(|variant| {
- let name = Ident::new(variant.name.as_ref(), Span::call_site());
-
- match &variant.ty {
- MetalTemplateParameterType::Type => {
- Ok((quote! { #[allow(non_snake_case)] #name: crate::DataType }, quote! { #name.metal_type() }))
- },
- MetalTemplateParameterType::Value(ty) => {
- let ty: Type = syn::parse_str(ty.as_ref())?;
- Ok((quote! { #[allow(non_snake_case)] #name: #ty }, quote! { #name.to_string() }))
- },
- }
- })
- .collect::<anyhow::Result<_>>()?
- } else {
- (Vec::new(), Vec::new())
+ let mut referenced_variants: BTreeSet<String> = BTreeSet::new();
+ let mut parse_expr_rewritten = |expr: &str| -> anyhow::Result<TokenStream> {
+ let e: Expr = syn::parse_str(expr)
+ .with_context(|| format!("cannot parse rust expression `{}` in kernel `{}`", expr, kernel_name))?;
+ let folded = fold_variant_idents(e, &variant_param_names, &mut referenced_variants);
+ Ok(quote! { #folded })
};
+ // Precomputed per-variant state shared across constructor args, mangling,
+ // and the post-dispatch struct-field pass; avoids re-zipping parallel arrays.
+ struct VariantBind {
+ param_ident: Ident,
+ field_ident: Ident,
+ parsed_ty: Option<Type>,
+ }
+
+ let variant_binds: Vec<VariantBind> = kernel
+ .variants
+ .as_deref()
+ .unwrap_or(&[])
+ .iter()
+ .map(|v| {
+ let param_ident = Ident::new(v.name.as_ref(), Span::call_site());
+ let field_ident = variant_field_ident(v.name.as_ref());
+ let parsed_ty = match &v.ty {
+ MetalTemplateParameterType::Type => None,
+ MetalTemplateParameterType::Value(ty) => Some(syn::parse_str::<Type>(ty.as_ref())?),
+ };
+ Ok(VariantBind {
+ param_ident,
+ field_ident,
+ parsed_ty,
+ })
+ })
+ .collect::<anyhow::Result<_>>()?;
+
+ let (variants_extra_arguments, variants_kernel_format): (Vec<TokenStream>, Vec<TokenStream>) = variant_binds
+ .iter()
+ .map(|v| {
+ let name = &v.param_ident;
+ match &v.parsed_ty {
+ None => (quote! { #[allow(non_snake_case)] #name: crate::DataType }, quote! { #name.metal_type() }),
+ Some(ty) => (quote! { #[allow(non_snake_case)] #name: #ty }, quote! { #name.to_string() }),
+ }
+ })
+ .unzip();
+
let entry_name = dynamic_mangle(kernel.name.as_ref(), variants_kernel_format);
let base_index = specialize_indices.get(&kernel.name).copied();
@@ -271,8 +342,8 @@ pub fn bindgen(
_ => None,
})
.map(|(threads_rexprs, threads_per_group_rexprs)| {
- let threads = parse_expr(&threads_rexprs)?;
- let threads_per_group = parse_expr(&threads_per_group_rexprs)?;
+ let threads = parse_expr_rewritten(&threads_rexprs)?;
+ let threads_per_group = parse_expr_rewritten(&threads_per_group_rexprs)?;
Ok((threads, threads_per_group))
})
.collect::<anyhow::Result<Vec<(TokenStream, TokenStream)>>>()?;
@@ -295,7 +366,7 @@ pub fn bindgen(
.iter()
.filter_map(|a| {
if let Ok(MetalArgumentType::Threads(rexprs)) = a.argument_type() {
- Some(parse_expr(&rexprs))
+ Some(parse_expr_rewritten(&rexprs))
} else {
None
}
@@ -324,7 +395,7 @@ pub fn bindgen(
.iter()
.filter_map(|a| {
if let Ok(MetalArgumentType::Groups(MetalGroupsType::Direct(rexprs))) = a.argument_type() {
- Some(parse_expr(&rexprs))
+ Some(parse_expr_rewritten(&rexprs))
} else {
None
}
@@ -385,10 +456,25 @@ pub fn bindgen(
quote! { pub(crate) }
};
+ // Emit struct fields only for variants actually referenced in THREADS/GROUPS expressions.
+ let (variant_fields, variant_inits): (Vec<TokenStream>, Vec<TokenStream>) = variant_binds
+ .iter()
+ .filter_map(|v| {
+ let ty = v.parsed_ty.as_ref()?;
+ if !referenced_variants.contains(&v.param_ident.to_string()) {
+ return None;
+ }
+ let field = &v.field_ident;
+ let param = &v.param_ident;
+ Some((quote! { #field: #ty }, quote! { #field: #param }))
+ })
+ .unzip();
+
let kernel = quote! {
pub struct #struct_name {
pipeline: Retained<ProtocolObject<dyn MTLComputePipelineState>>,
#(#conditional_buffer_fields,)*
+ #(#variant_fields,)*
}
impl #maybe_trait_impl #struct_name {
@@ -398,7 +484,7 @@ pub fn bindgen(
let entry_name = #entry_name;
#function_constants_init
let pipeline = context.compute_pipeline_state(#cache_key, &entry_name, #function_constants_arg)?;
- Ok(Self { pipeline #(, #conditional_buffer_sets)* })
+ Ok(Self { pipeline #(, #conditional_buffer_sets)* #(, #variant_inits)* })
}
#method_visibility fn encode<#(#encode_generics, )* 'encoder>(&self, #(#encode_args_defs, )* encoder: &'encoder mut crate::backends::common::Encoder<crate::backends::metal::Metal>) {
diff --git a/crates/uzu/build/metal/wrapper.rs b/crates/uzu/build/metal/wrapper.rs
index d31d3a580..82f145aa4 100644
--- a/crates/uzu/build/metal/wrapper.rs
+++ b/crates/uzu/build/metal/wrapper.rs
@@ -101,21 +101,8 @@ fn kernel_wrappers(
} else {
vec![None]
} {
- if let Some(type_variant) = &type_variant {
- let mut scope = rhai::Scope::with_capacity(type_variant.len());
- for (type_variant_name, type_variant_value) in type_variant {
- scope.push(
- type_variant_name.clone(),
- engine
- .eval_expression::<rhai::Dynamic>(type_variant_value.as_ref())
- .unwrap_or_else(|_| type_variant_value.clone().into()),
- );
- }
- if !kernel
- .constraints
- .iter()
- .all(|constraint_expr| engine.eval_expression_with_scope(&mut scope, constraint_expr.as_ref()).unwrap())
- {
+ if let Some(ref tv) = type_variant {
+ if !crate::common::constraints::satisfied(&engine, tv, &kernel.constraints) {
continue;
}
}
diff --git a/crates/uzu/src/backends/common/kernel/quant_matmul.rs b/crates/uzu/src/backends/common/kernel/quant_matmul.rs
index 56c7c6895..207f645db 100644
--- a/crates/uzu/src/backends/common/kernel/quant_matmul.rs
+++ b/crates/uzu/src/backends/common/kernel/quant_matmul.rs
@@ -3,10 +3,7 @@ use crate::{
backends::common::{
Backend, Encoder, Kernels,
gpu_types::QuantizationMode,
- kernel::{
- QuantizedMatmulQmmTransposed64x64Kernel, QuantizedMatmulQmmTransposedKernel,
- QuantizedMatmulQmmTransposedWideKernel, QuantizedMatmulQmvFastKernel, QuantizedMatmulQmvKernel,
- },
+ kernel::{QuantizedMatmulQmmTransposedKernel, QuantizedMatmulQmvFastKernel, QuantizedMatmulQmvKernel},
},
};
@@ -52,7 +49,7 @@ pub struct QuantizedMatmulArguments<'a, B: Backend> {
pub struct QuantizedMatmulKernelEncodable<B: Backend> {
matrix_vector: MatrixVectorKernel<B>,
- matrix_matrix: MatrixMatrixKernel<B>,
+ matrix_matrix: <B::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel,
input_dim: usize,
output_dim: usize,
quantization_type: QuantizedMatmulType,
@@ -63,12 +60,6 @@ enum MatrixVectorKernel<B: Backend> {
QmvFast(<B::Kernels as Kernels>::QuantizedMatmulQmvFastKernel),
}
-enum MatrixMatrixKernel<B: Backend> {
- QmmTransposed(<B::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel),
- QmmTransposed64x64(<B::Kernels as Kernels>::QuantizedMatmulQmmTransposed64x64Kernel),
- QmmTransposedWide(<B::Kernels as Kernels>::QuantizedMatmulQmmTransposedWideKernel),
-}
-
impl<B: Backend> QuantizedMatmulKernelEncodable<B> {
pub fn new(
context: &B::Context,
@@ -122,51 +113,34 @@ impl<B: Backend> QuantizedMatmulKernelEncodable<B> {
)
};
- // Matrix-matrix
+ // 64-tile (BM=BN=64): bf16 + output_dim%64==0; BK=64 if group_size≥64, BK=32 otherwise.
+ // 32-tile fallback otherwise.
let aligned_n_64 = configuration.output_dim % 64 == 0;
let is_bf16 = configuration.data_type == DataType::BF16;
+ let can_use_64_tile = aligned_n_64 && is_bf16;
- let matrix_matrix =
- if !configuration.use_hadamard && aligned_n_64 && is_bf16 && matches!(configuration.group_size, 64 | 128) {
- MatrixMatrixKernel::QmmTransposed64x64(
- <B::Kernels as Kernels>::QuantizedMatmulQmmTransposed64x64Kernel::new(
- context,
- configuration.data_type,
- group_size,
- bits,
- use_zero_points,
- use_mlx_quant,
- )
- .map_err(QuantizedMatmulError::BackendError)?,
- )
- } else if !configuration.use_hadamard && aligned_n_64 && is_bf16 {
- MatrixMatrixKernel::QmmTransposedWide(
- <B::Kernels as Kernels>::QuantizedMatmulQmmTransposedWideKernel::new(
- context,
- configuration.data_type,
- group_size,
- bits,
- use_zero_points,
- use_mlx_quant,
- )
- .map_err(QuantizedMatmulError::BackendError)?,
- )
- } else {
- let aligned_n = configuration.output_dim % 32 == 0;
- MatrixMatrixKernel::QmmTransposed(
- <B::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel::new(
- context,
- configuration.data_type,
- group_size,
- bits,
- use_zero_points,
- use_mlx_quant,
- configuration.use_hadamard,
- aligned_n,
- )
- .map_err(QuantizedMatmulError::BackendError)?,
- )
- };
+ let (bm, bk, bn, aligned_n) = if can_use_64_tile && matches!(configuration.group_size, 64 | 128) {
+ (64u32, 64u32, 64u32, true)
+ } else if can_use_64_tile {
+ (64u32, 32u32, 64u32, true)
+ } else {
+ (32u32, 32u32, 32u32, configuration.output_dim % 32 == 0)
+ };
+
+ let matrix_matrix = <B::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel::new(
+ context,
+ configuration.data_type,
+ group_size,
+ bits,
+ bm,
+ bk,
+ bn,
+ use_zero_points,
+ use_mlx_quant,
+ configuration.use_hadamard,
+ aligned_n,
+ )
+ .map_err(QuantizedMatmulError::BackendError)?;
Ok(Self {
matrix_vector,
@@ -211,11 +185,7 @@ impl<B: Backend> QuantizedMatmulKernelEncodable<B> {
MatrixVectorKernel::QmvFast(k) => encode_kernel!(k, arguments.hadamard_factors),
}
} else {
- match &self.matrix_matrix {
- MatrixMatrixKernel::QmmTransposed(k) => encode_kernel!(k, arguments.hadamard_factors),
- MatrixMatrixKernel::QmmTransposed64x64(k) => encode_kernel!(k,),
- MatrixMatrixKernel::QmmTransposedWide(k) => encode_kernel!(k,),
- }
+ encode_kernel!(&self.matrix_matrix, arguments.hadamard_factors);
}
Ok(())
diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/mod.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/mod.rs
index 84dcec8b9..c8b61dac4 100644
--- a/crates/uzu/src/backends/cpu/kernel/quant_matmul/mod.rs
+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/mod.rs
@@ -1,5 +1,3 @@
pub mod qmm_transposed;
-pub mod qmm_transposed_64x64;
-pub mod qmm_transposed_wide;
pub mod qmv;
pub mod qmv_fast;
diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rs
index c04bcb55f..a77a447cc 100644
--- a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rs
+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rs
@@ -88,7 +88,21 @@ pub fn qmm_transposed<T: ArrayElement + Float>(
#[variants(T, f32, f16, bf16)]
#[variants(GROUP_SIZE, 32, 64, 128)]
#[variants(BITS, 4, 8)]
-pub fn quantized_matmul_qmm_transposed<T: ArrayElement + Float, const GROUP_SIZE: u32, const BITS: u32>(
+#[variants(BM, 32, 64)]
+#[variants(BK, 32, 64)]
+#[variants(BN, 32, 64)]
+#[constraint(BK <= BM)]
+#[constraint(BK <= GROUP_SIZE)]
+#[constraint(BM == BN)]
+#[constraint(T != "f32" || BK < 64)]
+pub fn quantized_matmul_qmm_transposed<
+ T: ArrayElement + Float,
+ const GROUP_SIZE: u32,
+ const BITS: u32,
+ const BM: u32,
+ const BK: u32,
+ const BN: u32,
+>(
weights: *const u32,
scales: *const T,
#[optional(use_zero_points)] zero_points: Option<*const u8>,
@@ -104,6 +118,7 @@ pub fn quantized_matmul_qmm_transposed<T: ArrayElement + Float, const GROUP_SIZE
#[specialize] use_hadamard: bool,
#[specialize] aligned_n: bool,
) {
+ let _ = (BM, BK, BN); // tile params unused in tile-agnostic CPU reference
if use_hadamard {
unimplemented!("not supported yet");
}
diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_64x64.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_64x64.rs
deleted file mode 100644
index c28219b6a..000000000
--- a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_64x64.rs
+++ /dev/null
@@ -1,39 +0,0 @@
-use dsl::kernel;
-use half::bf16;
-use num_traits::Float;
-
-use crate::{ArrayElement, backends::cpu::kernel::quant_matmul::qmm_transposed::qmm_transposed};
-
-#[kernel(QuantizedMatmulQmmTransposed64x64)]
-#[variants(T, bf16)]
-#[variants(GROUP_SIZE, 64, 128)]
-#[variants(BITS, 4, 8)]
-pub fn quantized_matmul_qmm_transposed64x64<T: ArrayElement + Float, const GROUP_SIZE: u32, const BITS: u32>(
- weights: *const u32,
- scales: *const T,
- #[optional(use_zero_points)] zero_points: Option<*const u8>,
- #[optional(use_mlx_quant)] biases: Option<*const T>,
- input: *const T,
- output: *mut T,
- in_vec_size: u32,
- out_vec_size: u32,
- batch_size: u32,
- #[specialize] use_zero_points: bool,
- #[specialize] use_mlx_quant: bool,
-) {
- qmm_transposed::<T>(
- weights,
- scales,
- zero_points,
- biases,
- input,
- output,
- in_vec_size as usize,
- out_vec_size as usize,
- batch_size as usize,
- use_zero_points,
- use_mlx_quant,
- GROUP_SIZE as usize,
- BITS as usize,
- );
-}
diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_wide.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_wide.rs
deleted file mode 100644
index ad227a8d5..000000000
--- a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed_wide.rs
+++ /dev/null
@@ -1,39 +0,0 @@
-use dsl::kernel;
-use half::bf16;
-use num_traits::Float;
-
-use crate::{ArrayElement, backends::cpu::kernel::quant_matmul::qmm_transposed::qmm_transposed};
-
-#[kernel(QuantizedMatmulQmmTransposedWide)]
-#[variants(T, bf16)]
-#[variants(GROUP_SIZE, 32, 64, 128)]
-#[variants(BITS, 4, 8)]
-pub fn quantized_matmul_qmm_transposed_wide<T: ArrayElement + Float, const GROUP_SIZE: u32, const BITS: u32>(
- weights: *const u32,
- scales: *const T,
- #[optional(use_zero_points)] zero_points: Option<*const u8>,
- #[optional(use_mlx_quant)] biases: Option<*const T>,
- input: *const T,
- output: *mut T,
- in_vec_size: u32,
- out_vec_size: u32,
- batch_size: u32,
- #[specialize] use_zero_points: bool,
- #[specialize] use_mlx_quant: bool,
-) {
- qmm_transposed::<T>(
- weights,
- scales,
- zero_points,
- biases,
- input,
- output,
- in_vec_size as usize,
- out_vec_size as usize,
- batch_size as usize,
- use_zero_points,
- use_mlx_quant,
- GROUP_SIZE as usize,
- BITS as usize,
- );
-}
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal
index ab1ee36c0..2b00dac2d 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal
@@ -3,10 +3,19 @@
#include "../hadamard_transform/hadamard_transform.h"
#include "quant_matmul.h"
-template <typename T, uint GROUP_SIZE, uint BITS>
+// Transposed QMM kernel. BM = batch tile rows, BK = K-axis tile, BN = output
+// tile cols.
+template <typename T, uint GROUP_SIZE, uint BITS, uint BM, uint BK, uint BN>
VARIANTS(T, float, half, bfloat)
VARIANTS(GROUP_SIZE, 32, 64, 128)
VARIANTS(BITS, 4, 8)
+VARIANTS(BM, 32, 64)
+VARIANTS(BK, 32, 64)
+VARIANTS(BN, 32, 64)
+CONSTRAINT(BK <= BM)
+CONSTRAINT(BK <= GROUP_SIZE)
+CONSTRAINT(BM == BN)
+CONSTRAINT(T != "float" || BK < 64)
PUBLIC KERNEL(QuantizedMatmulQmmTransposed)(
const device uint32_t* weights,
const device T* scales,
@@ -18,20 +27,20 @@ PUBLIC KERNEL(QuantizedMatmulQmmTransposed)(
const constant uint& in_vec_size,
const constant uint& out_vec_size,
const constant uint& batch_size,
- threadgroup T Xs[32 * (32 + 16 / sizeof(T))],
- threadgroup T Ws[32 * (32 + 16 / sizeof(T))],
+ threadgroup T Xs[BM * (BK + 16 / sizeof(T))],
+ threadgroup T Ws[BN * (BK + 16 / sizeof(T))],
const bool use_zero_points SPECIALIZE,
const bool use_mlx_quant SPECIALIZE,
const bool use_hadamard SPECIALIZE,
const bool aligned_n SPECIALIZE,
- const uint out_block_idx GROUPS(out_vec_size.div_ceil(32)),
- const uint batch_block_idx GROUPS(batch_size.div_ceil(32)),
+ const uint out_block_idx GROUPS(out_vec_size.div_ceil(BN)),
+ const uint batch_block_idx GROUPS(batch_size.div_ceil(BM)),
const uint simd_lane THREADS(32),
const uint simd_group THREADS(4)
) {
if (use_mlx_quant) {
if (aligned_n) {
- qmm_transposed_impl<T, GROUP_SIZE, BITS, true, 32, 32, 32, true>(
+ qmm_transposed_impl<T, GROUP_SIZE, BITS, true, BM, BK, BN, true>(
weights,
scales,
zero_points,
@@ -49,7 +58,7 @@ PUBLIC KERNEL(QuantizedMatmulQmmTransposed)(
simd_lane
);
} else {
- qmm_transposed_impl<T, GROUP_SIZE, BITS, false, 32, 32, 32, true>(
+ qmm_transposed_impl<T, GROUP_SIZE, BITS, false, BM, BK, BN, true>(
weights,
scales,
zero_points,
@@ -69,7 +78,7 @@ PUBLIC KERNEL(QuantizedMatmulQmmTransposed)(
}
} else {
if (aligned_n) {
- qmm_transposed_impl<T, GROUP_SIZE, BITS, true, 32, 32, 32, false>(
+ qmm_transposed_impl<T, GROUP_SIZE, BITS, true, BM, BK, BN, false>(
weights,
scales,
zero_points,
@@ -87,7 +96,7 @@ PUBLIC KERNEL(QuantizedMatmulQmmTransposed)(
simd_lane
);
} else {
- qmm_transposed_impl<T, GROUP_SIZE, BITS, false, 32, 32, 32, false>(
+ qmm_transposed_impl<T, GROUP_SIZE, BITS, false, BM, BK, BN, false>(
weights,
scales,
zero_points,
@@ -110,26 +119,28 @@ PUBLIC KERNEL(QuantizedMatmulQmmTransposed)(
if (use_hadamard) {
threadgroup_barrier(mem_flags::mem_device);
- constexpr uint BLOCK_SIZE = 32;
+ // Each lane covers BN/32 output columns (1 at BN=32, 2 at BN=64).
constexpr uint NUM_SIMD_GROUPS = 4;
-
- constexpr uint ROWS_PER_SIMD_GROUP = BLOCK_SIZE / NUM_SIMD_GROUPS;
-
- const uint out_col = out_block_idx * BLOCK_SIZE + simd_lane;
- const int32_t factor = hadamard_factors[out_col];
+ constexpr uint ROWS_PER_SIMD_GROUP = BM / NUM_SIMD_GROUPS;
+ constexpr uint COLS_PER_LANE = BN / 32;
const uint row_start =
- batch_block_idx * BLOCK_SIZE + simd_group * ROWS_PER_SIMD_GROUP;
+ batch_block_idx * BM + simd_group * ROWS_PER_SIMD_GROUP;
const uint row_end = min(row_start + ROWS_PER_SIMD_GROUP, batch_size);
- for (uint row = row_start; row < row_end; row++) {
- uint output_index = row * out_vec_size + out_col;
+ for (uint c = 0; c < COLS_PER_LANE; c++) {
+ const uint out_col = out_block_idx * BN + c * 32 + simd_lane;
+ const int32_t factor = hadamard_factors[out_col];
- output[output_index] = simdgroup_random_hadamard_transform(
- static_cast<ushort>(simd_lane),
- output[output_index],
- factor
- );
+ for (uint row = row_start; row < row_end; row++) {
+ uint output_index = row * out_vec_size + out_col;
+
+ output[output_index] = simdgroup_random_hadamard_transform(
+ static_cast<ushort>(simd_lane),
+ output[output_index],
+ factor
+ );
+ }
}
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metal
deleted file mode 100644
index a11b6c023..000000000
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metal
+++ /dev/null
@@ -1,65 +0,0 @@
-#include <metal_stdlib>
-#include "../common/dsl.h"
-#include "quant_matmul.h"
-
-template <typename T, uint GROUP_SIZE, uint BITS>
-VARIANTS(T, bfloat)
-VARIANTS(GROUP_SIZE, 64, 128)
-VARIANTS(BITS, 4, 8)
-PUBLIC KERNEL(QuantizedMatmulQmmTransposed64x64)(
- const device uint32_t* weights,
- const device T* scales,
- const device uint8_t* zero_points OPTIONAL(use_zero_points),
- const device T* biases OPTIONAL(use_mlx_quant),
- const device T* input,
- device T* output,
- const constant uint& in_vec_size,
- const constant uint& out_vec_size,
- const constant uint& batch_size,
- threadgroup T Xs[64 * (64 + 16 / sizeof(T))],
- threadgroup T Ws[64 * (64 + 16 / sizeof(T))],
- const bool use_zero_points SPECIALIZE,
- const bool use_mlx_quant SPECIALIZE,
- const uint out_block_idx GROUPS(out_vec_size.div_ceil(64)),
- const uint batch_block_idx GROUPS(batch_size.div_ceil(64)),
- const uint simd_lane THREADS(32),
- const uint simd_group THREADS(4)
-) {
- if (use_mlx_quant) {
- qmm_transposed_impl<T, GROUP_SIZE, BITS, true, 64, 64, 64, true>(
- weights,
- scales,
- zero_points,
- biases,
- input,
- output,
- Xs,
- Ws,
- in_vec_size,
- out_vec_size,
- batch_size,
- out_block_idx,
- batch_block_idx,
- simd_group,
- simd_lane
- );
- } else {
- qmm_transposed_impl<T, GROUP_SIZE, BITS, true, 64, 64, 64, false>(
- weights,
- scales,
- zero_points,
- biases,
- input,
- output,
- Xs,
- Ws,
- in_vec_size,
- out_vec_size,
- batch_size,
- out_block_idx,
- batch_block_idx,
- simd_group,
- simd_lane
- );
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_wide.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_wide.metal
deleted file mode 100644
index 1c6464ba9..000000000
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_wide.metal
+++ /dev/null
@@ -1,69 +0,0 @@
-#include <metal_stdlib>
-#include "../common/dsl.h"
-#include "quant_matmul.h"
-
-// Wide QMM tile: BM=64, BK=32, BN=64.
-// Processes 4x more output elements per threadgroup than the 32x32 variant.
-// BK=32 satisfies the group_size >= BK constraint for all group sizes (32, 64,
-// 128).
-template <typename T, uint GROUP_SIZE, uint BITS>
-VARIANTS(T, bfloat)
-VARIANTS(GROUP_SIZE, 32, 64, 128)
-VARIANTS(BITS, 4, 8)
-PUBLIC KERNEL(QuantizedMatmulQmmTransposedWide)(
- const device uint32_t* weights,
- const device T* scales,
- const device uint8_t* zero_points OPTIONAL(use_zero_points),
- const device T* biases OPTIONAL(use_mlx_quant),
- const device T* input,
- device T* output,
- const constant uint& in_vec_size,
- const constant uint& out_vec_size,
- const constant uint& batch_size,
- threadgroup T Xs[64 * (32 + 16 / sizeof(T))],
- threadgroup T Ws[64 * (32 + 16 / sizeof(T))],
- const bool use_zero_points SPECIALIZE,
- const bool use_mlx_quant SPECIALIZE,
- const uint out_block_idx GROUPS(out_vec_size.div_ceil(64)),
- const uint batch_block_idx GROUPS(batch_size.div_ceil(64)),
- const uint simd_lane THREADS(32),
- const uint simd_group THREADS(4)
-) {
- if (use_mlx_quant) {
- qmm_transposed_impl<T, GROUP_SIZE, BITS, true, 64, 32, 64, true>(
- weights,
- scales,
- zero_points,
- biases,
- input,
- output,
- Xs,
- Ws,
- in_vec_size,
- out_vec_size,
- batch_size,
- out_block_idx,
- batch_block_idx,
- simd_group,
- simd_lane
- );
- } else {
- qmm_transposed_impl<T, GROUP_SIZE, BITS, true, 64, 32, 64, false>(
- weights,
- scales,
- zero_points,
- biases,
- input,
- output,
- Xs,
- Ws,
- in_vec_size,
- out_vec_size,
- batch_size,
- out_block_idx,
- batch_block_idx,
- simd_group,
- simd_lane
- );
- }
-}
diff --git a/crates/uzu/tests/unit/kernel/quant_matmul/mod.rs b/crates/uzu/tests/unit/kernel/quant_matmul/mod.rs
index b97526f4f..bef2b6ea4 100644
--- a/crates/uzu/tests/unit/kernel/quant_matmul/mod.rs
+++ b/crates/uzu/tests/unit/kernel/quant_matmul/mod.rs
@@ -2,9 +2,7 @@
#[path = "../../../common/mod.rs"]
mod common;
-mod qmm_transposed_64x64_test;
mod qmm_transposed_test;
-mod qmm_transposed_wide_test;
mod qmv_fast_test;
mod qmv_test;
diff --git a/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_64x64_test.rs b/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_64x64_test.rs
deleted file mode 100644
index 62bad063e..000000000
--- a/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_64x64_test.rs
+++ /dev/null
@@ -1,372 +0,0 @@
-use std::fmt::{Debug, Display};
-
-use half::bf16;
-use num_traits::Float;
-use uzu::{
- ArrayElement, DataType,
- backends::{
- common::{Backend, Buffer, Context, Encoder, Kernels, kernel::QuantizedMatmulQmmTransposed64x64Kernel},
- cpu::Cpu,
- },
-};
-
-use super::{Input, check_tolerance, pack_weights_u32, pack_zero_points};
-use crate::{common::helpers::alloc_buffer_with_data, uzu_test};
-
-fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
- let context = B::Context::new().expect("Failed to create Context");
- let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmmTransposed64x64Kernel::new(
- &context,
- T::data_type(),
- input.group_size,
- input.bits,
- input.use_zero_points,
- input.use_mlx_quant,
- )
- .expect("Failed to create QuantizedMatmulQmmTransposed64x64Kernel");
-
- let w_buf = alloc_buffer_with_data::<B, u32>(&context, &input.w_packed);
- let scales_buf = alloc_buffer_with_data::<B, T>(&context, &input.scales);
-
- let zp_buf = input.zero_points.as_ref().map(|zp| alloc_buffer_with_data::<B, u8>(&context, zp));
- let bias_buf = input.biases.as_ref().map(|b| alloc_buffer_with_data::<B, T>(&context, b));
-
- let x_buf = alloc_buffer_with_data::<B, T>(&context, &input.x);
- let output_size = (input.m as usize) * (input.n as usize) * T::data_type().size_in_bytes();
- let mut y_buf = context.create_buffer(output_size).expect("Failed to create buffer");
-
- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
- kernel.encode(
- &w_buf,
- &scales_buf,
- zp_buf.as_ref(),
- bias_buf.as_ref(),
- &x_buf,
- &mut y_buf,
- input.k,
- input.n,
- input.m,
- &mut encoder,
- );
-
- encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
-
- let y_ptr = y_buf.cpu_ptr().as_ptr() as *const T;
- let y_len = (input.m as usize) * (input.n as usize);
- unsafe { std::slice::from_raw_parts(y_ptr, y_len) }.to_vec()
-}
-
-/// Create test data for transposed 64x64 qmm: x[m×k] * w_transposed[n×k] = y[m×n]
-/// Weights are in transposed [n × k] layout.
-fn get_test_data_basic<T: ArrayElement + Float>(
- m: usize,
- k: usize,
- n: usize,
- group_size: u32,
- bits: u32,
- use_zero_points: bool,
- use_mlx_quant: bool,
-) -> (Input<T>, Vec<T>) {
- let num_groups_k = (k + group_size as usize - 1) / group_size as usize;
-
- // Generate quantized weights [n × k]
- let max_val = if bits == 4 {
- 15u8
- } else {
- 255u8
- };
- let mut weights_raw: Vec<u8> = Vec::with_capacity(n * k);
- for j in 0..n {
- for l in 0..k {
- weights_raw.push(((j * 3 + l * 7 + 1) % (max_val as usize + 1)) as u8);
- }
- }
- let w_packed = pack_weights_u32(&weights_raw, bits);
-
- // Scales: [n × num_groups_k]
- let mut scales_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);
- for j in 0..n {
- for g in 0..num_groups_k {
- scales_f32.push(0.5 + 0.1 * ((j + g) % 5) as f32);
- }
- }
- let scales: Vec<T> = scales_f32.iter().map(|&v| T::from(v).unwrap()).collect();
-
- // Zero points or biases
- let zp_stride = if bits == 4 {
- (num_groups_k + 1) / 2
- } else {
- num_groups_k
- };
-
- let (zero_points, biases) = if use_zero_points {
- let mut zp_raw: Vec<u8> = Vec::with_capacity(n * num_groups_k);
- for j in 0..n {
- for g in 0..num_groups_k {
- let zp_val = ((j * 2 + g * 3) % (max_val as usize + 1)) as u8;
- zp_raw.push(zp_val);
- }
- }
- // Pack zero points per row
- let mut zp_packed: Vec<u8> = Vec::with_capacity(n * zp_stride);
- for j in 0..n {
- let row = &zp_raw[j * num_groups_k..(j + 1) * num_groups_k];
- let packed_row = pack_zero_points(row, bits);
- let mut padded = packed_row;
- padded.resize(zp_stride, 0);
- zp_packed.extend_from_slice(&padded);
- }
- (Some(zp_packed), None)
- } else if use_mlx_quant {
- let mut biases_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);
- for j in 0..n {
- for g in 0..num_groups_k {
- biases_f32.push(0.01 * ((j + g * 2) % 7) as f32);
- }
- }
- let biases: Vec<T> = biases_f32.iter().map(|&v| T::from(v).unwrap()).collect();
- (None, Some(biases))
- } else {
- unreachable!("Must use either zero_points or mlx_quant");
- };
-
- // X: [m × k]
- let mut x_f32: Vec<f32> = Vec::with_capacity(m * k);
- for i in 0..m {
- for l in 0..k {
- x_f32.push(0.1 * f32::sin((i * k + l) as f32 * 0.05) + 0.5);
- }
- }
- let x: Vec<T> = x_f32.iter().map(|&v| T::from(v).unwrap()).collect();
-
- let input = Input {
- w_packed,
- scales,
- zero_points,
- biases,
- x,
- k: k as u32,
- n: n as u32,
- m: m as u32,
- group_size,
- bits,
- use_zero_points,
- use_mlx_quant,
- };
-
- let expected = get_output::<Cpu, T>(&input);
- (input, expected)
-}
-
-/// Small edge-case test data with known values
-fn get_test_data_edge<T: ArrayElement + Float>(
- group_size: u32,
- bits: u32,
- use_zero_points: bool,
- use_mlx_quant: bool,
-) -> (Input<T>, Vec<T>) {
- let m = 2usize;
- let k = group_size as usize;
- let n = 64usize;
- let num_groups_k = 1;
-
- // Simple weights [n × k]: all 1s
- let weights_raw: Vec<u8> = vec![1u8; n * k];
- let w_packed = pack_weights_u32(&weights_raw, bits);
-
- // Unit scales
- let scales: Vec<T> = vec![T::one(); n * num_groups_k];
-
- let zp_stride = if bits == 4 {
- (num_groups_k + 1) / 2
- } else {
- num_groups_k
- };
-
- let (zero_points, biases) = if use_zero_points {
- // Zero points all 0 -> bias = 0
- let zp_packed = vec![0u8; n * zp_stride];
- (Some(zp_packed), None)
- } else if use_mlx_quant {
- let biases: Vec<T> = vec![T::zero(); n * num_groups_k];
- (None, Some(biases))
- } else {
- unreachable!("Must use either zero_points or mlx_quant");
- };
-
- // X: simple increasing values
- let x_f32: Vec<f32> = (0..m * k).map(|i| (i + 1) as f32 * 0.1).collect();
- let x: Vec<T> = x_f32.iter().map(|&v| T::from(v).unwrap()).collect();
-
- let input = Input {
- w_packed,
- scales,
- zero_points,
- biases,
- x,
- k: k as u32,
- n: n as u32,
- m: m as u32,
- group_size,
- bits,
- use_zero_points,
- use_mlx_quant,
- };
-
- let expected = get_output::<Cpu, T>(&input);
- (input, expected)
-}
-
-fn test_internal<T: ArrayElement + Float + Debug + Display>(
- input: &Input<T>,
- expected: &[T],
-) {
- let (rel_tol, abs_tol): (f64, f64) = match T::data_type() {
- DataType::BF16 => (0.05, 0.5),
- DataType::F16 => (0.02, 0.5),
- _ => (0.01, 0.1),
- };
-
- for_each_non_cpu_backend!(|B| {
- let output = get_output::<B, T>(input);
- assert_eq!(expected.len(), output.len(), "Output length mismatch");
-
- let mut errors = 0;
- for (i, (&exp, &got)) in expected.iter().zip(output.iter()).enumerate() {
- let exp_f32 = exp.to_f32().unwrap();
- let got_f32 = got.to_f32().unwrap();
- if !check_tolerance(exp_f32, got_f32, rel_tol, abs_tol) {
- if errors < 5 {
- eprintln!(" idx={} expected={} got={} diff={}", i, exp_f32, got_f32, (exp_f32 - got_f32).abs());
- }
- errors += 1;
- }
- }
- assert_eq!(
- errors,
- 0,
- "QMM transposed 64x64 kernel: backend={}, m={}, k={}, n={}, gs={}, bits={}, zp={}, mlx={}: {} mismatches",
- std::any::type_name::<B>(),
- input.m,
- input.k,
- input.n,
- input.group_size,
- input.bits,
- input.use_zero_points,
- input.use_mlx_quant,
- errors,
- );
- });
-}
-
-fn get_test_dims(
- group_size: u32,
- bits: u32,
-) -> Vec<(usize, usize, usize)> {
- let gs = group_size as usize;
- // 64x64 kernel: BM=64, BK=64, BN=64.
- // K must be >= group_size (transposed grouping along K).
- // N must be a multiple of 64 (BN tile size).
- // bf16 8-bit: keep dims small to avoid catastrophic cancellation.
- if bits == 8 {
- vec![(2, gs, 64), (4, gs, 64)]
- } else {
- vec![(2, gs * 2, 64), (4, gs * 2, 128), (8, gs, 128)]
- }
-}
-
-fn test_basic(
- group_size: u32,
- bits: u32,
- use_zero_points: bool,
- use_mlx_quant: bool,
-) {
- let dims = get_test_dims(group_size, bits);
-
- for (m, k, n) in dims {
- let (input, expected) = get_test_data_basic::<bf16>(m, k, n, group_size, bits, use_zero_points, use_mlx_quant);
- test_internal::<bf16>(&input, &expected);
- }
-}
-
-fn test_edge(
- group_size: u32,
- bits: u32,
- use_zero_points: bool,
- use_mlx_quant: bool,
-) {
- let (input, expected) = get_test_data_edge::<bf16>(group_size, bits, use_zero_points, use_mlx_quant);
- test_internal::<bf16>(&input, &expected);
-}
-
-// -- 4-bit, zero points -------------------------------------------------------
-
-#[uzu_test]
-#[ignore]
-fn test_bf16_gs64_4bit_zp() {
- test_basic(64, 4, true, false);
-}
-
-#[uzu_test]
-#[ignore]
-fn test_bf16_gs128_4bit_zp() {
- test_basic(128, 4, true, false);
-}
-
-// -- 8-bit, zero points -------------------------------------------------------
-
-#[uzu_test]
-fn test_bf16_gs64_8bit_zp() {
- test_basic(64, 8, true, false);
-}
-
-#[uzu_test]
-fn test_bf16_gs128_8bit_zp() {
- test_basic(128, 8, true, false);
-}
-
-// -- 4-bit, mlx quant ----------------------------------------------------------
-
-#[uzu_test]
-fn test_bf16_gs64_4bit_mlx() {
- test_basic(64, 4, false, true);
-}
-
-#[uzu_test]
-fn test_bf16_gs128_4bit_mlx() {
- test_basic(128, 4, false, true);
-}
-
-// -- 8-bit, mlx quant ----------------------------------------------------------
-
-#[uzu_test]
-fn test_bf16_gs64_8bit_mlx() {
- test_basic(64, 8, false, true);
-}
-
-#[uzu_test]
-fn test_bf16_gs128_8bit_mlx() {
- test_basic(128, 8, false, true);
-}
-
-// -- Edge cases ----------------------------------------------------------------
-
-#[uzu_test]
-fn test_edge_bf16_4bit_zp() {
- test_edge(64, 4, true, false);
-}
-
-#[uzu_test]
-fn test_edge_bf16_8bit_zp() {
- test_edge(64, 8, true, false);
-}
-
-#[uzu_test]
-fn test_edge_bf16_4bit_mlx() {
- test_edge(64, 4, false, true);
-}
-
-#[uzu_test]
-fn test_edge_bf16_8bit_mlx() {
- test_edge(64, 8, false, true);
-}
diff --git a/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rs b/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rs
index fcbb00756..cf42d4161 100644
--- a/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rs
+++ b/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rs
@@ -13,56 +13,7 @@ use uzu::{
use super::{Input, check_tolerance, pack_weights_u32, pack_zero_points};
use crate::{common::helpers::alloc_buffer_with_data, uzu_test};
-fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
- let context = B::Context::new().expect("Failed to create Context");
- let aligned_n = (input.n % 32) == 0;
- let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel::new(
- &context,
- T::data_type(),
- input.group_size,
- input.bits,
- input.use_zero_points,
- input.use_mlx_quant,
- false,
- aligned_n,
- )
- .expect("Failed to create QuantizedMatmulQmmTransposedKernel");
-
- let w_buf = alloc_buffer_with_data::<B, u32>(&context, &input.w_packed);
- let scales_buf = alloc_buffer_with_data::<B, T>(&context, &input.scales);
-
- let zp_buf = input.zero_points.as_ref().map(|zp| alloc_buffer_with_data::<B, u8>(&context, zp));
- let bias_buf = input.biases.as_ref().map(|b| alloc_buffer_with_data::<B, T>(&context, b));
-
- let x_buf = alloc_buffer_with_data::<B, T>(&context, &input.x);
- let output_size = (input.m as usize) * (input.n as usize) * T::data_type().size_in_bytes();
- let mut y_buf = context.create_buffer(output_size).expect("Failed to create buffer");
-
- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
- kernel.encode(
- &w_buf,
- &scales_buf,
- zp_buf.as_ref(),
- bias_buf.as_ref(),
- &x_buf,
- &mut y_buf,
- None::<&B::Buffer>,
- input.k,
- input.n,
- input.m,
- &mut encoder,
- );
-
- encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
-
- let y_ptr = y_buf.cpu_ptr().as_ptr() as *const T;
- let y_len = (input.m as usize) * (input.n as usize);
- unsafe { std::slice::from_raw_parts(y_ptr, y_len) }.to_vec()
-}
-
-/// Create test data for transposed qmm: x[m×k] * w_transposed[n×k] = y[m×n]
-/// Weights are in transposed [n × k] layout.
-fn get_test_data_basic<T: ArrayElement + Float>(
+fn make_input_basic<T: ArrayElement + Float>(
m: usize,
k: usize,
n: usize,
@@ -70,10 +21,9 @@ fn get_test_data_basic<T: ArrayElement + Float>(
bits: u32,
use_zero_points: bool,
use_mlx_quant: bool,
-) -> (Input<T>, Vec<T>) {
+) -> Input<T> {
let num_groups_k = (k + group_size as usize - 1) / group_size as usize;
- // Generate quantized weights [n × k]
let max_val = if bits == 4 {
15u8
} else {
@@ -87,7 +37,6 @@ fn get_test_data_basic<T: ArrayElement + Float>(
}
let w_packed = pack_weights_u32(&weights_raw, bits);
- // Scales: [n × num_groups_k]
let mut scales_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);
for j in 0..n {
for g in 0..num_groups_k {
@@ -96,7 +45,6 @@ fn get_test_data_basic<T: ArrayElement + Float>(
}
let scales: Vec<T> = scales_f32.iter().map(|&v| T::from(v).unwrap()).collect();
- // Zero points or biases
let zp_stride = if bits == 4 {
(num_groups_k + 1) / 2
} else {
@@ -111,7 +59,6 @@ fn get_test_data_basic<T: ArrayElement + Float>(
zp_raw.push(zp_val);
}
}
- // Pack zero points per row
let mut zp_packed: Vec<u8> = Vec::with_capacity(n * zp_stride);
for j in 0..n {
let row = &zp_raw[j * num_groups_k..(j + 1) * num_groups_k];
@@ -134,7 +81,6 @@ fn get_test_data_basic<T: ArrayElement + Float>(
unreachable!("Must use either zero_points or mlx_quant");
};
- // X: [m × k]
let mut x_f32: Vec<f32> = Vec::with_capacity(m * k);
for i in 0..m {
for l in 0..k {
@@ -143,7 +89,7 @@ fn get_test_data_basic<T: ArrayElement + Float>(
}
let x: Vec<T> = x_f32.iter().map(|&v| T::from(v).unwrap()).collect();
- let input = Input {
+ Input {
w_packed,
scales,
zero_points,
@@ -156,29 +102,23 @@ fn get_test_data_basic<T: ArrayElement + Float>(
bits,
use_zero_points,
use_mlx_quant,
- };
-
- let expected = get_output::<Cpu, T>(&input);
- (input, expected)
+ }
}
-/// Small edge-case test data with known values
-fn get_test_data_edge<T: ArrayElement + Float>(
+fn make_input_edge<T: ArrayElement + Float>(
+ n: usize,
group_size: u32,
bits: u32,
use_zero_points: bool,
use_mlx_quant: bool,
-) -> (Input<T>, Vec<T>) {
+) -> Input<T> {
let m = 2usize;
let k = group_size as usize;
- let n = 32usize;
let num_groups_k = 1;
- // Simple weights [n × k]: all 1s
let weights_raw: Vec<u8> = vec![1u8; n * k];
let w_packed = pack_weights_u32(&weights_raw, bits);
- // Unit scales
let scales: Vec<T> = vec![T::one(); n * num_groups_k];
let zp_stride = if bits == 4 {
@@ -188,7 +128,6 @@ fn get_test_data_edge<T: ArrayElement + Float>(
};
let (zero_points, biases) = if use_zero_points {
- // Zero points all 0 -> bias = 0
let zp_packed = vec![0u8; n * zp_stride];
(Some(zp_packed), None)
} else if use_mlx_quant {
@@ -198,11 +137,10 @@ fn get_test_data_edge<T: ArrayElement + Float>(
unreachable!("Must use either zero_points or mlx_quant");
};
- // X: simple increasing values
let x_f32: Vec<f32> = (0..m * k).map(|i| (i + 1) as f32 * 0.1).collect();
let x: Vec<T> = x_f32.iter().map(|&v| T::from(v).unwrap()).collect();
- let input = Input {
+ Input {
w_packed,
scales,
zero_points,
@@ -215,20 +153,76 @@ fn get_test_data_edge<T: ArrayElement + Float>(
bits,
use_zero_points,
use_mlx_quant,
- };
+ }
+}
+
+fn get_output<B: Backend, T: ArrayElement + Float>(
+ input: &Input<T>,
+ bm: u32,
+ bk: u32,
+ bn: u32,
+ aligned_n: bool,
+ hadamard_factors: Option<&[i32]>,
+) -> Vec<T> {
+ let context = B::Context::new().expect("Failed to create Context");
+ let use_hadamard = hadamard_factors.is_some();
+ let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel::new(
+ &context,
+ T::data_type(),
+ input.group_size,
+ input.bits,
+ bm,
+ bk,
+ bn,
+ input.use_zero_points,
+ input.use_mlx_quant,
+ use_hadamard,
+ aligned_n,
+ )
+ .expect("Failed to create QuantizedMatmulQmmTransposedKernel");
+
+ let w_buf = alloc_buffer_with_data::<B, u32>(&context, &input.w_packed);
+ let scales_buf = alloc_buffer_with_data::<B, T>(&context, &input.scales);
+
+ let zp_buf = input.zero_points.as_ref().map(|zp| alloc_buffer_with_data::<B, u8>(&context, zp));
+ let bias_buf = input.biases.as_ref().map(|b| alloc_buffer_with_data::<B, T>(&context, b));
+ let had_buf = hadamard_factors.map(|f| alloc_buffer_with_data::<B, i32>(&context, f));
+
+ let x_buf = alloc_buffer_with_data::<B, T>(&context, &input.x);
+ let output_size = (input.m as usize) * (input.n as usize) * T::data_type().size_in_bytes();
+ let mut y_buf = context.create_buffer(output_size).expect("Failed to create buffer");
- let expected = get_output::<Cpu, T>(&input);
- (input, expected)
+ let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
+ kernel.encode(
+ &w_buf,
+ &scales_buf,
+ zp_buf.as_ref(),
+ bias_buf.as_ref(),
+ &x_buf,
+ &mut y_buf,
+ had_buf.as_ref(),
+ input.k,
+ input.n,
+ input.m,
+ &mut encoder,
+ );
+
+ encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
+
+ let y_ptr = y_buf.cpu_ptr().as_ptr() as *const T;
+ let y_len = (input.m as usize) * (input.n as usize);
+ unsafe { std::slice::from_raw_parts(y_ptr, y_len) }.to_vec()
}
-fn test_internal<T: ArrayElement + Float + Debug + Display>(
+fn check_against_expected<T: ArrayElement + Float + Debug + Display>(
input: &Input<T>,
expected: &[T],
+ bm: u32,
+ bk: u32,
+ bn: u32,
+ aligned_n: bool,
) {
- // GPU MMA units use different rounding than scalar CPU code.
- // bf16 has 7-bit mantissa (ULP ~0.8%), f16 has 10-bit (~0.1%).
- // Tiled accumulation reorders FMA ops, compounding rounding differences.
- // 8-bit weights (0-255) with ZP bias cause larger accumulated errors.
+ // GPU tiled accumulation rounds differently from CPU scalar reference.
let (rel_tol, abs_tol): (f64, f64) = match T::data_type() {
DataType::BF16 => (0.05, 0.5),
DataType::F16 => (0.02, 0.5),
@@ -236,7 +230,7 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(
};
for_each_non_cpu_backend!(|B| {
- let output = get_output::<B, T>(input);
+ let output = get_output::<B, T>(input, bm, bk, bn, aligned_n, None);
assert_eq!(expected.len(), output.len(), "Output length mismatch");
let mut errors = 0;
@@ -253,8 +247,11 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(
assert_eq!(
errors,
0,
- "QMM transposed kernel: backend={}, m={}, k={}, n={}, gs={}, bits={}, zp={}, mlx={}: {} mismatches",
+ "QMM transposed kernel: backend={}, shape=(BM={},BK={},BN={}), m={}, k={}, n={}, gs={}, bits={}, zp={}, mlx={}: {} mismatches",
std::any::type_name::<B>(),
+ bm,
+ bk,
+ bn,
input.m,
input.k,
input.n,
@@ -267,266 +264,328 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(
});
}
-fn get_test_dims(
- group_size: u32,
- bits: u32,
- is_half_precision: bool,
-) -> Vec<(usize, usize, usize)> {
- let gs = group_size as usize;
- // K must be >= group_size for the transposed grouping along K.
- // N must be >= 32 for the Metal tiled weight loader (BN=32).
- // For half-precision 8-bit, keep dimensions small to avoid
- // catastrophic cancellation from large weight values (0-255).
- if is_half_precision && bits == 8 {
- vec![(2, gs, gs), (4, gs, gs)]
+struct TileCfg {
+ bm: u32,
+ bk: u32,
+ bn: u32,
+}
+
+impl TileCfg {
+ const fn aligned_n_default(&self) -> bool {
+ // BN=64 tiles require output_dim % 64 == 0; BN=32 checks actual N at test time.
+ self.bn == 64
+ }
+}
+
+const CFG_32X32: TileCfg = TileCfg { bm: 32, bk: 32, bn: 32 };
+const CFG_WIDE: TileCfg = TileCfg { bm: 64, bk: 32, bn: 64 };
+const CFG_64X64: TileCfg = TileCfg { bm: 64, bk: 64, bn: 64 };
+
+fn pick_aligned_n(
+ cfg: &TileCfg,
+ n: usize,
+) -> bool {
+ if cfg.bn == 32 {
+ (n % 32) == 0
} else {
- vec![(2, gs * 2, gs), (4, gs * 2, gs * 2), (8, gs, gs * 2)]
+ cfg.aligned_n_default()
}
}
fn test_basic<T: ArrayElement + Float + Debug + Display>(
+ cfg: &TileCfg,
group_size: u32,
bits: u32,
use_zero_points: bool,
use_mlx_quant: bool,
) {
let is_half = matches!(T::data_type(), DataType::F16 | DataType::BF16);
- let dims = get_test_dims(group_size, bits, is_half);
+ let gs = group_size as usize;
+ let bn = cfg.bn as usize;
+ // half+8bit: smaller dims to cap accumulated rounding error.
+ let dims: Vec<(usize, usize, usize)> = if is_half && bits == 8 {
+ vec![(2, gs, bn), (4, gs, bn)]
+ } else {
+ vec![(2, gs * 2, bn), (4, gs * 2, bn * 2), (8, gs, bn * 2)]
+ };
for (m, k, n) in dims {
- let (input, expected) = get_test_data_basic::<T>(m, k, n, group_size, bits, use_zero_points, use_mlx_quant);
- test_internal::<T>(&input, &expected);
+ let input = make_input_basic::<T>(m, k, n, group_size, bits, use_zero_points, use_mlx_quant);
+ let aligned_n = pick_aligned_n(cfg, n);
+ let expected = get_output::<Cpu, T>(&input, cfg.bm, cfg.bk, cfg.bn, aligned_n, None);
+ check_against_expected::<T>(&input, &expected, cfg.bm, cfg.bk, cfg.bn, aligned_n);
}
}
fn test_edge<T: ArrayElement + Float + Debug + Display>(
+ cfg: &TileCfg,
group_size: u32,
bits: u32,
use_zero_points: bool,
use_mlx_quant: bool,
) {
- let (input, expected) = get_test_data_edge::<T>(group_size, bits, use_zero_points, use_mlx_quant);
- test_internal::<T>(&input, &expected);
-}
-
-// -- 4-bit, zero points -------------------------------------------------------
-
-#[uzu_test]
-fn test_f32_gs32_4bit_zp() {
- test_basic::<f32>(32, 4, true, false);
-}
-
-#[uzu_test]
-fn test_f32_gs64_4bit_zp() {
- test_basic::<f32>(64, 4, true, false);
-}
-
-#[uzu_test]
-fn test_f32_gs128_4bit_zp() {
- test_basic::<f32>(128, 4, true, false);
-}
-
-#[uzu_test]
-fn test_f16_gs32_4bit_zp() {
- test_basic::<f16>(32, 4, true, false);
-}
-
-#[uzu_test]
-fn test_f16_gs64_4bit_zp() {
- test_basic::<f16>(64, 4, true, false);
-}
-
-#[uzu_test]
-fn test_f16_gs128_4bit_zp() {
- test_basic::<f16>(128, 4, true, false);
-}
-
-#[uzu_test]
-fn test_bf16_gs32_4bit_zp() {
- test_basic::<bf16>(32, 4, true, false);
-}
-
-#[uzu_test]
-#[ignore]
-fn test_bf16_gs64_4bit_zp() {
- test_basic::<bf16>(64, 4, true, false);
-}
-
-#[uzu_test]
-#[ignore]
-fn test_bf16_gs128_4bit_zp() {
- test_basic::<bf16>(128, 4, true, false);
-}
-
-// -- 8-bit, zero points -------------------------------------------------------
-
-#[uzu_test]
-fn test_f32_gs32_8bit_zp() {
- test_basic::<f32>(32, 8, true, false);
-}
-
-#[uzu_test]
-fn test_f32_gs64_8bit_zp() {
- test_basic::<f32>(64, 8, true, false);
-}
-
-#[uzu_test]
-fn test_f32_gs128_8bit_zp() {
- test_basic::<f32>(128, 8, true, false);
-}
-
-#[uzu_test]
-fn test_f16_gs32_8bit_zp() {
- test_basic::<f16>(32, 8, true, false);
-}
-
-#[uzu_test]
-fn test_f16_gs64_8bit_zp() {
- test_basic::<f16>(64, 8, true, false);
-}
-
-#[uzu_test]
-fn test_f16_gs128_8bit_zp() {
- test_basic::<f16>(128, 8, true, false);
-}
-
-#[uzu_test]
-fn test_bf16_gs32_8bit_zp() {
- test_basic::<bf16>(32, 8, true, false);
-}
-
-#[uzu_test]
-fn test_bf16_gs64_8bit_zp() {
- test_basic::<bf16>(64, 8, true, false);
-}
-
-#[uzu_test]
-fn test_bf16_gs128_8bit_zp() {
- test_basic::<bf16>(128, 8, true, false);
-}
-
-// -- 4-bit, mlx quant ----------------------------------------------------------
-
-#[uzu_test]
-fn test_f32_gs32_4bit_mlx() {
- test_basic::<f32>(32, 4, false, true);
-}
-
-#[uzu_test]
-fn test_f32_gs64_4bit_mlx() {
- test_basic::<f32>(64, 4, false, true);
-}
-
-#[uzu_test]
-fn test_f32_gs128_4bit_mlx() {
- test_basic::<f32>(128, 4, false, true);
-}
-
-#[uzu_test]
-fn test_f16_gs32_4bit_mlx() {
- test_basic::<f16>(32, 4, false, true);
-}
-
-#[uzu_test]
-fn test_f16_gs64_4bit_mlx() {
- test_basic::<f16>(64, 4, false, true);
-}
-
-#[uzu_test]
-fn test_f16_gs128_4bit_mlx() {
- test_basic::<f16>(128, 4, false, true);
-}
-
-#[uzu_test]
-fn test_bf16_gs32_4bit_mlx() {
- test_basic::<bf16>(32, 4, false, true);
-}
-
-#[uzu_test]
-fn test_bf16_gs64_4bit_mlx() {
- test_basic::<bf16>(64, 4, false, true);
+ // Edge N is one BN tile wide — smallest legal N for the shape.
+ let n = cfg.bn as usize;
+ let input = make_input_edge::<T>(n, group_size, bits, use_zero_points, use_mlx_quant);
+ let aligned_n = pick_aligned_n(cfg, n);
+ let expected = get_output::<Cpu, T>(&input, cfg.bm, cfg.bk, cfg.bn, aligned_n, None);
+ check_against_expected::<T>(&input, &expected, cfg.bm, cfg.bk, cfg.bn, aligned_n);
}
-#[uzu_test]
-fn test_bf16_gs128_4bit_mlx() {
- test_basic::<bf16>(128, 4, false, true);
-}
+// No CPU Hadamard reference — verify all three tile shapes agree instead.
+fn test_hadamard_consistency<T: ArrayElement + Float + Debug + Display>(
+ group_size: u32,
+ bits: u32,
+ use_zero_points: bool,
+ use_mlx_quant: bool,
+) {
+ // m=n=64 satisfies both BN=32 and BN=64 alignment; group_size >= 64 for BK=64.
+ let m = 64usize;
+ let k = group_size as usize * 2;
+ let n = 64usize;
-// -- 8-bit, mlx quant ----------------------------------------------------------
+ let input = make_input_basic::<T>(m, k, n, group_size, bits, use_zero_points, use_mlx_quant);
-#[uzu_test]
-fn test_f32_gs32_8bit_mlx() {
- test_basic::<f32>(32, 8, false, true);
-}
+ // ±1 factors: non-trivial Hadamard, bounded output across all dtypes.
+ let factors: Vec<i32> = (0..n).map(|i| if (i * 13 + 7) % 2 == 0 { 1 } else { -1 }).collect();
-#[uzu_test]
-fn test_f32_gs64_8bit_mlx() {
- test_basic::<f32>(64, 8, false, true);
-}
+ // Hadamard amplifies by sqrt(32) ≈ 5.65; tolerance widened accordingly.
+ let (rel_tol, abs_tol): (f64, f64) = match T::data_type() {
+ DataType::BF16 => (0.1, 1.0),
+ DataType::F16 => (0.05, 1.0),
+ _ => (0.02, 0.5),
+ };
-#[uzu_test]
-fn test_f32_gs128_8bit_mlx() {
- test_basic::<f32>(128, 8, false, true);
+ for_each_non_cpu_backend!(|B| {
+ let shapes: [(u32, u32, u32, &str); 3] = [
+ (32, 32, 32, "32x32"),
+ (64, 32, 64, "wide"),
+ (64, 64, 64, "64x64"),
+ ];
+
+ let outputs: Vec<(&str, Vec<T>)> = shapes
+ .iter()
+ .map(|&(bm, bk, bn, name)| {
+ let out = get_output::<B, T>(&input, bm, bk, bn, true, Some(&factors));
+ (name, out)
+ })
+ .collect();
+
+ // 32x32 is the reference — its Hadamard path is byte-identical to the original.
+ let (_, ref reference) = outputs[0];
+
+ for (name, output) in outputs.iter().skip(1) {
+ let mut errors = 0;
+ for (i, (&r, &o)) in reference.iter().zip(output.iter()).enumerate() {
+ let r_f32 = r.to_f32().unwrap();
+ let o_f32 = o.to_f32().unwrap();
+ if !check_tolerance(r_f32, o_f32, rel_tol, abs_tol) {
+ if errors < 5 {
+ eprintln!(
+ " idx={} 32x32={} {}={} diff={}",
+ i,
+ r_f32,
+ name,
+ o_f32,
+ (r_f32 - o_f32).abs()
+ );
+ }
+ errors += 1;
+ }
+ }
+ assert_eq!(
+ errors,
+ 0,
+ "Hadamard cross-shape mismatch: backend={}, {} vs 32x32 reference, gs={}, bits={}, zp={}, mlx={}: {} mismatches",
+ std::any::type_name::<B>(),
+ name,
+ group_size,
+ bits,
+ use_zero_points,
+ use_mlx_quant,
+ errors,
+ );
+ }
+ });
}
-#[uzu_test]
-fn test_f16_gs32_8bit_mlx() {
- test_basic::<f16>(32, 8, false, true);
-}
+// ---- Test-function macros -----------------------------------------------
-#[uzu_test]
-fn test_f16_gs64_8bit_mlx() {
- test_basic::<f16>(64, 8, false, true);
+macro_rules! basic_tests {
+ ($cfg:expr; $(
+ $(#[$attr:meta])*
+ fn $name:ident : $t:ty, $gs:literal, $bits:literal, $zp:literal, $mlx:literal;
+ )*) => {
+ $(
+ $(#[$attr])*
+ #[uzu_test]
+ fn $name() {
+ test_basic::<$t>($cfg, $gs, $bits, $zp, $mlx);
+ }
+ )*
+ };
}
-#[uzu_test]
-fn test_f16_gs128_8bit_mlx() {
- test_basic::<f16>(128, 8, false, true);
+macro_rules! edge_tests {
+ ($cfg:expr; $(
+ $(#[$attr:meta])*
+ fn $name:ident : $t:ty, $gs:literal, $bits:literal, $zp:literal, $mlx:literal;
+ )*) => {
+ $(
+ $(#[$attr])*
+ #[uzu_test]
+ fn $name() {
+ test_edge::<$t>($cfg, $gs, $bits, $zp, $mlx);
+ }
+ )*
+ };
}
-#[uzu_test]
-fn test_bf16_gs32_8bit_mlx() {
- test_basic::<bf16>(32, 8, false, true);
+macro_rules! hadamard_tests {
+ ($(
+ $(#[$attr:meta])*
+ fn $name:ident : $t:ty, $gs:literal, $bits:literal, $zp:literal, $mlx:literal;
+ )*) => {
+ $(
+ $(#[$attr])*
+ #[uzu_test]
+ fn $name() {
+ test_hadamard_consistency::<$t>($gs, $bits, $zp, $mlx);
+ }
+ )*
+ };
}
-#[uzu_test]
-fn test_bf16_gs64_8bit_mlx() {
- test_basic::<bf16>(64, 8, false, true);
-}
+// ---- tile_32x32 — the original base shape, full dtype matrix -------------
+
+mod tile_32x32 {
+ use super::*;
+
+ basic_tests!(&CFG_32X32;
+ // 4-bit, zero points
+ fn test_f32_gs32_4bit_zp: f32, 32, 4, true, false;
+ fn test_f32_gs64_4bit_zp: f32, 64, 4, true, false;
+ fn test_f32_gs128_4bit_zp: f32, 128, 4, true, false;
+ fn test_f16_gs32_4bit_zp: f16, 32, 4, true, false;
+ fn test_f16_gs64_4bit_zp: f16, 64, 4, true, false;
+ fn test_f16_gs128_4bit_zp: f16, 128, 4, true, false;
+ fn test_bf16_gs32_4bit_zp: bf16, 32, 4, true, false;
+ #[ignore] fn test_bf16_gs64_4bit_zp: bf16, 64, 4, true, false;
+ #[ignore] fn test_bf16_gs128_4bit_zp: bf16, 128, 4, true, false;
+ // 8-bit, zero points
+ fn test_f32_gs32_8bit_zp: f32, 32, 8, true, false;
+ fn test_f32_gs64_8bit_zp: f32, 64, 8, true, false;
+ fn test_f32_gs128_8bit_zp: f32, 128, 8, true, false;
+ fn test_f16_gs32_8bit_zp: f16, 32, 8, true, false;
+ fn test_f16_gs64_8bit_zp: f16, 64, 8, true, false;
+ fn test_f16_gs128_8bit_zp: f16, 128, 8, true, false;
+ fn test_bf16_gs32_8bit_zp: bf16, 32, 8, true, false;
+ fn test_bf16_gs64_8bit_zp: bf16, 64, 8, true, false;
+ fn test_bf16_gs128_8bit_zp: bf16,128, 8, true, false;
+ // 4-bit, mlx quant
+ fn test_f32_gs32_4bit_mlx: f32, 32, 4, false, true;
+ fn test_f32_gs64_4bit_mlx: f32, 64, 4, false, true;
+ fn test_f32_gs128_4bit_mlx: f32, 128, 4, false, true;
+ fn test_f16_gs32_4bit_mlx: f16, 32, 4, false, true;
+ fn test_f16_gs64_4bit_mlx: f16, 64, 4, false, true;
+ fn test_f16_gs128_4bit_mlx: f16, 128, 4, false, true;
+ fn test_bf16_gs32_4bit_mlx: bf16, 32, 4, false, true;
+ fn test_bf16_gs64_4bit_mlx: bf16, 64, 4, false, true;
+ fn test_bf16_gs128_4bit_mlx: bf16,128, 4, false, true;
+ // 8-bit, mlx quant
+ fn test_f32_gs32_8bit_mlx: f32, 32, 8, false, true;
+ fn test_f32_gs64_8bit_mlx: f32, 64, 8, false, true;
+ fn test_f32_gs128_8bit_mlx: f32, 128, 8, false, true;
+ fn test_f16_gs32_8bit_mlx: f16, 32, 8, false, true;
+ fn test_f16_gs64_8bit_mlx: f16, 64, 8, false, true;
+ fn test_f16_gs128_8bit_mlx: f16, 128, 8, false, true;
+ fn test_bf16_gs32_8bit_mlx: bf16, 32, 8, false, true;
+ fn test_bf16_gs64_8bit_mlx: bf16, 64, 8, false, true;
+ fn test_bf16_gs128_8bit_mlx: bf16,128, 8, false, true;
+ );
-#[uzu_test]
-fn test_bf16_gs128_8bit_mlx() {
- test_basic::<bf16>(128, 8, false, true);
+ edge_tests!(&CFG_32X32;
+ fn test_edge_f32_4bit_zp: f32, 32, 4, true, false;
+ fn test_edge_f32_8bit_zp: f32, 32, 8, true, false;
+ fn test_edge_f16_4bit_zp: f16, 32, 4, true, false;
+ fn test_edge_f32_4bit_mlx: f32, 32, 4, false, true;
+ fn test_edge_f32_8bit_mlx: f32, 32, 8, false, true;
+ fn test_edge_f16_4bit_mlx: f16, 32, 4, false, true;
+ );
}
-// -- Edge cases ----------------------------------------------------------------
+// ---- tile_wide — BM=64, BK=32, BN=64, bf16 only --------------------------
+
+mod tile_wide {
+ use super::*;
+
+ basic_tests!(&CFG_WIDE;
+ // 4-bit, zero points
+ fn test_bf16_gs32_4bit_zp: bf16, 32, 4, true, false;
+ #[ignore] fn test_bf16_gs64_4bit_zp: bf16, 64, 4, true, false;
+ #[ignore] fn test_bf16_gs128_4bit_zp: bf16, 128, 4, true, false;
+ // 8-bit, zero points
+ fn test_bf16_gs32_8bit_zp: bf16, 32, 8, true, false;
+ fn test_bf16_gs64_8bit_zp: bf16, 64, 8, true, false;
+ fn test_bf16_gs128_8bit_zp: bf16, 128, 8, true, false;
+ // 4-bit, mlx quant
+ fn test_bf16_gs32_4bit_mlx: bf16, 32, 4, false, true;
+ fn test_bf16_gs64_4bit_mlx: bf16, 64, 4, false, true;
+ fn test_bf16_gs128_4bit_mlx: bf16, 128, 4, false, true;
+ // 8-bit, mlx quant
+ fn test_bf16_gs32_8bit_mlx: bf16, 32, 8, false, true;
+ fn test_bf16_gs64_8bit_mlx: bf16, 64, 8, false, true;
+ fn test_bf16_gs128_8bit_mlx: bf16, 128, 8, false, true;
+ );
-#[uzu_test]
-fn test_edge_f32_4bit_zp() {
- test_edge::<f32>(32, 4, true, false);
+ edge_tests!(&CFG_WIDE;
+ fn test_edge_bf16_4bit_zp: bf16, 32, 4, true, false;
+ fn test_edge_bf16_8bit_zp: bf16, 32, 8, true, false;
+ fn test_edge_bf16_4bit_mlx: bf16, 32, 4, false, true;
+ fn test_edge_bf16_8bit_mlx: bf16, 32, 8, false, true;
+ );
}
-#[uzu_test]
-fn test_edge_f32_8bit_zp() {
- test_edge::<f32>(32, 8, true, false);
-}
+// ---- tile_64x64 — BM=BK=BN=64, bf16 only, group_size >= 64 --------------
+
+mod tile_64x64 {
+ use super::*;
+
+ basic_tests!(&CFG_64X64;
+ // 4-bit, zero points
+ #[ignore] fn test_bf16_gs64_4bit_zp: bf16, 64, 4, true, false;
+ #[ignore] fn test_bf16_gs128_4bit_zp: bf16, 128, 4, true, false;
+ // 8-bit, zero points
+ fn test_bf16_gs64_8bit_zp: bf16, 64, 8, true, false;
+ fn test_bf16_gs128_8bit_zp: bf16, 128, 8, true, false;
+ // 4-bit, mlx quant
+ fn test_bf16_gs64_4bit_mlx: bf16, 64, 4, false, true;
+ fn test_bf16_gs128_4bit_mlx: bf16, 128, 4, false, true;
+ // 8-bit, mlx quant
+ fn test_bf16_gs64_8bit_mlx: bf16, 64, 8, false, true;
+ fn test_bf16_gs128_8bit_mlx: bf16, 128, 8, false, true;
+ );
-#[uzu_test]
-fn test_edge_f16_4bit_zp() {
- test_edge::<f16>(32, 4, true, false);
+ edge_tests!(&CFG_64X64;
+ fn test_edge_bf16_4bit_zp: bf16, 64, 4, true, false;
+ fn test_edge_bf16_8bit_zp: bf16, 64, 8, true, false;
+ fn test_edge_bf16_4bit_mlx: bf16, 64, 4, false, true;
+ fn test_edge_bf16_8bit_mlx: bf16, 64, 8, false, true;
+ );
}
-#[uzu_test]
-fn test_edge_f32_4bit_mlx() {
- test_edge::<f32>(32, 4, false, true);
-}
+// ---- hadamard — cross-shape consistency of the generalized HT loop -------
-#[uzu_test]
-fn test_edge_f32_8bit_mlx() {
- test_edge::<f32>(32, 8, false, true);
-}
+mod hadamard {
+ use super::*;
-#[uzu_test]
-fn test_edge_f16_4bit_mlx() {
- test_edge::<f16>(32, 4, false, true);
+ // f32 at (64,64,64) overflows threadgroup memory (34816 > 32768 bytes).
+ hadamard_tests!(
+ fn test_bf16_gs64_4bit_mlx: bf16, 64, 4, false, true;
+ fn test_bf16_gs128_4bit_mlx: bf16, 128, 4, false, true;
+ fn test_bf16_gs64_8bit_mlx: bf16, 64, 8, false, true;
+ fn test_bf16_gs128_8bit_mlx: bf16, 128, 8, false, true;
+ fn test_f16_gs64_4bit_mlx: f16, 64, 4, false, true;
+ fn test_f16_gs128_4bit_mlx: f16, 128, 4, false, true;
+ );
}
diff --git a/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_wide_test.rs b/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_wide_test.rs
deleted file mode 100644
index 59a74348b..000000000
--- a/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_wide_test.rs
+++ /dev/null
@@ -1,392 +0,0 @@
-use std::fmt::{Debug, Display};
-
-use half::bf16;
-use num_traits::Float;
-use uzu::{
- ArrayElement, DataType,
- backends::{
- common::{Backend, Buffer, Context, Encoder, Kernels, kernel::QuantizedMatmulQmmTransposedWideKernel},
- cpu::Cpu,
- },
-};
-
-use super::{Input, check_tolerance, pack_weights_u32, pack_zero_points};
-use crate::{common::helpers::alloc_buffer_with_data, uzu_test};
-
-fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
- let context = B::Context::new().expect("Failed to create Context");
- let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmmTransposedWideKernel::new(
- &context,
- T::data_type(),
- input.group_size,
- input.bits,
- input.use_zero_points,
- input.use_mlx_quant,
- )
- .expect("Failed to create QuantizedMatmulQmmTransposedWideKernel");
-
- let w_buf = alloc_buffer_with_data::<B, u32>(&context, &input.w_packed);
- let scales_buf = alloc_buffer_with_data::<B, T>(&context, &input.scales);
-
- let zp_buf = input.zero_points.as_ref().map(|zp| alloc_buffer_with_data::<B, u8>(&context, zp));
- let bias_buf = input.biases.as_ref().map(|b| alloc_buffer_with_data::<B, T>(&context, b));
-
- let x_buf = alloc_buffer_with_data::<B, T>(&context, &input.x);
- let output_size = (input.m as usize) * (input.n as usize) * T::data_type().size_in_bytes();
- let mut y_buf = context.create_buffer(output_size).expect("Failed to create buffer");
-
- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
- kernel.encode(
- &w_buf,
- &scales_buf,
- zp_buf.as_ref(),
- bias_buf.as_ref(),
- &x_buf,
- &mut y_buf,
- input.k,
- input.n,
- input.m,
- &mut encoder,
- );
-
- encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
-
- let y_ptr = y_buf.cpu_ptr().as_ptr() as *const T;
- let y_len = (input.m as usize) * (input.n as usize);
- unsafe { std::slice::from_raw_parts(y_ptr, y_len) }.to_vec()
-}
-
-/// Create test data for transposed wide qmm: x[m×k] * w_transposed[n×k] = y[m×n]
-/// Weights are in transposed [n × k] layout.
-fn get_test_data_basic<T: ArrayElement + Float>(
- m: usize,
- k: usize,
- n: usize,
- group_size: u32,
- bits: u32,
- use_zero_points: bool,
- use_mlx_quant: bool,
-) -> (Input<T>, Vec<T>) {
- let num_groups_k = (k + group_size as usize - 1) / group_size as usize;
-
- // Generate quantized weights [n × k]
- let max_val = if bits == 4 {
- 15u8
- } else {
- 255u8
- };
- let mut weights_raw: Vec<u8> = Vec::with_capacity(n * k);
- for j in 0..n {
- for l in 0..k {
- weights_raw.push(((j * 3 + l * 7 + 1) % (max_val as usize + 1)) as u8);
- }
- }
- let w_packed = pack_weights_u32(&weights_raw, bits);
-
- // Scales: [n × num_groups_k]
- let mut scales_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);
- for j in 0..n {
- for g in 0..num_groups_k {
- scales_f32.push(0.5 + 0.1 * ((j + g) % 5) as f32);
- }
- }
- let scales: Vec<T> = scales_f32.iter().map(|&v| T::from(v).unwrap()).collect();
-
- // Zero points or biases
- let zp_stride = if bits == 4 {
- (num_groups_k + 1) / 2
- } else {
- num_groups_k
- };
-
- let (zero_points, biases) = if use_zero_points {
- let mut zp_raw: Vec<u8> = Vec::with_capacity(n * num_groups_k);
- for j in 0..n {
- for g in 0..num_groups_k {
- let zp_val = ((j * 2 + g * 3) % (max_val as usize + 1)) as u8;
- zp_raw.push(zp_val);
- }
- }
- // Pack zero points per row
- let mut zp_packed: Vec<u8> = Vec::with_capacity(n * zp_stride);
- for j in 0..n {
- let row = &zp_raw[j * num_groups_k..(j + 1) * num_groups_k];
- let packed_row = pack_zero_points(row, bits);
- let mut padded = packed_row;
- padded.resize(zp_stride, 0);
- zp_packed.extend_from_slice(&padded);
- }
- (Some(zp_packed), None)
- } else if use_mlx_quant {
- let mut biases_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);
- for j in 0..n {
- for g in 0..num_groups_k {
- biases_f32.push(0.01 * ((j + g * 2) % 7) as f32);
- }
- }
- let biases: Vec<T> = biases_f32.iter().map(|&v| T::from(v).unwrap()).collect();
- (None, Some(biases))
- } else {
- unreachable!("Must use either zero_points or mlx_quant");
- };
-
- // X: [m × k]
- let mut x_f32: Vec<f32> = Vec::with_capacity(m * k);
- for i in 0..m {
- for l in 0..k {
- x_f32.push(0.1 * f32::sin((i * k + l) as f32 * 0.05) + 0.5);
- }
- }
- let x: Vec<T> = x_f32.iter().map(|&v| T::from(v).unwrap()).collect();
-
- let input = Input {
- w_packed,
- scales,
- zero_points,
- biases,
- x,
- k: k as u32,
- n: n as u32,
- m: m as u32,
- group_size,
- bits,
- use_zero_points,
- use_mlx_quant,
- };
-
- let expected = get_output::<Cpu, T>(&input);
- (input, expected)
-}
-
-/// Small edge-case test data with known values
-fn get_test_data_edge<T: ArrayElement + Float>(
- group_size: u32,
- bits: u32,
- use_zero_points: bool,
- use_mlx_quant: bool,
-) -> (Input<T>, Vec<T>) {
- let m = 2usize;
- let k = group_size as usize;
- let n = 64usize;
- let num_groups_k = 1;
-
- // Simple weights [n × k]: all 1s
- let weights_raw: Vec<u8> = vec![1u8; n * k];
- let w_packed = pack_weights_u32(&weights_raw, bits);
-
- // Unit scales
- let scales: Vec<T> = vec![T::one(); n * num_groups_k];
-
- let zp_stride = if bits == 4 {
- (num_groups_k + 1) / 2
- } else {
- num_groups_k
- };
-
- let (zero_points, biases) = if use_zero_points {
- // Zero points all 0 -> bias = 0
- let zp_packed = vec![0u8; n * zp_stride];
- (Some(zp_packed), None)
- } else if use_mlx_quant {
- let biases: Vec<T> = vec![T::zero(); n * num_groups_k];
- (None, Some(biases))
- } else {
- unreachable!("Must use either zero_points or mlx_quant");
- };
-
- // X: simple increasing values
- let x_f32: Vec<f32> = (0..m * k).map(|i| (i + 1) as f32 * 0.1).collect();
- let x: Vec<T> = x_f32.iter().map(|&v| T::from(v).unwrap()).collect();
-
- let input = Input {
- w_packed,
- scales,
- zero_points,
- biases,
- x,
- k: k as u32,
- n: n as u32,
- m: m as u32,
- group_size,
- bits,
- use_zero_points,
- use_mlx_quant,
- };
-
- let expected = get_output::<Cpu, T>(&input);
- (input, expected)
-}
-
-fn test_internal<T: ArrayElement + Float + Debug + Display>(
- input: &Input<T>,
- expected: &[T],
-) {
- let (rel_tol, abs_tol): (f64, f64) = match T::data_type() {
- DataType::BF16 => (0.05, 0.5),
- DataType::F16 => (0.02, 0.5),
- _ => (0.01, 0.1),
- };
-
- for_each_non_cpu_backend!(|B| {
- let output = get_output::<B, T>(input);
- assert_eq!(expected.len(), output.len(), "Output length mismatch");
-
- let mut errors = 0;
- for (i, (&exp, &got)) in expected.iter().zip(output.iter()).enumerate() {
- let exp_f32 = exp.to_f32().unwrap();
- let got_f32 = got.to_f32().unwrap();
- if !check_tolerance(exp_f32, got_f32, rel_tol, abs_tol) {
- if errors < 5 {
- eprintln!(" idx={} expected={} got={} diff={}", i, exp_f32, got_f32, (exp_f32 - got_f32).abs());
- }
- errors += 1;
- }
- }
- assert_eq!(
- errors,
- 0,
- "QMM transposed wide kernel: backend={}, m={}, k={}, n={}, gs={}, bits={}, zp={}, mlx={}: {} mismatches",
- std::any::type_name::<B>(),
- input.m,
- input.k,
- input.n,
- input.group_size,
- input.bits,
- input.use_zero_points,
- input.use_mlx_quant,
- errors,
- );
- });
-}
-
-fn get_test_dims(
- group_size: u32,
- bits: u32,
-) -> Vec<(usize, usize, usize)> {
- let gs = group_size as usize;
- // Wide kernel: BM=64, BK=32, BN=64.
- // K must be >= group_size (transposed grouping along K).
- // N must be a multiple of 64 (BN tile size).
- // bf16 8-bit: keep dims small to avoid catastrophic cancellation.
- if bits == 8 {
- vec![(2, gs, 64), (4, gs, 64)]
- } else {
- vec![(2, gs * 2, 64), (4, gs * 2, 128), (8, gs, 128)]
- }
-}
-
-fn test_basic(
- group_size: u32,
- bits: u32,
- use_zero_points: bool,
- use_mlx_quant: bool,
-) {
- let dims = get_test_dims(group_size, bits);
-
- for (m, k, n) in dims {
- let (input, expected) = get_test_data_basic::<bf16>(m, k, n, group_size, bits, use_zero_points, use_mlx_quant);
- test_internal::<bf16>(&input, &expected);
- }
-}
-
-fn test_edge(
- group_size: u32,
- bits: u32,
- use_zero_points: bool,
- use_mlx_quant: bool,
-) {
- let (input, expected) = get_test_data_edge::<bf16>(group_size, bits, use_zero_points, use_mlx_quant);
- test_internal::<bf16>(&input, &expected);
-}
-
-// -- 4-bit, zero points -------------------------------------------------------
-
-#[uzu_test]
-fn test_bf16_gs32_4bit_zp() {
- test_basic(32, 4, true, false);
-}
-
-#[uzu_test]
-#[ignore]
-fn test_bf16_gs64_4bit_zp() {
- test_basic(64, 4, true, false);
-}
-
-#[uzu_test]
-#[ignore]
-fn test_bf16_gs128_4bit_zp() {
- test_basic(128, 4, true, false);
-}
-
-// -- 8-bit, zero points -------------------------------------------------------
-
-#[uzu_test]
-fn test_bf16_gs32_8bit_zp() {
- test_basic(32, 8, true, false);
-}
-
-#[uzu_test]
-fn test_bf16_gs64_8bit_zp() {
- test_basic(64, 8, true, false);
-}
-
-#[uzu_test]
-fn test_bf16_gs128_8bit_zp() {
- test_basic(128, 8, true, false);
-}
-
-// -- 4-bit, mlx quant ----------------------------------------------------------
-
-#[uzu_test]
-fn test_bf16_gs32_4bit_mlx() {
- test_basic(32, 4, false, true);
-}
-
-#[uzu_test]
-fn test_bf16_gs64_4bit_mlx() {
- test_basic(64, 4, false, true);
-}
-
-#[uzu_test]
-fn test_bf16_gs128_4bit_mlx() {
- test_basic(128, 4, false, true);
-}
-
-// -- 8-bit, mlx quant ----------------------------------------------------------
-
-#[uzu_test]
-fn test_bf16_gs32_8bit_mlx() {
- test_basic(32, 8, false, true);
-}
-
-#[uzu_test]
-fn test_bf16_gs64_8bit_mlx() {
- test_basic(64, 8, false, true);
-}
-
-#[uzu_test]
-fn test_bf16_gs128_8bit_mlx() {
- test_basic(128, 8, false, true);
-}
-
-// -- Edge cases ----------------------------------------------------------------
-
-#[uzu_test]
-fn test_edge_bf16_4bit_zp() {
- test_edge(32, 4, true, false);
-}
-
-#[uzu_test]
-fn test_edge_bf16_8bit_zp() {
- test_edge(32, 8, true, false);
-}
-
-#[uzu_test]
-fn test_edge_bf16_4bit_mlx() {
- test_edge(32, 4, false, true);
-}
-
-#[uzu_test]
-fn test_edge_bf16_8bit_mlx() {
- test_edge(32, 8, false, true);
-}