A high-performance inference engine for AI models
Faster quantized matmul for small suffix_len (#340)
CCYehdf26ead54a84
diff --git a/crates/uzu/src/backends/common/kernel/quant_matmul.rs b/crates/uzu/src/backends/common/kernel/quant_matmul.rsindex 207f645db..54236aec4 100644--- a/crates/uzu/src/backends/common/kernel/quant_matmul.rs+++ b/crates/uzu/src/backends/common/kernel/quant_matmul.rs@@ -49,7 +49,7 @@ pub struct QuantizedMatmulArguments<'a, B: Backend> {pub struct QuantizedMatmulKernelEncodable<B: Backend> {matrix_vector: MatrixVectorKernel<B>,- matrix_matrix: <B::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel,+ matrix_matrix: MatrixMatrixKernel<B>,input_dim: usize,output_dim: usize,quantization_type: QuantizedMatmulType,@@ -60,6 +60,37 @@ enum MatrixVectorKernel<B: Backend> {QmvFast(<B::Kernels as Kernels>::QuantizedMatmulQmvFastKernel),}+type QmmKernel<B> = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel;++/// Big tile alone, or paired with a BM=8 small tile for `batch_dim < 48`.+enum MatrixMatrixKernel<B: Backend> {+ Big(QmmKernel<B>),+ BigAndSmall {+ big: QmmKernel<B>,+ small: QmmKernel<B>,+ },+}++impl<B: Backend> MatrixMatrixKernel<B> {+ /// `None` means fall back to `matrix_vector` (small `batch_dim`, no small tile available).+ fn pick(+ &self,+ batch_dim: usize,+ ) -> Option<&QmmKernel<B>> {+ match self {+ Self::Big(big) => (batch_dim >= 32).then_some(big),+ Self::BigAndSmall {+ big,+ small,+ } => Some(if batch_dim < 48 {+ small+ } else {+ big+ }),+ }+ }+}+impl<B: Backend> QuantizedMatmulKernelEncodable<B> {pub fn new(context: &B::Context,@@ -116,6 +147,7 @@ impl<B: Backend> QuantizedMatmulKernelEncodable<B> {// 64-tile (BM=BN=64): bf16 + output_dim%64==0; BK=64 if group_size≥64, BK=32 otherwise.// 32-tile fallback otherwise.let aligned_n_64 = configuration.output_dim % 64 == 0;+ let aligned_n_32 = configuration.output_dim % 32 == 0;let is_bf16 = configuration.data_type == DataType::BF16;let can_use_64_tile = aligned_n_64 && is_bf16;@@ -127,7 +159,7 @@ impl<B: Backend> QuantizedMatmulKernelEncodable<B> {(32u32, 32u32, 32u32, configuration.output_dim % 32 == 0)};- let matrix_matrix = <B::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel::new(+ let big = <B::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel::new(context,configuration.data_type,group_size,@@ -135,6 +167,8 @@ impl<B: Backend> QuantizedMatmulKernelEncodable<B> {bm,bk,bn,+ 2u32, // WM+ 2u32, // WNuse_zero_points,use_mlx_quant,configuration.use_hadamard,@@ -142,6 +176,34 @@ impl<B: Backend> QuantizedMatmulKernelEncodable<B> {).map_err(QuantizedMatmulError::BackendError)?;+ // Small-tile kernel: BM=8, BK=32, BN=32, WM=1, WN=1. Requires N%32==0+ // and supported bits/group_size. Works for any supported T via the unified kernel.+ // Used for batch_dim in [8, 48) at encode time.+ let matrix_matrix = if aligned_n_32 {+ let small = <B::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel::new(+ context,+ configuration.data_type,+ group_size,+ bits,+ 8u32, // BM+ 32u32, // BK+ 32u32, // BN+ 1u32, // WM+ 1u32, // WN+ use_zero_points,+ use_mlx_quant,+ false, // use_hadamard: BM=8 tile does not support hadamard+ true, // aligned_n: BN=32, always aligned for the small tile+ )+ .map_err(QuantizedMatmulError::BackendError)?;+ MatrixMatrixKernel::BigAndSmall {+ big,+ small,+ }+ } else {+ MatrixMatrixKernel::Big(big)+ };+Ok(Self {matrix_vector,matrix_matrix,@@ -179,13 +241,18 @@ impl<B: Backend> QuantizedMatmulKernelEncodable<B> {};}- if arguments.batch_dim < 32 || self.output_dim == 1 {- match &self.matrix_vector {- MatrixVectorKernel::Qmv(k) => encode_kernel!(k,),- MatrixVectorKernel::QmvFast(k) => encode_kernel!(k, arguments.hadamard_factors),+ if arguments.batch_dim >= 8 && self.output_dim > 1 {+ if let Some(kernel) = self.matrix_matrix.pick(arguments.batch_dim) {+ encode_kernel!(kernel, arguments.hadamard_factors);+ return Ok(());}- } else {- encode_kernel!(&self.matrix_matrix, arguments.hadamard_factors);+ }++ // Matrix-vector fallback: small batch_dim, scalar output, or Big-tile-only+ // configs that don't cover this batch_dim.+ match &self.matrix_vector {+ MatrixVectorKernel::Qmv(k) => encode_kernel!(k,),+ MatrixVectorKernel::QmvFast(k) => encode_kernel!(k, arguments.hadamard_factors),}Ok(())diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rsindex a77a447cc..256761b26 100644--- a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rs+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rs@@ -88,12 +88,18 @@ pub fn qmm_transposed<T: ArrayElement + Float>(#[variants(T, f32, f16, bf16)]#[variants(GROUP_SIZE, 32, 64, 128)]#[variants(BITS, 4, 8)]-#[variants(BM, 32, 64)]+#[variants(BM, 8, 32, 64)]#[variants(BK, 32, 64)]#[variants(BN, 32, 64)]-#[constraint(BK <= BM)]+#[variants(WM, 1, 2)]+#[variants(WN, 1, 2)]+#[constraint(+ (BM == 8 && BK == 32 && BN == 32 && WM == 1 && WN == 1) ||+ (BM == 32 && BK == 32 && BN == 32 && WM == 2 && WN == 2) ||+ (BM == 32 && BK == 64 && BN == 32 && WM == 2 && WN == 2) ||+ (BM == 64 && BK == 32 && BN == 64 && WM == 2 && WN == 2) ||+ (BM == 64 && BK == 64 && BN == 64 && WM == 2 && WN == 2))]#[constraint(BK <= GROUP_SIZE)]-#[constraint(BM == BN)]#[constraint(T != "f32" || BK < 64)]pub fn quantized_matmul_qmm_transposed<T: ArrayElement + Float,@@ -102,6 +108,8 @@ pub fn quantized_matmul_qmm_transposed<const BM: u32,const BK: u32,const BN: u32,+ const WM: u32,+ const WN: u32,>(weights: *const u32,scales: *const T,@@ -118,7 +126,7 @@ pub fn quantized_matmul_qmm_transposed<#[specialize] use_hadamard: bool,#[specialize] aligned_n: bool,) {- let _ = (BM, BK, BN); // tile params unused in tile-agnostic CPU reference+ let _ = (BM, BK, BN, WM, WN); // tile params unused in tile-agnostic CPU referenceif use_hadamard {unimplemented!("not supported yet");}diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metalindex 2b00dac2d..fcf438167 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal@@ -4,17 +4,23 @@#include "quant_matmul.h"// Transposed QMM kernel. BM = batch tile rows, BK = K-axis tile, BN = output-// tile cols.-template <typename T, uint GROUP_SIZE, uint BITS, uint BM, uint BK, uint BN>+// tile cols. WM, WN = simdgroup tile counts along M and N axes.+template <typename T, uint GROUP_SIZE, uint BITS, uint BM, uint BK, uint BN, uint WM, uint WN>VARIANTS(T, float, half, bfloat)VARIANTS(GROUP_SIZE, 32, 64, 128)VARIANTS(BITS, 4, 8)-VARIANTS(BM, 32, 64)+VARIANTS(BM, 8, 32, 64)VARIANTS(BK, 32, 64)VARIANTS(BN, 32, 64)-CONSTRAINT(BK <= BM)+VARIANTS(WM, 1, 2)+VARIANTS(WN, 1, 2)+CONSTRAINT(+ (BM == 8 && BK == 32 && BN == 32 && WM == 1 && WN == 1) ||+ (BM == 32 && BK == 32 && BN == 32 && WM == 2 && WN == 2) ||+ (BM == 32 && BK == 64 && BN == 32 && WM == 2 && WN == 2) ||+ (BM == 64 && BK == 32 && BN == 64 && WM == 2 && WN == 2) ||+ (BM == 64 && BK == 64 && BN == 64 && WM == 2 && WN == 2))CONSTRAINT(BK <= GROUP_SIZE)-CONSTRAINT(BM == BN)CONSTRAINT(T != "float" || BK < 64)PUBLIC KERNEL(QuantizedMatmulQmmTransposed)(const device uint32_t* weights,@@ -36,11 +42,11 @@ PUBLIC KERNEL(QuantizedMatmulQmmTransposed)(const uint out_block_idx GROUPS(out_vec_size.div_ceil(BN)),const uint batch_block_idx GROUPS(batch_size.div_ceil(BM)),const uint simd_lane THREADS(32),- const uint simd_group THREADS(4)+ const uint simd_group THREADS(WM * WN)) {if (use_mlx_quant) {if (aligned_n) {- qmm_transposed_impl<T, GROUP_SIZE, BITS, true, BM, BK, BN, true>(+ qmm_transposed_impl<T, GROUP_SIZE, BITS, true, BM, BK, BN, true, WM, WN>(weights,scales,zero_points,@@ -58,7 +64,7 @@ PUBLIC KERNEL(QuantizedMatmulQmmTransposed)(simd_lane);} else {- qmm_transposed_impl<T, GROUP_SIZE, BITS, false, BM, BK, BN, true>(+ qmm_transposed_impl<T, GROUP_SIZE, BITS, false, BM, BK, BN, true, WM, WN>(weights,scales,zero_points,@@ -78,7 +84,7 @@ PUBLIC KERNEL(QuantizedMatmulQmmTransposed)(}} else {if (aligned_n) {- qmm_transposed_impl<T, GROUP_SIZE, BITS, true, BM, BK, BN, false>(+ qmm_transposed_impl<T, GROUP_SIZE, BITS, true, BM, BK, BN, false, WM, WN>(weights,scales,zero_points,@@ -96,7 +102,17 @@ PUBLIC KERNEL(QuantizedMatmulQmmTransposed)(simd_lane);} else {- qmm_transposed_impl<T, GROUP_SIZE, BITS, false, BM, BK, BN, false>(+ qmm_transposed_impl<+ T,+ GROUP_SIZE,+ BITS,+ false,+ BM,+ BK,+ BN,+ false,+ WM,+ WN>(weights,scales,zero_points,@@ -120,7 +136,7 @@ PUBLIC KERNEL(QuantizedMatmulQmmTransposed)(threadgroup_barrier(mem_flags::mem_device);// Each lane covers BN/32 output columns (1 at BN=32, 2 at BN=64).- constexpr uint NUM_SIMD_GROUPS = 4;+ constexpr uint NUM_SIMD_GROUPS = WM * WN;constexpr uint ROWS_PER_SIMD_GROUP = BM / NUM_SIMD_GROUPS;constexpr uint COLS_PER_LANE = BN / 32;diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.h b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.hindex 801e26c18..5769293b7 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.h+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.h@@ -675,7 +675,9 @@ template <const int BM = 32,const int BK = 32,const int BN = 32,- const bool use_mlx_quant = false>+ const bool use_mlx_quant = false,+ const int WM = 2,+ const int WN = 2>void qmm_transposed_impl(const device uint32_t* weights,const device T* scales,@@ -695,9 +697,6 @@ void qmm_transposed_impl() {static_assert(BK >= 32, "BK should be larger than METAL_SIMD_SIZE");static_assert(BK % 32 == 0, "BK should be divisible by METAL_SIMD_SIZE");-- constexpr int WM = 2;- constexpr int WN = 2;constexpr int pack_factor = get_pack_factor<bits, 8>();constexpr int bytes_per_pack = get_bytes_per_pack<bits>();constexpr int BK_padded = (BK + 16 / sizeof(T));diff --git a/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rs b/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rsindex cf42d4161..587a5a12a 100644--- a/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rs+++ b/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rs@@ -161,6 +161,8 @@ fn get_output<B: Backend, T: ArrayElement + Float>(bm: u32,bk: u32,bn: u32,+ wm: u32,+ wn: u32,aligned_n: bool,hadamard_factors: Option<&[i32]>,) -> Vec<T> {@@ -174,6 +176,8 @@ fn get_output<B: Backend, T: ArrayElement + Float>(bm,bk,bn,+ wm,+ wn,input.use_zero_points,input.use_mlx_quant,use_hadamard,@@ -220,6 +224,8 @@ fn check_against_expected<T: ArrayElement + Float + Debug + Display>(bm: u32,bk: u32,bn: u32,+ wm: u32,+ wn: u32,aligned_n: bool,) {// GPU tiled accumulation rounds differently from CPU scalar reference.@@ -230,7 +236,7 @@ fn check_against_expected<T: ArrayElement + Float + Debug + Display>(};for_each_non_cpu_backend!(|B| {- let output = get_output::<B, T>(input, bm, bk, bn, aligned_n, None);+ let output = get_output::<B, T>(input, bm, bk, bn, wm, wn, aligned_n, None);assert_eq!(expected.len(), output.len(), "Output length mismatch");let mut errors = 0;@@ -268,6 +274,8 @@ struct TileCfg {bm: u32,bk: u32,bn: u32,+ wm: u32,+ wn: u32,}impl TileCfg {@@ -277,9 +285,10 @@ impl TileCfg {}}-const CFG_32X32: TileCfg = TileCfg { bm: 32, bk: 32, bn: 32 };-const CFG_WIDE: TileCfg = TileCfg { bm: 64, bk: 32, bn: 64 };-const CFG_64X64: TileCfg = TileCfg { bm: 64, bk: 64, bn: 64 };+const CFG_BM8: TileCfg = TileCfg { bm: 8, bk: 32, bn: 32, wm: 1, wn: 1 };+const CFG_32X32: TileCfg = TileCfg { bm: 32, bk: 32, bn: 32, wm: 2, wn: 2 };+const CFG_WIDE: TileCfg = TileCfg { bm: 64, bk: 32, bn: 64, wm: 2, wn: 2 };+const CFG_64X64: TileCfg = TileCfg { bm: 64, bk: 64, bn: 64, wm: 2, wn: 2 };fn pick_aligned_n(cfg: &TileCfg,@@ -312,8 +321,8 @@ fn test_basic<T: ArrayElement + Float + Debug + Display>(for (m, k, n) in dims {let input = make_input_basic::<T>(m, k, n, group_size, bits, use_zero_points, use_mlx_quant);let aligned_n = pick_aligned_n(cfg, n);- let expected = get_output::<Cpu, T>(&input, cfg.bm, cfg.bk, cfg.bn, aligned_n, None);- check_against_expected::<T>(&input, &expected, cfg.bm, cfg.bk, cfg.bn, aligned_n);+ let expected = get_output::<Cpu, T>(&input, cfg.bm, cfg.bk, cfg.bn, cfg.wm, cfg.wn, aligned_n, None);+ check_against_expected::<T>(&input, &expected, cfg.bm, cfg.bk, cfg.bn, cfg.wm, cfg.wn, aligned_n);}}@@ -328,8 +337,8 @@ fn test_edge<T: ArrayElement + Float + Debug + Display>(let n = cfg.bn as usize;let input = make_input_edge::<T>(n, group_size, bits, use_zero_points, use_mlx_quant);let aligned_n = pick_aligned_n(cfg, n);- let expected = get_output::<Cpu, T>(&input, cfg.bm, cfg.bk, cfg.bn, aligned_n, None);- check_against_expected::<T>(&input, &expected, cfg.bm, cfg.bk, cfg.bn, aligned_n);+ let expected = get_output::<Cpu, T>(&input, cfg.bm, cfg.bk, cfg.bn, cfg.wm, cfg.wn, aligned_n, None);+ check_against_expected::<T>(&input, &expected, cfg.bm, cfg.bk, cfg.bn, cfg.wm, cfg.wn, aligned_n);}// No CPU Hadamard reference — verify all three tile shapes agree instead.@@ -357,16 +366,16 @@ fn test_hadamard_consistency<T: ArrayElement + Float + Debug + Display>(};for_each_non_cpu_backend!(|B| {- let shapes: [(u32, u32, u32, &str); 3] = [- (32, 32, 32, "32x32"),- (64, 32, 64, "wide"),- (64, 64, 64, "64x64"),+ let shapes: [(u32, u32, u32, u32, u32, &str); 3] = [+ (32, 32, 32, 2, 2, "32x32"),+ (64, 32, 64, 2, 2, "wide"),+ (64, 64, 64, 2, 2, "64x64"),];let outputs: Vec<(&str, Vec<T>)> = shapes.iter()- .map(|&(bm, bk, bn, name)| {- let out = get_output::<B, T>(&input, bm, bk, bn, true, Some(&factors));+ .map(|&(bm, bk, bn, wm, wn, name)| {+ let out = get_output::<B, T>(&input, bm, bk, bn, wm, wn, true, Some(&factors));(name, out)}).collect();@@ -456,6 +465,28 @@ macro_rules! hadamard_tests {};}+mod tile_bm8 {+ use super::*;++ basic_tests!(&CFG_BM8;+ // bf16 4-bit ZP has known precision limits+ #[ignore] fn test_bf16_gs32_4bit_zp: bf16, 32, 4, true, false;+ #[ignore] fn test_bf16_gs64_4bit_zp: bf16, 64, 4, true, false;+ fn test_bf16_gs32_8bit_zp: bf16, 32, 8, true, false;+ fn test_bf16_gs64_8bit_zp: bf16, 64, 8, true, false;+ fn test_bf16_gs32_4bit_mlx: bf16, 32, 4, false, true;+ fn test_bf16_gs64_4bit_mlx: bf16, 64, 4, false, true;+ fn test_bf16_gs32_8bit_mlx: bf16, 32, 8, false, true;+ fn test_bf16_gs64_8bit_mlx: bf16, 64, 8, false, true;+ );++ edge_tests!(&CFG_BM8;+ fn test_edge_bf16_4bit_mlx: bf16, 32, 4, false, true;+ fn test_edge_bf16_8bit_mlx: bf16, 32, 8, false, true;+ fn test_edge_bf16_8bit_zp: bf16, 32, 8, true, false;+ );+}+// ---- tile_32x32 — the original base shape, full dtype matrix -------------mod tile_32x32 {