Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Faster quantized matmul for small suffix_len (#340)

CCYehdf26ead54a84
diff --git a/crates/uzu/src/backends/common/kernel/quant_matmul.rs b/crates/uzu/src/backends/common/kernel/quant_matmul.rs
index 207f645db..54236aec4 100644
--- a/crates/uzu/src/backends/common/kernel/quant_matmul.rs
+++ b/crates/uzu/src/backends/common/kernel/quant_matmul.rs
@@ -49,7 +49,7 @@ pub struct QuantizedMatmulArguments<'a, B: Backend> {
pub struct QuantizedMatmulKernelEncodable<B: Backend> {
matrix_vector: MatrixVectorKernel<B>,
- matrix_matrix: <B::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel,
+ matrix_matrix: MatrixMatrixKernel<B>,
input_dim: usize,
output_dim: usize,
quantization_type: QuantizedMatmulType,
@@ -60,6 +60,37 @@ enum MatrixVectorKernel<B: Backend> {
QmvFast(<B::Kernels as Kernels>::QuantizedMatmulQmvFastKernel),
}
+type QmmKernel<B> = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel;
+
+/// Big tile alone, or paired with a BM=8 small tile for `batch_dim < 48`.
+enum MatrixMatrixKernel<B: Backend> {
+ Big(QmmKernel<B>),
+ BigAndSmall {
+ big: QmmKernel<B>,
+ small: QmmKernel<B>,
+ },
+}
+
+impl<B: Backend> MatrixMatrixKernel<B> {
+ /// `None` means fall back to `matrix_vector` (small `batch_dim`, no small tile available).
+ fn pick(
+ &self,
+ batch_dim: usize,
+ ) -> Option<&QmmKernel<B>> {
+ match self {
+ Self::Big(big) => (batch_dim >= 32).then_some(big),
+ Self::BigAndSmall {
+ big,
+ small,
+ } => Some(if batch_dim < 48 {
+ small
+ } else {
+ big
+ }),
+ }
+ }
+}
+
impl<B: Backend> QuantizedMatmulKernelEncodable<B> {
pub fn new(
context: &B::Context,
@@ -116,6 +147,7 @@ impl<B: Backend> QuantizedMatmulKernelEncodable<B> {
// 64-tile (BM=BN=64): bf16 + output_dim%64==0; BK=64 if group_size≥64, BK=32 otherwise.
// 32-tile fallback otherwise.
let aligned_n_64 = configuration.output_dim % 64 == 0;
+ let aligned_n_32 = configuration.output_dim % 32 == 0;
let is_bf16 = configuration.data_type == DataType::BF16;
let can_use_64_tile = aligned_n_64 && is_bf16;
@@ -127,7 +159,7 @@ impl<B: Backend> QuantizedMatmulKernelEncodable<B> {
(32u32, 32u32, 32u32, configuration.output_dim % 32 == 0)
};
- let matrix_matrix = <B::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel::new(
+ let big = <B::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel::new(
context,
configuration.data_type,
group_size,
@@ -135,6 +167,8 @@ impl<B: Backend> QuantizedMatmulKernelEncodable<B> {
bm,
bk,
bn,
+ 2u32, // WM
+ 2u32, // WN
use_zero_points,
use_mlx_quant,
configuration.use_hadamard,
@@ -142,6 +176,34 @@ impl<B: Backend> QuantizedMatmulKernelEncodable<B> {
)
.map_err(QuantizedMatmulError::BackendError)?;
+ // Small-tile kernel: BM=8, BK=32, BN=32, WM=1, WN=1. Requires N%32==0
+ // and supported bits/group_size. Works for any supported T via the unified kernel.
+ // Used for batch_dim in [8, 48) at encode time.
+ let matrix_matrix = if aligned_n_32 {
+ let small = <B::Kernels as Kernels>::QuantizedMatmulQmmTransposedKernel::new(
+ context,
+ configuration.data_type,
+ group_size,
+ bits,
+ 8u32, // BM
+ 32u32, // BK
+ 32u32, // BN
+ 1u32, // WM
+ 1u32, // WN
+ use_zero_points,
+ use_mlx_quant,
+ false, // use_hadamard: BM=8 tile does not support hadamard
+ true, // aligned_n: BN=32, always aligned for the small tile
+ )
+ .map_err(QuantizedMatmulError::BackendError)?;
+ MatrixMatrixKernel::BigAndSmall {
+ big,
+ small,
+ }
+ } else {
+ MatrixMatrixKernel::Big(big)
+ };
+
Ok(Self {
matrix_vector,
matrix_matrix,
@@ -179,13 +241,18 @@ impl<B: Backend> QuantizedMatmulKernelEncodable<B> {
};
}
- if arguments.batch_dim < 32 || self.output_dim == 1 {
- match &self.matrix_vector {
- MatrixVectorKernel::Qmv(k) => encode_kernel!(k,),
- MatrixVectorKernel::QmvFast(k) => encode_kernel!(k, arguments.hadamard_factors),
+ if arguments.batch_dim >= 8 && self.output_dim > 1 {
+ if let Some(kernel) = self.matrix_matrix.pick(arguments.batch_dim) {
+ encode_kernel!(kernel, arguments.hadamard_factors);
+ return Ok(());
}
- } else {
- encode_kernel!(&self.matrix_matrix, arguments.hadamard_factors);
+ }
+
+ // Matrix-vector fallback: small batch_dim, scalar output, or Big-tile-only
+ // configs that don't cover this batch_dim.
+ match &self.matrix_vector {
+ MatrixVectorKernel::Qmv(k) => encode_kernel!(k,),
+ MatrixVectorKernel::QmvFast(k) => encode_kernel!(k, arguments.hadamard_factors),
}
Ok(())
diff --git a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rs b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rs
index a77a447cc..256761b26 100644
--- a/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rs
+++ b/crates/uzu/src/backends/cpu/kernel/quant_matmul/qmm_transposed.rs
@@ -88,12 +88,18 @@ pub fn qmm_transposed<T: ArrayElement + Float>(
#[variants(T, f32, f16, bf16)]
#[variants(GROUP_SIZE, 32, 64, 128)]
#[variants(BITS, 4, 8)]
-#[variants(BM, 32, 64)]
+#[variants(BM, 8, 32, 64)]
#[variants(BK, 32, 64)]
#[variants(BN, 32, 64)]
-#[constraint(BK <= BM)]
+#[variants(WM, 1, 2)]
+#[variants(WN, 1, 2)]
+#[constraint(
+ (BM == 8 && BK == 32 && BN == 32 && WM == 1 && WN == 1) ||
+ (BM == 32 && BK == 32 && BN == 32 && WM == 2 && WN == 2) ||
+ (BM == 32 && BK == 64 && BN == 32 && WM == 2 && WN == 2) ||
+ (BM == 64 && BK == 32 && BN == 64 && WM == 2 && WN == 2) ||
+ (BM == 64 && BK == 64 && BN == 64 && WM == 2 && WN == 2))]
#[constraint(BK <= GROUP_SIZE)]
-#[constraint(BM == BN)]
#[constraint(T != "f32" || BK < 64)]
pub fn quantized_matmul_qmm_transposed<
T: ArrayElement + Float,
@@ -102,6 +108,8 @@ pub fn quantized_matmul_qmm_transposed<
const BM: u32,
const BK: u32,
const BN: u32,
+ const WM: u32,
+ const WN: u32,
>(
weights: *const u32,
scales: *const T,
@@ -118,7 +126,7 @@ pub fn quantized_matmul_qmm_transposed<
#[specialize] use_hadamard: bool,
#[specialize] aligned_n: bool,
) {
- let _ = (BM, BK, BN); // tile params unused in tile-agnostic CPU reference
+ let _ = (BM, BK, BN, WM, WN); // tile params unused in tile-agnostic CPU reference
if use_hadamard {
unimplemented!("not supported yet");
}
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal
index 2b00dac2d..fcf438167 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal
@@ -4,17 +4,23 @@
#include "quant_matmul.h"
// Transposed QMM kernel. BM = batch tile rows, BK = K-axis tile, BN = output
-// tile cols.
-template <typename T, uint GROUP_SIZE, uint BITS, uint BM, uint BK, uint BN>
+// tile cols. WM, WN = simdgroup tile counts along M and N axes.
+template <typename T, uint GROUP_SIZE, uint BITS, uint BM, uint BK, uint BN, uint WM, uint WN>
VARIANTS(T, float, half, bfloat)
VARIANTS(GROUP_SIZE, 32, 64, 128)
VARIANTS(BITS, 4, 8)
-VARIANTS(BM, 32, 64)
+VARIANTS(BM, 8, 32, 64)
VARIANTS(BK, 32, 64)
VARIANTS(BN, 32, 64)
-CONSTRAINT(BK <= BM)
+VARIANTS(WM, 1, 2)
+VARIANTS(WN, 1, 2)
+CONSTRAINT(
+ (BM == 8 && BK == 32 && BN == 32 && WM == 1 && WN == 1) ||
+ (BM == 32 && BK == 32 && BN == 32 && WM == 2 && WN == 2) ||
+ (BM == 32 && BK == 64 && BN == 32 && WM == 2 && WN == 2) ||
+ (BM == 64 && BK == 32 && BN == 64 && WM == 2 && WN == 2) ||
+ (BM == 64 && BK == 64 && BN == 64 && WM == 2 && WN == 2))
CONSTRAINT(BK <= GROUP_SIZE)
-CONSTRAINT(BM == BN)
CONSTRAINT(T != "float" || BK < 64)
PUBLIC KERNEL(QuantizedMatmulQmmTransposed)(
const device uint32_t* weights,
@@ -36,11 +42,11 @@ PUBLIC KERNEL(QuantizedMatmulQmmTransposed)(
const uint out_block_idx GROUPS(out_vec_size.div_ceil(BN)),
const uint batch_block_idx GROUPS(batch_size.div_ceil(BM)),
const uint simd_lane THREADS(32),
- const uint simd_group THREADS(4)
+ const uint simd_group THREADS(WM * WN)
) {
if (use_mlx_quant) {
if (aligned_n) {
- qmm_transposed_impl<T, GROUP_SIZE, BITS, true, BM, BK, BN, true>(
+ qmm_transposed_impl<T, GROUP_SIZE, BITS, true, BM, BK, BN, true, WM, WN>(
weights,
scales,
zero_points,
@@ -58,7 +64,7 @@ PUBLIC KERNEL(QuantizedMatmulQmmTransposed)(
simd_lane
);
} else {
- qmm_transposed_impl<T, GROUP_SIZE, BITS, false, BM, BK, BN, true>(
+ qmm_transposed_impl<T, GROUP_SIZE, BITS, false, BM, BK, BN, true, WM, WN>(
weights,
scales,
zero_points,
@@ -78,7 +84,7 @@ PUBLIC KERNEL(QuantizedMatmulQmmTransposed)(
}
} else {
if (aligned_n) {
- qmm_transposed_impl<T, GROUP_SIZE, BITS, true, BM, BK, BN, false>(
+ qmm_transposed_impl<T, GROUP_SIZE, BITS, true, BM, BK, BN, false, WM, WN>(
weights,
scales,
zero_points,
@@ -96,7 +102,17 @@ PUBLIC KERNEL(QuantizedMatmulQmmTransposed)(
simd_lane
);
} else {
- qmm_transposed_impl<T, GROUP_SIZE, BITS, false, BM, BK, BN, false>(
+ qmm_transposed_impl<
+ T,
+ GROUP_SIZE,
+ BITS,
+ false,
+ BM,
+ BK,
+ BN,
+ false,
+ WM,
+ WN>(
weights,
scales,
zero_points,
@@ -120,7 +136,7 @@ PUBLIC KERNEL(QuantizedMatmulQmmTransposed)(
threadgroup_barrier(mem_flags::mem_device);
// Each lane covers BN/32 output columns (1 at BN=32, 2 at BN=64).
- constexpr uint NUM_SIMD_GROUPS = 4;
+ constexpr uint NUM_SIMD_GROUPS = WM * WN;
constexpr uint ROWS_PER_SIMD_GROUP = BM / NUM_SIMD_GROUPS;
constexpr uint COLS_PER_LANE = BN / 32;
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.h b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.h
index 801e26c18..5769293b7 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.h
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.h
@@ -675,7 +675,9 @@ template <
const int BM = 32,
const int BK = 32,
const int BN = 32,
- const bool use_mlx_quant = false>
+ const bool use_mlx_quant = false,
+ const int WM = 2,
+ const int WN = 2>
void qmm_transposed_impl(
const device uint32_t* weights,
const device T* scales,
@@ -695,9 +697,6 @@ void qmm_transposed_impl(
) {
static_assert(BK >= 32, "BK should be larger than METAL_SIMD_SIZE");
static_assert(BK % 32 == 0, "BK should be divisible by METAL_SIMD_SIZE");
-
- constexpr int WM = 2;
- constexpr int WN = 2;
constexpr int pack_factor = get_pack_factor<bits, 8>();
constexpr int bytes_per_pack = get_bytes_per_pack<bits>();
constexpr int BK_padded = (BK + 16 / sizeof(T));
diff --git a/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rs b/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rs
index cf42d4161..587a5a12a 100644
--- a/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rs
+++ b/crates/uzu/tests/unit/kernel/quant_matmul/qmm_transposed_test.rs
@@ -161,6 +161,8 @@ fn get_output<B: Backend, T: ArrayElement + Float>(
bm: u32,
bk: u32,
bn: u32,
+ wm: u32,
+ wn: u32,
aligned_n: bool,
hadamard_factors: Option<&[i32]>,
) -> Vec<T> {
@@ -174,6 +176,8 @@ fn get_output<B: Backend, T: ArrayElement + Float>(
bm,
bk,
bn,
+ wm,
+ wn,
input.use_zero_points,
input.use_mlx_quant,
use_hadamard,
@@ -220,6 +224,8 @@ fn check_against_expected<T: ArrayElement + Float + Debug + Display>(
bm: u32,
bk: u32,
bn: u32,
+ wm: u32,
+ wn: u32,
aligned_n: bool,
) {
// GPU tiled accumulation rounds differently from CPU scalar reference.
@@ -230,7 +236,7 @@ fn check_against_expected<T: ArrayElement + Float + Debug + Display>(
};
for_each_non_cpu_backend!(|B| {
- let output = get_output::<B, T>(input, bm, bk, bn, aligned_n, None);
+ let output = get_output::<B, T>(input, bm, bk, bn, wm, wn, aligned_n, None);
assert_eq!(expected.len(), output.len(), "Output length mismatch");
let mut errors = 0;
@@ -268,6 +274,8 @@ struct TileCfg {
bm: u32,
bk: u32,
bn: u32,
+ wm: u32,
+ wn: u32,
}
impl TileCfg {
@@ -277,9 +285,10 @@ impl TileCfg {
}
}
-const CFG_32X32: TileCfg = TileCfg { bm: 32, bk: 32, bn: 32 };
-const CFG_WIDE: TileCfg = TileCfg { bm: 64, bk: 32, bn: 64 };
-const CFG_64X64: TileCfg = TileCfg { bm: 64, bk: 64, bn: 64 };
+const CFG_BM8: TileCfg = TileCfg { bm: 8, bk: 32, bn: 32, wm: 1, wn: 1 };
+const CFG_32X32: TileCfg = TileCfg { bm: 32, bk: 32, bn: 32, wm: 2, wn: 2 };
+const CFG_WIDE: TileCfg = TileCfg { bm: 64, bk: 32, bn: 64, wm: 2, wn: 2 };
+const CFG_64X64: TileCfg = TileCfg { bm: 64, bk: 64, bn: 64, wm: 2, wn: 2 };
fn pick_aligned_n(
cfg: &TileCfg,
@@ -312,8 +321,8 @@ fn test_basic<T: ArrayElement + Float + Debug + Display>(
for (m, k, n) in dims {
let input = make_input_basic::<T>(m, k, n, group_size, bits, use_zero_points, use_mlx_quant);
let aligned_n = pick_aligned_n(cfg, n);
- let expected = get_output::<Cpu, T>(&input, cfg.bm, cfg.bk, cfg.bn, aligned_n, None);
- check_against_expected::<T>(&input, &expected, cfg.bm, cfg.bk, cfg.bn, aligned_n);
+ let expected = get_output::<Cpu, T>(&input, cfg.bm, cfg.bk, cfg.bn, cfg.wm, cfg.wn, aligned_n, None);
+ check_against_expected::<T>(&input, &expected, cfg.bm, cfg.bk, cfg.bn, cfg.wm, cfg.wn, aligned_n);
}
}
@@ -328,8 +337,8 @@ fn test_edge<T: ArrayElement + Float + Debug + Display>(
let n = cfg.bn as usize;
let input = make_input_edge::<T>(n, group_size, bits, use_zero_points, use_mlx_quant);
let aligned_n = pick_aligned_n(cfg, n);
- let expected = get_output::<Cpu, T>(&input, cfg.bm, cfg.bk, cfg.bn, aligned_n, None);
- check_against_expected::<T>(&input, &expected, cfg.bm, cfg.bk, cfg.bn, aligned_n);
+ let expected = get_output::<Cpu, T>(&input, cfg.bm, cfg.bk, cfg.bn, cfg.wm, cfg.wn, aligned_n, None);
+ check_against_expected::<T>(&input, &expected, cfg.bm, cfg.bk, cfg.bn, cfg.wm, cfg.wn, aligned_n);
}
// No CPU Hadamard reference — verify all three tile shapes agree instead.
@@ -357,16 +366,16 @@ fn test_hadamard_consistency<T: ArrayElement + Float + Debug + Display>(
};
for_each_non_cpu_backend!(|B| {
- let shapes: [(u32, u32, u32, &str); 3] = [
- (32, 32, 32, "32x32"),
- (64, 32, 64, "wide"),
- (64, 64, 64, "64x64"),
+ let shapes: [(u32, u32, u32, u32, u32, &str); 3] = [
+ (32, 32, 32, 2, 2, "32x32"),
+ (64, 32, 64, 2, 2, "wide"),
+ (64, 64, 64, 2, 2, "64x64"),
];
let outputs: Vec<(&str, Vec<T>)> = shapes
.iter()
- .map(|&(bm, bk, bn, name)| {
- let out = get_output::<B, T>(&input, bm, bk, bn, true, Some(&factors));
+ .map(|&(bm, bk, bn, wm, wn, name)| {
+ let out = get_output::<B, T>(&input, bm, bk, bn, wm, wn, true, Some(&factors));
(name, out)
})
.collect();
@@ -456,6 +465,28 @@ macro_rules! hadamard_tests {
};
}
+mod tile_bm8 {
+ use super::*;
+
+ basic_tests!(&CFG_BM8;
+ // bf16 4-bit ZP has known precision limits
+ #[ignore] fn test_bf16_gs32_4bit_zp: bf16, 32, 4, true, false;
+ #[ignore] fn test_bf16_gs64_4bit_zp: bf16, 64, 4, true, false;
+ fn test_bf16_gs32_8bit_zp: bf16, 32, 8, true, false;
+ fn test_bf16_gs64_8bit_zp: bf16, 64, 8, true, false;
+ fn test_bf16_gs32_4bit_mlx: bf16, 32, 4, false, true;
+ fn test_bf16_gs64_4bit_mlx: bf16, 64, 4, false, true;
+ fn test_bf16_gs32_8bit_mlx: bf16, 32, 8, false, true;
+ fn test_bf16_gs64_8bit_mlx: bf16, 64, 8, false, true;
+ );
+
+ edge_tests!(&CFG_BM8;
+ fn test_edge_bf16_4bit_mlx: bf16, 32, 4, false, true;
+ fn test_edge_bf16_8bit_mlx: bf16, 32, 8, false, true;
+ fn test_edge_bf16_8bit_zp: bf16, 32, 8, true, false;
+ );
+}
+
// ---- tile_32x32 — the original base shape, full dtype matrix -------------
mod tile_32x32 {