A high-performance inference engine for AI models
faster qmv (#330)
CCYehd96d08917ab5
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metalindex a35ed5b9d..eb53f49c7 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal@@ -96,7 +96,7 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(zp = static_cast<U>(zl[g]);}result[row] +=- qdot_zero_point<U, values_per_thread, BITS>(wl, x_thread, s, zp);+ qdot<U, values_per_thread, BITS>(wl, x_thread, s, -s * zp, sum);}}@@ -140,7 +140,7 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(zp = static_cast<U>(zl[g]);}result[row] +=- qdot_zero_point<U, values_per_thread, BITS>(wl, x_thread, s, zp);+ qdot<U, values_per_thread, BITS>(wl, x_thread, s, -s * zp, sum);}}}@@ -197,7 +197,7 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(zp = static_cast<U>(zl[g]);}result[row] +=- qdot_zero_point<U, values_per_thread, BITS>(wl, x_thread, s, zp);+ qdot<U, values_per_thread, BITS>(wl, x_thread, s, -s * zp, sum);}}@@ -246,7 +246,7 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(zp = static_cast<U>(zl[g]);}result[row] +=- qdot_zero_point<U, values_per_thread, BITS>(wl, x_thread, s, zp);+ qdot<U, values_per_thread, BITS>(wl, x_thread, s, -s * zp, sum);}}}diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metalindex 129af75b7..9277fc434 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal@@ -118,13 +118,13 @@ PUBLIC KERNEL(QuantizedMatmulQmvFast)(zp3 = static_cast<U>(zp_byte3);}result[0] +=- qdot_zero_point<U, values_per_thread, BITS>(wl0, x_thread, s0, zp0);+ qdot<U, values_per_thread, BITS>(wl0, x_thread, s0, -s0 * zp0, sum);result[1] +=- qdot_zero_point<U, values_per_thread, BITS>(wl1, x_thread, s1, zp1);+ qdot<U, values_per_thread, BITS>(wl1, x_thread, s1, -s1 * zp1, sum);result[2] +=- qdot_zero_point<U, values_per_thread, BITS>(wl2, x_thread, s2, zp2);+ qdot<U, values_per_thread, BITS>(wl2, x_thread, s2, -s2 * zp2, sum);result[3] +=- qdot_zero_point<U, values_per_thread, BITS>(wl3, x_thread, s3, zp3);+ qdot<U, values_per_thread, BITS>(wl3, x_thread, s3, -s3 * zp3, sum);}}diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.h b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.hindex c0122c0cb..801e26c18 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.h+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.h@@ -7,6 +7,37 @@ using namespace metal;#include "mma.h"+// Avoids air.convert (SFU) for int → float which is slower on Apple GPU.+template <typename U>+METAL_FUNC U uint_to_fp(uint32_t x) {+ return static_cast<U>(as_type<float>(x | 0x4B000000u) - 8388608.0f);+}++template <>+METAL_FUNC bfloat uint_to_fp<bfloat>(uint32_t x) {+ return as_type<bfloat>(uint16_t(x | 0x4300u)) - bfloat(128.0f);+}++template <typename U>+METAL_FUNC vec<U, 4> uint4_to_fp4(uint4 n);++template <>+METAL_FUNC float4 uint4_to_fp4<float>(uint4 n) {+ n &= uint4(0xFu);+ return as_type<float4>(n | uint4(0x4B000000u)) - float4(8388608.0f);+}++template <>+METAL_FUNC half4 uint4_to_fp4<half>(uint4 n) {+ return half4(uint4_to_fp4<float>(n));+}++template <>+METAL_FUNC bfloat4 uint4_to_fp4<bfloat>(uint4 n) {+ ushort4 narrow = ushort4(n & uint4(0xFu));+ return as_type<bfloat4>(narrow | ushort4(0x4300u)) - bfloat(128.0f);+}+template <int bits, int wsize = 8>inline constexpr short get_pack_factor() {return (bits == 3 || bits == 5) ? 8 : (bits == 6 ? 4 : wsize / bits);@@ -22,20 +53,13 @@ template <typename T, typename U, int values_per_thread, int bits>inline U load_vector(const device T* x, thread U* x_thread) {static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");+ using U4 = vec<U, 4>;U sum = 0;- if (bits == 4) {- for (int i = 0; i < values_per_thread; i += 4) {- sum += x[i] + x[i + 1] + x[i + 2] + x[i + 3];- x_thread[i] = x[i];- x_thread[i + 1] = x[i + 1] / 16.0f;- x_thread[i + 2] = x[i + 2] / 256.0f;- x_thread[i + 3] = x[i + 3] / 4096.0f;- }- } else if (bits == 8) {- for (int i = 0; i < values_per_thread; i++) {- sum += x[i];- x_thread[i] = x[i];- }+ thread U4* x4 = (thread U4*)x_thread;+ for (int i = 0; i < values_per_thread / 4; i++) {+ U4 v = U4(x[4 * i], x[4 * i + 1], x[4 * i + 2], x[4 * i + 3]);+ sum += v[0] + v[1] + v[2] + v[3];+ x4[i] = v;}return sum;}@@ -45,31 +69,13 @@ inline U load_vector_safe(const device T* x, thread U* x_thread, int N) {static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");U sum = 0;- if (bits == 4) {- const U scale_lut[4] = {- static_cast<U>(1.0f),- static_cast<U>(1.0f / 16.0f),- static_cast<U>(1.0f / 256.0f),- static_cast<U>(1.0f / 4096.0f)- };-- for (int i = 0; i < values_per_thread; ++i) {- x_thread[i] = 0;- }- for (int i = 0; i < N; ++i) {- U v = x[i];- sum += v;- x_thread[i] = v * scale_lut[i & 3];- }- } else if (bits == 8) {- for (int i = 0; i < N; ++i) {- U v = x[i];- sum += v;- x_thread[i] = v;- }- for (int i = N; i < values_per_thread; ++i) {- x_thread[i] = 0;- }+ for (int i = 0; i < values_per_thread; ++i) {+ x_thread[i] = 0;+ }+ for (int i = 0; i < N; ++i) {+ U v = x[i];+ sum += v;+ x_thread[i] = v;}return sum;}@@ -85,11 +91,10 @@ inline void qouter(static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");if (bits == 4) {- U s0 = scale;- U s1 = scale / 16.0f;for (int i = 0; i < (values_per_thread / 2); i++) {- result[2 * i] += x * (s0 * (w[i] & 0x0f) + bias);- result[2 * i + 1] += x * (s1 * (w[i] & 0xf0) + bias);+ result[2 * i] += x * (scale * uint_to_fp<U>(w[i] & 0x0fu) + bias);+ result[2 * i + 1] +=+ x * (scale * uint_to_fp<U>((w[i] >> 4) & 0x0fu) + bias);}} else if (bits == 8) {for (int i = 0; i < values_per_thread; i++) {@@ -110,13 +115,13 @@ inline U qdot(U accum = 0;if (bits == 4) {- const device uint16_t* ws = (const device uint16_t*)w;+ using U4 = vec<U, 4>;+ const device ushort* ws = (const device ushort*)w;+ const thread U4* x4 = (const thread U4*)x_thread;for (int i = 0; i < (values_per_thread / 4); i++) {- accum +=- (x_thread[4 * i] * (ws[i] & 0x000f) +- x_thread[4 * i + 1] * (ws[i] & 0x00f0) +- x_thread[4 * i + 2] * (ws[i] & 0x0f00) +- x_thread[4 * i + 3] * (ws[i] & 0xf000));+ uint wi = ws[i];+ U4 w_vec = uint4_to_fp4<U>(uint4(wi, wi >> 4, wi >> 8, wi >> 12));+ accum += dot(x4[i], w_vec);}} else if (bits == 8) {for (int i = 0; i < values_per_thread; i++) {@@ -126,50 +131,6 @@ inline U qdot(return scale * accum + sum * bias;}-template <typename U, int values_per_thread, int bits>-inline U qdot_zero_point(- const device uint8_t* w,- const thread U* x_thread,- U scale,- U zero_point-) {- static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");-- U accum = 0;- if (bits == 4) {- const device uint16_t* ws = (const device uint16_t*)w;- const uint16_t zp0 = static_cast<uint16_t>(zero_point);- const uint16_t zp1 = static_cast<uint16_t>(zero_point) * 16;- const uint16_t zp2 = static_cast<uint16_t>(zero_point) * 256;- const uint16_t zp3 = static_cast<uint16_t>(zero_point) * 4096;-- for (int i = 0; i < (values_per_thread / 4); i++) {- uint16_t word = ws[i];- accum += x_thread[4 * i] *- static_cast<U>(- static_cast<int>(word & 0x000f) - static_cast<int>(zp0)- );- accum += x_thread[4 * i + 1] *- static_cast<U>(- static_cast<int>(word & 0x00f0) - static_cast<int>(zp1)- );- accum += x_thread[4 * i + 2] *- static_cast<U>(- static_cast<int>(word & 0x0f00) - static_cast<int>(zp2)- );- accum += x_thread[4 * i + 3] *- static_cast<U>(- static_cast<int>(word & 0xf000) - static_cast<int>(zp3)- );- }- } else if (bits == 8) {- for (int i = 0; i < values_per_thread; i++) {- accum += x_thread[i] * (static_cast<U>(w[i]) - zero_point);- }- }- return scale * accum;-}-template <typename U, int values_per_thread, int bits>inline U qdot_safe(const device uint8_t* w,@@ -183,15 +144,15 @@ inline U qdot_safe(U accum = 0;if (bits == 4) {+ using U4 = vec<U, 4>;const device uint16_t* ws = (const device uint16_t*)w;+ const thread U4* x4 = (const thread U4*)x_thread;int full = N / 4;for (int i = 0; i < full; i++) {- accum +=- (x_thread[4 * i] * (ws[i] & 0x000f) +- x_thread[4 * i + 1] * (ws[i] & 0x00f0) +- x_thread[4 * i + 2] * (ws[i] & 0x0f00) +- x_thread[4 * i + 3] * (ws[i] & 0xf000));+ uint16_t wi = ws[i];+ U4 w_vec = uint4_to_fp4<U>(uint4(wi, wi >> 4, wi >> 8, wi >> 12));+ accum += dot(x4[i], w_vec);}int rem = N & 3;@@ -199,11 +160,11 @@ inline U qdot_safe(uint16_t wv = ws[full];int base = 4 * full;if (rem > 0)- accum += x_thread[base] * (wv & 0x000f);+ accum += x_thread[base] * uint_to_fp<U>(wv & 0xf);if (rem > 1)- accum += x_thread[base + 1] * (wv & 0x00f0);+ accum += x_thread[base + 1] * uint_to_fp<U>((wv >> 4) & 0xf);if (rem > 2)- accum += x_thread[base + 2] * (wv & 0x0f00);+ accum += x_thread[base + 2] * uint_to_fp<U>((wv >> 8) & 0xf);}} else if (bits == 8) {for (int i = 0; i < N; i++) {diff --git a/crates/uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs b/crates/uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rsindex 56aefbdbe..035dbfeb9 100644--- a/crates/uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs+++ b/crates/uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs@@ -1,6 +1,10 @@use std::fmt::{Debug, Display};+use criterion::{BenchmarkId, Criterion, Throughput};+use half::{bf16, f16};+use itertools::iproduct;use num_traits::Float;+use rand::{RngExt, SeedableRng, rngs::SmallRng};use uzu::{ArrayElement, DataType,backends::{@@ -13,7 +17,10 @@ use uzu::{};use super::{Input, check_tolerance, pack_weights_u32, pack_zero_points};-use crate::{common::helpers::alloc_buffer_with_data, uzu_test};+use crate::{+ common::{helpers::alloc_buffer_with_data, type_short_name},+ uzu_bench, uzu_test,+};fn get_expected<T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {let context = <Cpu as Backend>::Context::new().expect("Failed to create Context");@@ -236,8 +243,6 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(});}-/// The fast kernel requires in_vec_size to be a multiple of block_size-/// (512 for 4-bit, 256 for 8-bit) and out_vec_size to be a multiple of 32.fn get_test_dims(group_size: u32,bits: u32,@@ -273,94 +278,85 @@ fn test_basic<T: ArrayElement + Float + Debug + Display>(}}-// ── 4-bit, zero points ─────────────────────────────────────────────────────--#[uzu_test]-fn test_gs32_4bit_zp() {- for_each_float_type!(|F| {- test_basic::<F>(32, 4, true, false);- })-}--#[uzu_test]-fn test_gs64_4bit_zp() {- for_each_float_type!(|F| {- test_basic::<F>(64, 4, true, false);- })-}--#[uzu_test]-fn test_gs128_4bit_zp() {- for_each_float_type!(|F| {- test_basic::<F>(128, 4, true, false);- })-}--// ── 8-bit, zero points ─────────────────────────────────────────────────────--#[uzu_test]-fn test_gs32_8bit_zp() {- for_each_float_type!(|F| {- test_basic::<F>(32, 8, true, false);- })-}--#[uzu_test]-fn test_gs64_8bit_zp() {- for_each_float_type!(|F| {- test_basic::<F>(64, 8, true, false);- })-}--#[uzu_test]-fn test_gs128_8bit_zp() {- for_each_float_type!(|F| {- test_basic::<F>(128, 8, true, false);- })-}--// ── 4-bit, mlx quant ───────────────────────────────────────────────────────--#[uzu_test]-fn test_gs32_4bit_mlx() {- for_each_float_type!(|F| {- test_basic::<F>(32, 4, false, true);- })-}--#[uzu_test]-fn test_gs64_4bit_mlx() {- for_each_float_type!(|F| {- test_basic::<F>(64, 4, false, true);- })-}--#[uzu_test]-fn test_gs128_4bit_mlx() {- for_each_float_type!(|F| {- test_basic::<F>(128, 4, false, true);- })+macro_rules! qmv_fast_test {+ ($name:ident, gs=$gs:expr, bits=$bits:expr, zp=$zp:expr, mlx=$mlx:expr) => {+ #[uzu_test]+ fn $name() {+ for_each_float_type!(|F| { test_basic::<F>($gs, $bits, $zp, $mlx); })+ }+ };}-// ── 8-bit, mlx quant ───────────────────────────────────────────────────────--#[uzu_test]-fn test_gs32_8bit_mlx() {- for_each_float_type!(|F| {- test_basic::<F>(32, 8, false, true);- })+qmv_fast_test!(test_gs32_4bit_zp, gs=32, bits=4, zp=true, mlx=false);+qmv_fast_test!(test_gs64_4bit_zp, gs=64, bits=4, zp=true, mlx=false);+qmv_fast_test!(test_gs128_4bit_zp, gs=128, bits=4, zp=true, mlx=false);+qmv_fast_test!(test_gs32_8bit_zp, gs=32, bits=8, zp=true, mlx=false);+qmv_fast_test!(test_gs64_8bit_zp, gs=64, bits=8, zp=true, mlx=false);+qmv_fast_test!(test_gs128_8bit_zp, gs=128, bits=8, zp=true, mlx=false);+qmv_fast_test!(test_gs32_4bit_mlx, gs=32, bits=4, zp=false, mlx=true);+qmv_fast_test!(test_gs64_4bit_mlx, gs=64, bits=4, zp=false, mlx=true);+qmv_fast_test!(test_gs128_4bit_mlx, gs=128, bits=4, zp=false, mlx=true);+qmv_fast_test!(test_gs32_8bit_mlx, gs=32, bits=8, zp=false, mlx=true);+qmv_fast_test!(test_gs64_8bit_mlx, gs=64, bits=8, zp=false, mlx=true);+qmv_fast_test!(test_gs128_8bit_mlx, gs=128, bits=8, zp=false, mlx=true);++fn gen_random<T: rand::distr::uniform::SampleUniform + PartialOrd + Copy, R: rand::Rng>(rng: &mut R, range: std::ops::Range<T>, len: usize) -> Box<[T]> {+ (0..len).map(|_| rng.random_range(range.clone())).collect()}-#[uzu_test]-fn test_gs64_8bit_mlx() {- for_each_float_type!(|F| {- test_basic::<F>(64, 8, false, true);- })+fn bench_qmv_fast_typed<B: Backend, T: ArrayElement + Float>(+ c: &mut Criterion,+ context: &B::Context,+ label: &str,+ group_size: u32,+ bits: u32,+ use_zero_points: bool,+ use_mlx_quant: bool,+) {+ let mut group = c.benchmark_group(format!("{}/Kernel/QmvFast/{}", type_short_name::<B>(), label));+ let block_size: usize = if bits == 4 { 512 } else { 256 };++ for (m, n, k) in iproduct!([1, 2, 3, 4], [2048, 4096, 14336], [2048, 4096, 8192, 14336]) {+ if n % 8 != 0 || k % block_size != 0 { continue; }++ let num_groups = k.div_ceil(group_size as usize);+ let mut rng = SmallRng::seed_from_u64(42);++ let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmvFastKernel::new(+ context, T::data_type(), group_size, bits, use_zero_points, use_mlx_quant,+ ).unwrap();++ let w_buf = alloc_buffer_with_data::<B, u32>(context, &gen_random::<u32, _>(&mut rng, 0..u32::MAX, n * k * bits as usize / 32));+ let scales_buf = alloc_buffer_with_data::<B, T>(context, &gen_random::<f32, _>(&mut rng, 0.01..1.0, n * num_groups).iter().map(|&v| T::from(v).unwrap()).collect::<Vec<_>>());+ let x_buf = alloc_buffer_with_data::<B, T>(context, &gen_random::<f32, _>(&mut rng, -1.0..1.0, m * k).iter().map(|&v| T::from(v).unwrap()).collect::<Vec<_>>());+ let mut y_buf = context.create_buffer(m * n * std::mem::size_of::<T>()).unwrap();++ let zp_buf = use_zero_points.then(|| {+ let zp_stride = if bits == 4 { (num_groups + 1) / 2 } else { num_groups };+ alloc_buffer_with_data::<B, u8>(context, &gen_random::<u8, _>(&mut rng, 0..u8::MAX, n * zp_stride))+ });+ let bias_buf = use_mlx_quant.then(|| {+ alloc_buffer_with_data::<B, T>(context, &gen_random::<f32, _>(&mut rng, -0.5..0.5, n * num_groups).iter().map(|&v| T::from(v).unwrap()).collect::<Vec<_>>())+ });++ group.throughput(Throughput::Elements((m * n * k) as u64));+ group.bench_function(BenchmarkId::from_parameter(format!("M[{m}]N[{n}]K[{k}]")), |b| {+ b.iter_custom(|n_iters| {+ let mut encoder = Encoder::<B>::new(context).unwrap();+ for _ in 0..n_iters {+ kernel.encode(&w_buf, &scales_buf, zp_buf.as_ref(), bias_buf.as_ref(), &x_buf, &mut y_buf, k as u32, n as u32, m as u32, &mut encoder);+ }+ encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time().unwrap()+ })+ });+ }}-#[uzu_test]-fn test_gs128_8bit_mlx() {- for_each_float_type!(|F| {- test_basic::<F>(128, 8, false, true);- })+#[uzu_bench]+fn bench_qmv_fast(c: &mut Criterion) {+ for_each_backend!(|B| {+ let context = <B as Backend>::Context::new().unwrap();+ bench_qmv_fast_typed::<B, bf16>(c, &context, "Mlx_BF16_gs128", 128, 4, false, true);+ bench_qmv_fast_typed::<B, f16>(c, &context, "ZP_F16_gs64", 64, 4, true, false);+ });}