Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

faster qmv (#330)

CCYehd96d08917ab5
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal
index a35ed5b9d..eb53f49c7 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal
@@ -96,7 +96,7 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(
zp = static_cast<U>(zl[g]);
}
result[row] +=
- qdot_zero_point<U, values_per_thread, BITS>(wl, x_thread, s, zp);
+ qdot<U, values_per_thread, BITS>(wl, x_thread, s, -s * zp, sum);
}
}
@@ -140,7 +140,7 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(
zp = static_cast<U>(zl[g]);
}
result[row] +=
- qdot_zero_point<U, values_per_thread, BITS>(wl, x_thread, s, zp);
+ qdot<U, values_per_thread, BITS>(wl, x_thread, s, -s * zp, sum);
}
}
}
@@ -197,7 +197,7 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(
zp = static_cast<U>(zl[g]);
}
result[row] +=
- qdot_zero_point<U, values_per_thread, BITS>(wl, x_thread, s, zp);
+ qdot<U, values_per_thread, BITS>(wl, x_thread, s, -s * zp, sum);
}
}
@@ -246,7 +246,7 @@ PUBLIC KERNEL(QuantizedMatmulQmv)(
zp = static_cast<U>(zl[g]);
}
result[row] +=
- qdot_zero_point<U, values_per_thread, BITS>(wl, x_thread, s, zp);
+ qdot<U, values_per_thread, BITS>(wl, x_thread, s, -s * zp, sum);
}
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal
index 129af75b7..9277fc434 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal
@@ -118,13 +118,13 @@ PUBLIC KERNEL(QuantizedMatmulQmvFast)(
zp3 = static_cast<U>(zp_byte3);
}
result[0] +=
- qdot_zero_point<U, values_per_thread, BITS>(wl0, x_thread, s0, zp0);
+ qdot<U, values_per_thread, BITS>(wl0, x_thread, s0, -s0 * zp0, sum);
result[1] +=
- qdot_zero_point<U, values_per_thread, BITS>(wl1, x_thread, s1, zp1);
+ qdot<U, values_per_thread, BITS>(wl1, x_thread, s1, -s1 * zp1, sum);
result[2] +=
- qdot_zero_point<U, values_per_thread, BITS>(wl2, x_thread, s2, zp2);
+ qdot<U, values_per_thread, BITS>(wl2, x_thread, s2, -s2 * zp2, sum);
result[3] +=
- qdot_zero_point<U, values_per_thread, BITS>(wl3, x_thread, s3, zp3);
+ qdot<U, values_per_thread, BITS>(wl3, x_thread, s3, -s3 * zp3, sum);
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.h b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.h
index c0122c0cb..801e26c18 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.h
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.h
@@ -7,6 +7,37 @@ using namespace metal;
#include "mma.h"
+// Avoids air.convert (SFU) for int → float which is slower on Apple GPU.
+template <typename U>
+METAL_FUNC U uint_to_fp(uint32_t x) {
+ return static_cast<U>(as_type<float>(x | 0x4B000000u) - 8388608.0f);
+}
+
+template <>
+METAL_FUNC bfloat uint_to_fp<bfloat>(uint32_t x) {
+ return as_type<bfloat>(uint16_t(x | 0x4300u)) - bfloat(128.0f);
+}
+
+template <typename U>
+METAL_FUNC vec<U, 4> uint4_to_fp4(uint4 n);
+
+template <>
+METAL_FUNC float4 uint4_to_fp4<float>(uint4 n) {
+ n &= uint4(0xFu);
+ return as_type<float4>(n | uint4(0x4B000000u)) - float4(8388608.0f);
+}
+
+template <>
+METAL_FUNC half4 uint4_to_fp4<half>(uint4 n) {
+ return half4(uint4_to_fp4<float>(n));
+}
+
+template <>
+METAL_FUNC bfloat4 uint4_to_fp4<bfloat>(uint4 n) {
+ ushort4 narrow = ushort4(n & uint4(0xFu));
+ return as_type<bfloat4>(narrow | ushort4(0x4300u)) - bfloat(128.0f);
+}
+
template <int bits, int wsize = 8>
inline constexpr short get_pack_factor() {
return (bits == 3 || bits == 5) ? 8 : (bits == 6 ? 4 : wsize / bits);
@@ -22,20 +53,13 @@ template <typename T, typename U, int values_per_thread, int bits>
inline U load_vector(const device T* x, thread U* x_thread) {
static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");
+ using U4 = vec<U, 4>;
U sum = 0;
- if (bits == 4) {
- for (int i = 0; i < values_per_thread; i += 4) {
- sum += x[i] + x[i + 1] + x[i + 2] + x[i + 3];
- x_thread[i] = x[i];
- x_thread[i + 1] = x[i + 1] / 16.0f;
- x_thread[i + 2] = x[i + 2] / 256.0f;
- x_thread[i + 3] = x[i + 3] / 4096.0f;
- }
- } else if (bits == 8) {
- for (int i = 0; i < values_per_thread; i++) {
- sum += x[i];
- x_thread[i] = x[i];
- }
+ thread U4* x4 = (thread U4*)x_thread;
+ for (int i = 0; i < values_per_thread / 4; i++) {
+ U4 v = U4(x[4 * i], x[4 * i + 1], x[4 * i + 2], x[4 * i + 3]);
+ sum += v[0] + v[1] + v[2] + v[3];
+ x4[i] = v;
}
return sum;
}
@@ -45,31 +69,13 @@ inline U load_vector_safe(const device T* x, thread U* x_thread, int N) {
static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");
U sum = 0;
- if (bits == 4) {
- const U scale_lut[4] = {
- static_cast<U>(1.0f),
- static_cast<U>(1.0f / 16.0f),
- static_cast<U>(1.0f / 256.0f),
- static_cast<U>(1.0f / 4096.0f)
- };
-
- for (int i = 0; i < values_per_thread; ++i) {
- x_thread[i] = 0;
- }
- for (int i = 0; i < N; ++i) {
- U v = x[i];
- sum += v;
- x_thread[i] = v * scale_lut[i & 3];
- }
- } else if (bits == 8) {
- for (int i = 0; i < N; ++i) {
- U v = x[i];
- sum += v;
- x_thread[i] = v;
- }
- for (int i = N; i < values_per_thread; ++i) {
- x_thread[i] = 0;
- }
+ for (int i = 0; i < values_per_thread; ++i) {
+ x_thread[i] = 0;
+ }
+ for (int i = 0; i < N; ++i) {
+ U v = x[i];
+ sum += v;
+ x_thread[i] = v;
}
return sum;
}
@@ -85,11 +91,10 @@ inline void qouter(
static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");
if (bits == 4) {
- U s0 = scale;
- U s1 = scale / 16.0f;
for (int i = 0; i < (values_per_thread / 2); i++) {
- result[2 * i] += x * (s0 * (w[i] & 0x0f) + bias);
- result[2 * i + 1] += x * (s1 * (w[i] & 0xf0) + bias);
+ result[2 * i] += x * (scale * uint_to_fp<U>(w[i] & 0x0fu) + bias);
+ result[2 * i + 1] +=
+ x * (scale * uint_to_fp<U>((w[i] >> 4) & 0x0fu) + bias);
}
} else if (bits == 8) {
for (int i = 0; i < values_per_thread; i++) {
@@ -110,13 +115,13 @@ inline U qdot(
U accum = 0;
if (bits == 4) {
- const device uint16_t* ws = (const device uint16_t*)w;
+ using U4 = vec<U, 4>;
+ const device ushort* ws = (const device ushort*)w;
+ const thread U4* x4 = (const thread U4*)x_thread;
for (int i = 0; i < (values_per_thread / 4); i++) {
- accum +=
- (x_thread[4 * i] * (ws[i] & 0x000f) +
- x_thread[4 * i + 1] * (ws[i] & 0x00f0) +
- x_thread[4 * i + 2] * (ws[i] & 0x0f00) +
- x_thread[4 * i + 3] * (ws[i] & 0xf000));
+ uint wi = ws[i];
+ U4 w_vec = uint4_to_fp4<U>(uint4(wi, wi >> 4, wi >> 8, wi >> 12));
+ accum += dot(x4[i], w_vec);
}
} else if (bits == 8) {
for (int i = 0; i < values_per_thread; i++) {
@@ -126,50 +131,6 @@ inline U qdot(
return scale * accum + sum * bias;
}
-template <typename U, int values_per_thread, int bits>
-inline U qdot_zero_point(
- const device uint8_t* w,
- const thread U* x_thread,
- U scale,
- U zero_point
-) {
- static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");
-
- U accum = 0;
- if (bits == 4) {
- const device uint16_t* ws = (const device uint16_t*)w;
- const uint16_t zp0 = static_cast<uint16_t>(zero_point);
- const uint16_t zp1 = static_cast<uint16_t>(zero_point) * 16;
- const uint16_t zp2 = static_cast<uint16_t>(zero_point) * 256;
- const uint16_t zp3 = static_cast<uint16_t>(zero_point) * 4096;
-
- for (int i = 0; i < (values_per_thread / 4); i++) {
- uint16_t word = ws[i];
- accum += x_thread[4 * i] *
- static_cast<U>(
- static_cast<int>(word & 0x000f) - static_cast<int>(zp0)
- );
- accum += x_thread[4 * i + 1] *
- static_cast<U>(
- static_cast<int>(word & 0x00f0) - static_cast<int>(zp1)
- );
- accum += x_thread[4 * i + 2] *
- static_cast<U>(
- static_cast<int>(word & 0x0f00) - static_cast<int>(zp2)
- );
- accum += x_thread[4 * i + 3] *
- static_cast<U>(
- static_cast<int>(word & 0xf000) - static_cast<int>(zp3)
- );
- }
- } else if (bits == 8) {
- for (int i = 0; i < values_per_thread; i++) {
- accum += x_thread[i] * (static_cast<U>(w[i]) - zero_point);
- }
- }
- return scale * accum;
-}
-
template <typename U, int values_per_thread, int bits>
inline U qdot_safe(
const device uint8_t* w,
@@ -183,15 +144,15 @@ inline U qdot_safe(
U accum = 0;
if (bits == 4) {
+ using U4 = vec<U, 4>;
const device uint16_t* ws = (const device uint16_t*)w;
+ const thread U4* x4 = (const thread U4*)x_thread;
int full = N / 4;
for (int i = 0; i < full; i++) {
- accum +=
- (x_thread[4 * i] * (ws[i] & 0x000f) +
- x_thread[4 * i + 1] * (ws[i] & 0x00f0) +
- x_thread[4 * i + 2] * (ws[i] & 0x0f00) +
- x_thread[4 * i + 3] * (ws[i] & 0xf000));
+ uint16_t wi = ws[i];
+ U4 w_vec = uint4_to_fp4<U>(uint4(wi, wi >> 4, wi >> 8, wi >> 12));
+ accum += dot(x4[i], w_vec);
}
int rem = N & 3;
@@ -199,11 +160,11 @@ inline U qdot_safe(
uint16_t wv = ws[full];
int base = 4 * full;
if (rem > 0)
- accum += x_thread[base] * (wv & 0x000f);
+ accum += x_thread[base] * uint_to_fp<U>(wv & 0xf);
if (rem > 1)
- accum += x_thread[base + 1] * (wv & 0x00f0);
+ accum += x_thread[base + 1] * uint_to_fp<U>((wv >> 4) & 0xf);
if (rem > 2)
- accum += x_thread[base + 2] * (wv & 0x0f00);
+ accum += x_thread[base + 2] * uint_to_fp<U>((wv >> 8) & 0xf);
}
} else if (bits == 8) {
for (int i = 0; i < N; i++) {
diff --git a/crates/uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs b/crates/uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs
index 56aefbdbe..035dbfeb9 100644
--- a/crates/uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs
+++ b/crates/uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs
@@ -1,6 +1,10 @@
use std::fmt::{Debug, Display};
+use criterion::{BenchmarkId, Criterion, Throughput};
+use half::{bf16, f16};
+use itertools::iproduct;
use num_traits::Float;
+use rand::{RngExt, SeedableRng, rngs::SmallRng};
use uzu::{
ArrayElement, DataType,
backends::{
@@ -13,7 +17,10 @@ use uzu::{
};
use super::{Input, check_tolerance, pack_weights_u32, pack_zero_points};
-use crate::{common::helpers::alloc_buffer_with_data, uzu_test};
+use crate::{
+ common::{helpers::alloc_buffer_with_data, type_short_name},
+ uzu_bench, uzu_test,
+};
fn get_expected<T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
let context = <Cpu as Backend>::Context::new().expect("Failed to create Context");
@@ -236,8 +243,6 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(
});
}
-/// The fast kernel requires in_vec_size to be a multiple of block_size
-/// (512 for 4-bit, 256 for 8-bit) and out_vec_size to be a multiple of 32.
fn get_test_dims(
group_size: u32,
bits: u32,
@@ -273,94 +278,85 @@ fn test_basic<T: ArrayElement + Float + Debug + Display>(
}
}
-// ── 4-bit, zero points ─────────────────────────────────────────────────────
-
-#[uzu_test]
-fn test_gs32_4bit_zp() {
- for_each_float_type!(|F| {
- test_basic::<F>(32, 4, true, false);
- })
-}
-
-#[uzu_test]
-fn test_gs64_4bit_zp() {
- for_each_float_type!(|F| {
- test_basic::<F>(64, 4, true, false);
- })
-}
-
-#[uzu_test]
-fn test_gs128_4bit_zp() {
- for_each_float_type!(|F| {
- test_basic::<F>(128, 4, true, false);
- })
-}
-
-// ── 8-bit, zero points ─────────────────────────────────────────────────────
-
-#[uzu_test]
-fn test_gs32_8bit_zp() {
- for_each_float_type!(|F| {
- test_basic::<F>(32, 8, true, false);
- })
-}
-
-#[uzu_test]
-fn test_gs64_8bit_zp() {
- for_each_float_type!(|F| {
- test_basic::<F>(64, 8, true, false);
- })
-}
-
-#[uzu_test]
-fn test_gs128_8bit_zp() {
- for_each_float_type!(|F| {
- test_basic::<F>(128, 8, true, false);
- })
-}
-
-// ── 4-bit, mlx quant ───────────────────────────────────────────────────────
-
-#[uzu_test]
-fn test_gs32_4bit_mlx() {
- for_each_float_type!(|F| {
- test_basic::<F>(32, 4, false, true);
- })
-}
-
-#[uzu_test]
-fn test_gs64_4bit_mlx() {
- for_each_float_type!(|F| {
- test_basic::<F>(64, 4, false, true);
- })
-}
-
-#[uzu_test]
-fn test_gs128_4bit_mlx() {
- for_each_float_type!(|F| {
- test_basic::<F>(128, 4, false, true);
- })
+macro_rules! qmv_fast_test {
+ ($name:ident, gs=$gs:expr, bits=$bits:expr, zp=$zp:expr, mlx=$mlx:expr) => {
+ #[uzu_test]
+ fn $name() {
+ for_each_float_type!(|F| { test_basic::<F>($gs, $bits, $zp, $mlx); })
+ }
+ };
}
-// ── 8-bit, mlx quant ───────────────────────────────────────────────────────
-
-#[uzu_test]
-fn test_gs32_8bit_mlx() {
- for_each_float_type!(|F| {
- test_basic::<F>(32, 8, false, true);
- })
+qmv_fast_test!(test_gs32_4bit_zp, gs=32, bits=4, zp=true, mlx=false);
+qmv_fast_test!(test_gs64_4bit_zp, gs=64, bits=4, zp=true, mlx=false);
+qmv_fast_test!(test_gs128_4bit_zp, gs=128, bits=4, zp=true, mlx=false);
+qmv_fast_test!(test_gs32_8bit_zp, gs=32, bits=8, zp=true, mlx=false);
+qmv_fast_test!(test_gs64_8bit_zp, gs=64, bits=8, zp=true, mlx=false);
+qmv_fast_test!(test_gs128_8bit_zp, gs=128, bits=8, zp=true, mlx=false);
+qmv_fast_test!(test_gs32_4bit_mlx, gs=32, bits=4, zp=false, mlx=true);
+qmv_fast_test!(test_gs64_4bit_mlx, gs=64, bits=4, zp=false, mlx=true);
+qmv_fast_test!(test_gs128_4bit_mlx, gs=128, bits=4, zp=false, mlx=true);
+qmv_fast_test!(test_gs32_8bit_mlx, gs=32, bits=8, zp=false, mlx=true);
+qmv_fast_test!(test_gs64_8bit_mlx, gs=64, bits=8, zp=false, mlx=true);
+qmv_fast_test!(test_gs128_8bit_mlx, gs=128, bits=8, zp=false, mlx=true);
+
+fn gen_random<T: rand::distr::uniform::SampleUniform + PartialOrd + Copy, R: rand::Rng>(rng: &mut R, range: std::ops::Range<T>, len: usize) -> Box<[T]> {
+ (0..len).map(|_| rng.random_range(range.clone())).collect()
}
-#[uzu_test]
-fn test_gs64_8bit_mlx() {
- for_each_float_type!(|F| {
- test_basic::<F>(64, 8, false, true);
- })
+fn bench_qmv_fast_typed<B: Backend, T: ArrayElement + Float>(
+ c: &mut Criterion,
+ context: &B::Context,
+ label: &str,
+ group_size: u32,
+ bits: u32,
+ use_zero_points: bool,
+ use_mlx_quant: bool,
+) {
+ let mut group = c.benchmark_group(format!("{}/Kernel/QmvFast/{}", type_short_name::<B>(), label));
+ let block_size: usize = if bits == 4 { 512 } else { 256 };
+
+ for (m, n, k) in iproduct!([1, 2, 3, 4], [2048, 4096, 14336], [2048, 4096, 8192, 14336]) {
+ if n % 8 != 0 || k % block_size != 0 { continue; }
+
+ let num_groups = k.div_ceil(group_size as usize);
+ let mut rng = SmallRng::seed_from_u64(42);
+
+ let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmvFastKernel::new(
+ context, T::data_type(), group_size, bits, use_zero_points, use_mlx_quant,
+ ).unwrap();
+
+ let w_buf = alloc_buffer_with_data::<B, u32>(context, &gen_random::<u32, _>(&mut rng, 0..u32::MAX, n * k * bits as usize / 32));
+ let scales_buf = alloc_buffer_with_data::<B, T>(context, &gen_random::<f32, _>(&mut rng, 0.01..1.0, n * num_groups).iter().map(|&v| T::from(v).unwrap()).collect::<Vec<_>>());
+ let x_buf = alloc_buffer_with_data::<B, T>(context, &gen_random::<f32, _>(&mut rng, -1.0..1.0, m * k).iter().map(|&v| T::from(v).unwrap()).collect::<Vec<_>>());
+ let mut y_buf = context.create_buffer(m * n * std::mem::size_of::<T>()).unwrap();
+
+ let zp_buf = use_zero_points.then(|| {
+ let zp_stride = if bits == 4 { (num_groups + 1) / 2 } else { num_groups };
+ alloc_buffer_with_data::<B, u8>(context, &gen_random::<u8, _>(&mut rng, 0..u8::MAX, n * zp_stride))
+ });
+ let bias_buf = use_mlx_quant.then(|| {
+ alloc_buffer_with_data::<B, T>(context, &gen_random::<f32, _>(&mut rng, -0.5..0.5, n * num_groups).iter().map(|&v| T::from(v).unwrap()).collect::<Vec<_>>())
+ });
+
+ group.throughput(Throughput::Elements((m * n * k) as u64));
+ group.bench_function(BenchmarkId::from_parameter(format!("M[{m}]N[{n}]K[{k}]")), |b| {
+ b.iter_custom(|n_iters| {
+ let mut encoder = Encoder::<B>::new(context).unwrap();
+ for _ in 0..n_iters {
+ kernel.encode(&w_buf, &scales_buf, zp_buf.as_ref(), bias_buf.as_ref(), &x_buf, &mut y_buf, k as u32, n as u32, m as u32, &mut encoder);
+ }
+ encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time().unwrap()
+ })
+ });
+ }
}
-#[uzu_test]
-fn test_gs128_8bit_mlx() {
- for_each_float_type!(|F| {
- test_basic::<F>(128, 8, false, true);
- })
+#[uzu_bench]
+fn bench_qmv_fast(c: &mut Criterion) {
+ for_each_backend!(|B| {
+ let context = <B as Backend>::Context::new().unwrap();
+ bench_qmv_fast_typed::<B, bf16>(c, &context, "Mlx_BF16_gs128", 128, 4, false, true);
+ bench_qmv_fast_typed::<B, f16>(c, &context, "ZP_F16_gs64", 64, 4, true, false);
+ });
}