A high-performance inference engine for AI models
faster qdot for 8-bit (#347)
CCYehc7a54997c5b5
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.h b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.hindex 5769293b7..6e8d0eda2 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.h+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.h@@ -18,24 +18,47 @@ METAL_FUNC bfloat uint_to_fp<bfloat>(uint32_t x) {return as_type<bfloat>(uint16_t(x | 0x4300u)) - bfloat(128.0f);}-template <typename U>+// Unpack 4 lanes of `BITS`-wide packed uints (low bits of each uint) into+// vec<U, 4> of floats. `BITS` in [1, 23] to fit the float23 mantissa trick.+template <int BITS>+METAL_FUNC float4 _uint4_to_fp4_float(uint4 n) {+ static_assert(BITS > 0 && BITS <= 23, "BITS must fit in float23 mantissa");+ constexpr uint mask = (1u << BITS) - 1u;+ n &= uint4(mask);+ return as_type<float4>(n | uint4(0x4B000000u)) - float4(8388608.0f);+}++template <typename U, int BITS>METAL_FUNC vec<U, 4> uint4_to_fp4(uint4 n);template <>-METAL_FUNC float4 uint4_to_fp4<float>(uint4 n) {- n &= uint4(0xFu);- return as_type<float4>(n | uint4(0x4B000000u)) - float4(8388608.0f);+METAL_FUNC float4 uint4_to_fp4<float, 4>(uint4 n) {+ return _uint4_to_fp4_float<4>(n);+}++template <>+METAL_FUNC float4 uint4_to_fp4<float, 8>(uint4 n) {+ return _uint4_to_fp4_float<8>(n);+}++template <>+METAL_FUNC half4 uint4_to_fp4<half, 4>(uint4 n) {+ return half4(_uint4_to_fp4_float<4>(n));}template <>-METAL_FUNC half4 uint4_to_fp4<half>(uint4 n) {- return half4(uint4_to_fp4<float>(n));+METAL_FUNC half4 uint4_to_fp4<half, 8>(uint4 n) {+ return half4(_uint4_to_fp4_float<8>(n));}template <>-METAL_FUNC bfloat4 uint4_to_fp4<bfloat>(uint4 n) {- ushort4 narrow = ushort4(n & uint4(0xFu));- return as_type<bfloat4>(narrow | ushort4(0x4300u)) - bfloat(128.0f);+METAL_FUNC bfloat4 uint4_to_fp4<bfloat, 4>(uint4 n) {+ return bfloat4(_uint4_to_fp4_float<4>(n));+}++template <>+METAL_FUNC bfloat4 uint4_to_fp4<bfloat, 8>(uint4 n) {+ return bfloat4(_uint4_to_fp4_float<8>(n));}template <int bits, int wsize = 8>@@ -120,12 +143,17 @@ inline U qdot(const thread U4* x4 = (const thread U4*)x_thread;for (int i = 0; i < (values_per_thread / 4); i++) {uint wi = ws[i];- U4 w_vec = uint4_to_fp4<U>(uint4(wi, wi >> 4, wi >> 8, wi >> 12));+ U4 w_vec = uint4_to_fp4<U, 4>(uint4(wi, wi >> 4, wi >> 8, wi >> 12));accum += dot(x4[i], w_vec);}} else if (bits == 8) {- for (int i = 0; i < values_per_thread; i++) {- accum += x_thread[i] * w[i];+ using U4 = vec<U, 4>;+ const device uint* ws = (const device uint*)w;+ const thread U4* x4 = (const thread U4*)x_thread;+ for (int i = 0; i < (values_per_thread / 4); i++) {+ uint wi = ws[i];+ U4 w_vec = uint4_to_fp4<U, 8>(uint4(wi, wi >> 8, wi >> 16, wi >> 24));+ accum += dot(x4[i], w_vec);}}return scale * accum + sum * bias;@@ -151,7 +179,7 @@ inline U qdot_safe(int full = N / 4;for (int i = 0; i < full; i++) {uint16_t wi = ws[i];- U4 w_vec = uint4_to_fp4<U>(uint4(wi, wi >> 4, wi >> 8, wi >> 12));+ U4 w_vec = uint4_to_fp4<U, 4>(uint4(wi, wi >> 4, wi >> 8, wi >> 12));accum += dot(x4[i], w_vec);}diff --git a/crates/uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs b/crates/uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rsindex e9583aff4..bac371451 100644--- a/crates/uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs+++ b/crates/uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs@@ -328,7 +328,11 @@ fn bench_qmv_fast_typed<B: Backend, T: ArrayElement + Float>(256};- for (m, n, k) in iproduct!([1, 2, 3, 4], [2048, 4096, 14336], [2048, 4096, 8192, 14336]) {+ for (m, n, k) in iproduct!(+ [1, 2, 3, 4],+ [1024, 2048, 4096, 14336, 65536],+ [1024, 2048, 4096, 8192, 14336]+ ) {if n % 8 != 0 || k % block_size != 0 {continue;}@@ -413,5 +417,6 @@ fn bench_qmv_fast(c: &mut Criterion) {let context = <B as Backend>::Context::new().unwrap();bench_qmv_fast_typed::<B, bf16>(c, &context, "Mlx_BF16_gs128", 128, 4, false, true);bench_qmv_fast_typed::<B, f16>(c, &context, "ZP_F16_gs64", 64, 4, true, false);+ bench_qmv_fast_typed::<B, bf16>(c, &context, "ZP_BF16_gs64_8b", 64, 8, true, false);});}