Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

faster qdot for 8-bit (#347)

CCYehc7a54997c5b5
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.h b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.h
index 5769293b7..6e8d0eda2 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.h
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.h
@@ -18,24 +18,47 @@ METAL_FUNC bfloat uint_to_fp<bfloat>(uint32_t x) {
return as_type<bfloat>(uint16_t(x | 0x4300u)) - bfloat(128.0f);
}
-template <typename U>
+// Unpack 4 lanes of `BITS`-wide packed uints (low bits of each uint) into
+// vec<U, 4> of floats. `BITS` in [1, 23] to fit the float23 mantissa trick.
+template <int BITS>
+METAL_FUNC float4 _uint4_to_fp4_float(uint4 n) {
+ static_assert(BITS > 0 && BITS <= 23, "BITS must fit in float23 mantissa");
+ constexpr uint mask = (1u << BITS) - 1u;
+ n &= uint4(mask);
+ return as_type<float4>(n | uint4(0x4B000000u)) - float4(8388608.0f);
+}
+
+template <typename U, int BITS>
METAL_FUNC vec<U, 4> uint4_to_fp4(uint4 n);
template <>
-METAL_FUNC float4 uint4_to_fp4<float>(uint4 n) {
- n &= uint4(0xFu);
- return as_type<float4>(n | uint4(0x4B000000u)) - float4(8388608.0f);
+METAL_FUNC float4 uint4_to_fp4<float, 4>(uint4 n) {
+ return _uint4_to_fp4_float<4>(n);
+}
+
+template <>
+METAL_FUNC float4 uint4_to_fp4<float, 8>(uint4 n) {
+ return _uint4_to_fp4_float<8>(n);
+}
+
+template <>
+METAL_FUNC half4 uint4_to_fp4<half, 4>(uint4 n) {
+ return half4(_uint4_to_fp4_float<4>(n));
}
template <>
-METAL_FUNC half4 uint4_to_fp4<half>(uint4 n) {
- return half4(uint4_to_fp4<float>(n));
+METAL_FUNC half4 uint4_to_fp4<half, 8>(uint4 n) {
+ return half4(_uint4_to_fp4_float<8>(n));
}
template <>
-METAL_FUNC bfloat4 uint4_to_fp4<bfloat>(uint4 n) {
- ushort4 narrow = ushort4(n & uint4(0xFu));
- return as_type<bfloat4>(narrow | ushort4(0x4300u)) - bfloat(128.0f);
+METAL_FUNC bfloat4 uint4_to_fp4<bfloat, 4>(uint4 n) {
+ return bfloat4(_uint4_to_fp4_float<4>(n));
+}
+
+template <>
+METAL_FUNC bfloat4 uint4_to_fp4<bfloat, 8>(uint4 n) {
+ return bfloat4(_uint4_to_fp4_float<8>(n));
}
template <int bits, int wsize = 8>
@@ -120,12 +143,17 @@ inline U qdot(
const thread U4* x4 = (const thread U4*)x_thread;
for (int i = 0; i < (values_per_thread / 4); i++) {
uint wi = ws[i];
- U4 w_vec = uint4_to_fp4<U>(uint4(wi, wi >> 4, wi >> 8, wi >> 12));
+ U4 w_vec = uint4_to_fp4<U, 4>(uint4(wi, wi >> 4, wi >> 8, wi >> 12));
accum += dot(x4[i], w_vec);
}
} else if (bits == 8) {
- for (int i = 0; i < values_per_thread; i++) {
- accum += x_thread[i] * w[i];
+ using U4 = vec<U, 4>;
+ const device uint* ws = (const device uint*)w;
+ const thread U4* x4 = (const thread U4*)x_thread;
+ for (int i = 0; i < (values_per_thread / 4); i++) {
+ uint wi = ws[i];
+ U4 w_vec = uint4_to_fp4<U, 8>(uint4(wi, wi >> 8, wi >> 16, wi >> 24));
+ accum += dot(x4[i], w_vec);
}
}
return scale * accum + sum * bias;
@@ -151,7 +179,7 @@ inline U qdot_safe(
int full = N / 4;
for (int i = 0; i < full; i++) {
uint16_t wi = ws[i];
- U4 w_vec = uint4_to_fp4<U>(uint4(wi, wi >> 4, wi >> 8, wi >> 12));
+ U4 w_vec = uint4_to_fp4<U, 4>(uint4(wi, wi >> 4, wi >> 8, wi >> 12));
accum += dot(x4[i], w_vec);
}
diff --git a/crates/uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs b/crates/uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs
index e9583aff4..bac371451 100644
--- a/crates/uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs
+++ b/crates/uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs
@@ -328,7 +328,11 @@ fn bench_qmv_fast_typed<B: Backend, T: ArrayElement + Float>(
256
};
- for (m, n, k) in iproduct!([1, 2, 3, 4], [2048, 4096, 14336], [2048, 4096, 8192, 14336]) {
+ for (m, n, k) in iproduct!(
+ [1, 2, 3, 4],
+ [1024, 2048, 4096, 14336, 65536],
+ [1024, 2048, 4096, 8192, 14336]
+ ) {
if n % 8 != 0 || k % block_size != 0 {
continue;
}
@@ -413,5 +417,6 @@ fn bench_qmv_fast(c: &mut Criterion) {
let context = <B as Backend>::Context::new().unwrap();
bench_qmv_fast_typed::<B, bf16>(c, &context, "Mlx_BF16_gs128", 128, 4, false, true);
bench_qmv_fast_typed::<B, f16>(c, &context, "ZP_F16_gs64", 64, 4, true, false);
+ bench_qmv_fast_typed::<B, bf16>(c, &context, "ZP_BF16_gs64_8b", 64, 8, true, false);
});
}