Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Quantized gemm perfomance improved (#95)

Aleksei Savinb29d47f75954
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs
index 75d71378d..30cb489ac 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs
@@ -326,7 +326,10 @@ fn select_qmm_kernel_name(
if weights_transposed {
if output_dim % 32 != 0 {
kernel_name.push_str("_unaligned");
- } else if type_suffix == "bf16" && group_size == 128 && bits == 4 {
+ } else if type_suffix == "bf16"
+ && (group_size == 128 || group_size == 64)
+ && (bits == 4 || bits == 8)
+ {
kernel_name.push_str("_64x64");
}
} else if input_dim % 32 == 0 {
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal
index ce9591df8..1f11b9008 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal
@@ -3749,6 +3749,23 @@ qmm_transposed<bfloat, 64, 4, false>(
uint simd_lid [[thread_index_in_simdgroup]]
);
+template [[host_name("qmm_transposed_bf16_g64_b4_64x64")]] [[kernel]] void
+qmm_transposed<bfloat, 64, 4, false, 64, 32, 64>(
+ const device uint32_t* w [[buffer(0)]],
+ const device bfloat* scales [[buffer(1)]],
+ const device uint8_t* zero_points [[buffer(2)]],
+ const device bfloat* biases [[buffer(2)]],
+ const device bfloat* x [[buffer(3)]],
+ device bfloat* y [[buffer(4)]],
+ const constant int& K [[buffer(5)]],
+ const constant int& N [[buffer(6)]],
+ const constant int& M [[buffer(7)]],
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint lid [[thread_index_in_threadgroup]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
+
template [[host_name("qmv_bf16_g64_b4")]] [[kernel]] void qmv<bfloat, 64, 4>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
@@ -3853,6 +3870,23 @@ qmm_transposed<bfloat, 64, 8, false>(
uint simd_lid [[thread_index_in_simdgroup]]
);
+template [[host_name("qmm_transposed_bf16_g64_b8_64x64")]] [[kernel]] void
+qmm_transposed<bfloat, 64, 8, false, 64, 32, 64>(
+ const device uint32_t* w [[buffer(0)]],
+ const device bfloat* scales [[buffer(1)]],
+ const device uint8_t* zero_points [[buffer(2)]],
+ const device bfloat* biases [[buffer(2)]],
+ const device bfloat* x [[buffer(3)]],
+ device bfloat* y [[buffer(4)]],
+ const constant int& K [[buffer(5)]],
+ const constant int& N [[buffer(6)]],
+ const constant int& M [[buffer(7)]],
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint lid [[thread_index_in_threadgroup]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
+
template [[host_name("qmv_bf16_g64_b8")]] [[kernel]] void qmv<bfloat, 64, 8>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
@@ -4107,6 +4141,23 @@ qmm_transposed<bfloat, 128, 4, true, 64, 32, 64>(
uint simd_lid [[thread_index_in_simdgroup]]
);
+template [[host_name("qmm_transposed_bf16_g128_b8_64x64")]] [[kernel]] void
+qmm_transposed<bfloat, 128, 8, true, 64, 32, 64>(
+ const device uint32_t* w [[buffer(0)]],
+ const device bfloat* scales [[buffer(1)]],
+ const device uint8_t* zero_points [[buffer(2)]],
+ const device bfloat* biases [[buffer(2)]],
+ const device bfloat* x [[buffer(3)]],
+ device bfloat* y [[buffer(4)]],
+ const constant int& K [[buffer(5)]],
+ const constant int& N [[buffer(6)]],
+ const constant int& M [[buffer(7)]],
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint lid [[thread_index_in_threadgroup]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
+
template [[host_name("qmm_transposed_bf16_g128_b4_64x128")]] [[kernel]] void
qmm_transposed<bfloat, 128, 4, true, 64, 32, 128>(
const device uint32_t* w [[buffer(0)]],