A high-performance inference engine for AI models
Quantized gemm perfomance improved (#95)
Aleksei Savinb29d47f75954
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rsindex 75d71378d..30cb489ac 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs@@ -326,7 +326,10 @@ fn select_qmm_kernel_name(if weights_transposed {if output_dim % 32 != 0 {kernel_name.push_str("_unaligned");- } else if type_suffix == "bf16" && group_size == 128 && bits == 4 {+ } else if type_suffix == "bf16"+ && (group_size == 128 || group_size == 64)+ && (bits == 4 || bits == 8)+ {kernel_name.push_str("_64x64");}} else if input_dim % 32 == 0 {diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metalindex ce9591df8..1f11b9008 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal@@ -3749,6 +3749,23 @@ qmm_transposed<bfloat, 64, 4, false>(uint simd_lid [[thread_index_in_simdgroup]]);+template [[host_name("qmm_transposed_bf16_g64_b4_64x64")]] [[kernel]] void+qmm_transposed<bfloat, 64, 4, false, 64, 32, 64>(+ const device uint32_t* w [[buffer(0)]],+ const device bfloat* scales [[buffer(1)]],+ const device uint8_t* zero_points [[buffer(2)]],+ const device bfloat* biases [[buffer(2)]],+ const device bfloat* x [[buffer(3)]],+ device bfloat* y [[buffer(4)]],+ const constant int& K [[buffer(5)]],+ const constant int& N [[buffer(6)]],+ const constant int& M [[buffer(7)]],+ uint3 tid [[threadgroup_position_in_grid]],+ uint lid [[thread_index_in_threadgroup]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]+);+template [[host_name("qmv_bf16_g64_b4")]] [[kernel]] void qmv<bfloat, 64, 4>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],@@ -3853,6 +3870,23 @@ qmm_transposed<bfloat, 64, 8, false>(uint simd_lid [[thread_index_in_simdgroup]]);+template [[host_name("qmm_transposed_bf16_g64_b8_64x64")]] [[kernel]] void+qmm_transposed<bfloat, 64, 8, false, 64, 32, 64>(+ const device uint32_t* w [[buffer(0)]],+ const device bfloat* scales [[buffer(1)]],+ const device uint8_t* zero_points [[buffer(2)]],+ const device bfloat* biases [[buffer(2)]],+ const device bfloat* x [[buffer(3)]],+ device bfloat* y [[buffer(4)]],+ const constant int& K [[buffer(5)]],+ const constant int& N [[buffer(6)]],+ const constant int& M [[buffer(7)]],+ uint3 tid [[threadgroup_position_in_grid]],+ uint lid [[thread_index_in_threadgroup]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]+);+template [[host_name("qmv_bf16_g64_b8")]] [[kernel]] void qmv<bfloat, 64, 8>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],@@ -4107,6 +4141,23 @@ qmm_transposed<bfloat, 128, 4, true, 64, 32, 64>(uint simd_lid [[thread_index_in_simdgroup]]);+template [[host_name("qmm_transposed_bf16_g128_b8_64x64")]] [[kernel]] void+qmm_transposed<bfloat, 128, 8, true, 64, 32, 64>(+ const device uint32_t* w [[buffer(0)]],+ const device bfloat* scales [[buffer(1)]],+ const device uint8_t* zero_points [[buffer(2)]],+ const device bfloat* biases [[buffer(2)]],+ const device bfloat* x [[buffer(3)]],+ device bfloat* y [[buffer(4)]],+ const constant int& K [[buffer(5)]],+ const constant int& N [[buffer(6)]],+ const constant int& M [[buffer(7)]],+ uint3 tid [[threadgroup_position_in_grid]],+ uint lid [[thread_index_in_threadgroup]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]+);+template [[host_name("qmm_transposed_bf16_g128_b4_64x128")]] [[kernel]] voidqmm_transposed<bfloat, 128, 4, true, 64, 32, 128>(const device uint32_t* w [[buffer(0)]],