A high-performance inference engine for AI models
Fix swapped N/K bounds in quantized block loaders (#669)
CCYehb071b3703531
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.hindex 39e1b349f..c5056fb5c 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h@@ -95,26 +95,32 @@ struct QuantizedBlockLoaderScaleBias {}}- if constexpr (REDUCTION_DIMENSION == 1) {- if (tile_row_index >= src_tile_dim.x) {- for (int i = 0; i < READS_PER_THREAD * pack_factor; i++) {- dst[i] = T(0);- }- return;- }- } else {- if (tile_row_index >= src_tile_dim.y) {- for (int i = 0; i < READS_PER_THREAD * pack_factor; i++) {- dst[i] = T(0);- }- return;+ if (tile_row_index >= src_tile_dim.y) {+ for (int i = 0; i < READS_PER_THREAD * pack_factor; i++) {+ dst[i] = T(0);}+ return;}+ const int valid_cols = src_tile_dim.x;+ const int valid_packs = (valid_cols + pack_factor - 1) / pack_factor;T scale = *scales;T bias = *biases;for (int i = 0; i < READS_PER_THREAD; i++) {- dequantize<T, pack_factor, BITS>(src + i * bytes_per_pack, scale, bias, dst + i * pack_factor, signed_codes);+ const int pack_index = tile_col_index + i;+ if (pack_index < valid_packs) {+ dequantize<T, pack_factor, BITS>(src + i * bytes_per_pack, scale, bias, dst + i * pack_factor, signed_codes);+ if (pack_index == valid_packs - 1) {+ const int remaining = valid_cols - pack_index * pack_factor;+ for (int lane = remaining; lane < pack_factor; ++lane) {+ dst[i * pack_factor + lane] = T(0);+ }+ }+ } else {+ for (int lane = 0; lane < pack_factor; ++lane) {+ dst[i * pack_factor + lane] = T(0);+ }+ }}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.hindex 981197036..89b2e5031 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h@@ -129,53 +129,32 @@ struct QuantizedBlockLoaderScaleZeroPoint {}}- if constexpr (REDUCTION_DIMENSION == 1) {- if (tile_row_index >= src_tile_dim.x) {- for (int i = 0; i < READS_PER_THREAD * pack_factor; i++) {- dst[i] = T(0);- }- return;+ if (tile_row_index >= src_tile_dim.y) {+ for (int i = 0; i < READS_PER_THREAD * pack_factor; i++) {+ dst[i] = T(0);}+ return;+ }- int valid_cols = src_tile_dim.y;- int valid_packs = (valid_cols + pack_factor - 1) / pack_factor;-- T scale;- T bias;- current_scale_bias(scale, bias);- for (int i = 0; i < READS_PER_THREAD; i++) {- int pack_index = tile_col_index + i;- if (pack_index < valid_packs) {- dequantize<T, pack_factor, BITS>(src + i * bytes_per_pack, scale, bias, dst + i * pack_factor, signed_codes);-- if (pack_index == valid_packs - 1) {- int remaining = valid_cols - pack_index * pack_factor;- if (remaining < pack_factor) {- for (int r = remaining; r < pack_factor; ++r) {- dst[i * pack_factor + r] = T(0);- }- }- }- } else {- for (int j = 0; j < pack_factor; ++j) {- dst[i * pack_factor + j] = T(0);+ const int valid_cols = src_tile_dim.x;+ const int valid_packs = (valid_cols + pack_factor - 1) / pack_factor;+ T scale;+ T bias;+ current_scale_bias(scale, bias);+ for (int i = 0; i < READS_PER_THREAD; i++) {+ const int pack_index = tile_col_index + i;+ if (pack_index < valid_packs) {+ dequantize<T, pack_factor, BITS>(src + i * bytes_per_pack, scale, bias, dst + i * pack_factor, signed_codes);+ if (pack_index == valid_packs - 1) {+ const int remaining = valid_cols - pack_index * pack_factor;+ for (int lane = remaining; lane < pack_factor; ++lane) {+ dst[i * pack_factor + lane] = T(0);}}- }- return;- } else {- if (tile_row_index >= src_tile_dim.y) {- for (int i = 0; i < READS_PER_THREAD * pack_factor; i++) {- dst[i] = T(0);+ } else {+ for (int lane = 0; lane < pack_factor; ++lane) {+ dst[i * pack_factor + lane] = T(0);}- return;- }-- T scale;- T bias;- current_scale_bias(scale, bias);- for (int i = 0; i < READS_PER_THREAD; i++) {- dequantize<T, pack_factor, BITS>(src + i * bytes_per_pack, scale, bias, dst + i * pack_factor, signed_codes);}}}diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/quant_dispatch_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/quant_dispatch_test.rsindex bcce9d754..12186888a 100644--- a/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/quant_dispatch_test.rs+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/quant_dispatch_test.rs@@ -101,6 +101,10 @@ fn run_parity<T: ArrayElement + Float + Debug + Display>(#[case::gs16_4bit_zp_decode(8, 256, 64, 16, 4, QuantizationMethod::ScaleZeroPoint)]#[case::gs64_4bit_zp_decode(8, 256, 64, 64, 4, QuantizationMethod::ScaleZeroPoint)]#[case::gs32_unaligned_n(64, 256, 96, 32, 4, QuantizationMethod::ScaleBias)]+#[case::gs32_narrow_n_tail_zp(64, 256, 72, 32, 4, QuantizationMethod::ScaleZeroPoint)]+#[case::gs32_narrow_n_tail_mlx(64, 256, 72, 32, 4, QuantizationMethod::ScaleBias)]+#[case::gs32_narrow_n_tail_sym(64, 256, 40, 32, 4, QuantizationMethod::ScaleSymmetric)]+#[case::gs16_k_tail_mlx(64, 272, 64, 16, 4, QuantizationMethod::ScaleBias)]fn parity_bf16(#[case] m: usize,#[case] k: usize,