Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Fix swapped N/K bounds in quantized block loaders (#669)

CCYehb071b3703531
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h
index 39e1b349f..c5056fb5c 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h
@@ -95,26 +95,32 @@ struct QuantizedBlockLoaderScaleBias {
}
}
- if constexpr (REDUCTION_DIMENSION == 1) {
- if (tile_row_index >= src_tile_dim.x) {
- for (int i = 0; i < READS_PER_THREAD * pack_factor; i++) {
- dst[i] = T(0);
- }
- return;
- }
- } else {
- if (tile_row_index >= src_tile_dim.y) {
- for (int i = 0; i < READS_PER_THREAD * pack_factor; i++) {
- dst[i] = T(0);
- }
- return;
+ if (tile_row_index >= src_tile_dim.y) {
+ for (int i = 0; i < READS_PER_THREAD * pack_factor; i++) {
+ dst[i] = T(0);
}
+ return;
}
+ const int valid_cols = src_tile_dim.x;
+ const int valid_packs = (valid_cols + pack_factor - 1) / pack_factor;
T scale = *scales;
T bias = *biases;
for (int i = 0; i < READS_PER_THREAD; i++) {
- dequantize<T, pack_factor, BITS>(src + i * bytes_per_pack, scale, bias, dst + i * pack_factor, signed_codes);
+ const int pack_index = tile_col_index + i;
+ if (pack_index < valid_packs) {
+ dequantize<T, pack_factor, BITS>(src + i * bytes_per_pack, scale, bias, dst + i * pack_factor, signed_codes);
+ if (pack_index == valid_packs - 1) {
+ const int remaining = valid_cols - pack_index * pack_factor;
+ for (int lane = remaining; lane < pack_factor; ++lane) {
+ dst[i * pack_factor + lane] = T(0);
+ }
+ }
+ } else {
+ for (int lane = 0; lane < pack_factor; ++lane) {
+ dst[i * pack_factor + lane] = T(0);
+ }
+ }
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h
index 981197036..89b2e5031 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h
@@ -129,53 +129,32 @@ struct QuantizedBlockLoaderScaleZeroPoint {
}
}
- if constexpr (REDUCTION_DIMENSION == 1) {
- if (tile_row_index >= src_tile_dim.x) {
- for (int i = 0; i < READS_PER_THREAD * pack_factor; i++) {
- dst[i] = T(0);
- }
- return;
+ if (tile_row_index >= src_tile_dim.y) {
+ for (int i = 0; i < READS_PER_THREAD * pack_factor; i++) {
+ dst[i] = T(0);
}
+ return;
+ }
- int valid_cols = src_tile_dim.y;
- int valid_packs = (valid_cols + pack_factor - 1) / pack_factor;
-
- T scale;
- T bias;
- current_scale_bias(scale, bias);
- for (int i = 0; i < READS_PER_THREAD; i++) {
- int pack_index = tile_col_index + i;
- if (pack_index < valid_packs) {
- dequantize<T, pack_factor, BITS>(src + i * bytes_per_pack, scale, bias, dst + i * pack_factor, signed_codes);
-
- if (pack_index == valid_packs - 1) {
- int remaining = valid_cols - pack_index * pack_factor;
- if (remaining < pack_factor) {
- for (int r = remaining; r < pack_factor; ++r) {
- dst[i * pack_factor + r] = T(0);
- }
- }
- }
- } else {
- for (int j = 0; j < pack_factor; ++j) {
- dst[i * pack_factor + j] = T(0);
+ const int valid_cols = src_tile_dim.x;
+ const int valid_packs = (valid_cols + pack_factor - 1) / pack_factor;
+ T scale;
+ T bias;
+ current_scale_bias(scale, bias);
+ for (int i = 0; i < READS_PER_THREAD; i++) {
+ const int pack_index = tile_col_index + i;
+ if (pack_index < valid_packs) {
+ dequantize<T, pack_factor, BITS>(src + i * bytes_per_pack, scale, bias, dst + i * pack_factor, signed_codes);
+ if (pack_index == valid_packs - 1) {
+ const int remaining = valid_cols - pack_index * pack_factor;
+ for (int lane = remaining; lane < pack_factor; ++lane) {
+ dst[i * pack_factor + lane] = T(0);
}
}
- }
- return;
- } else {
- if (tile_row_index >= src_tile_dim.y) {
- for (int i = 0; i < READS_PER_THREAD * pack_factor; i++) {
- dst[i] = T(0);
+ } else {
+ for (int lane = 0; lane < pack_factor; ++lane) {
+ dst[i * pack_factor + lane] = T(0);
}
- return;
- }
-
- T scale;
- T bias;
- current_scale_bias(scale, bias);
- for (int i = 0; i < READS_PER_THREAD; i++) {
- dequantize<T, pack_factor, BITS>(src + i * bytes_per_pack, scale, bias, dst + i * pack_factor, signed_codes);
}
}
}
diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/quant_dispatch_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/quant_dispatch_test.rs
index bcce9d754..12186888a 100644
--- a/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/quant_dispatch_test.rs
+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/matmul/quant_dispatch_test.rs
@@ -101,6 +101,10 @@ fn run_parity<T: ArrayElement + Float + Debug + Display>(
#[case::gs16_4bit_zp_decode(8, 256, 64, 16, 4, QuantizationMethod::ScaleZeroPoint)]
#[case::gs64_4bit_zp_decode(8, 256, 64, 64, 4, QuantizationMethod::ScaleZeroPoint)]
#[case::gs32_unaligned_n(64, 256, 96, 32, 4, QuantizationMethod::ScaleBias)]
+#[case::gs32_narrow_n_tail_zp(64, 256, 72, 32, 4, QuantizationMethod::ScaleZeroPoint)]
+#[case::gs32_narrow_n_tail_mlx(64, 256, 72, 32, 4, QuantizationMethod::ScaleBias)]
+#[case::gs32_narrow_n_tail_sym(64, 256, 40, 32, 4, QuantizationMethod::ScaleSymmetric)]
+#[case::gs16_k_tail_mlx(64, 272, 64, 16, 4, QuantizationMethod::ScaleBias)]
fn parity_bf16(
#[case] m: usize,
#[case] k: usize,