Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

drop 8-bit scale+bias matmul kernels (#909)

eugenefbf74e150492
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal
index 5905cdb15..db6f59292 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/gemm.metal
@@ -78,6 +78,7 @@ CONSTRAINT((BITS == 0) == (GROUP_SIZE == 0))
CONSTRAINT(B_PROLOGUE == GemmBPrologueKind::FullPrecision || BT != "float")
CONSTRAINT(B_PROLOGUE != GemmBPrologueKind::ScaleZeroPointDequant || BITS == 4)
CONSTRAINT(B_PROLOGUE != GemmBPrologueKind::ScaleSymmetricDequant || BITS == 8)
+CONSTRAINT(B_PROLOGUE != GemmBPrologueKind::ScaleBiasDequant || BITS == 4)
CONSTRAINT(
B_PROLOGUE == GemmBPrologueKind::FullPrecision ||
(TRANSPOSE_B &&
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal
index 174d01d48..f953ee140 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal
@@ -51,6 +51,7 @@ CONSTRAINT((BITS == 0) == (GROUP_SIZE == 0))
CONSTRAINT(B_PROLOGUE == GemmBPrologueKind::FullPrecision || BT != "float")
CONSTRAINT(B_PROLOGUE != GemmBPrologueKind::ScaleZeroPointDequant || BITS == 4)
CONSTRAINT(B_PROLOGUE != GemmBPrologueKind::ScaleSymmetricDequant || BITS == 8)
+CONSTRAINT(B_PROLOGUE != GemmBPrologueKind::ScaleBiasDequant || BITS == 4)
CONSTRAINT(BITS == 0 || K_SPLIT == 1)
CONSTRAINT(BITS != 0 || (INPUT_ROW_TILE == 1 && REDUCTION_LANES == 32 && NUM_SIMDGROUPS == 8 && GROUP_LANES == 1))
CONSTRAINT(INPUT_ROW_TILE != 1 || REDUCTION_LANES == 32)
diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rs
index 0e7e48bc6..6a236908e 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rs
@@ -160,7 +160,7 @@ fn gemv_bf16(
#[rstest]
#[test_attr(uzu_test)]
#[case::w4_zero_point(4, QuantizationMethod::ScaleZeroPoint)]
-#[case::w8_bias(8, QuantizationMethod::ScaleBias)]
+#[case::w8_symmetric(8, QuantizationMethod::ScaleSymmetric)]
fn group_major_gemv_bf16(
#[case] bits: u32,
#[case] method: QuantizationMethod,
diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_dispatch_test.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_dispatch_test.rs
index 95cbc385f..24a903df2 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_dispatch_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_dispatch_test.rs
@@ -145,7 +145,7 @@ fn parity_bf16(
#[rstest]
#[test_attr(uzu_test)]
-#[case::gs32_8bit_mlx_prefill(64, 256, 64, 32, 8, QuantizationMethod::ScaleBias)]
+#[case::gs32_8bit_symmetric_prefill(64, 256, 64, 32, 8, QuantizationMethod::ScaleSymmetric)]
#[case::gs64_8bit_symmetric_prefill(64, 256, 64, 64, 8, QuantizationMethod::ScaleSymmetric)]
fn parity_bf16_8bit_splitk(
#[case] m: u32,
@@ -289,7 +289,7 @@ fn parity_gemv_partial_group_bf16(
#[test_attr(uzu_test)]
#[case::n12_gs32_4bit(1, 256, 12, 32, 4, QuantizationMethod::ScaleBias)]
#[case::n20_gs64_4bit_zp(2, 256, 20, 64, 4, QuantizationMethod::ScaleZeroPoint)]
-#[case::n36_gs32_8bit(1, 256, 36, 32, 8, QuantizationMethod::ScaleBias)]
+#[case::n36_gs32_8bit_sym(1, 256, 36, 32, 8, QuantizationMethod::ScaleSymmetric)]
fn parity_gemv_unaligned_width_bf16(
#[case] m: u32,
#[case] k: u32,
@@ -453,8 +453,8 @@ fn quant_gemm_accumulate_returns_unsupported_dop() {
#[case::output_group_single_group_gs64(OutputGroup, 64, 64, 64, 4, ScaleZeroPoint, 64)]
#[case::output_group_odd_groups(OutputGroup, 192, 64, 64, 4, ScaleZeroPoint, 64)]
#[case::group_output_padded_n(GroupOutput, 192, 66, 64, 4, ScaleZeroPoint, 66)]
-#[case::output_group_bias_prefix(OutputGroup, 128, 12, 32, 8, ScaleBias, 4)]
-#[case::group_output_bias_prefix(GroupOutput, 128, 12, 32, 8, ScaleBias, 4)]
+#[case::output_group_bias_prefix(OutputGroup, 128, 12, 32, 4, ScaleBias, 4)]
+#[case::group_output_bias_prefix(GroupOutput, 128, 12, 32, 4, ScaleBias, 4)]
fn quant_gemm_parameter_layout_prefix_matches_cpu(
#[case] params_layout: QuantParamsLayout,
#[case] k: u32,
@@ -535,7 +535,6 @@ fn mxu_quant_parity_bf16(
#[test_attr(uzu_test)]
#[case::sym_w8_gs64(16u32, 64u32, 8u32, QuantizationMethod::ScaleSymmetric)]
#[case::bias_w4_gs32(16u32, 32u32, 4u32, QuantizationMethod::ScaleBias)]
-#[case::bias_w8_gs64(16u32, 64u32, 8u32, QuantizationMethod::ScaleBias)]
#[case::zp_w4_gs32(16u32, 32u32, 4u32, QuantizationMethod::ScaleZeroPoint)]
#[case::m1_bias_w4_gs64(1u32, 64u32, 4u32, QuantizationMethod::ScaleBias)]
fn a8w_mxu_parity_bf16(
@@ -579,11 +578,7 @@ fn a8w_independent_activation_group_parity_bf16(#[case] m: u32) {
for (activation_group_size, weight_group_size) in
[(ACTIVATION_SCALE_GROUP_SIZE, 32u32), (ACTIVATION_SCALE_GROUP_SIZE, 64)]
{
- for (bits, method) in [
- (8, QuantizationMethod::ScaleSymmetric),
- (8, QuantizationMethod::ScaleBias),
- (4, QuantizationMethod::ScaleZeroPoint),
- ] {
+ for (bits, method) in [(8, QuantizationMethod::ScaleSymmetric), (4, QuantizationMethod::ScaleZeroPoint)] {
let (input, reference_input) = QuantInput::<bf16>::new(m, 256, 12, weight_group_size, bits, method, 0)
.with_prepared_a_and_reference(
activation_group_size,
@@ -625,7 +620,6 @@ fn a8w4_zero_point_tail_parity(#[case] m: u32) {
#[case::w4_bias(4u32, QuantizationMethod::ScaleBias, 256u32)]
#[case::w4_zp(4u32, QuantizationMethod::ScaleZeroPoint, 256u32)]
#[case::w8_sym(8u32, QuantizationMethod::ScaleSymmetric, 256u32)]
-#[case::w8_bias(8u32, QuantizationMethod::ScaleBias, 256u32)]
fn signed_weights_full_precision_activations_parity_bf16(
#[case] bits: u32,
#[case] method: QuantizationMethod,
diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_gemv_bench.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_gemv_bench.rs
index c6ef41e56..ee9bf49c3 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_gemv_bench.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/quant_gemv_bench.rs
@@ -61,6 +61,6 @@ fn bench_gemv(c: &mut Criterion) {
bench_gemv_typed::<B, bf16>(c, &context, "ZP_BF16_gs32", 32, 4, QuantizationMethod::ScaleZeroPoint);
bench_gemv_typed::<B, bf16>(c, &context, "ScaleBias_BF16_gs64", 64, 4, QuantizationMethod::ScaleBias);
bench_gemv_typed::<B, bf16>(c, &context, "ZP_BF16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);
- bench_gemv_typed::<B, bf16>(c, &context, "ZP_BF16_gs64_8b", 64, 8, QuantizationMethod::ScaleZeroPoint);
+ bench_gemv_typed::<B, bf16>(c, &context, "Sym_BF16_gs64_8b", 64, 8, QuantizationMethod::ScaleSymmetric);
});
}