A high-performance inference engine for AI models
Improve small-M INT8 GEMM scheduling (#764)
CCYeh040adf2fa9ed
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rsindex 31b855317..6b88dab0c 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs@@ -29,6 +29,7 @@ const SIMDGROUP_QUANT_WIDE_N_MIN: u32 = 6144;const SPLIT_K_TARGET_TILES_FP: u32 = 512;const SPLIT_K_TARGET_TILES_A8: u32 = 256;+const SPLIT_K_TARGET_TILES_A8_TILE16X32: u32 = 4 * SPLIT_K_TARGET_TILES_A8;const SPLIT_K_TARGET_TILES_A8_TILE32_W4: u32 = 512;const SPLIT_K_TARGET_TILES_A8_TILE32_W8: u32 = 1024;@@ -48,7 +49,7 @@ pub(super) fn mxu_mn_tile((_, _, 0) => GemmTiling::Tile64x32x256_Simdgroups4x1,(false, 0..=1, _) => GemmTiling::Tile32x64x256_Simdgroups2x2,(false, 3..=4, 2) => GemmTiling::Tile128x128x256_Simdgroups4x4,- (true, 0, _) => GemmTiling::Tile16x128x256_Simdgroups1x4,+ (true, 0, _) => GemmTiling::Tile16x32x256_Simdgroups1x1,(true, 1, _) => GemmTiling::Tile32x64x256_Simdgroups2x2,(true, 4, _) => GemmTiling::Tile128x128x256_Simdgroups4x4,_ => MXU_DEFAULT_TILE,@@ -137,6 +138,7 @@ pub(super) fn split_k_target_tiles(match (is_a_int8, tiling, b_bits) {(true, GemmTiling::Tile32x64x256_Simdgroups2x2, Some(4)) => SPLIT_K_TARGET_TILES_A8_TILE32_W4,(true, GemmTiling::Tile32x64x256_Simdgroups2x2, _) => SPLIT_K_TARGET_TILES_A8_TILE32_W8,+ (true, GemmTiling::Tile16x32x256_Simdgroups1x1, _) => SPLIT_K_TARGET_TILES_A8_TILE16X32,(true, _, _) => SPLIT_K_TARGET_TILES_A8,(false, _, _) => SPLIT_K_TARGET_TILES_FP,}diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rsindex 125aef215..c1a50399d 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs@@ -61,7 +61,7 @@ fn policy_boundaries_are_preserved() {use GemmTiling::*;for (is_a_int8, m, n, expected) in [- (true, 16, 4096, Tile16x128x256_Simdgroups1x4),+ (true, 16, 4096, Tile16x32x256_Simdgroups1x1),(true, 17, 4096, Tile32x64x256_Simdgroups2x2),(true, 64, 63, Tile64x32x256_Simdgroups4x1),(true, 512, 4096, Tile128x128x256_Simdgroups4x4),@@ -122,9 +122,17 @@ fn selection_fallbacks_and_split_k_are_preserved() {large_group.b_group_size = Some(128);assert_eq!(problem(large_group, DataType::BF16).select_plan().tiling, Tile64x64x256_Simdgroups2x2);- let mut a8 = quant(shape(16, 4096, 4096));- a8.a_full_precision = false;- assert_eq!(problem(a8, DataType::BF16).select_plan().split_k, 8);+ for (m, n, k, expected_tiling, expected_split_k) in [+ (16, 4096, 4096, Tile16x32x256_Simdgroups1x1, 8),+ (16, 1024, 4096, Tile16x32x256_Simdgroups1x1, 16),+ (16, 34816, 4096, Tile16x32x256_Simdgroups1x1, 1),+ ] {+ let mut a8 = quant(shape(m, n, k));+ a8.a_full_precision = false;+ let plan = problem(a8, DataType::BF16).select_plan();+ assert_eq!(plan.tiling, expected_tiling);+ assert_eq!(plan.split_k, expected_split_k);+ }let mut biased = quant(shape(16, 4096, 4096));biased.a_full_precision = false;