Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Improve small-M INT8 GEMM scheduling (#764)

CCYeh040adf2fa9ed
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs
index 31b855317..6b88dab0c 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/policy.rs
@@ -29,6 +29,7 @@ const SIMDGROUP_QUANT_WIDE_N_MIN: u32 = 6144;
const SPLIT_K_TARGET_TILES_FP: u32 = 512;
const SPLIT_K_TARGET_TILES_A8: u32 = 256;
+const SPLIT_K_TARGET_TILES_A8_TILE16X32: u32 = 4 * SPLIT_K_TARGET_TILES_A8;
const SPLIT_K_TARGET_TILES_A8_TILE32_W4: u32 = 512;
const SPLIT_K_TARGET_TILES_A8_TILE32_W8: u32 = 1024;
@@ -48,7 +49,7 @@ pub(super) fn mxu_mn_tile(
(_, _, 0) => GemmTiling::Tile64x32x256_Simdgroups4x1,
(false, 0..=1, _) => GemmTiling::Tile32x64x256_Simdgroups2x2,
(false, 3..=4, 2) => GemmTiling::Tile128x128x256_Simdgroups4x4,
- (true, 0, _) => GemmTiling::Tile16x128x256_Simdgroups1x4,
+ (true, 0, _) => GemmTiling::Tile16x32x256_Simdgroups1x1,
(true, 1, _) => GemmTiling::Tile32x64x256_Simdgroups2x2,
(true, 4, _) => GemmTiling::Tile128x128x256_Simdgroups4x4,
_ => MXU_DEFAULT_TILE,
@@ -137,6 +138,7 @@ pub(super) fn split_k_target_tiles(
match (is_a_int8, tiling, b_bits) {
(true, GemmTiling::Tile32x64x256_Simdgroups2x2, Some(4)) => SPLIT_K_TARGET_TILES_A8_TILE32_W4,
(true, GemmTiling::Tile32x64x256_Simdgroups2x2, _) => SPLIT_K_TARGET_TILES_A8_TILE32_W8,
+ (true, GemmTiling::Tile16x32x256_Simdgroups1x1, _) => SPLIT_K_TARGET_TILES_A8_TILE16X32,
(true, _, _) => SPLIT_K_TARGET_TILES_A8,
(false, _, _) => SPLIT_K_TARGET_TILES_FP,
}
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs
index 125aef215..c1a50399d 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs
@@ -61,7 +61,7 @@ fn policy_boundaries_are_preserved() {
use GemmTiling::*;
for (is_a_int8, m, n, expected) in [
- (true, 16, 4096, Tile16x128x256_Simdgroups1x4),
+ (true, 16, 4096, Tile16x32x256_Simdgroups1x1),
(true, 17, 4096, Tile32x64x256_Simdgroups2x2),
(true, 64, 63, Tile64x32x256_Simdgroups4x1),
(true, 512, 4096, Tile128x128x256_Simdgroups4x4),
@@ -122,9 +122,17 @@ fn selection_fallbacks_and_split_k_are_preserved() {
large_group.b_group_size = Some(128);
assert_eq!(problem(large_group, DataType::BF16).select_plan().tiling, Tile64x64x256_Simdgroups2x2);
- let mut a8 = quant(shape(16, 4096, 4096));
- a8.a_full_precision = false;
- assert_eq!(problem(a8, DataType::BF16).select_plan().split_k, 8);
+ for (m, n, k, expected_tiling, expected_split_k) in [
+ (16, 4096, 4096, Tile16x32x256_Simdgroups1x1, 8),
+ (16, 1024, 4096, Tile16x32x256_Simdgroups1x1, 16),
+ (16, 34816, 4096, Tile16x32x256_Simdgroups1x1, 1),
+ ] {
+ let mut a8 = quant(shape(m, n, k));
+ a8.a_full_precision = false;
+ let plan = problem(a8, DataType::BF16).select_plan();
+ assert_eq!(plan.tiling, expected_tiling);
+ assert_eq!(plan.split_k, expected_split_k);
+ }
let mut biased = quant(shape(16, 4096, 4096));
biased.a_full_precision = false;