A high-performance inference engine for AI models
Route short BF16 suffixes through grouped MXU attention (#879)
CCYeh74f5d52a0a51
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/policy.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/policy.rsindex bbb136427..a982050fc 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/policy.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/policy.rs@@ -5,7 +5,7 @@ use crate::{};const GEMM_GROUPED_HEAD_DIMS: [u32; 2] = [128, 256];-const GEMM_GROUPED_DECODE_SUFFIX_MIN: u32 = 16;+const GEMM_GROUPED_DECODE_SUFFIX_MIN: u32 = 2;const GEMM_GROUPED_DECODE_SUFFIX_MAX: u32 = 64;const GEMM_GROUPED_PREFILL_SUFFIX_MAX: u32 = 1024;const GEMM_GROUPED_MIN_KV_LENGTH: u32 = 1024;@@ -42,7 +42,8 @@ pub fn should_encode(type MeasuredSplits = (u32, u32, &'static [(u32, u32)]);-const MEASURED_SUFFIX_MIN: u32 = 16;+// Reuse S=16 tuning values to choose KV splits for suffixes 2..15.+const MEASURED_SUFFIX_MIN: u32 = 2;const MEASURED_SUFFIX_MAX: u32 = 64;// TODO: validate this table per chip (M1-M5) before changing the policy.diff --git a/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rsindex 74c972d9e..c486acfdc 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rs@@ -29,6 +29,9 @@ fn splits(#[uzu_test]fn measured_and_fallback_boundaries() {for (cores, head_dim, suffix, kv, expected) in [+ (40, 256, 2, 16_384, 18),+ (40, 256, 8, 16_384, 9),+ (40, 256, 12, 61_440, 6),(40, 256, 16, 5_120, 6),(40, 256, 32, 32_768, 10),(40, 128, 64, 32_768, 10),@@ -45,10 +48,14 @@ fn measured_and_fallback_boundaries() {#[uzu_test]fn should_encode_boundaries() {for (head_dim, mask, suffix_length, kv_length, expected) in [+ (256, MaskKind::Causal, 1, 150_001, false),+ (256, MaskKind::Causal, 2, 1_023, false),+ (256, MaskKind::Causal, 2, 1_024, true),+ (128, MaskKind::Trie, 15, 1_024, true),(128, MaskKind::Causal, 16, 1_024, true),(128, MaskKind::Causal, 32, 1_023, false),(256, MaskKind::Causal, 16, 1_023, false),- (256, MaskKind::Causal, 8, 150_001, false),+ (256, MaskKind::Causal, 8, 150_001, true),(256, MaskKind::Causal, 1_024, 1_024, true),(256, MaskKind::None, 1_024, 1_024, false),(256, MaskKind::Trie, 65, 1_024, false),diff --git a/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rsindex 906f655c1..31e9f540e 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs@@ -633,6 +633,8 @@ fn attention_kernel_matches_cpu() {for &(head_dim, num_q_heads, num_groups, suffix_length, prefix_length, causal, use_trie) in &[(512, 8, 8, 9, 0, false, false),(128, 8, 2, 16, 1024, false, false),+ (256, 6, 1, 2, 1024, false, false),+ (256, 6, 1, 15, 2048, true, false),(256, 6, 1, 32, 1024, true, false),(256, 6, 1, TRIE_SUFFIX_LENGTH, 1024, true, true),(512, 8, 8, 1, 0, false, false),