A high-performance inference engine for AI models
M5 attention route cleanup + speedup (#973)
CCYeh2c16e96796de
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rsindex c922132e4..02cc63fb1 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs@@ -250,9 +250,8 @@ impl AttentionGemmGrouped {&self,mask: MaskKind,suffix_length: u32,- kv_length: u32,) -> bool {- policy::should_encode(self.head_dim, mask, suffix_length, kv_length)+ policy::should_encode(self.head_dim, mask, suffix_length)}pub fn encode(diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/policy.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/policy.rsindex 4762d687f..c28a61c6f 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/policy.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/policy.rs@@ -11,7 +11,6 @@ const GEMM_GROUPED_HEAD_DIMS: [u32; 2] = [128, 256];const GEMM_GROUPED_DECODE_SUFFIX_MIN: u32 = 2;const GEMM_GROUPED_DECODE_SUFFIX_MAX: u32 = 64;const GEMM_GROUPED_PREFILL_SUFFIX_MAX: u32 = 1024;-const GEMM_GROUPED_MIN_KV_LENGTH: u32 = 1024;pub const MAX_TRIE_SUFFIX: u32 = 64;const SMALL_SUFFIX_MAX: u32 = 4;const SMALL_SUFFIX_MAX_SPLITS: u32 = 16;@@ -36,7 +35,6 @@ pub fn should_encode(head_dim: u32,mask: MaskKind,suffix_length: u32,- kv_length: u32,) -> bool {let decode = (GEMM_GROUPED_DECODE_SUFFIX_MIN..=GEMM_GROUPED_DECODE_SUFFIX_MAX).contains(&suffix_length);// The grouped prefill path is only tuned for head_dim 256.@@ -44,9 +42,7 @@ pub fn should_encode(let causal_prefill = prefill_tuned&& mask == MaskKind::Causal&& (GEMM_GROUPED_DECODE_SUFFIX_MAX + 1..=GEMM_GROUPED_PREFILL_SUFFIX_MAX).contains(&suffix_length);- // Trie (tree-structured) attention is fastest on the grouped kernel at any KV length.- let enough_kv = mask == MaskKind::Trie || kv_length >= GEMM_GROUPED_MIN_KV_LENGTH;- enough_kv && (decode || causal_prefill)+ decode || causal_prefill}type MeasuredSplits = (u32, u32, &'static [(u32, u32)]);diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rsindex a947192f2..2b8e2b4ca 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs@@ -51,8 +51,6 @@ impl MaskKind {}const SINGLE_PASS_KV_THRESHOLD: u32 = 1_024;-const D256_SINGLE_PASS_KV_THRESHOLD: u32 = 150_000;-const D128_SHORT_KV_GEMM_THRESHOLD: u32 = 2_048;pub struct AttentionMetalKernel {config: AttentionKernelConfig,@@ -166,26 +164,9 @@ impl AttentionMetalKernel {if !is_ring&& let Some(mask) = mask&& let Some(grouped) = &self.grouped+ && grouped.should_encode(mask, suffix_length){- if grouped.should_encode(mask, suffix_length, kv_length) {- return grouped.encode(mask, input, arguments, command_buffer).map_err(AttentionMetalError::Backend);- }-- // These measured sibling exceptions apply only when the- // grouped core was built for this layer.- if self.config.head_dim == 256- && (1..=16).contains(&suffix_length)- && kv_length > D256_SINGLE_PASS_KV_THRESHOLD- {- return Ok((self.single_pass.encode(input, arguments, command_buffer)?, false));- }- if (self.config.head_dim == 256 && (9..=16).contains(&suffix_length))- || (self.config.head_dim == 128- && (9..=16).contains(&suffix_length)- && kv_length > D128_SHORT_KV_GEMM_THRESHOLD)- {- return Ok((self.two_pass.encode(input, arguments, command_buffer)?, gate_applied_by_two_pass));- }+ return grouped.encode(mask, input, arguments, command_buffer).map_err(AttentionMetalError::Backend);}if suffix_length > 8 {diff --git a/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rsindex a66352667..6524021d4 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rs@@ -47,20 +47,17 @@ fn measured_and_fallback_boundaries() {#[uzu_test]fn should_encode_boundaries() {- for (head_dim, mask, suffix_length, kv_length, expected) in [- (256, MaskKind::Causal, 1, 150_001, false),- (256, MaskKind::Causal, 2, 1_023, false),- (256, MaskKind::Causal, 2, 1_024, true),- (128, MaskKind::Trie, 15, 1_024, true),- (128, MaskKind::Causal, 16, 1_024, true),- (128, MaskKind::Causal, 32, 1_023, false),- (256, MaskKind::Causal, 16, 1_023, false),- (256, MaskKind::Causal, 8, 150_001, true),- (256, MaskKind::Causal, 1_024, 1_024, true),- (256, MaskKind::None, 1_024, 1_024, false),- (256, MaskKind::Trie, 65, 1_024, false),+ for (head_dim, mask, suffix_length, expected) in [+ (256, MaskKind::Causal, 1, false),+ (256, MaskKind::Causal, 2, true),+ (128, MaskKind::Trie, 64, true),+ (128, MaskKind::Causal, 65, false),+ (256, MaskKind::Causal, 1_024, true),+ (256, MaskKind::Causal, 1_025, false),+ (256, MaskKind::None, 1_024, false),+ (256, MaskKind::Trie, 65, false),] {- assert_eq!(should_encode(head_dim, mask, suffix_length, kv_length), expected);+ assert_eq!(should_encode(head_dim, mask, suffix_length), expected);}}