Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

M5 attention route cleanup + speedup (#973)

CCYeh2c16e96796de
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs
index c922132e4..02cc63fb1 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs
@@ -250,9 +250,8 @@ impl AttentionGemmGrouped {
&self,
mask: MaskKind,
suffix_length: u32,
- kv_length: u32,
) -> bool {
- policy::should_encode(self.head_dim, mask, suffix_length, kv_length)
+ policy::should_encode(self.head_dim, mask, suffix_length)
}
pub fn encode(
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/policy.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/policy.rs
index 4762d687f..c28a61c6f 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/policy.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/policy.rs
@@ -11,7 +11,6 @@ const GEMM_GROUPED_HEAD_DIMS: [u32; 2] = [128, 256];
const GEMM_GROUPED_DECODE_SUFFIX_MIN: u32 = 2;
const GEMM_GROUPED_DECODE_SUFFIX_MAX: u32 = 64;
const GEMM_GROUPED_PREFILL_SUFFIX_MAX: u32 = 1024;
-const GEMM_GROUPED_MIN_KV_LENGTH: u32 = 1024;
pub const MAX_TRIE_SUFFIX: u32 = 64;
const SMALL_SUFFIX_MAX: u32 = 4;
const SMALL_SUFFIX_MAX_SPLITS: u32 = 16;
@@ -36,7 +35,6 @@ pub fn should_encode(
head_dim: u32,
mask: MaskKind,
suffix_length: u32,
- kv_length: u32,
) -> bool {
let decode = (GEMM_GROUPED_DECODE_SUFFIX_MIN..=GEMM_GROUPED_DECODE_SUFFIX_MAX).contains(&suffix_length);
// The grouped prefill path is only tuned for head_dim 256.
@@ -44,9 +42,7 @@ pub fn should_encode(
let causal_prefill = prefill_tuned
&& mask == MaskKind::Causal
&& (GEMM_GROUPED_DECODE_SUFFIX_MAX + 1..=GEMM_GROUPED_PREFILL_SUFFIX_MAX).contains(&suffix_length);
- // Trie (tree-structured) attention is fastest on the grouped kernel at any KV length.
- let enough_kv = mask == MaskKind::Trie || kv_length >= GEMM_GROUPED_MIN_KV_LENGTH;
- enough_kv && (decode || causal_prefill)
+ decode || causal_prefill
}
type MeasuredSplits = (u32, u32, &'static [(u32, u32)]);
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs
index a947192f2..2b8e2b4ca 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs
@@ -51,8 +51,6 @@ impl MaskKind {
}
const SINGLE_PASS_KV_THRESHOLD: u32 = 1_024;
-const D256_SINGLE_PASS_KV_THRESHOLD: u32 = 150_000;
-const D128_SHORT_KV_GEMM_THRESHOLD: u32 = 2_048;
pub struct AttentionMetalKernel {
config: AttentionKernelConfig,
@@ -166,26 +164,9 @@ impl AttentionMetalKernel {
if !is_ring
&& let Some(mask) = mask
&& let Some(grouped) = &self.grouped
+ && grouped.should_encode(mask, suffix_length)
{
- if grouped.should_encode(mask, suffix_length, kv_length) {
- return grouped.encode(mask, input, arguments, command_buffer).map_err(AttentionMetalError::Backend);
- }
-
- // These measured sibling exceptions apply only when the
- // grouped core was built for this layer.
- if self.config.head_dim == 256
- && (1..=16).contains(&suffix_length)
- && kv_length > D256_SINGLE_PASS_KV_THRESHOLD
- {
- return Ok((self.single_pass.encode(input, arguments, command_buffer)?, false));
- }
- if (self.config.head_dim == 256 && (9..=16).contains(&suffix_length))
- || (self.config.head_dim == 128
- && (9..=16).contains(&suffix_length)
- && kv_length > D128_SHORT_KV_GEMM_THRESHOLD)
- {
- return Ok((self.two_pass.encode(input, arguments, command_buffer)?, gate_applied_by_two_pass));
- }
+ return grouped.encode(mask, input, arguments, command_buffer).map_err(AttentionMetalError::Backend);
}
if suffix_length > 8 {
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rs
index a66352667..6524021d4 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rs
@@ -47,20 +47,17 @@ fn measured_and_fallback_boundaries() {
#[uzu_test]
fn should_encode_boundaries() {
- for (head_dim, mask, suffix_length, kv_length, expected) in [
- (256, MaskKind::Causal, 1, 150_001, false),
- (256, MaskKind::Causal, 2, 1_023, false),
- (256, MaskKind::Causal, 2, 1_024, true),
- (128, MaskKind::Trie, 15, 1_024, true),
- (128, MaskKind::Causal, 16, 1_024, true),
- (128, MaskKind::Causal, 32, 1_023, false),
- (256, MaskKind::Causal, 16, 1_023, false),
- (256, MaskKind::Causal, 8, 150_001, true),
- (256, MaskKind::Causal, 1_024, 1_024, true),
- (256, MaskKind::None, 1_024, 1_024, false),
- (256, MaskKind::Trie, 65, 1_024, false),
+ for (head_dim, mask, suffix_length, expected) in [
+ (256, MaskKind::Causal, 1, false),
+ (256, MaskKind::Causal, 2, true),
+ (128, MaskKind::Trie, 64, true),
+ (128, MaskKind::Causal, 65, false),
+ (256, MaskKind::Causal, 1_024, true),
+ (256, MaskKind::Causal, 1_025, false),
+ (256, MaskKind::None, 1_024, false),
+ (256, MaskKind::Trie, 65, false),
] {
- assert_eq!(should_encode(head_dim, mask, suffix_length, kv_length), expected);
+ assert_eq!(should_encode(head_dim, mask, suffix_length), expected);
}
}