Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Route short BF16 suffixes through grouped MXU attention (#879)

CCYeh74f5d52a0a51
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/policy.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/policy.rs
index bbb136427..a982050fc 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/policy.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/policy.rs
@@ -5,7 +5,7 @@ use crate::{
};
const GEMM_GROUPED_HEAD_DIMS: [u32; 2] = [128, 256];
-const GEMM_GROUPED_DECODE_SUFFIX_MIN: u32 = 16;
+const GEMM_GROUPED_DECODE_SUFFIX_MIN: u32 = 2;
const GEMM_GROUPED_DECODE_SUFFIX_MAX: u32 = 64;
const GEMM_GROUPED_PREFILL_SUFFIX_MAX: u32 = 1024;
const GEMM_GROUPED_MIN_KV_LENGTH: u32 = 1024;
@@ -42,7 +42,8 @@ pub fn should_encode(
type MeasuredSplits = (u32, u32, &'static [(u32, u32)]);
-const MEASURED_SUFFIX_MIN: u32 = 16;
+// Reuse S=16 tuning values to choose KV splits for suffixes 2..15.
+const MEASURED_SUFFIX_MIN: u32 = 2;
const MEASURED_SUFFIX_MAX: u32 = 64;
// TODO: validate this table per chip (M1-M5) before changing the policy.
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rs
index 74c972d9e..c486acfdc 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rs
@@ -29,6 +29,9 @@ fn splits(
#[uzu_test]
fn measured_and_fallback_boundaries() {
for (cores, head_dim, suffix, kv, expected) in [
+ (40, 256, 2, 16_384, 18),
+ (40, 256, 8, 16_384, 9),
+ (40, 256, 12, 61_440, 6),
(40, 256, 16, 5_120, 6),
(40, 256, 32, 32_768, 10),
(40, 128, 64, 32_768, 10),
@@ -45,10 +48,14 @@ fn measured_and_fallback_boundaries() {
#[uzu_test]
fn should_encode_boundaries() {
for (head_dim, mask, suffix_length, kv_length, expected) in [
+ (256, MaskKind::Causal, 1, 150_001, false),
+ (256, MaskKind::Causal, 2, 1_023, false),
+ (256, MaskKind::Causal, 2, 1_024, true),
+ (128, MaskKind::Trie, 15, 1_024, true),
(128, MaskKind::Causal, 16, 1_024, true),
(128, MaskKind::Causal, 32, 1_023, false),
(256, MaskKind::Causal, 16, 1_023, false),
- (256, MaskKind::Causal, 8, 150_001, false),
+ (256, MaskKind::Causal, 8, 150_001, true),
(256, MaskKind::Causal, 1_024, 1_024, true),
(256, MaskKind::None, 1_024, 1_024, false),
(256, MaskKind::Trie, 65, 1_024, false),
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs
index 906f655c1..31e9f540e 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs
@@ -633,6 +633,8 @@ fn attention_kernel_matches_cpu() {
for &(head_dim, num_q_heads, num_groups, suffix_length, prefix_length, causal, use_trie) in &[
(512, 8, 8, 9, 0, false, false),
(128, 8, 2, 16, 1024, false, false),
+ (256, 6, 1, 2, 1024, false, false),
+ (256, 6, 1, 15, 2048, true, false),
(256, 6, 1, 32, 1024, true, false),
(256, 6, 1, TRIE_SUFFIX_LENGTH, 1024, true, true),
(512, 8, 8, 1, 0, false, false),