A high-performance inference engine for AI models
Split long-cache prefill attention (#875)
CCYeh71d7d1889cec
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rsindex efe870e83..01fd9b0e7 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs@@ -18,7 +18,7 @@ use crate::{};mod policy;-use policy::{MAX_TRIE_SUFFIX, choose_splits};+use policy::{MAX_TRIE_SUFFIX, SplitGeometry, choose_splits};use super::MaskKind;@@ -264,11 +264,15 @@ impl AttentionGemmGrouped {let gpu_core_count = encoder.context().gpu_core_count;let core = self.get_or_create(encoder.context(), mask)?;let num_splits = choose_splits(- core.head_dim,+ SplitGeometry {+ head_dim: core.head_dim,+ num_q_heads: core.num_q_heads,+ num_groups: core.num_groups,+ block_rows: core.block_rows,+ block_k: BLOCK_K,+ },suffix_length,kv_length,- (core.num_q_heads / core.num_groups * suffix_length).div_ceil(core.block_rows) * core.num_groups,- BLOCK_K,gpu_core_count,);core.encode(diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/policy.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/policy.rsindex 762f36e7e..bbb136427 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/policy.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/policy.rs@@ -23,6 +23,7 @@ pub fn is_supported(&& arguments.sliding_window_size.is_none()&& arguments.scale.is_none_or(|scale| scale > 0.0)&& arguments.num_groups > 0+ && arguments.num_q_heads > 0&& arguments.num_q_heads.is_multiple_of(arguments.num_groups)}@@ -57,6 +58,19 @@ const MEASURED_TG_PER_CORE_TENTHS: &[MeasuredSplits] = &[];const TG_PER_CORE_FALLBACK: u32 = 6;const TENTHS_PER_RATIO: u32 = 10;+const PREFILL_SPLIT_KEYS: u32 = 2048;+const PREFILL_SPLIT_MIN_KV: u32 = 16384;+const PREFILL_SPLIT_SCRATCH_BYTES: u64 = 256 << 20;+const PREFILL_MIN_SPLITS: u32 = 2;+const PARTIAL_STATS_PER_ROW: u64 = 2; // max and sum++pub struct SplitGeometry {+ pub head_dim: u32,+ pub num_q_heads: u32,+ pub num_groups: u32,+ pub block_rows: u32,+ pub block_k: u32,+}fn tg_per_core_tenths_for(steps: &[(u32, u32)],@@ -68,16 +82,16 @@ fn tg_per_core_tenths_for(fn splits_for_ratio(tg_per_core_tenths: u32,gpu_core_count: u32,- unsplit_threadgroups: u32,+ threadgroups_per_kv_split: u32,) -> u32 {- (tg_per_core_tenths * gpu_core_count.max(1)).div_ceil(TENTHS_PER_RATIO * unsplit_threadgroups.max(1))+ (tg_per_core_tenths * gpu_core_count.max(1)).div_ceil(TENTHS_PER_RATIO * threadgroups_per_kv_split.max(1))}fn tabled_splits(head_dim: u32,suffix_length: u32,kv_length: u32,- unsplit_threadgroups: u32,+ threadgroups_per_kv_split: u32,gpu_core_count: u32,) -> Option<u32> {if !(MEASURED_SUFFIX_MIN..=MEASURED_SUFFIX_MAX).contains(&suffix_length) {@@ -88,23 +102,42 @@ fn tabled_splits(.filter(|(row_head_dim, _, _)| *row_head_dim == head_dim).min_by_key(|(_, row_suffix, _)| (row_suffix.abs_diff(suffix_length), u32::MAX - row_suffix)).map(|(_, _, steps)| {- splits_for_ratio(tg_per_core_tenths_for(steps, kv_length), gpu_core_count, unsplit_threadgroups)+ splits_for_ratio(tg_per_core_tenths_for(steps, kv_length), gpu_core_count, threadgroups_per_kv_split)})}pub fn choose_splits(- head_dim: u32,+ geometry: SplitGeometry,suffix_length: u32,kv_length: u32,- unsplit_threadgroups: u32,- block_k: u32,gpu_core_count: u32,) -> u32 {- let splits = tabled_splits(head_dim, suffix_length, kv_length, unsplit_threadgroups, gpu_core_count)- .unwrap_or_else(|| {- splits_for_ratio(TENTHS_PER_RATIO * TG_PER_CORE_FALLBACK, gpu_core_count, unsplit_threadgroups)- });- splits.clamp(1, kv_length.div_ceil(block_k).max(1))+ let threadgroups_per_kv_split = (geometry.num_q_heads / geometry.num_groups * suffix_length)+ .div_ceil(geometry.block_rows)+ * geometry.num_groups;+ let max_splits = kv_length.div_ceil(geometry.block_k).max(1);++ if suffix_length > GEMM_GROUPED_DECODE_SUFFIX_MAX && kv_length >= PREFILL_SPLIT_MIN_KV {+ // Each padded row has head_dim partial values, one max, and one sum, all F32.+ let bytes_per_split = threadgroups_per_kv_split as u64+ * geometry.block_rows as u64+ * (geometry.head_dim as u64 + PARTIAL_STATS_PER_ROW)+ * DataType::F32.size_in_bytes() as u64;+ let requested_splits = kv_length.div_ceil(PREFILL_SPLIT_KEYS).min(max_splits);+ let scratch_splits = PREFILL_SPLIT_SCRATCH_BYTES / bytes_per_split;+ let splits = (requested_splits as u64).min(scratch_splits) as u32;+ if splits >= PREFILL_MIN_SPLITS {+ return splits;+ }+ }++ let measured_splits =+ tabled_splits(geometry.head_dim, suffix_length, kv_length, threadgroups_per_kv_split, gpu_core_count);+ let splits = match measured_splits {+ Some(splits) => splits,+ None => splits_for_ratio(TENTHS_PER_RATIO * TG_PER_CORE_FALLBACK, gpu_core_count, threadgroups_per_kv_split),+ };+ splits.clamp(1, max_splits)}#[cfg(test)]diff --git a/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rsindex 33b42c2c5..74c972d9e 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rs@@ -1,6 +1,21 @@use uzu_engine_macros::uzu_test;-use super::{MaskKind, choose_splits, should_encode};+use super::{MaskKind, SplitGeometry, choose_splits, should_encode};++fn geometry(head_dim: u32) -> SplitGeometry {+ let (num_q_heads, block_rows, num_groups) = if head_dim == 128 {+ (32, 64, 8)+ } else {+ (24, 32, 4)+ };+ SplitGeometry {+ head_dim,+ num_q_heads,+ num_groups,+ block_rows,+ block_k: 32,+ }+}fn splits(cores: u32,@@ -8,13 +23,7 @@ fn splits(suffix_length: u32,kv_length: u32,) -> u32 {- let (gqa, rows, groups) = if head_dim == 128 {- (4, 64, 8)- } else {- (6, 32, 4)- };- let threadgroups = (gqa * suffix_length).div_ceil(rows) * groups;- choose_splits(head_dim, suffix_length, kv_length, threadgroups, 32, cores)+ choose_splits(geometry(head_dim), suffix_length, kv_length, cores)}#[uzu_test]@@ -24,10 +33,10 @@ fn measured_and_fallback_boundaries() {(40, 256, 32, 32_768, 10),(40, 128, 64, 32_768, 10),(40, 256, 17, 262_144, 10),- (40, 256, 128, 262_144, 3),+ (40, 256, 128, 16_383, 3),(40, 256, 16, 128, 4),(10, 256, 32, 262_144, 3),- (10, 256, 128, 262_144, 1),+ (10, 256, 128, 16_383, 1),] {assert_eq!(splits(cores, head_dim, suffix, kv), expected);}@@ -47,3 +56,23 @@ fn should_encode_boundaries() {assert_eq!(should_encode(head_dim, mask, suffix_length, kv_length), expected);}}++#[uzu_test]+fn long_prefill_split_selection_boundaries() {+ for (suffix, kv, expected) in [+ (1024, 16_383, 1),+ (64, 61_440, 5),+ (1024, 16_384, 8),+ (256, 61_440, 30),+ (1024, 20_480, 10),+ (1024, 102_400, 10),+ ] {+ assert_eq!(splits(40, 256, suffix, kv), expected, "suffix={suffix}, kv={kv}");+ }++ let many_heads = SplitGeometry {+ num_q_heads: 240,+ ..geometry(256)+ };+ assert_eq!(choose_splits(many_heads, 1024, 61_440, 40), 1, "fewer than two prefill splits fit");+}