Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Split long-cache prefill attention (#875)

CCYeh71d7d1889cec
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs
index efe870e83..01fd9b0e7 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs
@@ -18,7 +18,7 @@ use crate::{
};
mod policy;
-use policy::{MAX_TRIE_SUFFIX, choose_splits};
+use policy::{MAX_TRIE_SUFFIX, SplitGeometry, choose_splits};
use super::MaskKind;
@@ -264,11 +264,15 @@ impl AttentionGemmGrouped {
let gpu_core_count = encoder.context().gpu_core_count;
let core = self.get_or_create(encoder.context(), mask)?;
let num_splits = choose_splits(
- core.head_dim,
+ SplitGeometry {
+ head_dim: core.head_dim,
+ num_q_heads: core.num_q_heads,
+ num_groups: core.num_groups,
+ block_rows: core.block_rows,
+ block_k: BLOCK_K,
+ },
suffix_length,
kv_length,
- (core.num_q_heads / core.num_groups * suffix_length).div_ceil(core.block_rows) * core.num_groups,
- BLOCK_K,
gpu_core_count,
);
core.encode(
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/policy.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/policy.rs
index 762f36e7e..bbb136427 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/policy.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/policy.rs
@@ -23,6 +23,7 @@ pub fn is_supported(
&& arguments.sliding_window_size.is_none()
&& arguments.scale.is_none_or(|scale| scale > 0.0)
&& arguments.num_groups > 0
+ && arguments.num_q_heads > 0
&& arguments.num_q_heads.is_multiple_of(arguments.num_groups)
}
@@ -57,6 +58,19 @@ const MEASURED_TG_PER_CORE_TENTHS: &[MeasuredSplits] = &[
];
const TG_PER_CORE_FALLBACK: u32 = 6;
const TENTHS_PER_RATIO: u32 = 10;
+const PREFILL_SPLIT_KEYS: u32 = 2048;
+const PREFILL_SPLIT_MIN_KV: u32 = 16384;
+const PREFILL_SPLIT_SCRATCH_BYTES: u64 = 256 << 20;
+const PREFILL_MIN_SPLITS: u32 = 2;
+const PARTIAL_STATS_PER_ROW: u64 = 2; // max and sum
+
+pub struct SplitGeometry {
+ pub head_dim: u32,
+ pub num_q_heads: u32,
+ pub num_groups: u32,
+ pub block_rows: u32,
+ pub block_k: u32,
+}
fn tg_per_core_tenths_for(
steps: &[(u32, u32)],
@@ -68,16 +82,16 @@ fn tg_per_core_tenths_for(
fn splits_for_ratio(
tg_per_core_tenths: u32,
gpu_core_count: u32,
- unsplit_threadgroups: u32,
+ threadgroups_per_kv_split: u32,
) -> u32 {
- (tg_per_core_tenths * gpu_core_count.max(1)).div_ceil(TENTHS_PER_RATIO * unsplit_threadgroups.max(1))
+ (tg_per_core_tenths * gpu_core_count.max(1)).div_ceil(TENTHS_PER_RATIO * threadgroups_per_kv_split.max(1))
}
fn tabled_splits(
head_dim: u32,
suffix_length: u32,
kv_length: u32,
- unsplit_threadgroups: u32,
+ threadgroups_per_kv_split: u32,
gpu_core_count: u32,
) -> Option<u32> {
if !(MEASURED_SUFFIX_MIN..=MEASURED_SUFFIX_MAX).contains(&suffix_length) {
@@ -88,23 +102,42 @@ fn tabled_splits(
.filter(|(row_head_dim, _, _)| *row_head_dim == head_dim)
.min_by_key(|(_, row_suffix, _)| (row_suffix.abs_diff(suffix_length), u32::MAX - row_suffix))
.map(|(_, _, steps)| {
- splits_for_ratio(tg_per_core_tenths_for(steps, kv_length), gpu_core_count, unsplit_threadgroups)
+ splits_for_ratio(tg_per_core_tenths_for(steps, kv_length), gpu_core_count, threadgroups_per_kv_split)
})
}
pub fn choose_splits(
- head_dim: u32,
+ geometry: SplitGeometry,
suffix_length: u32,
kv_length: u32,
- unsplit_threadgroups: u32,
- block_k: u32,
gpu_core_count: u32,
) -> u32 {
- let splits = tabled_splits(head_dim, suffix_length, kv_length, unsplit_threadgroups, gpu_core_count)
- .unwrap_or_else(|| {
- splits_for_ratio(TENTHS_PER_RATIO * TG_PER_CORE_FALLBACK, gpu_core_count, unsplit_threadgroups)
- });
- splits.clamp(1, kv_length.div_ceil(block_k).max(1))
+ let threadgroups_per_kv_split = (geometry.num_q_heads / geometry.num_groups * suffix_length)
+ .div_ceil(geometry.block_rows)
+ * geometry.num_groups;
+ let max_splits = kv_length.div_ceil(geometry.block_k).max(1);
+
+ if suffix_length > GEMM_GROUPED_DECODE_SUFFIX_MAX && kv_length >= PREFILL_SPLIT_MIN_KV {
+ // Each padded row has head_dim partial values, one max, and one sum, all F32.
+ let bytes_per_split = threadgroups_per_kv_split as u64
+ * geometry.block_rows as u64
+ * (geometry.head_dim as u64 + PARTIAL_STATS_PER_ROW)
+ * DataType::F32.size_in_bytes() as u64;
+ let requested_splits = kv_length.div_ceil(PREFILL_SPLIT_KEYS).min(max_splits);
+ let scratch_splits = PREFILL_SPLIT_SCRATCH_BYTES / bytes_per_split;
+ let splits = (requested_splits as u64).min(scratch_splits) as u32;
+ if splits >= PREFILL_MIN_SPLITS {
+ return splits;
+ }
+ }
+
+ let measured_splits =
+ tabled_splits(geometry.head_dim, suffix_length, kv_length, threadgroups_per_kv_split, gpu_core_count);
+ let splits = match measured_splits {
+ Some(splits) => splits,
+ None => splits_for_ratio(TENTHS_PER_RATIO * TG_PER_CORE_FALLBACK, gpu_core_count, threadgroups_per_kv_split),
+ };
+ splits.clamp(1, max_splits)
}
#[cfg(test)]
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rs
index 33b42c2c5..74c972d9e 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rs
@@ -1,6 +1,21 @@
use uzu_engine_macros::uzu_test;
-use super::{MaskKind, choose_splits, should_encode};
+use super::{MaskKind, SplitGeometry, choose_splits, should_encode};
+
+fn geometry(head_dim: u32) -> SplitGeometry {
+ let (num_q_heads, block_rows, num_groups) = if head_dim == 128 {
+ (32, 64, 8)
+ } else {
+ (24, 32, 4)
+ };
+ SplitGeometry {
+ head_dim,
+ num_q_heads,
+ num_groups,
+ block_rows,
+ block_k: 32,
+ }
+}
fn splits(
cores: u32,
@@ -8,13 +23,7 @@ fn splits(
suffix_length: u32,
kv_length: u32,
) -> u32 {
- let (gqa, rows, groups) = if head_dim == 128 {
- (4, 64, 8)
- } else {
- (6, 32, 4)
- };
- let threadgroups = (gqa * suffix_length).div_ceil(rows) * groups;
- choose_splits(head_dim, suffix_length, kv_length, threadgroups, 32, cores)
+ choose_splits(geometry(head_dim), suffix_length, kv_length, cores)
}
#[uzu_test]
@@ -24,10 +33,10 @@ fn measured_and_fallback_boundaries() {
(40, 256, 32, 32_768, 10),
(40, 128, 64, 32_768, 10),
(40, 256, 17, 262_144, 10),
- (40, 256, 128, 262_144, 3),
+ (40, 256, 128, 16_383, 3),
(40, 256, 16, 128, 4),
(10, 256, 32, 262_144, 3),
- (10, 256, 128, 262_144, 1),
+ (10, 256, 128, 16_383, 1),
] {
assert_eq!(splits(cores, head_dim, suffix, kv), expected);
}
@@ -47,3 +56,23 @@ fn should_encode_boundaries() {
assert_eq!(should_encode(head_dim, mask, suffix_length, kv_length), expected);
}
}
+
+#[uzu_test]
+fn long_prefill_split_selection_boundaries() {
+ for (suffix, kv, expected) in [
+ (1024, 16_383, 1),
+ (64, 61_440, 5),
+ (1024, 16_384, 8),
+ (256, 61_440, 30),
+ (1024, 20_480, 10),
+ (1024, 102_400, 10),
+ ] {
+ assert_eq!(splits(40, 256, suffix, kv), expected, "suffix={suffix}, kv={kv}");
+ }
+
+ let many_heads = SplitGeometry {
+ num_q_heads: 240,
+ ..geometry(256)
+ };
+ assert_eq!(choose_splits(many_heads, 1024, 61_440, 40), 1, "fewer than two prefill splits fit");
+}