A high-performance inference engine for AI models
grouped attention for pre M5 chips (#979)
CCYeh2968097ab20c
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/attention.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/attention.metalindex 3fb235c83..a764b3053 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/attention.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/attention.metal@@ -14,19 +14,22 @@ using namespace uzu::matmul;using namespace uzu::attention;using namespace uzu::trie;-// TODO: support more BD and should tune more to replace AttentionGEMM-template <typename T, uint BK, uint BD, uint D_SLICES>+template <typename T, uint BK, uint BD, uint HEAD_DIM_SLICES, bool USE_MXU>VARIANTS(T, bfloat)-VARIANTS(BK, 32)+VARIANTS(BK, 16, 32)VARIANTS(BD, 128, 256)-VARIANTS(D_SLICES, 2, 4)-CONSTRAINT(D_SLICES == BD / 64)+VARIANTS(HEAD_DIM_SLICES, 1, 2, 4, 8)+VARIANTS(USE_MXU, false, true)+CONSTRAINT((BK == 16) == (HEAD_DIM_SLICES == 1))+CONSTRAINT(HEAD_DIM_SLICES != 1 || BD == 128)+CONSTRAINT(!USE_MXU || HEAD_DIM_SLICES == BD / 64)+CONSTRAINT(USE_MXU || BD == 256 || HEAD_DIM_SLICES == 1 || HEAD_DIM_SLICES == BD / 32)KERNEL(AttentionGemmGrouped)(const device T* q,const device T* k,const device T* v,device T* o OPTIONAL(!split_kv),- device float* partials OPTIONAL(split_kv),+ device T* partials OPTIONAL(split_kv),device float* partial_maxs OPTIONAL(split_kv),device float* partial_sums OPTIONAL(split_kv),const device TrieNode* trie OPTIONAL(is_trie),@@ -37,15 +40,17 @@ KERNEL(AttentionGemmGrouped)(const bool split_kv SPECIALIZE,const bool is_causal SPECIALIZE,const bool is_trie SPECIALIZE,- threadgroup float score_exchange[AttentionGemmGroupedLayout<BK, BD, D_SLICES>::SCORE_EXCHANGE_SIZE],+ threadgroup float scratch[AttentionGemmGroupedLayout<BK, BD, HEAD_DIM_SLICES, USE_MXU>::THREADGROUP_SCRATCH_FLOATS],const ThreadContext thread_context,const uint tile_split_flat GROUPS(query_tiles * num_splits),const uint kv_head_idx GROUPS(kv_heads),const uint lid THREADS(ATTENTION_GQA_SIMDGROUPS * METAL_SIMD_SIZE)) {- using Layout = AttentionGemmGroupedLayout<BK, BD, D_SLICES>;+ using Layout = AttentionGemmGroupedLayout<BK, BD, HEAD_DIM_SLICES, USE_MXU>;using Ops = typename Layout::Ops;using AccumType = float;+ constexpr bool GROUP_FOUR_Q_HEADS = HEAD_DIM_SLICES == 1;+ using InputType = metal::conditional_t<USE_MXU || GROUP_FOUR_Q_HEADS, T, AccumType>;constexpr int TILE_ROWS = Layout::TILE_ROWS;constexpr int SLICE_COLS = Layout::SLICE_COLS;@@ -54,30 +59,41 @@ KERNEL(AttentionGemmGrouped)(constexpr int KEY_COL_FRAGMENTS = int(BK) / Ops::FRAGMENT_COLS;constexpr int HEAD_DIM_FRAGMENTS_PER_SLICE = SLICE_COLS / Ops::FRAGMENT_ROWS;constexpr int VALUE_COL_FRAGMENTS = SLICE_COLS / Ops::FRAGMENT_COLS;+ constexpr int PV_COL_FRAGMENTS = VALUE_COL_FRAGMENTS < 8 ? VALUE_COL_FRAGMENTS : 8;+ constexpr int PV_CHUNKS = VALUE_COL_FRAGMENTS / PV_COL_FRAGMENTS;constexpr int ROWS_PER_LANE = Ops::THREAD_ELEMENT_ROWS;static_assert(Layout::SIMDGROUPS == 8, "requires eight simdgroups");static_assert(KEY_COL_FRAGMENTS % 2 == 0, "QK fragments must be even");static_assert(VALUE_COL_FRAGMENTS % 2 == 0, "PV fragments must be even");+ static_assert(VALUE_COL_FRAGMENTS % PV_COL_FRAGMENTS == 0, "PV fragments must divide into chunks");- using QueryFragment = OperandFragment<T, 1, 1, Ops>;- using KeyFragment = OperandFragment<T, 1, KEY_COL_FRAGMENTS, Ops, ReadTranspose>;- using ValueFragment = OperandFragment<T, KEY_COL_FRAGMENTS, VALUE_COL_FRAGMENTS, Ops>;+ using QueryFragment = OperandFragment<InputType, 1, 1, Ops>;+ using KeyFragment = OperandFragment<InputType, 1, KEY_COL_FRAGMENTS, Ops, ReadTranspose>;+ using ValueFragment = OperandFragment<InputType, KEY_COL_FRAGMENTS, PV_COL_FRAGMENTS, Ops>;using ScoreFragment = Fragment<AccumType, 1, KEY_COL_FRAGMENTS, Ops>;using OutputFragment = Fragment<AccumType, 1, VALUE_COL_FRAGMENTS, Ops>;+ using PvOutputFragment = Fragment<AccumType, 1, PV_COL_FRAGMENTS, Ops>;+ using KvBlockStorage = metal::conditional_t<GROUP_FOUR_Q_HEADS, ThreadgroupBlockStorage, DeviceBlockStorage>;+ using KvBlockSource =+ BlockSource<T, BK, BD, Layout::SHARED_KV_ROW_STRIDE, Layout::SIMDGROUPS * METAL_SIMD_SIZE, KvBlockStorage>;const ushort lane = ushort(thread_context.simd_lane_id);const ushort simdgroup_index = ushort(thread_context.simdgroup_index);- const ushort band = simdgroup_index / ushort(D_SLICES);- const ushort slice = simdgroup_index % ushort(D_SLICES);+ const ushort band = simdgroup_index / ushort(HEAD_DIM_SLICES);+ const ushort slice = simdgroup_index % ushort(HEAD_DIM_SLICES);const uint query_tile_idx = split_kv ? (tile_split_flat % query_tiles) : tile_split_flat;const uint split_idx = split_kv ? (tile_split_flat / query_tiles) : 0u;const uint grouped_rows = params.gqa_factor * params.q_len;- const uint row_base = query_tile_idx * uint(BLOCK_ROWS) + uint(band) * uint(TILE_ROWS);- const uint head_in_group = row_base / params.q_len;- const uint query_base = row_base - head_in_group * params.q_len;+ const uint head_in_group = GROUP_FOUR_Q_HEADS+ ? uint(band / 2)+ : (query_tile_idx * uint(BLOCK_ROWS) + uint(band) * uint(TILE_ROWS)) / params.q_len;+ const uint query_base = GROUP_FOUR_Q_HEADS+ ? query_tile_idx * uint(2 * TILE_ROWS) + uint(band % 2) * uint(TILE_ROWS)+ : (query_tile_idx * uint(BLOCK_ROWS) + uint(band) * uint(TILE_ROWS)) % params.q_len;+ const uint row_base = head_in_group * params.q_len + query_base;const uint q_head_idx = kv_head_idx * params.gqa_factor + head_in_group;const uint query_row_stride = params.q_strides[2];@@ -104,6 +120,12 @@ KERNEL(AttentionGemmGrouped)(lane,query_source.advanced(head_dim_fragment_idx * Ops::FRAGMENT_ROWS));+ if constexpr (GROUP_FOUR_Q_HEADS) {+ const InputType query_scale = static_cast<InputType>(params.scale * M_LOG2E_F);+ query_frags[head_dim_fragment_idx].map_coords(lane, [&](short, short, InputType value) {+ return value * query_scale;+ });+ }}}@@ -155,8 +177,9 @@ KERNEL(AttentionGemmGrouped)(}}- threadgroup float* exchange_slot = score_exchange + simdgroup_index * SCORE_TILE_ELEMENTS;- const ushort band_exchange_base = band * ushort(D_SLICES);+ threadgroup float* exchange_slot = scratch + simdgroup_index * SCORE_TILE_ELEMENTS;+ threadgroup T* kv_shared = reinterpret_cast<threadgroup T*>(scratch);+ const ushort band_exchange_base = band * ushort(HEAD_DIM_SLICES);const uint key_block_stride = uint(BK) * key_row_stride;const uint value_block_stride = uint(BK) * value_row_stride;@@ -166,9 +189,11 @@ KERNEL(AttentionGemmGrouped)(if (is_causal) {const uint tile_first_row = query_tile_idx * uint(BLOCK_ROWS);const uint tile_last_row = min(tile_first_row + uint(BLOCK_ROWS) - 1u, grouped_rows - 1u);- const uint max_query_index = (tile_first_row / params.q_len == tile_last_row / params.q_len)- ? (tile_last_row % params.q_len)- : (params.q_len - 1u);+ const uint max_query_index =+ GROUP_FOUR_Q_HEADS+ ? min((query_tile_idx + 1u) * uint(2 * TILE_ROWS) - 1u, params.q_len - 1u)+ : ((tile_first_row / params.q_len == tile_last_row / params.q_len) ? (tile_last_row % params.q_len)+ : (params.q_len - 1u));const uint causal_blocks = (uint(prefix_length) + max_query_index + uint(BK)) / uint(BK);block_end = min(block_end, causal_blocks);}@@ -181,8 +206,7 @@ KERNEL(AttentionGemmGrouped)(k += size_t(block_begin) * key_block_stride;v += size_t(block_begin) * value_block_stride;- const AccumType score_scale = AccumType(params.scale) * AccumType(M_LOG2E_F);- // Masked scores are multiplied by score_scale below, so pre-divide this value.+ const AccumType score_scale = GROUP_FOUR_Q_HEADS ? AccumType(1) : AccumType(params.scale) * AccumType(M_LOG2E_F);const AccumType masked_score_pre_scale = (static_cast<AccumType>(-1e9f) * M_LOG2E_F) / score_scale;const uint first_key = block_begin * uint(BK);@@ -192,34 +216,35 @@ KERNEL(AttentionGemmGrouped)(ScoreFragment score;score.clear();{- auto key_source = fragment_source(k + size_t(kb) * key_block_stride, int(key_row_stride));- if constexpr (IS_TAIL) {- key_source = key_source.bounded(short(tail_keys), short(Ops::FRAGMENT_COLS));- }+ KvBlockSource key_source(+ k + size_t(kb) * key_block_stride,+ int(key_row_stride),+ kv_shared,+ IS_TAIL ? short(tail_keys) : short(BK),+ thread_context+ );+ key_source.prepare();METAL_PRAGMA_UNROLLfor (short head_dim_fragment_idx = 0; head_dim_fragment_idx < HEAD_DIM_FRAGMENTS_PER_SLICE;++head_dim_fragment_idx) {KeyFragment key_fragment;- key_fragment.load_from(lane, key_source.advanced(head_dim_fragment_idx * Ops::FRAGMENT_ROWS));+ key_source.load(key_fragment, head_dim_fragment_idx * Ops::FRAGMENT_ROWS);simdgroup_barrier(mem_flags::mem_none);fragment_mma(score, query_frags[head_dim_fragment_idx], key_fragment);}}- if constexpr (D_SLICES > 1) {+ if constexpr (HEAD_DIM_SLICES > 1) {threadgroup_barrier(mem_flags::mem_threadgroup);score.store(lane, exchange_slot, int(BK));threadgroup_barrier(mem_flags::mem_threadgroup);thread AccumType* score_elements = score.elements();METAL_PRAGMA_UNROLL- for (ushort j = 1; j < ushort(D_SLICES); ++j) {- const ushort other = (slice + j) % ushort(D_SLICES);+ for (ushort j = 1; j < ushort(HEAD_DIM_SLICES); ++j) {+ const ushort other = (slice + j) % ushort(HEAD_DIM_SLICES);ScoreFragment partial;- partial.load_from(- lane,- fragment_source(score_exchange + (band_exchange_base + other) * SCORE_TILE_ELEMENTS, int(BK))- );+ partial.load_from(lane, fragment_source(scratch + (band_exchange_base + other) * SCORE_TILE_ELEMENTS, int(BK)));thread AccumType* partial_elements = partial.elements();METAL_PRAGMA_UNROLLfor (ushort e = 0; e < ScoreFragment::ELEMENTS_PER_FRAGMENT; ++e) {@@ -276,14 +301,33 @@ KERNEL(AttentionGemmGrouped)(output.map_rows(rescale, [](AccumType value, AccumType f) { return value * f; });{- auto value_source = fragment_source(v + size_t(kb) * value_block_stride, int(value_row_stride));- if constexpr (IS_TAIL) {- value_source = value_source.bounded(short(tail_keys), short(SLICE_COLS));+ KvBlockSource value_block(+ v + size_t(kb) * value_block_stride,+ int(value_row_stride),+ kv_shared,+ IS_TAIL ? short(tail_keys) : short(BK),+ thread_context+ );+ value_block.prepare();+ METAL_PRAGMA_UNROLL+ for (ushort chunk = 0; chunk < PV_CHUNKS; ++chunk) {+ const ushort first_col = chunk * PV_COL_FRAGMENTS;+ const int chunk_offset = first_col * Ops::FRAGMENT_COLS;+ ValueFragment value_fragment;+ value_block.load(value_fragment, chunk_offset);+ simdgroup_barrier(mem_flags::mem_none);++ PvOutputFragment chunk_output;+ METAL_PRAGMA_UNROLL+ for (ushort col = 0; col < PV_COL_FRAGMENTS; ++col) {+ chunk_output.fragment_at(0, col) = output.fragment_at(0, first_col + col);+ }+ fragment_mma(chunk_output, score, value_fragment);+ METAL_PRAGMA_UNROLL+ for (ushort col = 0; col < PV_COL_FRAGMENTS; ++col) {+ output.fragment_at(0, first_col + col) = chunk_output.fragment_at(0, col);+ }}- ValueFragment value_fragment;- value_fragment.load_from(lane, value_source);- simdgroup_barrier(mem_flags::mem_none);- fragment_mma(output, score, value_fragment);}};@@ -295,14 +339,15 @@ KERNEL(AttentionGemmGrouped)(}if (split_kv) {- const uint tile_index = (split_idx * kv_heads + kv_head_idx) * query_tiles + query_tile_idx;- device float* partial_tile = partials + size_t(tile_index) * BLOCK_ROWS * BD + size_t(band) * TILE_ROWS * BD;+ const uint split_head_row = (split_idx * kv_heads + kv_head_idx) * query_tiles * uint(BLOCK_ROWS);+ const uint partial_row = split_head_row + row_base;+ device T* partial_tile = partials + size_t(partial_row) * BD;output.store(lane, partial_tile + slice_offset, int(BD));if (slice == 0) {const short2 position = OutputFragment::get_position(lane);if (position.x == 0) {- const uint row_base_in_tile = uint(tile_index) * uint(BLOCK_ROWS) + uint(band) * uint(TILE_ROWS);+ const uint row_base_in_tile = partial_row;METAL_PRAGMA_UNROLLfor (int r = 0; r < ROWS_PER_LANE; ++r) {const uint row = row_base_in_tile + uint(position.y + r * Ops::THREAD_ELEMENT_ROW_STRIDE);diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/combine.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/combine.metalindex 1a70e2f0b..2aedb9757 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/combine.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/combine.metal@@ -10,7 +10,7 @@ template <typename T, uint BD>VARIANTS(T, bfloat)VARIANTS(BD, 128, 256)KERNEL(AttentionGemmGroupedCombine)(- const device float* partials,+ const device T* partials,const device float* partial_maxs,const device float* partial_sums,device T* o,@@ -33,7 +33,7 @@ KERNEL(AttentionGemmGroupedCombine)(const uint q_head_idx = kv_head_idx * params.gqa_factor + head_in_group;const device float* row_maxs = partial_maxs + row;const device float* row_sums = partial_sums + row;- const device float* row_partials = partials + row * BD + column;+ const device T* row_partials = partials + row * BD + column;float numerator = 0.0f;float denominator = 0.0f;@@ -52,7 +52,7 @@ KERNEL(AttentionGemmGroupedCombine)(const float rescale = fast::exp2(row_maxs[s * split_rows] - global_max);const float weight = rescale * split_sum;denominator += weight;- numerator += rescale * row_partials[size_t(s) * split_rows * BD];+ numerator += rescale * float(row_partials[size_t(s) * split_rows * BD]);}}const float mean = denominator > 0.0f ? numerator / denominator : 0.0f;diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/layout.h b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/layout.hindex a9d312840..26e3feb95 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/layout.h+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/layout.h@@ -4,20 +4,23 @@#include "../../common/defines.h"#include "../../common/dsl.h"#include "../../matmul/common/mxu_fragment/ops.h"+#include "../../matmul/common/simdgroup_fragment_ops.h"using namespace metal;#define ATTENTION_GQA_SIMDGROUPS 8-template <uint BK, uint BD, uint D_SLICES>+template <uint BK, uint BD, uint HEAD_DIM_SLICES, bool USE_MXU = true>struct AttentionGemmGroupedLayout {- using Ops = uzu::matmul::MxuFragmentOps<>;+ using Ops = metal::conditional_t<USE_MXU, uzu::matmul::MxuFragmentOps<>, uzu::matmul::SimdgroupFragmentOps>;UZU_CONST int SIMDGROUPS = ATTENTION_GQA_SIMDGROUPS;UZU_CONST int TILE_ROWS = Ops::FRAGMENT_ROWS;- UZU_CONST int SLICE_COLS = int(BD) / int(D_SLICES);- UZU_CONST int BANDS = SIMDGROUPS / int(D_SLICES);+ UZU_CONST int SLICE_COLS = int(BD) / int(HEAD_DIM_SLICES);+ UZU_CONST int BANDS = SIMDGROUPS / int(HEAD_DIM_SLICES);UZU_CONST int BLOCK_ROWS = BANDS * TILE_ROWS;UZU_CONST int SCORE_TILE_ELEMENTS = TILE_ROWS * int(BK);- UZU_CONST int SCORE_EXCHANGE_SIZE = SIMDGROUPS * SCORE_TILE_ELEMENTS;+ UZU_CONST int SHARED_KV_ROW_STRIDE = int(BD) + 8;+ UZU_CONST int THREADGROUP_SCRATCH_FLOATS =+ HEAD_DIM_SLICES == 1 ? int(BK) * SHARED_KV_ROW_STRIDE / 2 : SIMDGROUPS * SCORE_TILE_ELEMENTS;};diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rsindex 02cc63fb1..ba8818d3e 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs@@ -1,3 +1,5 @@+use std::collections::{HashMap, hash_map::Entry};+use parking_lot::{MappedMutexGuard, Mutex, MutexGuard};use super::FullPrecisionInput;@@ -26,11 +28,12 @@ use super::MaskKind;const SIMDGROUPS_PER_THREADGROUP: u32 = 8;const SLICE_COLS: u32 = 64;-const TILE_ROWS: u32 = 16;+const MXU_TILE_ROWS: u32 = 16;+const SIMDGROUP_TILE_ROWS: u32 = 8;const BLOCK_K: u32 = 32;pub fn query_rows_per_threadgroup(head_dim: u32) -> u32 {- SIMDGROUPS_PER_THREADGROUP / (head_dim / SLICE_COLS) * TILE_ROWS+ SIMDGROUPS_PER_THREADGROUP / (head_dim / SLICE_COLS) * MXU_TILE_ROWS}struct AttentionGemmGroupedMetal {@@ -43,46 +46,59 @@ struct AttentionGemmGroupedMetal {scale: Option<f32>,gate_row_stride: Option<u32>,query_rows_per_threadgroup: u32,+ block_k: u32,mask: MaskKind,}impl AttentionGemmGroupedMetal {fn new_fixed(context: &MetalContext,- head_dim: u32,- num_groups: u32,- num_q_heads: u32,- scale: Option<f32>,- gate_row_stride: Option<u32>,+ config: &AttentionKernelConfig,+ head_dim_slices: u32,mask: MaskKind,) -> Result<Self, MetalError> {- assert!(matches!(head_dim, 128 | 256), "head_dim must be 128 or 256");- assert_eq!(num_q_heads % num_groups, 0, "num_q_heads must be divisible by num_groups");- let slices = head_dim / SLICE_COLS;+ let tile_rows = if context.supports_mxu {+ MXU_TILE_ROWS+ } else {+ SIMDGROUP_TILE_ROWS+ };+ let block_k = if head_dim_slices == 1 {+ 16+ } else {+ BLOCK_K+ };+ assert!(matches!(config.head_dim, 128 | 256), "head_dim must be 128 or 256");+ assert_eq!(config.num_q_heads % config.num_groups, 0, "num_q_heads must be divisible by num_groups");let new_kernel = |split| {AttentionGemmGroupedMetalKernel::new(context,DataType::BF16,- BLOCK_K,- head_dim,- slices,+ block_k,+ config.head_dim,+ head_dim_slices,+ context.supports_mxu,split,mask.is_causal(),mask.is_trie(),)};- let combine =- AttentionGemmGroupedCombineMetalKernel::new(context, DataType::BF16, head_dim, gate_row_stride.is_some())?;+ let combine = AttentionGemmGroupedCombineMetalKernel::new(+ context,+ DataType::BF16,+ config.head_dim,+ config.gate_row_stride.is_some(),+ )?;Ok(Self {kernel: new_kernel(false)?,split_kernel: new_kernel(true)?,combine,- head_dim,- num_groups,- num_q_heads,- scale,- gate_row_stride,- query_rows_per_threadgroup: query_rows_per_threadgroup(head_dim),+ head_dim: config.head_dim,+ num_groups: config.num_groups,+ num_q_heads: config.num_q_heads,+ scale: config.scale,+ gate_row_stride: config.gate_row_stride,+ query_rows_per_threadgroup: SIMDGROUPS_PER_THREADGROUP / head_dim_slices * tile_rows,+ block_k,mask,})}@@ -125,7 +141,7 @@ impl AttentionGemmGroupedMetal {}let partial_rows = self.num_groups * query_tiles * num_splits * self.query_rows_per_threadgroup;- let mut partials = command_buffer.allocate_scratch_for_shape(&[partial_rows, self.head_dim], DataType::F32)?;+ let mut partials = command_buffer.allocate_scratch_for_shape(&[partial_rows, self.head_dim], DataType::BF16)?;let mut partial_maxs = command_buffer.allocate_scratch_for_shape(&[partial_rows], DataType::F32)?;let mut partial_sums = command_buffer.allocate_scratch_for_shape(&[partial_rows], DataType::F32)?;@@ -184,21 +200,16 @@ impl AttentionGemmGroupedMetal {q_off: kv_length - suffix_length,nq_aligned: 0,q_rem: 0,- nk: kv_length.div_ceil(BLOCK_K),- nk_aligned: kv_length / BLOCK_K,- k_rem: kv_length % BLOCK_K,+ nk: kv_length.div_ceil(self.block_k),+ nk_aligned: kv_length / self.block_k,+ k_rem: kv_length % self.block_k,}}}pub struct AttentionGemmGrouped {- non_trie: Mutex<Option<AttentionGemmGroupedMetal>>,- trie: Mutex<Option<AttentionGemmGroupedMetal>>,- head_dim: u32,- num_groups: u32,- num_q_heads: u32,- scale: Option<f32>,- gate_row_stride: Option<u32>,+ variants: Mutex<HashMap<(MaskKind, u32), AttentionGemmGroupedMetal>>,+ config: AttentionKernelConfig,}impl AttentionGemmGrouped {@@ -206,43 +217,26 @@ impl AttentionGemmGrouped {&self,context: &MetalContext,mask: MaskKind,+ head_dim_slices: u32,) -> Result<MappedMutexGuard<'_, AttentionGemmGroupedMetal>, MetalError> {- let cache = if mask.is_trie() {- &self.trie- } else {- &self.non_trie- };- let mut cache = cache.lock();- if cache.is_none() {- *cache = Some(AttentionGemmGroupedMetal::new_fixed(- context,- self.head_dim,- self.num_groups,- self.num_q_heads,- self.scale,- self.gate_row_stride,- mask,- )?);+ let key = (mask, head_dim_slices);+ let mut variants = self.variants.lock();+ if let Entry::Vacant(slot) = variants.entry(key) {+ slot.insert(AttentionGemmGroupedMetal::new_fixed(context, &self.config, head_dim_slices, mask)?);}- Ok(MutexGuard::map(cache, |cache| cache.as_mut().expect("attention pipeline was just initialized")))+ Ok(MutexGuard::map(variants, |variants| {+ variants.get_mut(&key).expect("attention variant was just initialized")+ }))}- pub fn is_supported(- config: &AttentionKernelConfig,- context: &MetalContext,- ) -> bool {- policy::is_supported(config, context)+ pub fn is_supported(config: &AttentionKernelConfig) -> bool {+ policy::is_supported(config)}pub fn new(config: &AttentionKernelConfig) -> Self {Self {- non_trie: Mutex::new(None),- trie: Mutex::new(None),- head_dim: config.head_dim,- num_groups: config.num_groups,- num_q_heads: config.num_q_heads,- scale: config.scale,- gate_row_stride: config.gate_row_stride,+ variants: Mutex::new(HashMap::new()),+ config: *config,}}@@ -250,8 +244,11 @@ impl AttentionGemmGrouped {&self,mask: MaskKind,suffix_length: u32,+ kv_length: u32,+ use_mxu: bool,) -> bool {- policy::should_encode(self.head_dim, mask, suffix_length)+ let slices = policy::select_head_dim_slices(self.config.head_dim, mask, suffix_length, kv_length, use_mxu);+ slices.is_some_and(|slices| slices != 1 || self.config.num_q_heads == self.config.num_groups * 4)}pub fn encode(@@ -270,22 +267,30 @@ impl AttentionGemmGrouped {assert!(arguments.sinks.is_none(), "attention sinks are unsupported");let kv_length = arguments.cache.prefix_len() + suffix_length;- let mut output = command_buffer- .allocate_scratch_for_shape(&[suffix_length, self.num_q_heads, self.head_dim], DataType::BF16)?;- let gpu_core_count = command_buffer.context().gpu_core_count;- let core = self.get_or_create(command_buffer.context(), mask)?;- let num_splits = choose_splits(- SplitGeometry {- head_dim: core.head_dim,- num_q_heads: core.num_q_heads,- num_groups: core.num_groups,- query_rows_per_threadgroup: core.query_rows_per_threadgroup,- block_k: BLOCK_K,- },+ let head_dim_slices = policy::select_head_dim_slices(+ self.config.head_dim,+ mask,suffix_length,kv_length,- gpu_core_count,- );+ command_buffer.context().supports_mxu,+ )+ .expect("grouped attention request must be supported");+ assert!(head_dim_slices != 1 || self.config.num_q_heads == self.config.num_groups * 4);+ let mut output = command_buffer.allocate_scratch_for_shape(+ &[suffix_length, self.config.num_q_heads, self.config.head_dim],+ DataType::BF16,+ )?;+ let gpu_core_count = command_buffer.context().gpu_core_count;+ let core = self.get_or_create(command_buffer.context(), mask, head_dim_slices)?;+ let geometry = SplitGeometry {+ head_dim: core.head_dim,+ num_q_heads: core.num_q_heads,+ num_groups: core.num_groups,+ query_rows_per_threadgroup: core.query_rows_per_threadgroup,+ block_k: core.block_k,+ };+ let num_splits =+ choose_splits(geometry, suffix_length, kv_length, gpu_core_count, command_buffer.context().supports_mxu);let gate_applied = core.encode(input.queries,input.keys,diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/policy.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/policy.rsindex c28a61c6f..8a0395ce0 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/policy.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/policy.rs@@ -1,27 +1,17 @@use super::MaskKind;use crate::{- backends::{- common::{gpu_types::kv_layout, kernel::AttentionKernelConfig},- metal::context::MetalContext,- },+ backends::common::{gpu_types::kv_layout, kernel::AttentionKernelConfig},data_type::DataType,};-const GEMM_GROUPED_HEAD_DIMS: [u32; 2] = [128, 256];-const GEMM_GROUPED_DECODE_SUFFIX_MIN: u32 = 2;-const GEMM_GROUPED_DECODE_SUFFIX_MAX: u32 = 64;-const GEMM_GROUPED_PREFILL_SUFFIX_MAX: u32 = 1024;pub const MAX_TRIE_SUFFIX: u32 = 64;-const SMALL_SUFFIX_MAX: u32 = 4;-const SMALL_SUFFIX_MAX_SPLITS: u32 = 16;--pub fn is_supported(- arguments: &AttentionKernelConfig,- context: &MetalContext,-) -> bool {- GEMM_GROUPED_HEAD_DIMS.contains(&arguments.head_dim)++const MXU_DECODE_MAX_SUFFIX: u32 = 64;+const SHORT_SUFFIX_MAX: u32 = 4;++pub fn is_supported(arguments: &AttentionKernelConfig) -> bool {+ matches!(arguments.head_dim, 128 | 256)&& arguments.data_type == DataType::BF16- && context.supports_mxu&& !arguments.has_sinks&& !arguments.is_kv_cache_ring&& arguments.sliding_window_size.is_none()@@ -31,29 +21,34 @@ pub fn is_supported(&& arguments.num_q_heads.is_multiple_of(arguments.num_groups)}-pub fn should_encode(+pub fn select_head_dim_slices(head_dim: u32,mask: MaskKind,suffix_length: u32,-) -> bool {- let decode = (GEMM_GROUPED_DECODE_SUFFIX_MIN..=GEMM_GROUPED_DECODE_SUFFIX_MAX).contains(&suffix_length);- // The grouped prefill path is only tuned for head_dim 256.- let prefill_tuned = head_dim == 256;- let causal_prefill = prefill_tuned- && mask == MaskKind::Causal- && (GEMM_GROUPED_DECODE_SUFFIX_MAX + 1..=GEMM_GROUPED_PREFILL_SUFFIX_MAX).contains(&suffix_length);- decode || causal_prefill+ kv_length: u32,+ use_mxu: bool,+) -> Option<u32> {+ const SIMDGROUP_DECODE_MIN_KV_LENGTH: u32 = 16_384;+ const SIMDGROUP_PREFILL_MIN_KV_LENGTH: u32 = 61_440;++ match (use_mxu, mask, head_dim, suffix_length) {+ (true, _, _, 2..=64) => Some(head_dim / kv_layout::SLICE_DIMS),+ (true, MaskKind::Causal, 256, 65..=1024) => Some(4),+ (false, MaskKind::Causal, 256, 1 | 2 | 4) if kv_length >= SIMDGROUP_DECODE_MIN_KV_LENGTH => {+ Some(8 / suffix_length)+ },+ (false, MaskKind::Causal, 128, 1 | 2 | 4) if kv_length >= SIMDGROUP_DECODE_MIN_KV_LENGTH => Some(4),+ (false, MaskKind::Causal, 128, 512 | 1024) if kv_length >= 30_720 => Some(1),+ (false, MaskKind::Causal, 256, 512 | 1024) if kv_length >= SIMDGROUP_PREFILL_MIN_KV_LENGTH => Some(2),+ _ => None,+ }}type MeasuredSplits = (u32, u32, &'static [(u32, u32)]);-// Reuse S=16 tuning values to choose KV splits for suffixes 2..15.const MEASURED_SUFFIX_MIN: u32 = 2;const MEASURED_SUFFIX_MAX: u32 = 64;-// TODO: validate this table per chip (M1-M5) before changing the policy.-// TODO: add a simdgroup/non-MXU implementation before widening availability.-// TODO: measure the D128/D256 crossover before widening suffix ranges.const MEASURED_TG_PER_CORE_TENTHS: &[MeasuredSplits] = &[(256, 16, &[(0, 24), (5120, 18), (131072, 39)]),(256, 32, &[(0, 24), (5120, 18), (32768, 60)]),@@ -68,7 +63,12 @@ const PREFILL_SPLIT_KEYS: u32 = 2048;const PREFILL_SPLIT_MIN_KV: u32 = 16384;const PREFILL_SPLIT_SCRATCH_BYTES: u64 = 256 << 20;const PREFILL_MIN_SPLITS: u32 = 2;-const PARTIAL_STATS_PER_ROW: u64 = 2; // max and sum+const PARTIAL_STATS_PER_ROW: u64 = 2;++fn partial_row_bytes(head_dim: u32) -> u64 {+ head_dim as u64 * DataType::BF16.size_in_bytes() as u64+ + PARTIAL_STATS_PER_ROW * DataType::F32.size_in_bytes() as u64+}pub struct SplitGeometry {pub head_dim: u32,@@ -117,20 +117,34 @@ pub fn choose_splits(suffix_length: u32,kv_length: u32,gpu_core_count: u32,+ use_mxu: bool,) -> u32 {let threadgroups_per_kv_split = (geometry.num_q_heads / geometry.num_groups * suffix_length).div_ceil(geometry.query_rows_per_threadgroup)* geometry.num_groups;let max_splits = kv_length.div_ceil(geometry.block_k).max(1);- if suffix_length > GEMM_GROUPED_DECODE_SUFFIX_MAX && kv_length >= PREFILL_SPLIT_MIN_KV {- // Each padded row has head_dim partial values, one max, and one sum, all F32.- let bytes_per_split = threadgroups_per_kv_split as u64+ if !use_mxu {+ const MAX_SPLITS: u32 = 16;+ let threadgroups_per_split = threadgroups_per_kv_split.max(1);+ let splits = if suffix_length > SHORT_SUFFIX_MAX {+ let bytes_per_split = threadgroups_per_split as u64+ * geometry.query_rows_per_threadgroup as u64+ * partial_row_bytes(geometry.head_dim);+ (PREFILL_SPLIT_SCRATCH_BYTES / bytes_per_split) as u32+ } else {+ (gpu_core_count.max(1) * 4).div_ceil(threadgroups_per_split)+ };+ return splits.clamp(1, max_splits.min(MAX_SPLITS));+ }++ if suffix_length > MXU_DECODE_MAX_SUFFIX && kv_length >= PREFILL_SPLIT_MIN_KV {+ let f32_estimated_bytes_per_split = threadgroups_per_kv_split as u64* geometry.query_rows_per_threadgroup as u64* (geometry.head_dim as u64 + PARTIAL_STATS_PER_ROW)* DataType::F32.size_in_bytes() as u64;let requested_splits = kv_length.div_ceil(PREFILL_SPLIT_KEYS).min(max_splits);- let scratch_splits = PREFILL_SPLIT_SCRATCH_BYTES / bytes_per_split;+ let scratch_splits = PREFILL_SPLIT_SCRATCH_BYTES / f32_estimated_bytes_per_split;let splits = (requested_splits as u64).min(scratch_splits) as u32;if splits >= PREFILL_MIN_SPLITS {return splits;@@ -152,25 +166,21 @@ pub fn int8_splits(suffix_length: u32,kv_length: u32,) -> u32 {+ const SHORT_SUFFIX_MAX_SPLITS: u32 = 16;let kv_blocks = kv_length.div_ceil(kv_layout::TILE_KEYS);- let splits = if suffix_length <= GEMM_GROUPED_DECODE_SUFFIX_MAX {- // Estimate split count using at least 16 query tokens.+ let splits = if suffix_length <= MXU_DECODE_MAX_SUFFIX {let query_tokens = suffix_length.max(kv_layout::TILE_QUERIES);let splits = (kv_length as f32 / query_tokens as f32).sqrt().round() as u32;- // Tuned on M5 Max.- if suffix_length <= SMALL_SUFFIX_MAX {- splits.min(SMALL_SUFFIX_MAX_SPLITS)+ if suffix_length <= SHORT_SUFFIX_MAX {+ splits.min(SHORT_SUFFIX_MAX_SPLITS)} else {splits}} else if kv_length >= PREFILL_SPLIT_MIN_KV {- let row_bytes = head_dim as u64 * DataType::BF16.size_in_bytes() as u64- + PARTIAL_STATS_PER_ROW * DataType::F32.size_in_bytes() as u64;- let bytes_per_split = split_rows as u64 * row_bytes;+ let bytes_per_split = split_rows as u64 * partial_row_bytes(head_dim);let scratch_splits = (PREFILL_SPLIT_SCRATCH_BYTES / bytes_per_split) as u32;- // Round to the nearest split count.- let requested_splits = (kv_length + PREFILL_SPLIT_KEYS / 2) / PREFILL_SPLIT_KEYS;- requested_splits.min(scratch_splits)+ let rounded_splits = (kv_length + PREFILL_SPLIT_KEYS / 2) / PREFILL_SPLIT_KEYS;+ rounded_splits.min(scratch_splits)} else {1};diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/int8/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/int8/mod.rsindex 47427026d..344c86200 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/int8/mod.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/int8/mod.rs@@ -34,7 +34,7 @@ impl Int8Attention {config: &AttentionKernelConfig,context: &MetalContext,) -> bool {- policy::is_supported(config, context) && config.is_causal+ policy::is_supported(config) && context.supports_mxu && config.is_causal}pub fn new(diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rsindex 2b8e2b4ca..1c9270a93 100644--- a/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs@@ -21,7 +21,7 @@ use crate::backends::{metal::{Metal, command_buffer::MetalCommandBufferEncoding, context::MetalContext, kernel::SigmoidGateMetalKernel},};-#[derive(Clone, Copy, Debug, PartialEq, Eq)]+#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)]enum MaskKind {None,Causal,@@ -71,7 +71,7 @@ impl AttentionKernel for AttentionMetalKernel {context: &MetalContext,config: AttentionKernelConfig,) -> Result<Self, AttentionMetalError> {- let grouped = gemm_grouped::AttentionGemmGrouped::is_supported(&config, context)+ let grouped = gemm_grouped::AttentionGemmGrouped::is_supported(&config).then(|| gemm_grouped::AttentionGemmGrouped::new(&config));let int8 = int8::Int8Attention::is_supported(&config, context).then(|| int8::Int8Attention::new(context, &config))@@ -164,7 +164,7 @@ impl AttentionMetalKernel {if !is_ring&& let Some(mask) = mask&& let Some(grouped) = &self.grouped- && grouped.should_encode(mask, suffix_length)+ && grouped.should_encode(mask, suffix_length, kv_length, command_buffer.context().supports_mxu){return grouped.encode(mask, input, arguments, command_buffer).map_err(AttentionMetalError::Backend);}diff --git a/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rsindex 6524021d4..baa363990 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rs@@ -1,6 +1,6 @@use uzu_engine_macros::uzu_test;-use super::{MaskKind, SplitGeometry, choose_splits, int8_splits, should_encode};+use super::{MaskKind, SplitGeometry, choose_splits, int8_splits, select_head_dim_slices};fn geometry(head_dim: u32) -> SplitGeometry {let (num_q_heads, query_rows_per_threadgroup, num_groups) = if head_dim == 128 {@@ -17,68 +17,76 @@ fn geometry(head_dim: u32) -> SplitGeometry {}}-fn splits(- cores: u32,- head_dim: u32,- suffix_length: u32,- kv_length: u32,-) -> u32 {- choose_splits(geometry(head_dim), suffix_length, kv_length, cores)-}-#[uzu_test]fn measured_and_fallback_boundaries() {- for (cores, head_dim, suffix, kv, expected) in [- (40, 256, 2, 16_384, 18),- (40, 256, 8, 16_384, 9),- (40, 256, 12, 61_440, 6),- (40, 256, 16, 5_120, 6),- (40, 256, 32, 32_768, 10),- (40, 128, 64, 32_768, 10),- (40, 256, 17, 262_144, 10),- (40, 256, 128, 16_383, 3),- (40, 256, 16, 128, 4),- (10, 256, 32, 262_144, 3),- (10, 256, 128, 16_383, 1),+ for (cores, head_dim, suffix, kv, query_rows, use_mxu, expected) in [+ (40, 256, 2, 16_384, 32, true, 18),+ (40, 256, 8, 16_384, 32, true, 9),+ (40, 256, 12, 61_440, 32, true, 6),+ (40, 256, 16, 5_120, 32, true, 6),+ (40, 256, 32, 32_768, 32, true, 10),+ (40, 128, 64, 32_768, 64, true, 10),+ (40, 256, 17, 262_144, 32, true, 10),+ (40, 256, 128, 16_383, 32, true, 3),+ (40, 256, 16, 128, 32, true, 4),+ (10, 256, 32, 262_144, 32, true, 3),+ (10, 256, 128, 16_383, 32, true, 1),+ (10, 256, 1, 61_440, 8, false, 10),] {- assert_eq!(splits(cores, head_dim, suffix, kv), expected);+ let geometry = SplitGeometry {+ query_rows_per_threadgroup: query_rows,+ ..geometry(head_dim)+ };+ assert_eq!(choose_splits(geometry, suffix, kv, cores, use_mxu), expected);}}#[uzu_test]fn should_encode_boundaries() {- for (head_dim, mask, suffix_length, expected) in [- (256, MaskKind::Causal, 1, false),- (256, MaskKind::Causal, 2, true),- (128, MaskKind::Trie, 64, true),- (128, MaskKind::Causal, 65, false),- (256, MaskKind::Causal, 1_024, true),- (256, MaskKind::Causal, 1_025, false),- (256, MaskKind::None, 1_024, false),- (256, MaskKind::Trie, 65, false),+ for (head_dim, mask, suffix, kv, use_mxu, expected) in [+ (256, MaskKind::Causal, 1, 0, true, None),+ (256, MaskKind::Causal, 2, 0, true, Some(4)),+ (128, MaskKind::Trie, 64, 0, true, Some(2)),+ (128, MaskKind::Causal, 65, 0, true, None),+ (256, MaskKind::Causal, 1_024, 0, true, Some(4)),+ (256, MaskKind::Causal, 1_025, 0, true, None),+ (256, MaskKind::None, 1_024, 0, true, None),+ (256, MaskKind::Trie, 65, 0, true, None),+ (256, MaskKind::Causal, 1, 16_383, false, None),+ (256, MaskKind::Causal, 1, 16_384, false, Some(8)),+ (256, MaskKind::Causal, 2, 16_384, false, Some(4)),+ (256, MaskKind::Causal, 4, 16_384, false, Some(2)),+ (128, MaskKind::Causal, 1, 16_384, false, Some(4)),+ (128, MaskKind::Causal, 512, 30_719, false, None),+ (128, MaskKind::Causal, 512, 30_720, false, Some(1)),+ (256, MaskKind::Causal, 3, 61_440, false, None),+ (256, MaskKind::None, 1, 61_440, false, None),+ (256, MaskKind::Causal, 512, 61_439, false, None),+ (256, MaskKind::Causal, 512, 61_440, false, Some(2)),] {- assert_eq!(should_encode(head_dim, mask, suffix_length), expected);+ assert_eq!(select_head_dim_slices(head_dim, mask, suffix, kv, use_mxu), expected);}}#[uzu_test]fn long_prefill_split_selection_boundaries() {- for (suffix, kv, expected) in [- (1024, 16_383, 1),- (64, 61_440, 5),- (1024, 16_384, 8),- (256, 61_440, 30),- (1024, 20_480, 10),- (1024, 102_400, 10),+ for (suffix, kv, cores, use_mxu, expected) in [+ (1024, 16_383, 40, true, 1),+ (64, 61_440, 40, true, 5),+ (1024, 16_384, 40, true, 8),+ (256, 61_440, 40, true, 30),+ (1024, 20_480, 40, true, 10),+ (1024, 102_400, 40, true, 10),+ (512, 61_440, 10, false, 16),] {- assert_eq!(splits(40, 256, suffix, kv), expected, "suffix={suffix}, kv={kv}");+ assert_eq!(choose_splits(geometry(256), suffix, kv, cores, use_mxu), expected, "suffix={suffix}, kv={kv}");}let many_heads = SplitGeometry {num_q_heads: 240,..geometry(256)};- assert_eq!(choose_splits(many_heads, 1024, 61_440, 40), 1, "fewer than two prefill splits fit");+ assert_eq!(choose_splits(many_heads, 1024, 61_440, 40, true), 1, "fewer than two prefill splits fit");}#[uzu_test]diff --git a/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rsindex 8f6c20e32..d76b31263 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs@@ -125,6 +125,15 @@ fn attention_kernel_matches_cpu() {(64, 8, 2, 8, 0, false, false),(64, 8, 8, 8, 1024, false, false),(64, 8, 2, 8, 1024, false, false),+ (256, 24, 4, 1, 16_383, true, false),+ (256, 24, 4, 2, 16_383, true, false),+ (256, 24, 4, 4, 16_380, true, false),+ (128, 32, 8, 1, 16_383, true, false),+ (256, 6, 1, 512, 60_928, true, false),+ (256, 1, 1, 1024, 60_417, true, false),+ (128, 4, 1, 512, 30_208, true, false),+ (128, 4, 1, 1024, 29_696, true, false),+ (128, 8, 2, 512, 30_208, true, false),] {let nodes = use_trie.then_some(trie.as_slice());let shape = (head_dim, num_q_heads, num_groups, suffix_length, prefix_length, causal);@@ -265,7 +274,7 @@ fn attention_gate_matches_standalone_gate_on_cpu() {fn attention_gate_matches_standalone_gate_on_metal() {let context = block_on(<Metal as Backend>::Context::new()).expect("Metal attention context");for (suffix_length, prefix_length) in- [(1, 100), (1, 2000), (200, 800), (2, 1022), (16, 16368), (65, 959), (65, 16319), (1024, 0)]+ [(1, 100), (1, 2000), (200, 800), (2, 1022), (16, 16368), (65, 959), (65, 16319), (1024, 0), (512, 60_928)]{check_attention_gate::<Metal>(context.as_ref(), suffix_length, prefix_length);}