Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

grouped attention for pre M5 chips (#979)

CCYeh2968097ab20c
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/attention.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/attention.metal
index 3fb235c83..a764b3053 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/attention.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/attention.metal
@@ -14,19 +14,22 @@ using namespace uzu::matmul;
using namespace uzu::attention;
using namespace uzu::trie;
-// TODO: support more BD and should tune more to replace AttentionGEMM
-template <typename T, uint BK, uint BD, uint D_SLICES>
+template <typename T, uint BK, uint BD, uint HEAD_DIM_SLICES, bool USE_MXU>
VARIANTS(T, bfloat)
-VARIANTS(BK, 32)
+VARIANTS(BK, 16, 32)
VARIANTS(BD, 128, 256)
-VARIANTS(D_SLICES, 2, 4)
-CONSTRAINT(D_SLICES == BD / 64)
+VARIANTS(HEAD_DIM_SLICES, 1, 2, 4, 8)
+VARIANTS(USE_MXU, false, true)
+CONSTRAINT((BK == 16) == (HEAD_DIM_SLICES == 1))
+CONSTRAINT(HEAD_DIM_SLICES != 1 || BD == 128)
+CONSTRAINT(!USE_MXU || HEAD_DIM_SLICES == BD / 64)
+CONSTRAINT(USE_MXU || BD == 256 || HEAD_DIM_SLICES == 1 || HEAD_DIM_SLICES == BD / 32)
KERNEL(AttentionGemmGrouped)(
const device T* q,
const device T* k,
const device T* v,
device T* o OPTIONAL(!split_kv),
- device float* partials OPTIONAL(split_kv),
+ device T* partials OPTIONAL(split_kv),
device float* partial_maxs OPTIONAL(split_kv),
device float* partial_sums OPTIONAL(split_kv),
const device TrieNode* trie OPTIONAL(is_trie),
@@ -37,15 +40,17 @@ KERNEL(AttentionGemmGrouped)(
const bool split_kv SPECIALIZE,
const bool is_causal SPECIALIZE,
const bool is_trie SPECIALIZE,
- threadgroup float score_exchange[AttentionGemmGroupedLayout<BK, BD, D_SLICES>::SCORE_EXCHANGE_SIZE],
+ threadgroup float scratch[AttentionGemmGroupedLayout<BK, BD, HEAD_DIM_SLICES, USE_MXU>::THREADGROUP_SCRATCH_FLOATS],
const ThreadContext thread_context,
const uint tile_split_flat GROUPS(query_tiles * num_splits),
const uint kv_head_idx GROUPS(kv_heads),
const uint lid THREADS(ATTENTION_GQA_SIMDGROUPS * METAL_SIMD_SIZE)
) {
- using Layout = AttentionGemmGroupedLayout<BK, BD, D_SLICES>;
+ using Layout = AttentionGemmGroupedLayout<BK, BD, HEAD_DIM_SLICES, USE_MXU>;
using Ops = typename Layout::Ops;
using AccumType = float;
+ constexpr bool GROUP_FOUR_Q_HEADS = HEAD_DIM_SLICES == 1;
+ using InputType = metal::conditional_t<USE_MXU || GROUP_FOUR_Q_HEADS, T, AccumType>;
constexpr int TILE_ROWS = Layout::TILE_ROWS;
constexpr int SLICE_COLS = Layout::SLICE_COLS;
@@ -54,30 +59,41 @@ KERNEL(AttentionGemmGrouped)(
constexpr int KEY_COL_FRAGMENTS = int(BK) / Ops::FRAGMENT_COLS;
constexpr int HEAD_DIM_FRAGMENTS_PER_SLICE = SLICE_COLS / Ops::FRAGMENT_ROWS;
constexpr int VALUE_COL_FRAGMENTS = SLICE_COLS / Ops::FRAGMENT_COLS;
+ constexpr int PV_COL_FRAGMENTS = VALUE_COL_FRAGMENTS < 8 ? VALUE_COL_FRAGMENTS : 8;
+ constexpr int PV_CHUNKS = VALUE_COL_FRAGMENTS / PV_COL_FRAGMENTS;
constexpr int ROWS_PER_LANE = Ops::THREAD_ELEMENT_ROWS;
static_assert(Layout::SIMDGROUPS == 8, "requires eight simdgroups");
static_assert(KEY_COL_FRAGMENTS % 2 == 0, "QK fragments must be even");
static_assert(VALUE_COL_FRAGMENTS % 2 == 0, "PV fragments must be even");
+ static_assert(VALUE_COL_FRAGMENTS % PV_COL_FRAGMENTS == 0, "PV fragments must divide into chunks");
- using QueryFragment = OperandFragment<T, 1, 1, Ops>;
- using KeyFragment = OperandFragment<T, 1, KEY_COL_FRAGMENTS, Ops, ReadTranspose>;
- using ValueFragment = OperandFragment<T, KEY_COL_FRAGMENTS, VALUE_COL_FRAGMENTS, Ops>;
+ using QueryFragment = OperandFragment<InputType, 1, 1, Ops>;
+ using KeyFragment = OperandFragment<InputType, 1, KEY_COL_FRAGMENTS, Ops, ReadTranspose>;
+ using ValueFragment = OperandFragment<InputType, KEY_COL_FRAGMENTS, PV_COL_FRAGMENTS, Ops>;
using ScoreFragment = Fragment<AccumType, 1, KEY_COL_FRAGMENTS, Ops>;
using OutputFragment = Fragment<AccumType, 1, VALUE_COL_FRAGMENTS, Ops>;
+ using PvOutputFragment = Fragment<AccumType, 1, PV_COL_FRAGMENTS, Ops>;
+ using KvBlockStorage = metal::conditional_t<GROUP_FOUR_Q_HEADS, ThreadgroupBlockStorage, DeviceBlockStorage>;
+ using KvBlockSource =
+ BlockSource<T, BK, BD, Layout::SHARED_KV_ROW_STRIDE, Layout::SIMDGROUPS * METAL_SIMD_SIZE, KvBlockStorage>;
const ushort lane = ushort(thread_context.simd_lane_id);
const ushort simdgroup_index = ushort(thread_context.simdgroup_index);
- const ushort band = simdgroup_index / ushort(D_SLICES);
- const ushort slice = simdgroup_index % ushort(D_SLICES);
+ const ushort band = simdgroup_index / ushort(HEAD_DIM_SLICES);
+ const ushort slice = simdgroup_index % ushort(HEAD_DIM_SLICES);
const uint query_tile_idx = split_kv ? (tile_split_flat % query_tiles) : tile_split_flat;
const uint split_idx = split_kv ? (tile_split_flat / query_tiles) : 0u;
const uint grouped_rows = params.gqa_factor * params.q_len;
- const uint row_base = query_tile_idx * uint(BLOCK_ROWS) + uint(band) * uint(TILE_ROWS);
- const uint head_in_group = row_base / params.q_len;
- const uint query_base = row_base - head_in_group * params.q_len;
+ const uint head_in_group = GROUP_FOUR_Q_HEADS
+ ? uint(band / 2)
+ : (query_tile_idx * uint(BLOCK_ROWS) + uint(band) * uint(TILE_ROWS)) / params.q_len;
+ const uint query_base = GROUP_FOUR_Q_HEADS
+ ? query_tile_idx * uint(2 * TILE_ROWS) + uint(band % 2) * uint(TILE_ROWS)
+ : (query_tile_idx * uint(BLOCK_ROWS) + uint(band) * uint(TILE_ROWS)) % params.q_len;
+ const uint row_base = head_in_group * params.q_len + query_base;
const uint q_head_idx = kv_head_idx * params.gqa_factor + head_in_group;
const uint query_row_stride = params.q_strides[2];
@@ -104,6 +120,12 @@ KERNEL(AttentionGemmGrouped)(
lane,
query_source.advanced(head_dim_fragment_idx * Ops::FRAGMENT_ROWS)
);
+ if constexpr (GROUP_FOUR_Q_HEADS) {
+ const InputType query_scale = static_cast<InputType>(params.scale * M_LOG2E_F);
+ query_frags[head_dim_fragment_idx].map_coords(lane, [&](short, short, InputType value) {
+ return value * query_scale;
+ });
+ }
}
}
@@ -155,8 +177,9 @@ KERNEL(AttentionGemmGrouped)(
}
}
- threadgroup float* exchange_slot = score_exchange + simdgroup_index * SCORE_TILE_ELEMENTS;
- const ushort band_exchange_base = band * ushort(D_SLICES);
+ threadgroup float* exchange_slot = scratch + simdgroup_index * SCORE_TILE_ELEMENTS;
+ threadgroup T* kv_shared = reinterpret_cast<threadgroup T*>(scratch);
+ const ushort band_exchange_base = band * ushort(HEAD_DIM_SLICES);
const uint key_block_stride = uint(BK) * key_row_stride;
const uint value_block_stride = uint(BK) * value_row_stride;
@@ -166,9 +189,11 @@ KERNEL(AttentionGemmGrouped)(
if (is_causal) {
const uint tile_first_row = query_tile_idx * uint(BLOCK_ROWS);
const uint tile_last_row = min(tile_first_row + uint(BLOCK_ROWS) - 1u, grouped_rows - 1u);
- const uint max_query_index = (tile_first_row / params.q_len == tile_last_row / params.q_len)
- ? (tile_last_row % params.q_len)
- : (params.q_len - 1u);
+ const uint max_query_index =
+ GROUP_FOUR_Q_HEADS
+ ? min((query_tile_idx + 1u) * uint(2 * TILE_ROWS) - 1u, params.q_len - 1u)
+ : ((tile_first_row / params.q_len == tile_last_row / params.q_len) ? (tile_last_row % params.q_len)
+ : (params.q_len - 1u));
const uint causal_blocks = (uint(prefix_length) + max_query_index + uint(BK)) / uint(BK);
block_end = min(block_end, causal_blocks);
}
@@ -181,8 +206,7 @@ KERNEL(AttentionGemmGrouped)(
k += size_t(block_begin) * key_block_stride;
v += size_t(block_begin) * value_block_stride;
- const AccumType score_scale = AccumType(params.scale) * AccumType(M_LOG2E_F);
- // Masked scores are multiplied by score_scale below, so pre-divide this value.
+ const AccumType score_scale = GROUP_FOUR_Q_HEADS ? AccumType(1) : AccumType(params.scale) * AccumType(M_LOG2E_F);
const AccumType masked_score_pre_scale = (static_cast<AccumType>(-1e9f) * M_LOG2E_F) / score_scale;
const uint first_key = block_begin * uint(BK);
@@ -192,34 +216,35 @@ KERNEL(AttentionGemmGrouped)(
ScoreFragment score;
score.clear();
{
- auto key_source = fragment_source(k + size_t(kb) * key_block_stride, int(key_row_stride));
- if constexpr (IS_TAIL) {
- key_source = key_source.bounded(short(tail_keys), short(Ops::FRAGMENT_COLS));
- }
+ KvBlockSource key_source(
+ k + size_t(kb) * key_block_stride,
+ int(key_row_stride),
+ kv_shared,
+ IS_TAIL ? short(tail_keys) : short(BK),
+ thread_context
+ );
+ key_source.prepare();
METAL_PRAGMA_UNROLL
for (short head_dim_fragment_idx = 0; head_dim_fragment_idx < HEAD_DIM_FRAGMENTS_PER_SLICE;
++head_dim_fragment_idx) {
KeyFragment key_fragment;
- key_fragment.load_from(lane, key_source.advanced(head_dim_fragment_idx * Ops::FRAGMENT_ROWS));
+ key_source.load(key_fragment, head_dim_fragment_idx * Ops::FRAGMENT_ROWS);
simdgroup_barrier(mem_flags::mem_none);
fragment_mma(score, query_frags[head_dim_fragment_idx], key_fragment);
}
}
- if constexpr (D_SLICES > 1) {
+ if constexpr (HEAD_DIM_SLICES > 1) {
threadgroup_barrier(mem_flags::mem_threadgroup);
score.store(lane, exchange_slot, int(BK));
threadgroup_barrier(mem_flags::mem_threadgroup);
thread AccumType* score_elements = score.elements();
METAL_PRAGMA_UNROLL
- for (ushort j = 1; j < ushort(D_SLICES); ++j) {
- const ushort other = (slice + j) % ushort(D_SLICES);
+ for (ushort j = 1; j < ushort(HEAD_DIM_SLICES); ++j) {
+ const ushort other = (slice + j) % ushort(HEAD_DIM_SLICES);
ScoreFragment partial;
- partial.load_from(
- lane,
- fragment_source(score_exchange + (band_exchange_base + other) * SCORE_TILE_ELEMENTS, int(BK))
- );
+ partial.load_from(lane, fragment_source(scratch + (band_exchange_base + other) * SCORE_TILE_ELEMENTS, int(BK)));
thread AccumType* partial_elements = partial.elements();
METAL_PRAGMA_UNROLL
for (ushort e = 0; e < ScoreFragment::ELEMENTS_PER_FRAGMENT; ++e) {
@@ -276,14 +301,33 @@ KERNEL(AttentionGemmGrouped)(
output.map_rows(rescale, [](AccumType value, AccumType f) { return value * f; });
{
- auto value_source = fragment_source(v + size_t(kb) * value_block_stride, int(value_row_stride));
- if constexpr (IS_TAIL) {
- value_source = value_source.bounded(short(tail_keys), short(SLICE_COLS));
+ KvBlockSource value_block(
+ v + size_t(kb) * value_block_stride,
+ int(value_row_stride),
+ kv_shared,
+ IS_TAIL ? short(tail_keys) : short(BK),
+ thread_context
+ );
+ value_block.prepare();
+ METAL_PRAGMA_UNROLL
+ for (ushort chunk = 0; chunk < PV_CHUNKS; ++chunk) {
+ const ushort first_col = chunk * PV_COL_FRAGMENTS;
+ const int chunk_offset = first_col * Ops::FRAGMENT_COLS;
+ ValueFragment value_fragment;
+ value_block.load(value_fragment, chunk_offset);
+ simdgroup_barrier(mem_flags::mem_none);
+
+ PvOutputFragment chunk_output;
+ METAL_PRAGMA_UNROLL
+ for (ushort col = 0; col < PV_COL_FRAGMENTS; ++col) {
+ chunk_output.fragment_at(0, col) = output.fragment_at(0, first_col + col);
+ }
+ fragment_mma(chunk_output, score, value_fragment);
+ METAL_PRAGMA_UNROLL
+ for (ushort col = 0; col < PV_COL_FRAGMENTS; ++col) {
+ output.fragment_at(0, first_col + col) = chunk_output.fragment_at(0, col);
+ }
}
- ValueFragment value_fragment;
- value_fragment.load_from(lane, value_source);
- simdgroup_barrier(mem_flags::mem_none);
- fragment_mma(output, score, value_fragment);
}
};
@@ -295,14 +339,15 @@ KERNEL(AttentionGemmGrouped)(
}
if (split_kv) {
- const uint tile_index = (split_idx * kv_heads + kv_head_idx) * query_tiles + query_tile_idx;
- device float* partial_tile = partials + size_t(tile_index) * BLOCK_ROWS * BD + size_t(band) * TILE_ROWS * BD;
+ const uint split_head_row = (split_idx * kv_heads + kv_head_idx) * query_tiles * uint(BLOCK_ROWS);
+ const uint partial_row = split_head_row + row_base;
+ device T* partial_tile = partials + size_t(partial_row) * BD;
output.store(lane, partial_tile + slice_offset, int(BD));
if (slice == 0) {
const short2 position = OutputFragment::get_position(lane);
if (position.x == 0) {
- const uint row_base_in_tile = uint(tile_index) * uint(BLOCK_ROWS) + uint(band) * uint(TILE_ROWS);
+ const uint row_base_in_tile = partial_row;
METAL_PRAGMA_UNROLL
for (int r = 0; r < ROWS_PER_LANE; ++r) {
const uint row = row_base_in_tile + uint(position.y + r * Ops::THREAD_ELEMENT_ROW_STRIDE);
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/combine.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/combine.metal
index 1a70e2f0b..2aedb9757 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/combine.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/combine.metal
@@ -10,7 +10,7 @@ template <typename T, uint BD>
VARIANTS(T, bfloat)
VARIANTS(BD, 128, 256)
KERNEL(AttentionGemmGroupedCombine)(
- const device float* partials,
+ const device T* partials,
const device float* partial_maxs,
const device float* partial_sums,
device T* o,
@@ -33,7 +33,7 @@ KERNEL(AttentionGemmGroupedCombine)(
const uint q_head_idx = kv_head_idx * params.gqa_factor + head_in_group;
const device float* row_maxs = partial_maxs + row;
const device float* row_sums = partial_sums + row;
- const device float* row_partials = partials + row * BD + column;
+ const device T* row_partials = partials + row * BD + column;
float numerator = 0.0f;
float denominator = 0.0f;
@@ -52,7 +52,7 @@ KERNEL(AttentionGemmGroupedCombine)(
const float rescale = fast::exp2(row_maxs[s * split_rows] - global_max);
const float weight = rescale * split_sum;
denominator += weight;
- numerator += rescale * row_partials[size_t(s) * split_rows * BD];
+ numerator += rescale * float(row_partials[size_t(s) * split_rows * BD]);
}
}
const float mean = denominator > 0.0f ? numerator / denominator : 0.0f;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/layout.h b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/layout.h
index a9d312840..26e3feb95 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/layout.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/layout.h
@@ -4,20 +4,23 @@
#include "../../common/defines.h"
#include "../../common/dsl.h"
#include "../../matmul/common/mxu_fragment/ops.h"
+#include "../../matmul/common/simdgroup_fragment_ops.h"
using namespace metal;
#define ATTENTION_GQA_SIMDGROUPS 8
-template <uint BK, uint BD, uint D_SLICES>
+template <uint BK, uint BD, uint HEAD_DIM_SLICES, bool USE_MXU = true>
struct AttentionGemmGroupedLayout {
- using Ops = uzu::matmul::MxuFragmentOps<>;
+ using Ops = metal::conditional_t<USE_MXU, uzu::matmul::MxuFragmentOps<>, uzu::matmul::SimdgroupFragmentOps>;
UZU_CONST int SIMDGROUPS = ATTENTION_GQA_SIMDGROUPS;
UZU_CONST int TILE_ROWS = Ops::FRAGMENT_ROWS;
- UZU_CONST int SLICE_COLS = int(BD) / int(D_SLICES);
- UZU_CONST int BANDS = SIMDGROUPS / int(D_SLICES);
+ UZU_CONST int SLICE_COLS = int(BD) / int(HEAD_DIM_SLICES);
+ UZU_CONST int BANDS = SIMDGROUPS / int(HEAD_DIM_SLICES);
UZU_CONST int BLOCK_ROWS = BANDS * TILE_ROWS;
UZU_CONST int SCORE_TILE_ELEMENTS = TILE_ROWS * int(BK);
- UZU_CONST int SCORE_EXCHANGE_SIZE = SIMDGROUPS * SCORE_TILE_ELEMENTS;
+ UZU_CONST int SHARED_KV_ROW_STRIDE = int(BD) + 8;
+ UZU_CONST int THREADGROUP_SCRATCH_FLOATS =
+ HEAD_DIM_SLICES == 1 ? int(BK) * SHARED_KV_ROW_STRIDE / 2 : SIMDGROUPS * SCORE_TILE_ELEMENTS;
};
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs
index 02cc63fb1..ba8818d3e 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs
@@ -1,3 +1,5 @@
+use std::collections::{HashMap, hash_map::Entry};
+
use parking_lot::{MappedMutexGuard, Mutex, MutexGuard};
use super::FullPrecisionInput;
@@ -26,11 +28,12 @@ use super::MaskKind;
const SIMDGROUPS_PER_THREADGROUP: u32 = 8;
const SLICE_COLS: u32 = 64;
-const TILE_ROWS: u32 = 16;
+const MXU_TILE_ROWS: u32 = 16;
+const SIMDGROUP_TILE_ROWS: u32 = 8;
const BLOCK_K: u32 = 32;
pub fn query_rows_per_threadgroup(head_dim: u32) -> u32 {
- SIMDGROUPS_PER_THREADGROUP / (head_dim / SLICE_COLS) * TILE_ROWS
+ SIMDGROUPS_PER_THREADGROUP / (head_dim / SLICE_COLS) * MXU_TILE_ROWS
}
struct AttentionGemmGroupedMetal {
@@ -43,46 +46,59 @@ struct AttentionGemmGroupedMetal {
scale: Option<f32>,
gate_row_stride: Option<u32>,
query_rows_per_threadgroup: u32,
+ block_k: u32,
mask: MaskKind,
}
impl AttentionGemmGroupedMetal {
fn new_fixed(
context: &MetalContext,
- head_dim: u32,
- num_groups: u32,
- num_q_heads: u32,
- scale: Option<f32>,
- gate_row_stride: Option<u32>,
+ config: &AttentionKernelConfig,
+ head_dim_slices: u32,
mask: MaskKind,
) -> Result<Self, MetalError> {
- assert!(matches!(head_dim, 128 | 256), "head_dim must be 128 or 256");
- assert_eq!(num_q_heads % num_groups, 0, "num_q_heads must be divisible by num_groups");
- let slices = head_dim / SLICE_COLS;
+ let tile_rows = if context.supports_mxu {
+ MXU_TILE_ROWS
+ } else {
+ SIMDGROUP_TILE_ROWS
+ };
+ let block_k = if head_dim_slices == 1 {
+ 16
+ } else {
+ BLOCK_K
+ };
+ assert!(matches!(config.head_dim, 128 | 256), "head_dim must be 128 or 256");
+ assert_eq!(config.num_q_heads % config.num_groups, 0, "num_q_heads must be divisible by num_groups");
let new_kernel = |split| {
AttentionGemmGroupedMetalKernel::new(
context,
DataType::BF16,
- BLOCK_K,
- head_dim,
- slices,
+ block_k,
+ config.head_dim,
+ head_dim_slices,
+ context.supports_mxu,
split,
mask.is_causal(),
mask.is_trie(),
)
};
- let combine =
- AttentionGemmGroupedCombineMetalKernel::new(context, DataType::BF16, head_dim, gate_row_stride.is_some())?;
+ let combine = AttentionGemmGroupedCombineMetalKernel::new(
+ context,
+ DataType::BF16,
+ config.head_dim,
+ config.gate_row_stride.is_some(),
+ )?;
Ok(Self {
kernel: new_kernel(false)?,
split_kernel: new_kernel(true)?,
combine,
- head_dim,
- num_groups,
- num_q_heads,
- scale,
- gate_row_stride,
- query_rows_per_threadgroup: query_rows_per_threadgroup(head_dim),
+ head_dim: config.head_dim,
+ num_groups: config.num_groups,
+ num_q_heads: config.num_q_heads,
+ scale: config.scale,
+ gate_row_stride: config.gate_row_stride,
+ query_rows_per_threadgroup: SIMDGROUPS_PER_THREADGROUP / head_dim_slices * tile_rows,
+ block_k,
mask,
})
}
@@ -125,7 +141,7 @@ impl AttentionGemmGroupedMetal {
}
let partial_rows = self.num_groups * query_tiles * num_splits * self.query_rows_per_threadgroup;
- let mut partials = command_buffer.allocate_scratch_for_shape(&[partial_rows, self.head_dim], DataType::F32)?;
+ let mut partials = command_buffer.allocate_scratch_for_shape(&[partial_rows, self.head_dim], DataType::BF16)?;
let mut partial_maxs = command_buffer.allocate_scratch_for_shape(&[partial_rows], DataType::F32)?;
let mut partial_sums = command_buffer.allocate_scratch_for_shape(&[partial_rows], DataType::F32)?;
@@ -184,21 +200,16 @@ impl AttentionGemmGroupedMetal {
q_off: kv_length - suffix_length,
nq_aligned: 0,
q_rem: 0,
- nk: kv_length.div_ceil(BLOCK_K),
- nk_aligned: kv_length / BLOCK_K,
- k_rem: kv_length % BLOCK_K,
+ nk: kv_length.div_ceil(self.block_k),
+ nk_aligned: kv_length / self.block_k,
+ k_rem: kv_length % self.block_k,
}
}
}
pub struct AttentionGemmGrouped {
- non_trie: Mutex<Option<AttentionGemmGroupedMetal>>,
- trie: Mutex<Option<AttentionGemmGroupedMetal>>,
- head_dim: u32,
- num_groups: u32,
- num_q_heads: u32,
- scale: Option<f32>,
- gate_row_stride: Option<u32>,
+ variants: Mutex<HashMap<(MaskKind, u32), AttentionGemmGroupedMetal>>,
+ config: AttentionKernelConfig,
}
impl AttentionGemmGrouped {
@@ -206,43 +217,26 @@ impl AttentionGemmGrouped {
&self,
context: &MetalContext,
mask: MaskKind,
+ head_dim_slices: u32,
) -> Result<MappedMutexGuard<'_, AttentionGemmGroupedMetal>, MetalError> {
- let cache = if mask.is_trie() {
- &self.trie
- } else {
- &self.non_trie
- };
- let mut cache = cache.lock();
- if cache.is_none() {
- *cache = Some(AttentionGemmGroupedMetal::new_fixed(
- context,
- self.head_dim,
- self.num_groups,
- self.num_q_heads,
- self.scale,
- self.gate_row_stride,
- mask,
- )?);
+ let key = (mask, head_dim_slices);
+ let mut variants = self.variants.lock();
+ if let Entry::Vacant(slot) = variants.entry(key) {
+ slot.insert(AttentionGemmGroupedMetal::new_fixed(context, &self.config, head_dim_slices, mask)?);
}
- Ok(MutexGuard::map(cache, |cache| cache.as_mut().expect("attention pipeline was just initialized")))
+ Ok(MutexGuard::map(variants, |variants| {
+ variants.get_mut(&key).expect("attention variant was just initialized")
+ }))
}
- pub fn is_supported(
- config: &AttentionKernelConfig,
- context: &MetalContext,
- ) -> bool {
- policy::is_supported(config, context)
+ pub fn is_supported(config: &AttentionKernelConfig) -> bool {
+ policy::is_supported(config)
}
pub fn new(config: &AttentionKernelConfig) -> Self {
Self {
- non_trie: Mutex::new(None),
- trie: Mutex::new(None),
- head_dim: config.head_dim,
- num_groups: config.num_groups,
- num_q_heads: config.num_q_heads,
- scale: config.scale,
- gate_row_stride: config.gate_row_stride,
+ variants: Mutex::new(HashMap::new()),
+ config: *config,
}
}
@@ -250,8 +244,11 @@ impl AttentionGemmGrouped {
&self,
mask: MaskKind,
suffix_length: u32,
+ kv_length: u32,
+ use_mxu: bool,
) -> bool {
- policy::should_encode(self.head_dim, mask, suffix_length)
+ let slices = policy::select_head_dim_slices(self.config.head_dim, mask, suffix_length, kv_length, use_mxu);
+ slices.is_some_and(|slices| slices != 1 || self.config.num_q_heads == self.config.num_groups * 4)
}
pub fn encode(
@@ -270,22 +267,30 @@ impl AttentionGemmGrouped {
assert!(arguments.sinks.is_none(), "attention sinks are unsupported");
let kv_length = arguments.cache.prefix_len() + suffix_length;
- let mut output = command_buffer
- .allocate_scratch_for_shape(&[suffix_length, self.num_q_heads, self.head_dim], DataType::BF16)?;
- let gpu_core_count = command_buffer.context().gpu_core_count;
- let core = self.get_or_create(command_buffer.context(), mask)?;
- let num_splits = choose_splits(
- SplitGeometry {
- head_dim: core.head_dim,
- num_q_heads: core.num_q_heads,
- num_groups: core.num_groups,
- query_rows_per_threadgroup: core.query_rows_per_threadgroup,
- block_k: BLOCK_K,
- },
+ let head_dim_slices = policy::select_head_dim_slices(
+ self.config.head_dim,
+ mask,
suffix_length,
kv_length,
- gpu_core_count,
- );
+ command_buffer.context().supports_mxu,
+ )
+ .expect("grouped attention request must be supported");
+ assert!(head_dim_slices != 1 || self.config.num_q_heads == self.config.num_groups * 4);
+ let mut output = command_buffer.allocate_scratch_for_shape(
+ &[suffix_length, self.config.num_q_heads, self.config.head_dim],
+ DataType::BF16,
+ )?;
+ let gpu_core_count = command_buffer.context().gpu_core_count;
+ let core = self.get_or_create(command_buffer.context(), mask, head_dim_slices)?;
+ let geometry = SplitGeometry {
+ head_dim: core.head_dim,
+ num_q_heads: core.num_q_heads,
+ num_groups: core.num_groups,
+ query_rows_per_threadgroup: core.query_rows_per_threadgroup,
+ block_k: core.block_k,
+ };
+ let num_splits =
+ choose_splits(geometry, suffix_length, kv_length, gpu_core_count, command_buffer.context().supports_mxu);
let gate_applied = core.encode(
input.queries,
input.keys,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/policy.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/policy.rs
index c28a61c6f..8a0395ce0 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/policy.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/policy.rs
@@ -1,27 +1,17 @@
use super::MaskKind;
use crate::{
- backends::{
- common::{gpu_types::kv_layout, kernel::AttentionKernelConfig},
- metal::context::MetalContext,
- },
+ backends::common::{gpu_types::kv_layout, kernel::AttentionKernelConfig},
data_type::DataType,
};
-const GEMM_GROUPED_HEAD_DIMS: [u32; 2] = [128, 256];
-const GEMM_GROUPED_DECODE_SUFFIX_MIN: u32 = 2;
-const GEMM_GROUPED_DECODE_SUFFIX_MAX: u32 = 64;
-const GEMM_GROUPED_PREFILL_SUFFIX_MAX: u32 = 1024;
pub const MAX_TRIE_SUFFIX: u32 = 64;
-const SMALL_SUFFIX_MAX: u32 = 4;
-const SMALL_SUFFIX_MAX_SPLITS: u32 = 16;
-
-pub fn is_supported(
- arguments: &AttentionKernelConfig,
- context: &MetalContext,
-) -> bool {
- GEMM_GROUPED_HEAD_DIMS.contains(&arguments.head_dim)
+
+const MXU_DECODE_MAX_SUFFIX: u32 = 64;
+const SHORT_SUFFIX_MAX: u32 = 4;
+
+pub fn is_supported(arguments: &AttentionKernelConfig) -> bool {
+ matches!(arguments.head_dim, 128 | 256)
&& arguments.data_type == DataType::BF16
- && context.supports_mxu
&& !arguments.has_sinks
&& !arguments.is_kv_cache_ring
&& arguments.sliding_window_size.is_none()
@@ -31,29 +21,34 @@ pub fn is_supported(
&& arguments.num_q_heads.is_multiple_of(arguments.num_groups)
}
-pub fn should_encode(
+pub fn select_head_dim_slices(
head_dim: u32,
mask: MaskKind,
suffix_length: u32,
-) -> bool {
- let decode = (GEMM_GROUPED_DECODE_SUFFIX_MIN..=GEMM_GROUPED_DECODE_SUFFIX_MAX).contains(&suffix_length);
- // The grouped prefill path is only tuned for head_dim 256.
- let prefill_tuned = head_dim == 256;
- let causal_prefill = prefill_tuned
- && mask == MaskKind::Causal
- && (GEMM_GROUPED_DECODE_SUFFIX_MAX + 1..=GEMM_GROUPED_PREFILL_SUFFIX_MAX).contains(&suffix_length);
- decode || causal_prefill
+ kv_length: u32,
+ use_mxu: bool,
+) -> Option<u32> {
+ const SIMDGROUP_DECODE_MIN_KV_LENGTH: u32 = 16_384;
+ const SIMDGROUP_PREFILL_MIN_KV_LENGTH: u32 = 61_440;
+
+ match (use_mxu, mask, head_dim, suffix_length) {
+ (true, _, _, 2..=64) => Some(head_dim / kv_layout::SLICE_DIMS),
+ (true, MaskKind::Causal, 256, 65..=1024) => Some(4),
+ (false, MaskKind::Causal, 256, 1 | 2 | 4) if kv_length >= SIMDGROUP_DECODE_MIN_KV_LENGTH => {
+ Some(8 / suffix_length)
+ },
+ (false, MaskKind::Causal, 128, 1 | 2 | 4) if kv_length >= SIMDGROUP_DECODE_MIN_KV_LENGTH => Some(4),
+ (false, MaskKind::Causal, 128, 512 | 1024) if kv_length >= 30_720 => Some(1),
+ (false, MaskKind::Causal, 256, 512 | 1024) if kv_length >= SIMDGROUP_PREFILL_MIN_KV_LENGTH => Some(2),
+ _ => None,
+ }
}
type MeasuredSplits = (u32, u32, &'static [(u32, u32)]);
-// Reuse S=16 tuning values to choose KV splits for suffixes 2..15.
const MEASURED_SUFFIX_MIN: u32 = 2;
const MEASURED_SUFFIX_MAX: u32 = 64;
-// TODO: validate this table per chip (M1-M5) before changing the policy.
-// TODO: add a simdgroup/non-MXU implementation before widening availability.
-// TODO: measure the D128/D256 crossover before widening suffix ranges.
const MEASURED_TG_PER_CORE_TENTHS: &[MeasuredSplits] = &[
(256, 16, &[(0, 24), (5120, 18), (131072, 39)]),
(256, 32, &[(0, 24), (5120, 18), (32768, 60)]),
@@ -68,7 +63,12 @@ const PREFILL_SPLIT_KEYS: u32 = 2048;
const PREFILL_SPLIT_MIN_KV: u32 = 16384;
const PREFILL_SPLIT_SCRATCH_BYTES: u64 = 256 << 20;
const PREFILL_MIN_SPLITS: u32 = 2;
-const PARTIAL_STATS_PER_ROW: u64 = 2; // max and sum
+const PARTIAL_STATS_PER_ROW: u64 = 2;
+
+fn partial_row_bytes(head_dim: u32) -> u64 {
+ head_dim as u64 * DataType::BF16.size_in_bytes() as u64
+ + PARTIAL_STATS_PER_ROW * DataType::F32.size_in_bytes() as u64
+}
pub struct SplitGeometry {
pub head_dim: u32,
@@ -117,20 +117,34 @@ pub fn choose_splits(
suffix_length: u32,
kv_length: u32,
gpu_core_count: u32,
+ use_mxu: bool,
) -> u32 {
let threadgroups_per_kv_split = (geometry.num_q_heads / geometry.num_groups * suffix_length)
.div_ceil(geometry.query_rows_per_threadgroup)
* geometry.num_groups;
let max_splits = kv_length.div_ceil(geometry.block_k).max(1);
- if suffix_length > GEMM_GROUPED_DECODE_SUFFIX_MAX && kv_length >= PREFILL_SPLIT_MIN_KV {
- // Each padded row has head_dim partial values, one max, and one sum, all F32.
- let bytes_per_split = threadgroups_per_kv_split as u64
+ if !use_mxu {
+ const MAX_SPLITS: u32 = 16;
+ let threadgroups_per_split = threadgroups_per_kv_split.max(1);
+ let splits = if suffix_length > SHORT_SUFFIX_MAX {
+ let bytes_per_split = threadgroups_per_split as u64
+ * geometry.query_rows_per_threadgroup as u64
+ * partial_row_bytes(geometry.head_dim);
+ (PREFILL_SPLIT_SCRATCH_BYTES / bytes_per_split) as u32
+ } else {
+ (gpu_core_count.max(1) * 4).div_ceil(threadgroups_per_split)
+ };
+ return splits.clamp(1, max_splits.min(MAX_SPLITS));
+ }
+
+ if suffix_length > MXU_DECODE_MAX_SUFFIX && kv_length >= PREFILL_SPLIT_MIN_KV {
+ let f32_estimated_bytes_per_split = threadgroups_per_kv_split as u64
* geometry.query_rows_per_threadgroup as u64
* (geometry.head_dim as u64 + PARTIAL_STATS_PER_ROW)
* DataType::F32.size_in_bytes() as u64;
let requested_splits = kv_length.div_ceil(PREFILL_SPLIT_KEYS).min(max_splits);
- let scratch_splits = PREFILL_SPLIT_SCRATCH_BYTES / bytes_per_split;
+ let scratch_splits = PREFILL_SPLIT_SCRATCH_BYTES / f32_estimated_bytes_per_split;
let splits = (requested_splits as u64).min(scratch_splits) as u32;
if splits >= PREFILL_MIN_SPLITS {
return splits;
@@ -152,25 +166,21 @@ pub fn int8_splits(
suffix_length: u32,
kv_length: u32,
) -> u32 {
+ const SHORT_SUFFIX_MAX_SPLITS: u32 = 16;
let kv_blocks = kv_length.div_ceil(kv_layout::TILE_KEYS);
- let splits = if suffix_length <= GEMM_GROUPED_DECODE_SUFFIX_MAX {
- // Estimate split count using at least 16 query tokens.
+ let splits = if suffix_length <= MXU_DECODE_MAX_SUFFIX {
let query_tokens = suffix_length.max(kv_layout::TILE_QUERIES);
let splits = (kv_length as f32 / query_tokens as f32).sqrt().round() as u32;
- // Tuned on M5 Max.
- if suffix_length <= SMALL_SUFFIX_MAX {
- splits.min(SMALL_SUFFIX_MAX_SPLITS)
+ if suffix_length <= SHORT_SUFFIX_MAX {
+ splits.min(SHORT_SUFFIX_MAX_SPLITS)
} else {
splits
}
} else if kv_length >= PREFILL_SPLIT_MIN_KV {
- let row_bytes = head_dim as u64 * DataType::BF16.size_in_bytes() as u64
- + PARTIAL_STATS_PER_ROW * DataType::F32.size_in_bytes() as u64;
- let bytes_per_split = split_rows as u64 * row_bytes;
+ let bytes_per_split = split_rows as u64 * partial_row_bytes(head_dim);
let scratch_splits = (PREFILL_SPLIT_SCRATCH_BYTES / bytes_per_split) as u32;
- // Round to the nearest split count.
- let requested_splits = (kv_length + PREFILL_SPLIT_KEYS / 2) / PREFILL_SPLIT_KEYS;
- requested_splits.min(scratch_splits)
+ let rounded_splits = (kv_length + PREFILL_SPLIT_KEYS / 2) / PREFILL_SPLIT_KEYS;
+ rounded_splits.min(scratch_splits)
} else {
1
};
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/int8/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/int8/mod.rs
index 47427026d..344c86200 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/int8/mod.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/int8/mod.rs
@@ -34,7 +34,7 @@ impl Int8Attention {
config: &AttentionKernelConfig,
context: &MetalContext,
) -> bool {
- policy::is_supported(config, context) && config.is_causal
+ policy::is_supported(config) && context.supports_mxu && config.is_causal
}
pub fn new(
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs
index 2b8e2b4ca..1c9270a93 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/mod.rs
@@ -21,7 +21,7 @@ use crate::backends::{
metal::{Metal, command_buffer::MetalCommandBufferEncoding, context::MetalContext, kernel::SigmoidGateMetalKernel},
};
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)]
enum MaskKind {
None,
Causal,
@@ -71,7 +71,7 @@ impl AttentionKernel for AttentionMetalKernel {
context: &MetalContext,
config: AttentionKernelConfig,
) -> Result<Self, AttentionMetalError> {
- let grouped = gemm_grouped::AttentionGemmGrouped::is_supported(&config, context)
+ let grouped = gemm_grouped::AttentionGemmGrouped::is_supported(&config)
.then(|| gemm_grouped::AttentionGemmGrouped::new(&config));
let int8 = int8::Int8Attention::is_supported(&config, context)
.then(|| int8::Int8Attention::new(context, &config))
@@ -164,7 +164,7 @@ impl AttentionMetalKernel {
if !is_ring
&& let Some(mask) = mask
&& let Some(grouped) = &self.grouped
- && grouped.should_encode(mask, suffix_length)
+ && grouped.should_encode(mask, suffix_length, kv_length, command_buffer.context().supports_mxu)
{
return grouped.encode(mask, input, arguments, command_buffer).map_err(AttentionMetalError::Backend);
}
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rs
index 6524021d4..baa363990 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rs
@@ -1,6 +1,6 @@
use uzu_engine_macros::uzu_test;
-use super::{MaskKind, SplitGeometry, choose_splits, int8_splits, should_encode};
+use super::{MaskKind, SplitGeometry, choose_splits, int8_splits, select_head_dim_slices};
fn geometry(head_dim: u32) -> SplitGeometry {
let (num_q_heads, query_rows_per_threadgroup, num_groups) = if head_dim == 128 {
@@ -17,68 +17,76 @@ fn geometry(head_dim: u32) -> SplitGeometry {
}
}
-fn splits(
- cores: u32,
- head_dim: u32,
- suffix_length: u32,
- kv_length: u32,
-) -> u32 {
- choose_splits(geometry(head_dim), suffix_length, kv_length, cores)
-}
-
#[uzu_test]
fn measured_and_fallback_boundaries() {
- for (cores, head_dim, suffix, kv, expected) in [
- (40, 256, 2, 16_384, 18),
- (40, 256, 8, 16_384, 9),
- (40, 256, 12, 61_440, 6),
- (40, 256, 16, 5_120, 6),
- (40, 256, 32, 32_768, 10),
- (40, 128, 64, 32_768, 10),
- (40, 256, 17, 262_144, 10),
- (40, 256, 128, 16_383, 3),
- (40, 256, 16, 128, 4),
- (10, 256, 32, 262_144, 3),
- (10, 256, 128, 16_383, 1),
+ for (cores, head_dim, suffix, kv, query_rows, use_mxu, expected) in [
+ (40, 256, 2, 16_384, 32, true, 18),
+ (40, 256, 8, 16_384, 32, true, 9),
+ (40, 256, 12, 61_440, 32, true, 6),
+ (40, 256, 16, 5_120, 32, true, 6),
+ (40, 256, 32, 32_768, 32, true, 10),
+ (40, 128, 64, 32_768, 64, true, 10),
+ (40, 256, 17, 262_144, 32, true, 10),
+ (40, 256, 128, 16_383, 32, true, 3),
+ (40, 256, 16, 128, 32, true, 4),
+ (10, 256, 32, 262_144, 32, true, 3),
+ (10, 256, 128, 16_383, 32, true, 1),
+ (10, 256, 1, 61_440, 8, false, 10),
] {
- assert_eq!(splits(cores, head_dim, suffix, kv), expected);
+ let geometry = SplitGeometry {
+ query_rows_per_threadgroup: query_rows,
+ ..geometry(head_dim)
+ };
+ assert_eq!(choose_splits(geometry, suffix, kv, cores, use_mxu), expected);
}
}
#[uzu_test]
fn should_encode_boundaries() {
- for (head_dim, mask, suffix_length, expected) in [
- (256, MaskKind::Causal, 1, false),
- (256, MaskKind::Causal, 2, true),
- (128, MaskKind::Trie, 64, true),
- (128, MaskKind::Causal, 65, false),
- (256, MaskKind::Causal, 1_024, true),
- (256, MaskKind::Causal, 1_025, false),
- (256, MaskKind::None, 1_024, false),
- (256, MaskKind::Trie, 65, false),
+ for (head_dim, mask, suffix, kv, use_mxu, expected) in [
+ (256, MaskKind::Causal, 1, 0, true, None),
+ (256, MaskKind::Causal, 2, 0, true, Some(4)),
+ (128, MaskKind::Trie, 64, 0, true, Some(2)),
+ (128, MaskKind::Causal, 65, 0, true, None),
+ (256, MaskKind::Causal, 1_024, 0, true, Some(4)),
+ (256, MaskKind::Causal, 1_025, 0, true, None),
+ (256, MaskKind::None, 1_024, 0, true, None),
+ (256, MaskKind::Trie, 65, 0, true, None),
+ (256, MaskKind::Causal, 1, 16_383, false, None),
+ (256, MaskKind::Causal, 1, 16_384, false, Some(8)),
+ (256, MaskKind::Causal, 2, 16_384, false, Some(4)),
+ (256, MaskKind::Causal, 4, 16_384, false, Some(2)),
+ (128, MaskKind::Causal, 1, 16_384, false, Some(4)),
+ (128, MaskKind::Causal, 512, 30_719, false, None),
+ (128, MaskKind::Causal, 512, 30_720, false, Some(1)),
+ (256, MaskKind::Causal, 3, 61_440, false, None),
+ (256, MaskKind::None, 1, 61_440, false, None),
+ (256, MaskKind::Causal, 512, 61_439, false, None),
+ (256, MaskKind::Causal, 512, 61_440, false, Some(2)),
] {
- assert_eq!(should_encode(head_dim, mask, suffix_length), expected);
+ assert_eq!(select_head_dim_slices(head_dim, mask, suffix, kv, use_mxu), expected);
}
}
#[uzu_test]
fn long_prefill_split_selection_boundaries() {
- for (suffix, kv, expected) in [
- (1024, 16_383, 1),
- (64, 61_440, 5),
- (1024, 16_384, 8),
- (256, 61_440, 30),
- (1024, 20_480, 10),
- (1024, 102_400, 10),
+ for (suffix, kv, cores, use_mxu, expected) in [
+ (1024, 16_383, 40, true, 1),
+ (64, 61_440, 40, true, 5),
+ (1024, 16_384, 40, true, 8),
+ (256, 61_440, 40, true, 30),
+ (1024, 20_480, 40, true, 10),
+ (1024, 102_400, 40, true, 10),
+ (512, 61_440, 10, false, 16),
] {
- assert_eq!(splits(40, 256, suffix, kv), expected, "suffix={suffix}, kv={kv}");
+ assert_eq!(choose_splits(geometry(256), suffix, kv, cores, use_mxu), expected, "suffix={suffix}, kv={kv}");
}
let many_heads = SplitGeometry {
num_q_heads: 240,
..geometry(256)
};
- assert_eq!(choose_splits(many_heads, 1024, 61_440, 40), 1, "fewer than two prefill splits fit");
+ assert_eq!(choose_splits(many_heads, 1024, 61_440, 40, true), 1, "fewer than two prefill splits fit");
}
#[uzu_test]
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs
index 8f6c20e32..d76b31263 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs
@@ -125,6 +125,15 @@ fn attention_kernel_matches_cpu() {
(64, 8, 2, 8, 0, false, false),
(64, 8, 8, 8, 1024, false, false),
(64, 8, 2, 8, 1024, false, false),
+ (256, 24, 4, 1, 16_383, true, false),
+ (256, 24, 4, 2, 16_383, true, false),
+ (256, 24, 4, 4, 16_380, true, false),
+ (128, 32, 8, 1, 16_383, true, false),
+ (256, 6, 1, 512, 60_928, true, false),
+ (256, 1, 1, 1024, 60_417, true, false),
+ (128, 4, 1, 512, 30_208, true, false),
+ (128, 4, 1, 1024, 29_696, true, false),
+ (128, 8, 2, 512, 30_208, true, false),
] {
let nodes = use_trie.then_some(trie.as_slice());
let shape = (head_dim, num_q_heads, num_groups, suffix_length, prefix_length, causal);
@@ -265,7 +274,7 @@ fn attention_gate_matches_standalone_gate_on_cpu() {
fn attention_gate_matches_standalone_gate_on_metal() {
let context = block_on(<Metal as Backend>::Context::new()).expect("Metal attention context");
for (suffix_length, prefix_length) in
- [(1, 100), (1, 2000), (200, 800), (2, 1022), (16, 16368), (65, 959), (65, 16319), (1024, 0)]
+ [(1, 100), (1, 2000), (200, 800), (2, 1022), (16, 16368), (65, 959), (65, 16319), (1024, 0), (512, 60_928)]
{
check_attention_gate::<Metal>(context.as_ref(), suffix_length, prefix_length);
}