A high-performance inference engine for AI models
Refactor fragment (#558)
CCYeh45bfe5874e07
diff --git a/crates/backend-uzu/src/backends/common/context.rs b/crates/backend-uzu/src/backends/common/context.rsindex 102897d1b..f76e7678c 100644--- a/crates/backend-uzu/src/backends/common/context.rs+++ b/crates/backend-uzu/src/backends/common/context.rs@@ -49,4 +49,8 @@ pub trait Context: Sized {fn stop_capture(&self) -> Result<(), <Self::Backend as Backend>::Error>;fn sparse_buffers_supported(&self) -> bool;++ fn supports_mxu(&self) -> bool {+ false+ }}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_gemm.rs b/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_gemm.rsindex 0f685b838..ccdecba2f 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_gemm.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_gemm.rs@@ -11,7 +11,8 @@ use crate::{#[variants(T, f32, f16, bf16)]#[variants(BK, 16, 32)]#[variants(BD, 64, 128, 256)]-pub fn attention_gemm<T: ArrayElement + Float, const BK: u32, const BD: u32>(+#[variants(USE_MXU, false)]+pub fn attention_gemm<T: ArrayElement + Float, const BK: u32, const BD: u32, const USE_MXU: bool>(q: *const T,k: *const T,v: *const T,diff --git a/crates/backend-uzu/src/backends/metal/context.rs b/crates/backend-uzu/src/backends/metal/context.rsindex 210113317..c82022df5 100644--- a/crates/backend-uzu/src/backends/metal/context.rs+++ b/crates/backend-uzu/src/backends/metal/context.rs@@ -262,4 +262,8 @@ impl Context for MetalContext {fn sparse_buffers_supported(&self) -> bool {self.device.supports_placement_sparse_resources()}++ fn supports_mxu(&self) -> bool {+ self.supports_mxu()+ }}diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metalindex 252d45687..426a66134 100644--- a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal@@ -1,8 +1,10 @@#include <metal_stdlib>#include "../common/dsl.h"#include "../common/thread_context.h"+#include "../matmul/common/fragment.h"#include "../matmul/common/loader.h"-#include "../matmul/common/simdgroup_fragment.h"+#include "../matmul/common/mxu_fragment_ops.h"+#include "../matmul/common/simdgroup_fragment_ops.h"#include "../generated/ring.h"#include "../generated/trie.h"#include "../generated/attention.h"@@ -21,30 +23,27 @@ struct TransformScale {METAL_FUNC T apply(T x) const { return scale * x; }};-template <typename T>-METAL_FUNC T row_reduce_max(T v) {- // SimdgroupMultiplyAccumulate::get_lane_coordinates mapping groups lanes for- // a row as: {lane, lane^1, lane^8, (lane^1)^8}. Reduce in two steps.- v = metal::max(v, simd_shuffle_xor(v, 1));- v = metal::max(v, simd_shuffle_xor(v, 8));- return v;-}--template <typename T>-METAL_FUNC T row_reduce_sum(T v) {- v += simd_shuffle_xor(v, 1);- v += simd_shuffle_xor(v, 8);- return v;-}--#define BLOCK_QUERY_ROWS 32-#define SIMDGROUPS_PER_ROW 4-#define SIMDGROUPS_PER_COLUMN 1+template <typename T, uint BK, uint BD, bool USE_MXU>+struct AttentionGemmLayout {+ using Ops = metal::conditional_t<USE_MXU, MxuFragmentOps, SimdgroupFragmentOps>;++ METAL_CONST int FRAGMENT_ROWS = Ops::FRAGMENT_ROWS;+ METAL_CONST int SIMDGROUPS_PER_THREADGROUP = 4;+ METAL_CONST int BLOCK_QUERY_ROWS = SIMDGROUPS_PER_THREADGROUP * FRAGMENT_ROWS;+ METAL_CONST int ROW_ALIGNMENT_BYTES = 16;+ METAL_CONST int ROW_PADDING_ELEMENTS = ROW_ALIGNMENT_BYTES / int(sizeof(T));+ METAL_CONST int Q_SMEM_SIZE = USE_MXU ? 1 : (BLOCK_QUERY_ROWS * (int(BD) + ROW_PADDING_ELEMENTS));+ METAL_CONST int KV_SMEM_SIZE = USE_MXU ? 1 : (int(BK) * (int(BD) + ROW_PADDING_ELEMENTS));+};-template <typename T, uint BK, uint BD>+template <typename T, uint BK, uint BD, bool USE_MXU>VARIANTS(T, float, half, bfloat)VARIANTS(BK, 16, 32)VARIANTS(BD, 64, 128, 256)+VARIANTS(USE_MXU, false, true)+CONSTRAINT(!USE_MXU || BK == 32)+CONSTRAINT(!USE_MXU || T != "float")+CONSTRAINT(!USE_MXU || BD != 256)PUBLIC KERNEL(AttentionGemm)(const device T* q,const device T* k,@@ -64,19 +63,24 @@ PUBLIC KERNEL(AttentionGemm)(const bool is_trie SPECIALIZE,const bool is_sliding_window SPECIALIZE,const bool has_sinks SPECIALIZE,- threadgroup T q_smem[BLOCK_QUERY_ROWS * (BD + 16 / sizeof(T))],- threadgroup T kv_smem[BK * (BD + 16 / sizeof(T))],+ threadgroup T q_smem[AttentionGemmLayout<T, BK, BD, USE_MXU>::Q_SMEM_SIZE],+ threadgroup T kv_smem[AttentionGemmLayout<T, BK, BD, USE_MXU>::KV_SMEM_SIZE],const ThreadContext thread_context,- const uint q_tile_idx GROUPS(suffix_length.div_ceil(BLOCK_QUERY_ROWS)),+ const uint q_tile_idx GROUPS(suffix_length.div_ceil(if USE_MXU { 64 } else { 32 })),const uint head_idx GROUPS(num_heads),const uint batch_idx GROUPS(1),const uint lid THREADS(128)) {- // -------------------------------------------------------------------------- // Pointer setup (all strides are in elements)- // q_tile_idx: query tile index (BLOCK_QUERY_ROWS rows)- // head_idx: query head index- // batch_idx: batch index (currently 1 in uzu, but kept for completeness)+ using AccumType = float;+ using Layout = AttentionGemmLayout<T, BK, BD, USE_MXU>;+ using Ops = typename Layout::Ops;+ constexpr short FRAGMENT_ROWS = Layout::FRAGMENT_ROWS;+ constexpr int BLOCK_QUERY_ROWS = Layout::BLOCK_QUERY_ROWS;+ constexpr int ROW_PADDING_ELEMENTS = Layout::ROW_PADDING_ELEMENTS;+ constexpr int SIMDGROUPS_PER_THREADGROUP = Layout::SIMDGROUPS_PER_THREADGROUP;+ constexpr bool QUERY_IS_RESIDENT = USE_MXU;++ using InputType = metal::conditional_t<USE_MXU, T, AccumType>;q += batch_idx * params.q_strides[0] + head_idx * params.q_strides[1] +q_tile_idx * int64_t(BLOCK_QUERY_ROWS) * params.q_strides[2];@@ -92,329 +96,249 @@ PUBLIC KERNEL(AttentionGemm)(trie += batch_idx * suffix_length;}- // -------------------------------------------------------------------------- // Threadgroup memory- constexpr short query_padding = 16 / sizeof(T);- constexpr short key_padding = 16 / sizeof(T);- constexpr short value_padding = 16 / sizeof(T);-- constexpr short query_leading_dimension = BD + query_padding;- constexpr short key_leading_dimension = BD + key_padding;- constexpr short value_leading_dimension = BD + value_padding;-- threadgroup T* query_shared = q_smem;- threadgroup T* key_shared = kv_smem;- threadgroup T* value_shared = kv_smem;-- //- // -------------------------------------------------------------------------- // Block loaders- using QueryLoader = ThreadgroupLoader<- T,- BLOCK_QUERY_ROWS,- BD,- query_leading_dimension,- 1,- SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * 32>;-- using KeyLoader =- ThreadgroupLoader<T, BK, BD, key_leading_dimension, 0, SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * 32>;-- using ValueLoader =- ThreadgroupLoader<T, BK, BD, value_leading_dimension, 0, SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * 32>;-const int query_source_stride = int(params.q_strides[2]);const int key_source_stride = int(params.k_strides[2]);const int value_source_stride = int(params.v_strides[2]);- thread QueryLoader query_loader(q, query_source_stride, query_shared, thread_context);- thread KeyLoader key_loader(k, key_source_stride, key_shared, thread_context);- thread ValueLoader value_loader(v, value_source_stride, value_shared, thread_context);-- TransformScale<T> ts(static_cast<T>(params.scale * M_LOG2E_F));-- // -------------------------------------------------------------------------- // MMA tiles- constexpr short SIMDGROUP_BLOCK_SIZE = 8;- using AccumType = float;- using SimdgroupMultiplyAccumulateType =- SimdgroupMultiplyAccumulate<AccumType, SIMDGROUP_BLOCK_SIZE, SIMDGROUP_BLOCK_SIZE>;-- constexpr int SIMDGROUPS_PER_THREADGROUP = SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN;- static_assert(- BLOCK_QUERY_ROWS >= (SIMDGROUPS_PER_THREADGROUP * SIMDGROUP_BLOCK_SIZE) &&- BLOCK_QUERY_ROWS % (SIMDGROUPS_PER_THREADGROUP * SIMDGROUP_BLOCK_SIZE) == 0,- "Each simdgroup must host at least 1 simdgroup matrix along Q sequence."- );-- // Q sequence multiply-accumulate blocks per simdgroup (QUERY_GRID_ROWS == 1- // for the 32-row block layout)- constexpr int QUERY_GRID_ROWS = BLOCK_QUERY_ROWS / (SIMDGROUPS_PER_THREADGROUP * SIMDGROUP_BLOCK_SIZE);- constexpr int KEY_GRID_COLS = BK / SIMDGROUP_BLOCK_SIZE;- constexpr int HEAD_DIM_GRID_COLS = BD / SIMDGROUP_BLOCK_SIZE;-- static_assert(QUERY_GRID_ROWS == 1, "Expected QUERY_GRID_ROWS == 1");-- SimdgroupFragment<AccumType, QUERY_GRID_ROWS, 1, SimdgroupMultiplyAccumulateType> query_fragment;- SimdgroupFragment<AccumType, 1, KEY_GRID_COLS, SimdgroupMultiplyAccumulateType> key_fragment;- SimdgroupFragment<AccumType, QUERY_GRID_ROWS, KEY_GRID_COLS, SimdgroupMultiplyAccumulateType> score_fragment;- SimdgroupFragment<AccumType, 1, 1, SimdgroupMultiplyAccumulateType> value_fragment;- SimdgroupFragment<AccumType, QUERY_GRID_ROWS, HEAD_DIM_GRID_COLS, SimdgroupMultiplyAccumulateType> output_fragment;-- output_fragment.clear();-- // -------------------------------------------------------------------------- // Lane coordinates and pointer offsets- const short2 lane_coordinates = SimdgroupMultiplyAccumulateType::get_lane_coordinates(thread_context.simd_lane_id);- const short lane_row = lane_coordinates.y;- const short lane_col = lane_coordinates.x;+ constexpr int QUERY_ROW_FRAGMENTS = 1;+ constexpr int KEY_COL_FRAGMENTS = BK / FRAGMENT_ROWS;+ constexpr int HEAD_DIM_FRAGMENTS = BD / FRAGMENT_ROWS;++ static_assert(!USE_MXU || KEY_COL_FRAGMENTS % 2 == 0, "MXU QK needs even N (KEY_COL_FRAGMENTS)");++ constexpr int QUERY_FRAGMENT_COUNT = QUERY_IS_RESIDENT ? HEAD_DIM_FRAGMENTS : 1;+ using QueryFragment = OperandFragment<InputType, QUERY_ROW_FRAGMENTS, 1, Ops>;+ using KeyFragment = OperandFragment<InputType, 1, KEY_COL_FRAGMENTS, Ops, ReadTranspose>;+ using ScoreFragment = Fragment<AccumType, QUERY_ROW_FRAGMENTS, KEY_COL_FRAGMENTS, Ops>;+ QueryFragment query_frags[QUERY_FRAGMENT_COUNT];+ ScoreFragment score_fragment;+ constexpr int VALUE_COL_FRAGMENTS = USE_MXU ? 2 : HEAD_DIM_FRAGMENTS;+ static_assert(HEAD_DIM_FRAGMENTS % VALUE_COL_FRAGMENTS == 0, "head-dim must split into VALUE_COL_FRAGMENTS frags");+ static_assert(!USE_MXU || VALUE_COL_FRAGMENTS % 2 == 0, "MXU PV needs even N (VALUE_COL_FRAGMENTS)");+ constexpr int OUTPUT_CHUNKS = HEAD_DIM_FRAGMENTS / VALUE_COL_FRAGMENTS;+ constexpr short OUTPUT_CHUNK_COLS = VALUE_COL_FRAGMENTS * FRAGMENT_ROWS;+ using ValueFragment = OperandFragment<InputType, KEY_COL_FRAGMENTS, VALUE_COL_FRAGMENTS, Ops>;+ using OutputFragment = Fragment<AccumType, QUERY_ROW_FRAGMENTS, VALUE_COL_FRAGMENTS, Ops>;+ OutputFragment output_chunks[OUTPUT_CHUNKS];- const short simdgroup_row_base = SIMDGROUP_BLOCK_SIZE * QUERY_GRID_ROWS * short(thread_context.simdgroup_index);-- const short query_shared_offset = (simdgroup_row_base + lane_row) * query_leading_dimension + lane_col;- constexpr short query_tile_stride = SIMDGROUP_BLOCK_SIZE;-- const short key_shared_offset = lane_col * key_leading_dimension + lane_row;- constexpr short key_tile_stride = SIMDGROUP_BLOCK_SIZE;+ METAL_PRAGMA_UNROLL+ for (int c = 0; c < OUTPUT_CHUNKS; ++c) {+ output_chunks[c].clear();+ }- const short value_shared_offset = lane_row * value_leading_dimension + lane_col;+ const short simdgroup_row_base = FRAGMENT_ROWS * QUERY_ROW_FRAGMENTS * short(thread_context.simdgroup_index);+ constexpr short head_dim_fragment_stride = FRAGMENT_ROWS;- // -------------------------------------------------------------------------- // Load Q block once (and apply scaling)- threadgroup_barrier(mem_flags::mem_threadgroup);+ const bool ragged_q = (!align_q && int(q_tile_idx) == params.nq_aligned);+ constexpr short query_leading_dimension = BD + ROW_PADDING_ELEMENTS;+ threadgroup T* query_shared = q_smem;+ const short query_shared_offset = simdgroup_row_base * query_leading_dimension;- if (!align_q && int(q_tile_idx) == params.nq_aligned) {- query_loader.load_safe(short2(BD, params.q_rem));+ if constexpr (QUERY_IS_RESIDENT) {+ auto q_src = fragment_source(q + int64_t(simdgroup_row_base) * query_source_stride, query_source_stride);+ if (ragged_q) {+ q_src = q_src.bounded(params.q_rem - simdgroup_row_base, FRAGMENT_ROWS);+ }+ const InputType query_scale = static_cast<InputType>(params.scale * M_LOG2E_F);+ METAL_PRAGMA_UNROLL+ for (short dd = 0; dd < HEAD_DIM_FRAGMENTS; dd++) {+ query_frags[dd].load_from(thread_context.simd_lane_id, q_src.advanced(dd * head_dim_fragment_stride));+ query_frags[dd].map_coords(thread_context.simd_lane_id, [&](short, short, InputType v) {+ return v * query_scale;+ });+ }} else {- query_loader.load_unsafe();+ using QueryLoader = ThreadgroupLoader<+ T,+ BLOCK_QUERY_ROWS,+ BD,+ query_leading_dimension,+ 1,+ SIMDGROUPS_PER_THREADGROUP * METAL_SIMD_SIZE>;+ thread QueryLoader query_loader(q, query_source_stride, query_shared, thread_context);+ if (ragged_q) {+ query_loader.load_safe(short2(BD, params.q_rem));+ } else {+ query_loader.load_unsafe();+ }+ query_loader.apply_inplace_op(TransformScale<T>(static_cast<T>(params.scale * M_LOG2E_F)));+ threadgroup_barrier(mem_flags::mem_threadgroup);}- query_loader.apply_inplace_op(ts);-- // -------------------------------------------------------------------------- // Streaming softmax state for this row (shared across lanes in a row)- const AccumType neg_inf = static_cast<AccumType>(-1e9f) * M_LOG2E_F;- AccumType max_score = -INFINITY;- AccumType sum_score = AccumType(0);- if (has_sinks) {- max_score = M_LOG2E_F * static_cast<AccumType>(sinks[head_idx]);- sum_score = AccumType(1);+ const AccumType masked_score = static_cast<AccumType>(-1e9f) * M_LOG2E_F;+ constexpr int ROWS_PER_LANE = QUERY_ROW_FRAGMENTS * Ops::THREAD_ELEMENT_ROWS;+ const AccumType init_max = has_sinks ? AccumType(M_LOG2E_F * static_cast<AccumType>(sinks[head_idx])) : -INFINITY;+ const AccumType init_sum = has_sinks ? AccumType(1) : AccumType(0);+ AccumType max_score[ROWS_PER_LANE];+ AccumType sum_score[ROWS_PER_LANE];+ METAL_PRAGMA_UNROLL+ for (int r = 0; r < ROWS_PER_LANE; ++r) {+ max_score[r] = init_max;+ sum_score[r] = init_sum;}- // Determine K block loop limit (causal can early-stop)uint32_t kb_lim = params.nk;if (is_causal) {const int q_max = (int(q_tile_idx) + 1) * BLOCK_QUERY_ROWS + params.q_off;- kb_lim = (q_max + BK - 1) / BK;- kb_lim = min(params.nk, kb_lim);+ kb_lim = min(params.nk, uint32_t((q_max + BK - 1) / BK));}- const int q_rel = int(q_tile_idx) * BLOCK_QUERY_ROWS + int(simdgroup_row_base) + int(lane_row); // [0, q_len)-+ const int q_base = int(q_tile_idx) * BLOCK_QUERY_ROWS + int(simdgroup_row_base);const int prefix_length = params.q_off;-const int suffix_position = is_kv_cache_ring ? int(ring_params.ring_length) : prefix_length;+ using KVBlockSource = BlockSource<+ T,+ BK,+ BD,+ BD + ROW_PADDING_ELEMENTS,+ SIMDGROUPS_PER_THREADGROUP * METAL_SIMD_SIZE,+ typename Ops::BlockStorage>;+ threadgroup T* kv_shared = kv_smem;- const int query_position =- (is_trie && q_rel < params.q_len) ? suffix_position + int(trie[q_rel].height) : suffix_position + q_rel;-- // Loop over KV blocksfor (int kb = 0; kb < kb_lim; kb++) {- // Load K block- threadgroup_barrier(mem_flags::mem_threadgroup);- if (!align_k && kb == params.nk_aligned) {- key_loader.load_safe(short2(BD, params.k_rem));- } else {- key_loader.load_unsafe();- }-- // Compute S = Q @ K^T for this block+ const bool tail_k = (!align_k && kb == params.nk_aligned);+ const short valid_k_rows = tail_k ? short(params.k_rem) : short(BK);+ const device T* k_block = k + int64_t(kb) * int(BK) * key_source_stride;score_fragment.clear();- threadgroup_barrier(mem_flags::mem_threadgroup);+ KVBlockSource key_source{+ k_block,+ key_source_stride,+ kv_shared,+ valid_k_rows,+ thread_context,+ };+ key_source.prepare();+ auto q_shared_src = fragment_source(query_shared + query_shared_offset, query_leading_dimension);METAL_PRAGMA_UNROLL- for (short dd = 0; dd < HEAD_DIM_GRID_COLS; dd++) {- simdgroup_barrier(mem_flags::mem_none);-- query_fragment.template load<T, 1, 1, query_leading_dimension, 1>(- &query_shared[query_shared_offset + dd * query_tile_stride]- );- key_fragment.template load<T, 1, 1, 1, key_leading_dimension>(- &key_shared[key_shared_offset + dd * key_tile_stride]- );+ for (short dd = 0; dd < HEAD_DIM_FRAGMENTS; dd++) {+ if constexpr (!QUERY_IS_RESIDENT) {+ query_frags[0].load_from(thread_context.simd_lane_id, q_shared_src.advanced(dd * head_dim_fragment_stride));+ }+ const short q_idx = QUERY_IS_RESIDENT ? dd : 0;+ KeyFragment key_fragment;+ key_source.load(key_fragment, dd * head_dim_fragment_stride);simdgroup_barrier(mem_flags::mem_none);- tile_multiply_accumulate(score_fragment, query_fragment, key_fragment, score_fragment);+ fragment_mma(score_fragment, query_frags[q_idx], key_fragment);}- // Mask out tail keys for the last (unaligned) K block- if (!align_k && kb == params.nk_aligned) {- const int k_rem = params.k_rem;- METAL_PRAGMA_UNROLL- for (short j = 0; j < KEY_GRID_COLS; j++) {- thread auto& frag = score_fragment.multiply_accumulate_at(0, j);- const int col0 = int(lane_col) + int(j) * SIMDGROUP_BLOCK_SIZE;- if (col0 >= k_rem) {- frag[0] = neg_inf;- }- if ((col0 + 1) >= k_rem) {- frag[1] = neg_inf;- }- }- }-- // Unified masking: causal, trie, ring, sliding window- if (q_rel < params.q_len) {- METAL_PRAGMA_UNROLL- for (short j = 0; j < KEY_GRID_COLS; j++) {- thread auto& frag = score_fragment.multiply_accumulate_at(0, j);- const int col_base = kb * BK + int(lane_col) + int(j) * SIMDGROUP_BLOCK_SIZE;-- const int k0 = col_base;- const int k1 = col_base + 1;-- if (k0 < params.k_len && !should_use_key(- ring_params,- trie,- sliding_window_size,- q_rel,- prefix_length,- suffix_position,- query_position,- k0,- is_kv_cache_ring,- is_causal,- is_trie,- is_sliding_window- ))- frag[0] = neg_inf;-- if (k1 < params.k_len && !should_use_key(- ring_params,- trie,- sliding_window_size,- q_rel,- prefix_length,- suffix_position,- query_position,- k1,- is_kv_cache_ring,- is_causal,- is_trie,- is_sliding_window- ))- frag[1] = neg_inf;+ const int k_rem = params.k_rem;+ if (!is_trie && !is_kv_cache_ring && !is_sliding_window) {+ const bool diag = is_causal && ((int(kb) + 1) * int(BK) - 1 > prefix_length + q_base);+ if (tail_k || diag) {+ score_fragment.map_coords(thread_context.simd_lane_id, [&](short row, short col, AccumType v) {+ if (tail_k && int(col) >= k_rem) {+ return masked_score;+ }+ if (diag && (kb * int(BK) + int(col)) > prefix_length + q_base + int(row)) {+ return masked_score;+ }+ return v;+ });}- }-- // Load V block (overwriting K in shared memory)- threadgroup_barrier(mem_flags::mem_threadgroup);- if (!align_k && kb == params.nk_aligned) {- value_loader.load_safe(short2(BD, params.k_rem));} else {- value_loader.load_unsafe();+ score_fragment.map_coords(thread_context.simd_lane_id, [&](short row, short col, AccumType v) {+ if (tail_k && int(col) >= k_rem) {+ return masked_score;+ }+ const int q_rel = q_base + int(row);+ if (q_rel >= int(params.q_len)) {+ return v;+ }+ const int key = kb * BK + int(col);+ if (key >= int(params.k_len)) {+ return v;+ }+ const int query_position = is_trie ? suffix_position + int(trie[q_rel].height) : suffix_position + q_rel;+ if (!should_use_key(+ ring_params,+ trie,+ sliding_window_size,+ q_rel,+ prefix_length,+ suffix_position,+ query_position,+ key,+ is_kv_cache_ring,+ is_causal,+ is_trie,+ is_sliding_window+ )) {+ return masked_score;+ }+ return v;+ });}- // ------------------------------------------------------------------------ // Streaming softmax update for this block+ AccumType block_max[ROWS_PER_LANE];+ score_fragment.row_reduce(block_max, -INFINITY, [](AccumType a, AccumType b) { return metal::max(a, b); });- // Row max for this block- AccumType block_max_local = -INFINITY;+ AccumType new_max[ROWS_PER_LANE];+ AccumType factor[ROWS_PER_LANE];METAL_PRAGMA_UNROLL- for (short j = 0; j < KEY_GRID_COLS; j++) {- const thread auto& frag = score_fragment.multiply_accumulate_at(0, j);- block_max_local = metal::max(block_max_local, frag[0]);- block_max_local = metal::max(block_max_local, frag[1]);+ for (int r = 0; r < ROWS_PER_LANE; ++r) {+ new_max[r] = metal::max(max_score[r], block_max[r]);+ factor[r] = fast::exp2(max_score[r] - new_max[r]);+ max_score[r] = new_max[r];+ sum_score[r] *= factor[r];}- const AccumType block_max = row_reduce_max(block_max_local);-- const AccumType new_max = metal::max(max_score, block_max);- const AccumType factor = fast::exp2(max_score - new_max);- max_score = new_max;- // Rescale running sum- sum_score *= factor;+ score_fragment.map_rows(new_max, [](AccumType v, AccumType m) { return fast::exp2(v - m); });- // exp2(S - new_max) and row sum- AccumType block_sum_local = AccumType(0);+ AccumType block_sum[ROWS_PER_LANE];+ score_fragment.row_reduce(block_sum, AccumType(0), [](AccumType a, AccumType b) { return a + b; });METAL_PRAGMA_UNROLL- for (short j = 0; j < KEY_GRID_COLS; j++) {- thread auto& frag = score_fragment.multiply_accumulate_at(0, j);- frag[0] = fast::exp2(frag[0] - new_max);- frag[1] = fast::exp2(frag[1] - new_max);- block_sum_local += frag[0] + frag[1];+ for (int r = 0; r < ROWS_PER_LANE; ++r) {+ sum_score[r] += block_sum[r];}- const AccumType block_sum = row_reduce_sum(block_sum_local);- sum_score += block_sum;- // Rescale output accumulatorMETAL_PRAGMA_UNROLL- for (short id = 0; id < HEAD_DIM_GRID_COLS; id++) {- thread auto& frag = output_fragment.multiply_accumulate_at(0, id);- frag[0] *= factor;- frag[1] *= factor;+ for (int c = 0; c < OUTPUT_CHUNKS; ++c) {+ output_chunks[c].map_rows(factor, [](AccumType v, AccumType f) { return v * f; });}- // Accumulate output: output_fragment += score_fragment * Vblock- threadgroup_barrier(mem_flags::mem_threadgroup);+ const device T* v_block = v + int64_t(kb) * int(BK) * value_source_stride;+ KVBlockSource value_source{+ v_block,+ value_source_stride,+ kv_shared,+ valid_k_rows,+ thread_context,+ };+ value_source.prepare();METAL_PRAGMA_UNROLL- for (short id = 0; id < HEAD_DIM_GRID_COLS; id++) {- METAL_PRAGMA_UNROLL- for (short ik = 0; ik < KEY_GRID_COLS; ik++) {- if constexpr (BD == 128) {- simdgroup_barrier(mem_flags::mem_none);- }-- const short kk = ik * SIMDGROUP_BLOCK_SIZE;- const short dd = id * SIMDGROUP_BLOCK_SIZE;-- value_fragment.template load<T, 1, 1, value_leading_dimension, 1>(- &value_shared[value_shared_offset + kk * value_leading_dimension + dd]- );-- if constexpr (BD == 128) {- simdgroup_barrier(mem_flags::mem_none);- }-- SimdgroupMultiplyAccumulateType::multiply_accumulate(- output_fragment.multiply_accumulate_at(0, id),- score_fragment.multiply_accumulate_at(0, ik),- value_fragment.multiply_accumulate_at(0, 0),- output_fragment.multiply_accumulate_at(0, id)- );- }+ for (int c = 0; c < OUTPUT_CHUNKS; ++c) {+ ValueFragment value_chunk;+ value_source.load(value_chunk, c * OUTPUT_CHUNK_COLS);+ fragment_mma(output_chunks[c], score_fragment, value_chunk);}-- // Prepare for next iteration- key_loader.next();- value_loader.next();}- // -------------------------------------------------------------------------- // Normalize output by sum_score (avoid div-by-zero for masked-out rows)- const AccumType inv_sum = AccumType(1) / sum_score;+ AccumType inv_sum[ROWS_PER_LANE];METAL_PRAGMA_UNROLL- for (short id = 0; id < HEAD_DIM_GRID_COLS; id++) {- thread auto& frag = output_fragment.multiply_accumulate_at(0, id);- frag[0] *= inv_sum;- frag[1] *= inv_sum;+ for (int r = 0; r < ROWS_PER_LANE; ++r) {+ inv_sum[r] = AccumType(1) / sum_score[r];+ }+ METAL_PRAGMA_UNROLL+ for (int c = 0; c < OUTPUT_CHUNKS; ++c) {+ output_chunks[c].map_rows(inv_sum, [](AccumType v, AccumType s) { return v * s; });}- threadgroup_barrier(mem_flags::mem_none);-- // Store results (O is row-major with row-stride params.o_strides[2])- o += int64_t(simdgroup_row_base + lane_row) * params.o_strides[2] + int64_t(lane_col);-- if (!align_q && int(q_tile_idx) == params.nq_aligned) {- const short2 dst_tile_dims = short2(BD - lane_col, params.q_rem - (simdgroup_row_base + lane_row));+ o += int64_t(simdgroup_row_base) * params.o_strides[2];- if (dst_tile_dims.x <= 0 || dst_tile_dims.y <= 0) {- return;+ if (ragged_q && params.q_rem <= int(simdgroup_row_base)) {+ return;+ }+ METAL_PRAGMA_UNROLL+ for (int c = 0; c < OUTPUT_CHUNKS; ++c) {+ device T* o_chunk = o + c * OUTPUT_CHUNK_COLS;+ if (ragged_q) {+ const short2 dst_tile_dims = short2(OUTPUT_CHUNK_COLS, params.q_rem - simdgroup_row_base);+ output_chunks[c].store_safe(thread_context.simd_lane_id, o_chunk, int(params.o_strides[2]), dst_tile_dims);+ } else {+ output_chunks[c].store(thread_context.simd_lane_id, o_chunk, int(params.o_strides[2]));}-- output_fragment.template store_safe<T, 1, 1>(o, int(params.o_strides[2]), dst_tile_dims);- } else {- output_fragment.template store<T, 1, 1>(o, int(params.o_strides[2]));}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.hindex 6c3f08682..a12b2a5cb 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h@@ -11,13 +11,45 @@ using namespace metal;namespace uzu {namespace matmul {-///////////////////////////////////////////////////////////////////////////////-// Fragment - a thread-private tile of values arranged as TILE_ROWS x TILE_COLS-// sub-tiles. Ops provides the sub-tile shape constants and the MMA primitive;-// Fragment itself owns lane positioning and device<->register transfer.-///////////////////////////////////////////////////////////////////////////////+template <typename Ptr>+struct FragmentSource {+ Ptr base;+ int row_stride;+ int col_stride;+ short row_bound;+ short col_bound;+ bool ragged;-template <typename T, ushort TILE_ROWS_, ushort TILE_COLS_, class Ops>+ METAL_FUNC FragmentSource advanced(int elements) const thread {+ FragmentSource out = *this;+ out.base += elements;+ return out;+ }++ METAL_FUNC FragmentSource bounded(short rows, short cols) const thread {+ FragmentSource out = *this;+ out.row_bound = rows;+ out.col_bound = cols;+ out.ragged = true;+ return out;+ }+};++template <typename Ptr>+METAL_FUNC FragmentSource<Ptr> fragment_source(Ptr base, int row_stride, int col_stride = 1) {+ return FragmentSource<Ptr>{base, row_stride, col_stride, 0, 0, false};+}++struct ReadDirect {};+struct ReadTranspose {};++template <+ typename T,+ ushort ROW_FRAGMENTS_,+ ushort COL_FRAGMENTS_,+ class Ops,+ class ReadPolicy = ReadDirect,+ bool MMA_TRANSPOSE_ = false>struct Fragment {using FragmentOpsType = Ops;using ElementType = T;@@ -38,33 +70,27 @@ struct Fragment {"(elements() relies on it)");- METAL_CONST ushort TILE_ROWS = TILE_ROWS_;- METAL_CONST ushort TILE_COLS = TILE_COLS_;+ METAL_CONST ushort ROW_FRAGMENTS = ROW_FRAGMENTS_;+ METAL_CONST ushort COL_FRAGMENTS = COL_FRAGMENTS_;METAL_CONST ushort FRAGMENT_ROWS = ushort(Ops::FRAGMENT_ROWS);METAL_CONST ushort FRAGMENT_COLS = ushort(Ops::FRAGMENT_COLS);- METAL_CONST ushort NUM_FRAGS = TILE_ROWS * TILE_COLS;- METAL_CONST ushort ELEMENTS_PER_TILE = NUM_FRAGS * Ops::ELEMENTS_PER_THREAD;+ METAL_CONST ushort NUM_FRAGS = ROW_FRAGMENTS * COL_FRAGMENTS;+ METAL_CONST ushort ELEMENTS_PER_FRAGMENT = NUM_FRAGS * Ops::ELEMENTS_PER_THREAD;+ METAL_CONST ushort ROW_REDUCE_LANE_XOR_0 = 1;+ METAL_CONST ushort ROW_REDUCE_LANE_XOR_1 = 8;+ METAL_CONST bool MMA_TRANSPOSE = MMA_TRANSPOSE_;ThreadVectorType fragment_data[NUM_FRAGS];- ThreadContext thread_context;- METAL_FUNC Fragment(const thread ThreadContext& thread_context) thread : thread_context(thread_context) {}-- // Lane origin (row, col) within a single FRAGMENT_ROWS x FRAGMENT_COLS- // sub-tile. The mapping depends only on Ops::THREAD_ELEMENT_COLS, so it is- // identical for every Fragment sharing that Ops type.- METAL_FUNC static constexpr short2 get_position(const thread ThreadContext& thread_context) {- const ushort simd_lane_id = ushort(thread_context.simd_lane_id);+ METAL_FUNC static constexpr short2 get_position(const ushort simd_lane_id) {const short quad = simd_lane_id / 4;const short row = (quad & 4) + (simd_lane_id / 2) % 4;const short col = ((quad & 2) + simd_lane_id % 2) * Ops::THREAD_ELEMENT_COLS;return short2{col, row};}- METAL_FUNC short2 get_position() const thread { return get_position(thread_context); }-METAL_FUNC constexpr void clear() {METAL_PRAGMA_UNROLLfor (ushort index = 0; index < NUM_FRAGS; ++index) {@@ -73,7 +99,7 @@ struct Fragment {}METAL_FUNC constexpr thread ThreadVectorType& fragment_at(const ushort row_index, const ushort col_index) {- return fragment_data[row_index * TILE_COLS + col_index];+ return fragment_data[row_index * COL_FRAGMENTS + col_index];}template <bool transpose>@@ -91,81 +117,161 @@ struct Fragment {METAL_FUNC thread ElementType* elements() { return reinterpret_cast<thread ElementType*>(fragment_data); }- // Unsafe load: copy a (TILE_ROWS * FRAGMENT_ROWS) x (TILE_COLS *- // FRAGMENT_COLS) block from device memory into fragment registers. ColStride- // defaults to the compile-time constant 1 so that the row-major fast path- // triggers when the caller omits it. Tile strides control the gap between- // sub-tiles.- template <- class Ptr,- class RowStride,- class ColStride = Int<1>,- class TileRowStride = Int<1>,- class TileColStride = Int<1>>- METAL_FUNC void load(- Ptr source,- RowStride row_stride,- ColStride col_stride = {},- TileRowStride tile_row_stride = {},- TileColStride tile_col_stride = {}- ) thread {- transfer<LOAD, UNSAFE>(source, row_stride, col_stride, Int<0>{}, Int<0>{}, tile_row_stride, tile_col_stride);+ template <class Fn>+ METAL_FUNC void map(Fn fn) thread {+ thread ElementType* data = elements();+ METAL_PRAGMA_UNROLL+ for (ushort i = 0; i < ELEMENTS_PER_FRAGMENT; i++) {+ data[i] = ElementType(fn(data[i]));+ }}- // Safe load: col_stride is implicitly 1; out-of-bounds elements become T(0).- template <class Ptr, class TileRowStride = Int<1>, class TileColStride = Int<1>>- METAL_FUNC void load_safe(- Ptr source,- const int leading_dimension,- const short2 tile_dimensions,- TileRowStride tile_row_stride = {},- TileColStride tile_col_stride = {}- ) thread {- transfer<LOAD, SAFE>(- source,- leading_dimension,- Int<1>{},- tile_dimensions.y,- tile_dimensions.x,- tile_row_stride,- tile_col_stride- );+ template <class Fn>+ METAL_FUNC void map_coords(const ushort simd_lane_id, Fn fn) thread {+ const short2 position = get_position(simd_lane_id);+ thread ElementType* data = elements();+ for_each_fragment([&](auto fragment_row, auto fragment_col) {+ const short row_base = position.y + short(fragment_row.value) * FRAGMENT_ROWS;+ const short col_base = position.x + short(fragment_col.value) * FRAGMENT_COLS;+ const ushort frag_base = (fragment_row.value * COL_FRAGMENTS + fragment_col.value) * Ops::ELEMENTS_PER_THREAD;+ METAL_PRAGMA_UNROLL+ for (ushort i = 0; i < Ops::THREAD_ELEMENT_ROWS; ++i) {+ METAL_PRAGMA_UNROLL+ for (ushort j = 0; j < Ops::THREAD_ELEMENT_COLS; ++j) {+ const short row = row_base + short(i) * Ops::THREAD_ELEMENT_ROW_STRIDE;+ const short col = col_base + short(j);+ thread ElementType& value = data[frag_base + i * Ops::THREAD_ELEMENT_COLS + j];+ value = ElementType(fn(row, col, value));+ }+ }+ });+ }++ // Row lanes differ in bits {0,3} for both simdgroup and MXU layouts.+ template <typename Acc, class Fn>+ METAL_FUNC void row_reduce(thread Acc* out, const Acc identity, Fn op) thread {+ thread ElementType* data = elements();+ METAL_PRAGMA_UNROLL+ for (ushort tr = 0; tr < ROW_FRAGMENTS; ++tr) {+ METAL_PRAGMA_UNROLL+ for (ushort i = 0; i < Ops::THREAD_ELEMENT_ROWS; ++i) {+ Acc acc = identity;+ METAL_PRAGMA_UNROLL+ for (ushort tc = 0; tc < COL_FRAGMENTS; ++tc) {+ const ushort frag_base = (tr * COL_FRAGMENTS + tc) * Ops::ELEMENTS_PER_THREAD;+ METAL_PRAGMA_UNROLL+ for (ushort j = 0; j < Ops::THREAD_ELEMENT_COLS; ++j) {+ acc = op(acc, Acc(data[frag_base + i * Ops::THREAD_ELEMENT_COLS + j]));+ }+ }+ acc = op(acc, simd_shuffle_xor(acc, ROW_REDUCE_LANE_XOR_0));+ acc = op(acc, simd_shuffle_xor(acc, ROW_REDUCE_LANE_XOR_1));+ out[tr * Ops::THREAD_ELEMENT_ROWS + i] = acc;+ }+ }+ }++ template <typename Acc, class Fn>+ METAL_FUNC void map_rows(const thread Acc* row_vals, Fn fn) thread {+ thread ElementType* data = elements();+ METAL_PRAGMA_UNROLL+ for (ushort tr = 0; tr < ROW_FRAGMENTS; ++tr) {+ METAL_PRAGMA_UNROLL+ for (ushort i = 0; i < Ops::THREAD_ELEMENT_ROWS; ++i) {+ const Acc rv = row_vals[tr * Ops::THREAD_ELEMENT_ROWS + i];+ METAL_PRAGMA_UNROLL+ for (ushort tc = 0; tc < COL_FRAGMENTS; ++tc) {+ const ushort frag_base = (tr * COL_FRAGMENTS + tc) * Ops::ELEMENTS_PER_THREAD;+ METAL_PRAGMA_UNROLL+ for (ushort j = 0; j < Ops::THREAD_ELEMENT_COLS; ++j) {+ const ushort e = frag_base + i * Ops::THREAD_ELEMENT_COLS + j;+ data[e] = ElementType(fn(Acc(data[e]), rv));+ }+ }+ }+ }+ }++ template <class Ptr>+ METAL_FUNC void load_from(const ushort simd_lane_id, FragmentSource<Ptr> src) thread {+ if constexpr (metal::is_same_v<ReadPolicy, ReadTranspose> && Ops::READ_TRANSPOSE_SWAPS_SOURCE_STRIDES) {+ const int row_stride = src.row_stride;+ src.row_stride = src.col_stride;+ src.col_stride = row_stride;+ const short row_bound = src.row_bound;+ src.row_bound = src.col_bound;+ src.col_bound = row_bound;+ }++ if (src.ragged) {+ transfer<LOAD, SAFE>(+ simd_lane_id,+ src.base,+ src.row_stride,+ src.col_stride,+ src.row_bound,+ src.col_bound,+ Int<1>{},+ Int<1>{}+ );+ } else {+ transfer<LOAD, UNSAFE>(+ simd_lane_id,+ src.base,+ src.row_stride,+ src.col_stride,+ Int<0>{},+ Int<0>{},+ Int<1>{},+ Int<1>{}+ );+ }}- // Unsafe store: mirror of load. Same defaults apply.template <class Ptr,class RowStride,class ColStride = Int<1>,- class TileRowStride = Int<1>,- class TileColStride = Int<1>>+ class FragmentRowStride = Int<1>,+ class FragmentColStride = Int<1>>METAL_FUNC void store(+ const ushort simd_lane_id,Ptr destination,RowStride row_stride,ColStride col_stride = {},- TileRowStride tile_row_stride = {},- TileColStride tile_col_stride = {}+ FragmentRowStride fragment_row_stride = {},+ FragmentColStride fragment_col_stride = {}) thread {- transfer<STORE, UNSAFE>(destination, row_stride, col_stride, Int<0>{}, Int<0>{}, tile_row_stride, tile_col_stride);+ transfer<STORE, UNSAFE>(+ simd_lane_id,+ destination,+ row_stride,+ col_stride,+ Int<0>{},+ Int<0>{},+ fragment_row_stride,+ fragment_col_stride+ );}- // Safe store: col_stride is implicitly 1; out-of-bounds elements are skipped.- template <class Ptr, class TileRowStride = Int<1>, class TileColStride = Int<1>>+ template <class Ptr, class FragmentRowStride = Int<1>, class FragmentColStride = Int<1>>METAL_FUNC void store_safe(+ const ushort simd_lane_id,Ptr destination,const int leading_dimension,const short2 tile_dimensions,- TileRowStride tile_row_stride = {},- TileColStride tile_col_stride = {}+ FragmentRowStride fragment_row_stride = {},+ FragmentColStride fragment_col_stride = {}) thread {transfer<STORE, SAFE>(+ simd_lane_id,destination,leading_dimension,Int<1>{},tile_dimensions.y,tile_dimensions.x,- tile_row_stride,- tile_col_stride+ fragment_row_stride,+ fragment_col_stride);}@@ -177,20 +283,11 @@ private:template <class Fn>METAL_FUNC void for_each_fragment(Fn fn) thread {- const_for_loop<0, TILE_ROWS, 1>([&](auto row_index) {- const_for_loop<0, TILE_COLS, 1>([&](auto col_index) { fn(row_index, col_index); });+ const_for_loop<0, ROW_FRAGMENTS, 1>([&](auto row_index) {+ const_for_loop<0, COL_FRAGMENTS, 1>([&](auto col_index) { fn(row_index, col_index); });});}- // Unified memory transfer between device memory and fragment registers.- //- // IS_LOAD - true to read into fragment_data, false to write back.- // IS_SAFE - true to bounds-check each element against (row_limit,- // col_limit);- // on load, out-of-bounds elements become T(0); on store, skipped.- //- // When ColStride is a compile-time 1, the column-stride multiplication is- // elided to help the compiler collapse strength-reduced addressing.template <bool IS_LOAD,bool IS_SAFE,@@ -199,29 +296,30 @@ private:class ColStride,class RowLimit,class ColLimit,- class TileRowStride,- class TileColStride>+ class FragmentRowStride,+ class FragmentColStride>METAL_FUNC void transfer(+ const ushort simd_lane_id,Ptr ptr,RowStride row_stride,ColStride col_stride,RowLimit row_limit,ColLimit col_limit,- TileRowStride tile_row_stride,- TileColStride tile_col_stride+ FragmentRowStride fragment_row_stride,+ FragmentColStride fragment_col_stride) thread {using U = PointerElementType<Ptr>;constexpr bool col_stride_is_one = metal::is_same_v<ColStride, Int<1>>;- const short2 position = get_position();+ const short2 position = get_position(simd_lane_id);ptr += position.y * row_stride + position.x * col_stride;const auto local_row_limit = row_limit - position.y;const auto local_col_limit = col_limit - position.x;- for_each_fragment([&](auto tile_row, auto tile_col) {- thread auto& frag = fragment_at(tile_row.value, tile_col.value);- const auto row_base = tile_row.value * FRAGMENT_ROWS * tile_row_stride;- const auto col_base = tile_col.value * FRAGMENT_COLS * tile_col_stride;+ for_each_fragment([&](auto fragment_row, auto fragment_col) {+ thread auto& frag = fragment_at(fragment_row.value, fragment_col.value);+ const auto row_base = fragment_row.value * FRAGMENT_ROWS * fragment_row_stride;+ const auto col_base = fragment_col.value * FRAGMENT_COLS * fragment_col_stride;METAL_PRAGMA_UNROLLfor (ushort i = 0; i < Ops::THREAD_ELEMENT_ROWS; i++) {@@ -254,5 +352,34 @@ private:}};+template <class Ops, class Read>+struct OperandFragmentTraits {+ METAL_CONST bool READ_TRANSPOSE = metal::is_same_v<Read, ReadTranspose>;+ METAL_CONST bool MMA_TRANSPOSE = READ_TRANSPOSE && !Ops::READ_TRANSPOSE_SWAPS_SOURCE_STRIDES;+};++template <typename T, ushort ROW_FRAGMENTS, ushort COL_FRAGMENTS, class Ops, class Read = ReadDirect>+using OperandFragment = Fragment<+ T,+ OperandFragmentTraits<Ops, Read>::MMA_TRANSPOSE ? COL_FRAGMENTS : ROW_FRAGMENTS,+ OperandFragmentTraits<Ops, Read>::MMA_TRANSPOSE ? ROW_FRAGMENTS : COL_FRAGMENTS,+ Ops,+ Read,+ OperandFragmentTraits<Ops, Read>::MMA_TRANSPOSE>;++template <class OutputFragment, class LeftFragment, class RightFragment>+METAL_FUNC void fragment_mma(thread OutputFragment& output, thread LeftFragment& left, thread RightFragment& right) {+ static_assert(+ metal::is_same_v<typename OutputFragment::FragmentOpsType, typename LeftFragment::FragmentOpsType> &&+ metal::is_same_v<typename OutputFragment::FragmentOpsType, typename RightFragment::FragmentOpsType>,+ "fragment_mma requires output, left, and right fragments to use the same FragmentOps"+ );+ OutputFragment::FragmentOpsType::template fragment_mma<LeftFragment::MMA_TRANSPOSE, RightFragment::MMA_TRANSPOSE>(+ output,+ left,+ right+ );+}+} // namespace matmul} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/loader.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/loader.hindex abf7c3c6a..81eec16f0 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/loader.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/loader.h@@ -2,16 +2,13 @@#include "../../common/defines.h"#include "../../common/thread_context.h"+#include "fragment.h"using namespace metal;namespace uzu {namespace matmul {-///////////////////////////////////////////////////////////////////////////////-// Threadgroup Loader - loads tiles from device memory to threadgroup memory-///////////////////////////////////////////////////////////////////////////////-template <typename T,ushort THREADGROUP_TILE_ROWS,@@ -117,5 +114,73 @@ struct ThreadgroupLoader {METAL_FUNC void next() { source += tile_stride; }};+struct DeviceBlockStorage {};+struct ThreadgroupBlockStorage {};++template <typename T, ushort ROWS, ushort COLS, ushort SHARED_LEADING_DIMENSION, ushort THREADGROUP_SIZE, class Storage>+struct BlockSource;++template <typename T, ushort ROWS, ushort COLS, ushort SHARED_LEADING_DIMENSION, ushort THREADGROUP_SIZE>+struct BlockSource<T, ROWS, COLS, SHARED_LEADING_DIMENSION, THREADGROUP_SIZE, DeviceBlockStorage> {+ FragmentSource<const device T*> source;+ ushort simd_lane_id;++ METAL_FUNC BlockSource(+ const device T* source_,+ const int source_leading_dimension_,+ threadgroup T*,+ const short valid_rows_,+ const thread ThreadContext& thread_context_+ )+ : source(fragment_source(source_, source_leading_dimension_)), simd_lane_id(thread_context_.simd_lane_id) {+ if (valid_rows_ < short(ROWS)) {+ source = source.bounded(valid_rows_, COLS);+ }+ }++ METAL_FUNC void prepare() const thread {}++ template <class FragmentT>+ METAL_FUNC void load(thread FragmentT& frag, const int offset) const thread {+ frag.load_from(simd_lane_id, source.advanced(offset));+ }+};++template <typename T, ushort ROWS, ushort COLS, ushort SHARED_LEADING_DIMENSION, ushort THREADGROUP_SIZE>+struct BlockSource<T, ROWS, COLS, SHARED_LEADING_DIMENSION, THREADGROUP_SIZE, ThreadgroupBlockStorage> {+ const device T* source;+ int source_leading_dimension;+ short valid_rows;+ ThreadContext thread_context;+ FragmentSource<threadgroup T*> shared_source;++ METAL_FUNC BlockSource(+ const device T* source_,+ const int source_leading_dimension_,+ threadgroup T* shared_,+ const short valid_rows_,+ const thread ThreadContext& thread_context_+ )+ : source(source_), source_leading_dimension(source_leading_dimension_), valid_rows(valid_rows_),+ thread_context(thread_context_), shared_source(fragment_source(shared_, SHARED_LEADING_DIMENSION)) {}++ METAL_FUNC void prepare() const thread {+ threadgroup_barrier(mem_flags::mem_threadgroup);+ ThreadgroupLoader<T, ROWS, COLS, SHARED_LEADING_DIMENSION, 0, THREADGROUP_SIZE>+ loader(source, source_leading_dimension, shared_source.base, thread_context);+ if (valid_rows < short(ROWS)) {+ loader.load_safe(short2(COLS, valid_rows));+ } else {+ loader.load_unsafe();+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }++ template <class FragmentT>+ METAL_FUNC void load(thread FragmentT& frag, const int offset) const thread {+ frag.load_from(thread_context.simd_lane_id, shared_source.advanced(offset));+ }+};+} // namespace matmul} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment_ops.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment_ops.hindex cce0638d8..99fbadad9 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment_ops.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment_ops.h@@ -9,6 +9,7 @@using namespace uzu;#include "defines.h"+#include "loader.h"#include <MetalPerformancePrimitives/MetalPerformancePrimitives.h>@@ -17,16 +18,21 @@ using namespace metal;namespace uzu {namespace matmul {+METAL_CONST ushort MXU_MMA_ROWS = 16;+METAL_CONST ushort MXU_MMA_COLS = 16;+struct MxuFragmentOps {- METAL_CONST ushort FRAGMENT_ROWS = 16;- METAL_CONST ushort FRAGMENT_COLS = 16;+ METAL_CONST ushort FRAGMENT_ROWS = MXU_MMA_ROWS;+ METAL_CONST ushort FRAGMENT_COLS = MXU_MMA_COLS;+ METAL_CONST bool READ_TRANSPOSE_SWAPS_SOURCE_STRIDES = false;+ using BlockStorage = DeviceBlockStorage;METAL_CONST ushort ELEMENTS_PER_THREAD = (FRAGMENT_ROWS * FRAGMENT_COLS) / METAL_SIMD_SIZE;METAL_CONST ushort THREAD_ELEMENT_ROWS = 2;METAL_CONST ushort THREAD_ELEMENT_COLS = 4;- METAL_CONST ushort THREAD_ELEMENT_ROW_STRIDE = 8;+ METAL_CONST ushort THREAD_ELEMENT_ROW_STRIDE = FRAGMENT_ROWS / THREAD_ELEMENT_ROWS;static_assert(THREAD_ELEMENT_ROWS * THREAD_ELEMENT_COLS == ELEMENTS_PER_THREAD,@@ -36,10 +42,7 @@ struct MxuFragmentOps {template <typename U>using ThreadVector = typename metal::vec<U, ELEMENTS_PER_THREAD>;- // Runs a 16x32 MPP matmul accumulating into (output_0 | output_1). The caller- // supplies a `marshal_inputs` callable that copies its fragments into the- // cooperative left/right tensors; the descriptor and output marshalling are- // shared between all call shapes.+ // MPP has no valid 16x16x16 op; fragment_mma pairs fragments into 16x32.template <typename CType, typename AType, typename BType, bool transpose_a, bool transpose_b, typename MarshalInputs>METAL_FUNC static void mma_impl(thread ThreadVector<CType>& output_0,@@ -82,7 +85,6 @@ struct MxuFragmentOps {}}- // N-paired: output is two columns, left is one fragment, right is two.template <typename CType, typename AType, typename BType, bool transpose_a = false, bool transpose_b = false>METAL_FUNC static void mma(thread ThreadVector<CType>& output_col_0,@@ -107,7 +109,6 @@ struct MxuFragmentOps {);}- // M-paired: output is two rows, left is two fragments, right is one.template <typename CType, typename AType, typename BType, bool transpose_a = false, bool transpose_b = false>METAL_FUNC static void mma(thread ThreadVector<CType>& output_row_0,@@ -132,33 +133,42 @@ struct MxuFragmentOps {);}- template <bool transpose_a, bool transpose_b, class OutputTile, class LeftTile, class RightTile>- METAL_FUNC static void tile_matmul(thread OutputTile& output, thread LeftTile& left, thread RightTile& right) {- constexpr ushort left_tile_m = transpose_a ? LeftTile::TILE_COLS : LeftTile::TILE_ROWS;- constexpr ushort tile_m = OutputTile::TILE_ROWS;- static_assert(left_tile_m == tile_m, "tile matmul: M dimensions do not match");+ template <bool transpose_a, bool transpose_b, class OutputFragment, class LeftFragment, class RightFragment>+ METAL_FUNC static void fragment_mma(+ thread OutputFragment& output,+ thread LeftFragment& left,+ thread RightFragment& right+ ) {+ constexpr ushort left_rows = transpose_a ? LeftFragment::COL_FRAGMENTS : LeftFragment::ROW_FRAGMENTS;+ constexpr ushort rows = OutputFragment::ROW_FRAGMENTS;+ static_assert(left_rows == rows, "fragment matmul: M dimensions do not match");++ constexpr ushort right_cols = transpose_b ? RightFragment::ROW_FRAGMENTS : RightFragment::COL_FRAGMENTS;+ constexpr ushort cols = OutputFragment::COL_FRAGMENTS;+ static_assert(right_cols == cols, "fragment matmul: N dimensions do not match");- constexpr ushort right_tile_n = transpose_b ? RightTile::TILE_ROWS : RightTile::TILE_COLS;- constexpr ushort tile_n = OutputTile::TILE_COLS;- static_assert(right_tile_n == tile_n, "tile matmul: N dimensions do not match");+ constexpr ushort left_depth = transpose_a ? LeftFragment::ROW_FRAGMENTS : LeftFragment::COL_FRAGMENTS;+ constexpr ushort depth = transpose_b ? RightFragment::COL_FRAGMENTS : RightFragment::ROW_FRAGMENTS;+ static_assert(left_depth == depth, "fragment matmul: K dimensions do not match");- constexpr ushort left_tile_k = transpose_a ? LeftTile::TILE_ROWS : LeftTile::TILE_COLS;- constexpr ushort tile_k = transpose_b ? RightTile::TILE_COLS : RightTile::TILE_ROWS;- static_assert(left_tile_k == tile_k, "tile matmul: K dimensions do not match");+ static_assert(+ (cols % 2 == 0) || (cols == 1 && rows % 2 == 0),+ "MXU fragment_mma requires even N, or N==1 with even M (MPP pairing)"+ );constexpr auto transpose_left = metal::bool_constant<transpose_a>{};constexpr auto transpose_right = metal::bool_constant<transpose_b>{};- if constexpr (tile_n == 1 && tile_m % 2 == 0) {+ if constexpr (cols == 1 && rows % 2 == 0) {METAL_PRAGMA_UNROLL- for (ushort row = 0; row < tile_m; row += 2) {+ for (ushort row = 0; row < rows; row += 2) {METAL_PRAGMA_UNROLL- for (ushort col = 0; col < tile_n; ++col) {+ for (ushort col = 0; col < cols; ++col) {METAL_PRAGMA_UNROLL- for (ushort k = 0; k < tile_k; ++k) {- mma<typename OutputTile::ElementType,- typename LeftTile::ElementType,- typename RightTile::ElementType,+ for (ushort k = 0; k < depth; ++k) {+ mma<typename OutputFragment::ElementType,+ typename LeftFragment::ElementType,+ typename RightFragment::ElementType,transpose_a,transpose_b>(output.fragment_at(row, col),@@ -172,16 +182,16 @@ struct MxuFragmentOps {}}}- } else if constexpr (tile_n % 2 == 0) {+ } else if constexpr (cols % 2 == 0) {METAL_PRAGMA_UNROLL- for (ushort row = 0; row < tile_m; ++row) {+ for (ushort row = 0; row < rows; ++row) {METAL_PRAGMA_UNROLL- for (ushort col = 0; col < tile_n; col += 2) {+ for (ushort col = 0; col < cols; col += 2) {METAL_PRAGMA_UNROLL- for (ushort k = 0; k < tile_k; ++k) {- mma<typename OutputTile::ElementType,- typename LeftTile::ElementType,- typename RightTile::ElementType,+ for (ushort k = 0; k < depth; ++k) {+ mma<typename OutputFragment::ElementType,+ typename LeftFragment::ElementType,+ typename RightFragment::ElementType,transpose_a,transpose_b>(output.fragment_at(row, col),diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.hindex 4fd31c489..ad5c92331 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.h@@ -43,7 +43,7 @@ METAL_FUNC auto gemm_loop(constexpr ushort RIGHT_TILE_ROWS = transpose_b ? TILES_N : TILES_K;constexpr ushort RIGHT_TILE_COLS = transpose_b ? TILES_K : TILES_N;- Fragment<AccumulatorType, TILES_M, TILES_N, MxuFragmentOps> accumulator(thread_context);+ Fragment<AccumulatorType, TILES_M, TILES_N, MxuFragmentOps> accumulator;accumulator.clear();METAL_PRAGMA_NO_UNROLL@@ -52,31 +52,31 @@ METAL_FUNC auto gemm_loop(METAL_PRAGMA_NO_UNROLLfor (int inner_k = 0; inner_k < BLOCK_K; inner_k += SIMDGROUP_BLOCK_K) {- Fragment<AT, LEFT_TILE_ROWS, LEFT_TILE_COLS, MxuFragmentOps> left_tile(thread_context);- Fragment<BT, RIGHT_TILE_ROWS, RIGHT_TILE_COLS, MxuFragmentOps> right_tile(thread_context);+ Fragment<AT, LEFT_TILE_ROWS, LEFT_TILE_COLS, MxuFragmentOps> left_tile;+ Fragment<BT, RIGHT_TILE_ROWS, RIGHT_TILE_COLS, MxuFragmentOps> right_tile;volatile int mxu_iteration_fence;const int left_offset = transpose_a ? inner_k * leading_dimension_a : inner_k;const int right_offset = transpose_b ? inner_k : inner_k * leading_dimension_b;- if constexpr (aligned_m) {- left_tile.load(left_ptr + left_offset, leading_dimension_a);- } else {+ auto left_src = fragment_source(left_ptr + left_offset, leading_dimension_a);+ if constexpr (!aligned_m) {const short row_limit = transpose_a ? SIMDGROUP_BLOCK_K : simdgroup_limit_m;const short col_limit = transpose_a ? simdgroup_limit_m : SIMDGROUP_BLOCK_K;- left_tile.load_safe(left_ptr + left_offset, leading_dimension_a, short2(col_limit, row_limit));+ left_src = left_src.bounded(row_limit, col_limit);}+ left_tile.load_from(thread_context.simd_lane_id, left_src);- if constexpr (aligned_n) {- right_tile.load(right_ptr + right_offset, leading_dimension_b);- } else {+ auto right_src = fragment_source(right_ptr + right_offset, leading_dimension_b);+ if constexpr (!aligned_n) {const short row_limit = transpose_b ? simdgroup_limit_n : SIMDGROUP_BLOCK_K;const short col_limit = transpose_b ? SIMDGROUP_BLOCK_K : simdgroup_limit_n;- right_tile.load_safe(right_ptr + right_offset, leading_dimension_b, short2(col_limit, row_limit));+ right_src = right_src.bounded(row_limit, col_limit);}+ right_tile.load_from(thread_context.simd_lane_id, right_src);- MxuFragmentOps::template tile_matmul<transpose_a, transpose_b>(accumulator, left_tile, right_tile);+ MxuFragmentOps::template fragment_mma<transpose_a, transpose_b>(accumulator, left_tile, right_tile);(void)mxu_iteration_fence;}@@ -92,8 +92,8 @@ METAL_FUNC auto gemm_loop(METAL_PRAGMA_NO_UNROLLfor (int inner_k = 0; inner_k < remaining_k; inner_k += SIMDGROUP_BLOCK_K) {- Fragment<AT, LEFT_TILE_ROWS, LEFT_TILE_COLS, MxuFragmentOps> left_tile(thread_context);- Fragment<BT, RIGHT_TILE_ROWS, RIGHT_TILE_COLS, MxuFragmentOps> right_tile(thread_context);+ Fragment<AT, LEFT_TILE_ROWS, LEFT_TILE_COLS, MxuFragmentOps> left_tile;+ Fragment<BT, RIGHT_TILE_ROWS, RIGHT_TILE_COLS, MxuFragmentOps> right_tile;const short safe_k = max(short(0), short(remaining_k - inner_k));@@ -103,10 +103,16 @@ METAL_FUNC auto gemm_loop(const int left_offset = transpose_a ? inner_k * leading_dimension_a : inner_k;const int right_offset = transpose_b ? inner_k : inner_k * leading_dimension_b;- left_tile.load_safe(left_ptr + left_offset, leading_dimension_a, left_limits);- right_tile.load_safe(right_ptr + right_offset, leading_dimension_b, right_limits);+ left_tile.load_from(+ thread_context.simd_lane_id,+ fragment_source(left_ptr + left_offset, leading_dimension_a).bounded(left_limits.y, left_limits.x)+ );+ right_tile.load_from(+ thread_context.simd_lane_id,+ fragment_source(right_ptr + right_offset, leading_dimension_b).bounded(right_limits.y, right_limits.x)+ );- MxuFragmentOps::template tile_matmul<transpose_a, transpose_b>(accumulator, left_tile, right_tile);+ MxuFragmentOps::template fragment_mma<transpose_a, transpose_b>(accumulator, left_tile, right_tile);}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment.hdeleted file mode 100644index cae407263..000000000--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment.h+++ /dev/null@@ -1,154 +0,0 @@-#pragma once--#include "simdgroup_multiply_accumulate.h"--using namespace metal;--namespace uzu {-namespace matmul {--///////////////////////////////////////////////////////////////////////////////-// SimdgroupFragment - thread's portion of simdgroup tile data-///////////////////////////////////////////////////////////////////////////////--template <- typename T,- int GRID_ROWS_,- int GRID_COLS_,- class SimdgroupMultiplyAccumulateOps_ = SimdgroupMultiplyAccumulate<T, 8, 8>>-struct SimdgroupFragment {- using SimdgroupMultiplyAccumulateOpsType = SimdgroupMultiplyAccumulateOps_;- using ElementType = T;- METAL_CONST int MULTIPLY_ACCUMULATE_ROWS = SimdgroupMultiplyAccumulateOpsType::ROWS;- METAL_CONST int MULTIPLY_ACCUMULATE_COLS = SimdgroupMultiplyAccumulateOpsType::COLS;- METAL_CONST int ELEMENTS_PER_MULTIPLY_ACCUMULATE = SimdgroupMultiplyAccumulateOpsType::THREAD_ELEMENT_COUNT;-- METAL_CONST int GRID_ROWS = GRID_ROWS_;- METAL_CONST int GRID_COLS = GRID_COLS_;-- METAL_CONST int TOTAL_ROWS = GRID_ROWS * MULTIPLY_ACCUMULATE_ROWS;- METAL_CONST int TOTAL_COLS = GRID_COLS * MULTIPLY_ACCUMULATE_COLS;-- METAL_CONST int MULTIPLY_ACCUMULATE_COUNT = GRID_ROWS * GRID_COLS;- METAL_CONST int ELEMENTS_PER_FRAGMENT = MULTIPLY_ACCUMULATE_COUNT * ELEMENTS_PER_MULTIPLY_ACCUMULATE;-- typedef typename SimdgroupMultiplyAccumulateOpsType::SimdgroupMatrixType SimdgroupMatrixType;- typedef typename SimdgroupMultiplyAccumulateOpsType::ThreadDataType ThreadDataType;-- ThreadDataType multiply_accumulate_data[MULTIPLY_ACCUMULATE_COUNT] = {ThreadDataType(0)};-- METAL_FUNC SimdgroupFragment() thread {}-- METAL_FUNC constexpr void clear() {- METAL_PRAGMA_UNROLL- for (ushort i = 0; i < MULTIPLY_ACCUMULATE_COUNT; ++i) {- multiply_accumulate_data[i] = ThreadDataType(0);- }- }-- METAL_FUNC constexpr thread ThreadDataType& multiply_accumulate_at(const ushort i, const ushort j) {- return multiply_accumulate_data[i * GRID_COLS + j];- }-- METAL_FUNC constexpr const thread ThreadDataType& multiply_accumulate_at(const ushort i, const ushort j) const {- return multiply_accumulate_data[i * GRID_COLS + j];- }-- METAL_FUNC thread ElementType* elements() { return reinterpret_cast<thread ElementType*>(multiply_accumulate_data); }-- METAL_FUNC const thread ElementType* elements() const {- return reinterpret_cast<const thread ElementType*>(multiply_accumulate_data);- }-- template <typename U, int SIMDGROUP_STRIDE_X, int SIMDGROUP_STRIDE_Y, int STRIDE_X, int STRIDE_Y>- METAL_FUNC void load(const threadgroup U* source) {- METAL_PRAGMA_UNROLL- for (ushort i = 0; i < GRID_ROWS; ++i) {- METAL_PRAGMA_UNROLL- for (ushort j = 0; j < GRID_COLS; ++j) {- SimdgroupMultiplyAccumulateOpsType::load(- multiply_accumulate_at(i, j),- &(source- [(i * MULTIPLY_ACCUMULATE_ROWS) * SIMDGROUP_STRIDE_X * STRIDE_X +- (j * MULTIPLY_ACCUMULATE_COLS) * SIMDGROUP_STRIDE_Y * STRIDE_Y]),- STRIDE_X,- STRIDE_Y- );- }- }- }-- template <typename U, int SIMDGROUP_STRIDE_X, int SIMDGROUP_STRIDE_Y>- METAL_FUNC void store(device U* destination, const int leading_dimension) const {- METAL_PRAGMA_UNROLL- for (ushort i = 0; i < GRID_ROWS; ++i) {- METAL_PRAGMA_UNROLL- for (ushort j = 0; j < GRID_COLS; ++j) {- SimdgroupMultiplyAccumulateOpsType::store(- multiply_accumulate_at(i, j),- &(destination- [(i * MULTIPLY_ACCUMULATE_ROWS) * SIMDGROUP_STRIDE_X * leading_dimension +- (j * MULTIPLY_ACCUMULATE_COLS) * SIMDGROUP_STRIDE_Y]),- leading_dimension,- 1- );- }- }- }-- template <typename U, int SIMDGROUP_STRIDE_X, int SIMDGROUP_STRIDE_Y>- METAL_FUNC void store_safe(- device U* destination,- const int leading_dimension,- const short2 destination_tile_dimensions- ) const {- METAL_PRAGMA_UNROLL- for (int i = 0; i < GRID_ROWS; ++i) {- METAL_PRAGMA_UNROLL- for (int j = 0; j < GRID_COLS; ++j) {- SimdgroupMultiplyAccumulateOpsType::store_safe(- multiply_accumulate_at(i, j),- destination,- leading_dimension,- 1,- destination_tile_dimensions.y,- destination_tile_dimensions.x,- (i * MULTIPLY_ACCUMULATE_ROWS) * SIMDGROUP_STRIDE_X,- (j * MULTIPLY_ACCUMULATE_COLS) * SIMDGROUP_STRIDE_Y- );- }- }- }-};--///////////////////////////////////////////////////////////////////////////////-// Tile multiply-accumulate-///////////////////////////////////////////////////////////////////////////////--template <typename T, typename U, int M, int N, int K>-METAL_FUNC void tile_multiply_accumulate(- thread SimdgroupFragment<T, M, N>& D,- thread SimdgroupFragment<U, M, K>& A,- thread SimdgroupFragment<U, K, N>& B,- thread SimdgroupFragment<T, M, N>& C-) {- METAL_PRAGMA_UNROLL- for (ushort m = 0; m < M; ++m) {- METAL_PRAGMA_UNROLL- for (ushort n = 0; n < N; ++n) {- const ushort column_serpentine = (m % 2) ? (N - 1 - n) : n;- METAL_PRAGMA_UNROLL- for (ushort k = 0; k < K; ++k) {- SimdgroupFragment<T, M, N>::SimdgroupMultiplyAccumulateOpsType::multiply_accumulate(- D.multiply_accumulate_at(m, column_serpentine),- A.multiply_accumulate_at(m, k),- B.multiply_accumulate_at(k, column_serpentine),- C.multiply_accumulate_at(m, column_serpentine)- );- }- }- }-}--} // namespace matmul-} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment_ops.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment_ops.hnew file mode 100644index 000000000..b96d6c67f--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment_ops.h@@ -0,0 +1,132 @@+#pragma once++#include "defines.h"+#include "loader.h"+#include "simdgroup_multiply_accumulate.h"++using namespace metal;++namespace uzu {+namespace matmul {++struct SimdgroupFragmentOps {+ METAL_CONST ushort FRAGMENT_ROWS = SIMDGROUP_MMA_ROWS;+ METAL_CONST ushort FRAGMENT_COLS = SIMDGROUP_MMA_COLS;+ METAL_CONST bool READ_TRANSPOSE_SWAPS_SOURCE_STRIDES = true;+ using BlockStorage = ThreadgroupBlockStorage;++ METAL_CONST ushort ELEMENTS_PER_THREAD = (FRAGMENT_ROWS * FRAGMENT_COLS) / METAL_SIMD_SIZE;+ METAL_CONST ushort THREAD_ELEMENT_ROWS = 1;+ METAL_CONST ushort THREAD_ELEMENT_COLS = 2;+ METAL_CONST ushort THREAD_ELEMENT_ROW_STRIDE = FRAGMENT_ROWS / THREAD_ELEMENT_ROWS;++ template <typename U>+ using ThreadVector = typename metal::vec<U, ELEMENTS_PER_THREAD>;++ // Serpentine order matches the chain kernel's register reuse.+ template <bool transpose_a, bool transpose_b, class OutputFragment, class LeftFragment, class RightFragment>+ METAL_FUNC static void fragment_mma(+ thread OutputFragment& output,+ thread LeftFragment& left,+ thread RightFragment& right+ ) {+ static_assert(!transpose_a && !transpose_b, "SimdgroupFragmentOps::fragment_mma: transpose not supported");+ constexpr ushort rows = OutputFragment::ROW_FRAGMENTS;+ constexpr ushort cols = OutputFragment::COL_FRAGMENTS;+ constexpr ushort depth = LeftFragment::COL_FRAGMENTS;+ static_assert(LeftFragment::ROW_FRAGMENTS == rows, "fragment matmul: M dimensions do not match");+ static_assert(RightFragment::COL_FRAGMENTS == cols, "fragment matmul: N dimensions do not match");+ static_assert(RightFragment::ROW_FRAGMENTS == depth, "fragment matmul: K dimensions do not match");+ using Sg = SimdgroupMMA<typename OutputFragment::ElementType, FRAGMENT_ROWS, FRAGMENT_COLS>;++ METAL_PRAGMA_UNROLL+ for (ushort m = 0; m < rows; ++m) {+ METAL_PRAGMA_UNROLL+ for (ushort n = 0; n < cols; ++n) {+ const ushort col = (m % 2) ? (cols - 1 - n) : n;+ METAL_PRAGMA_UNROLL+ for (ushort k = 0; k < depth; ++k) {+ Sg::mma(+ output.fragment_at(m, col),+ left.fragment_at(m, k),+ right.fragment_at(k, col),+ output.fragment_at(m, col)+ );+ }+ }+ }+ }+};++template <typename U, int SIMDGROUP_STRIDE_X, int SIMDGROUP_STRIDE_Y, int STRIDE_X, int STRIDE_Y, typename FragT>+METAL_FUNC void fragment_load(thread FragT& frag, const threadgroup U* source) {+ constexpr ushort fragment_rows = SimdgroupFragmentOps::FRAGMENT_ROWS;+ constexpr ushort fragment_cols = SimdgroupFragmentOps::FRAGMENT_COLS;+ using Sg = SimdgroupMMA<typename FragT::ElementType, fragment_rows, fragment_cols>;+ METAL_PRAGMA_UNROLL+ for (ushort i = 0; i < FragT::ROW_FRAGMENTS; ++i) {+ METAL_PRAGMA_UNROLL+ for (ushort j = 0; j < FragT::COL_FRAGMENTS; ++j) {+ Sg::load(+ frag.fragment_at(i, j),+ &(source+ [(i * fragment_rows) * SIMDGROUP_STRIDE_X * STRIDE_X ++ (j * fragment_cols) * SIMDGROUP_STRIDE_Y * STRIDE_Y]),+ STRIDE_X,+ STRIDE_Y+ );+ }+ }+}++template <typename U, int SIMDGROUP_STRIDE_X, int SIMDGROUP_STRIDE_Y, typename FragT>+METAL_FUNC void fragment_store(thread FragT& frag, device U* destination, const int leading_dimension) {+ constexpr ushort fragment_rows = SimdgroupFragmentOps::FRAGMENT_ROWS;+ constexpr ushort fragment_cols = SimdgroupFragmentOps::FRAGMENT_COLS;+ using Sg = SimdgroupMMA<typename FragT::ElementType, fragment_rows, fragment_cols>;+ METAL_PRAGMA_UNROLL+ for (ushort i = 0; i < FragT::ROW_FRAGMENTS; ++i) {+ METAL_PRAGMA_UNROLL+ for (ushort j = 0; j < FragT::COL_FRAGMENTS; ++j) {+ Sg::store(+ frag.fragment_at(i, j),+ &(destination+ [(i * fragment_rows) * SIMDGROUP_STRIDE_X * leading_dimension ++ (j * fragment_cols) * SIMDGROUP_STRIDE_Y]),+ leading_dimension,+ 1+ );+ }+ }+}++template <typename U, int SIMDGROUP_STRIDE_X, int SIMDGROUP_STRIDE_Y, typename FragT>+METAL_FUNC void fragment_store_safe(+ thread FragT& frag,+ device U* destination,+ const int leading_dimension,+ const short2 destination_tile_dimensions+) {+ constexpr ushort fragment_rows = SimdgroupFragmentOps::FRAGMENT_ROWS;+ constexpr ushort fragment_cols = SimdgroupFragmentOps::FRAGMENT_COLS;+ using Sg = SimdgroupMMA<typename FragT::ElementType, fragment_rows, fragment_cols>;+ METAL_PRAGMA_UNROLL+ for (int i = 0; i < FragT::ROW_FRAGMENTS; ++i) {+ METAL_PRAGMA_UNROLL+ for (int j = 0; j < FragT::COL_FRAGMENTS; ++j) {+ Sg::store_safe(+ frag.fragment_at(i, j),+ destination,+ leading_dimension,+ 1,+ destination_tile_dimensions.y,+ destination_tile_dimensions.x,+ (i * fragment_rows) * SIMDGROUP_STRIDE_X,+ (j * fragment_cols) * SIMDGROUP_STRIDE_Y+ );+ }+ }+}++} // namespace matmul+} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_multiply_accumulate.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_multiply_accumulate.hindex 939635b80..e82edcc49 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_multiply_accumulate.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_multiply_accumulate.h@@ -7,27 +7,30 @@ using namespace metal;namespace uzu {namespace matmul {+METAL_CONST int SIMDGROUP_MMA_ROWS = 8;+METAL_CONST int SIMDGROUP_MMA_COLS = 8;+///////////////////////////////////////////////////////////////////////////////-// SimdgroupMultiplyAccumulate - stateless traits/ops for 8x8 simdgroup_matrix+// SimdgroupMMA - stateless traits/ops for 8x8 simdgroup_matrix///////////////////////////////////////////////////////////////////////////////template <typename T, int ROWS_, int COLS_>-struct SimdgroupMultiplyAccumulate {- static_assert(ROWS_ == 8, "Only 8 x 8 fragment matrices are currently supported");- static_assert(COLS_ == 8, "Only 8 x 8 fragment matrices are currently supported");+struct SimdgroupMMA {+ static_assert(ROWS_ == SIMDGROUP_MMA_ROWS, "Only 8 x 8 fragment matrices are currently supported");+ static_assert(COLS_ == SIMDGROUP_MMA_COLS, "Only 8 x 8 fragment matrices are currently supported");};template <typename T>-struct SimdgroupMultiplyAccumulate<T, 8, 8> {- METAL_CONST int ROWS = 8;- METAL_CONST int COLS = 8;+struct SimdgroupMMA<T, SIMDGROUP_MMA_ROWS, SIMDGROUP_MMA_COLS> {+ METAL_CONST int ROWS = SIMDGROUP_MMA_ROWS;+ METAL_CONST int COLS = SIMDGROUP_MMA_COLS;METAL_CONST int THREAD_ELEMENT_COUNT = (ROWS * COLS) / METAL_SIMD_SIZE;METAL_CONST int THREAD_ELEMENT_ROWS = 1;METAL_CONST int THREAD_ELEMENT_COLS = 2;static_assert(THREAD_ELEMENT_ROWS * THREAD_ELEMENT_COLS == THREAD_ELEMENT_COUNT,- "SimdgroupMultiplyAccumulate shape is not consistent with element count"+ "SimdgroupMMA shape is not consistent with element count");typedef metal::simdgroup_matrix<T, ROWS, COLS> SimdgroupMatrixType;@@ -138,7 +141,7 @@ struct SimdgroupMultiplyAccumulate<T, 8, 8> {}}- METAL_FUNC static constexpr void multiply_accumulate(+ METAL_FUNC static constexpr void mma(thread ThreadDataType& D,thread ThreadDataType& A,thread ThreadDataType& B,@@ -153,12 +156,12 @@ struct SimdgroupMultiplyAccumulate<T, 8, 8> {reinterpret_cast<thread ThreadDataType&>(B_mat.thread_elements()) = B;reinterpret_cast<thread ThreadDataType&>(C_mat.thread_elements()) = C;- multiply_accumulate(D_mat, A_mat, B_mat, C_mat);+ mma(D_mat, A_mat, B_mat, C_mat);D = reinterpret_cast<thread ThreadDataType&>(D_mat.thread_elements());}- METAL_FUNC static constexpr void multiply_accumulate(+ METAL_FUNC static constexpr void mma(thread SimdgroupMatrixType& D,thread SimdgroupMatrixType& A,thread SimdgroupMatrixType& B,diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.hindex 451bc066b..f59214c2d 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h@@ -1,17 +1,14 @@#pragma once#include "../../common/thread_context.h"-#include "simdgroup_fragment.h"+#include "fragment.h"+#include "simdgroup_fragment_ops.h"using namespace metal;namespace uzu {namespace matmul {-///////////////////////////////////////////////////////////////////////////////-// ThreadgroupTile - manages the GEMM computation for a threadgroup-///////////////////////////////////////////////////////////////////////////////-template <typename AT,typename BT,@@ -29,8 +26,7 @@ template <typename Epilogue = TransformNone<DT, AccumulatorType>>struct ThreadgroupTile {METAL_CONST ushort SIMDGROUP_BLOCK_SIZE = 8;- using SimdgroupMultiplyAccumulateType =- SimdgroupMultiplyAccumulate<AccumulatorType, SIMDGROUP_BLOCK_SIZE, SIMDGROUP_BLOCK_SIZE>;+ using SimdgroupMMAType = SimdgroupMMA<AccumulatorType, SIMDGROUP_BLOCK_SIZE, SIMDGROUP_BLOCK_SIZE>;METAL_CONST ushort TILE_ROW_STRIDE = SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW;METAL_CONST ushort TILE_COL_STRIDE = SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN;@@ -47,9 +43,9 @@ struct ThreadgroupTile {METAL_CONST ushort TILE_STRIDE_A = SIMDGROUP_BLOCK_SIZE * A_STRIDE_INNER;METAL_CONST ushort TILE_STRIDE_B = SIMDGROUP_BLOCK_SIZE * B_STRIDE_INNER;- SimdgroupFragment<AccumulatorType, TILE_ROWS, 1, SimdgroupMultiplyAccumulateType> a_fragment;- SimdgroupFragment<AccumulatorType, 1, TILE_COLS, SimdgroupMultiplyAccumulateType> b_fragment;- SimdgroupFragment<AccumulatorType, TILE_ROWS, TILE_COLS, SimdgroupMultiplyAccumulateType> c_fragment;+ Fragment<AccumulatorType, TILE_ROWS, 1, SimdgroupFragmentOps> a_fragment;+ Fragment<AccumulatorType, 1, TILE_COLS, SimdgroupFragmentOps> b_fragment;+ Fragment<AccumulatorType, TILE_ROWS, TILE_COLS, SimdgroupFragmentOps> c_fragment;ushort simdgroup_row_offset;ushort simdgroup_col_offset;@@ -61,8 +57,7 @@ struct ThreadgroupTile {ushort tile_row_base = SIMDGROUP_BLOCK_SIZE * (thread_context.simdgroup_index / SIMDGROUPS_PER_COLUMN);ushort tile_col_base = SIMDGROUP_BLOCK_SIZE * (thread_context.simdgroup_index % SIMDGROUPS_PER_COLUMN);- const ushort2 simdgroup_coordinates =- ushort2(SimdgroupMultiplyAccumulateType::get_lane_coordinates(thread_context.simd_lane_id));+ const ushort2 simdgroup_coordinates = ushort2(SimdgroupMMAType::get_lane_coordinates(thread_context.simd_lane_id));simdgroup_row_offset = simdgroup_coordinates.y;simdgroup_col_offset = simdgroup_coordinates.x;@@ -73,17 +68,17 @@ struct ThreadgroupTile {simdgroup_col_offset += tile_col_base;}- METAL_FUNC void multiply_accumulate(const threadgroup AT* a_shared, const threadgroup BT* b_shared) {+ METAL_FUNC void matmul(const threadgroup AT* a_shared, const threadgroup BT* b_shared) {a_shared += a_shared_offset;b_shared += b_shared_offset;METAL_PRAGMA_UNROLLfor (ushort k_block_index = 0; k_block_index < BLOCK_DEPTH; k_block_index += SIMDGROUP_BLOCK_SIZE) {- a_fragment.template load<AT, SIMDGROUPS_PER_ROW, 1, A_STRIDE_ROW, A_STRIDE_INNER>(a_shared);+ fragment_load<AT, SIMDGROUPS_PER_ROW, 1, A_STRIDE_ROW, A_STRIDE_INNER>(a_fragment, a_shared);- b_fragment.template load<BT, 1, SIMDGROUPS_PER_COLUMN, B_STRIDE_INNER, B_STRIDE_COL>(b_shared);+ fragment_load<BT, 1, SIMDGROUPS_PER_COLUMN, B_STRIDE_INNER, B_STRIDE_COL>(b_fragment, b_shared);- tile_multiply_accumulate(c_fragment, a_fragment, b_fragment, c_fragment);+ fragment_mma(c_fragment, a_fragment, b_fragment);a_shared += TILE_STRIDE_A;b_shared += TILE_STRIDE_B;@@ -91,21 +86,15 @@ struct ThreadgroupTile {}METAL_FUNC void store_result(device DT* D, const int leading_dimension_d) {- METAL_PRAGMA_UNROLL- for (ushort i = 0; i < decltype(c_fragment)::ELEMENTS_PER_FRAGMENT; i++) {- c_fragment.elements()[i] = Epilogue::apply(c_fragment.elements()[i]);- }+ c_fragment.map([](AccumulatorType value) { return Epilogue::apply(value); });D += simdgroup_row_offset * leading_dimension_d + simdgroup_col_offset;- c_fragment.template store<DT, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN>(D, leading_dimension_d);+ fragment_store<DT, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN>(c_fragment, D, leading_dimension_d);}METAL_FUNC void store_result_safe(device DT* D, const int leading_dimension_d, short2 destination_tile_dimensions) {- METAL_PRAGMA_UNROLL- for (ushort i = 0; i < decltype(c_fragment)::ELEMENTS_PER_FRAGMENT; i++) {- c_fragment.elements()[i] = Epilogue::apply(c_fragment.elements()[i]);- }+ c_fragment.map([](AccumulatorType value) { return Epilogue::apply(value); });D += simdgroup_row_offset * leading_dimension_d + simdgroup_col_offset;destination_tile_dimensions -= short2(simdgroup_col_offset, simdgroup_row_offset);@@ -113,105 +102,70 @@ struct ThreadgroupTile {if (destination_tile_dimensions.x <= 0 || destination_tile_dimensions.y <= 0)return;- c_fragment.template store_safe<DT, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN>(+ fragment_store_safe<DT, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN>(+ c_fragment,D,leading_dimension_d,destination_tile_dimensions);}- template <typename EpilogueOp>- METAL_FUNC void apply_epilogue(- const device DT* C,- const int leading_dimension_c,- const int column_stride_c,- thread const EpilogueOp& epilogue_operation- ) {- const device DT* c_pointer =- C + simdgroup_row_offset * leading_dimension_c + simdgroup_col_offset * column_stride_c;-+ template <class Fn>+ METAL_FUNC void for_each_output(Fn fn) {+ thread AccumulatorType* data = c_fragment.elements();+ constexpr ushort epm = SimdgroupFragmentOps::ELEMENTS_PER_THREAD;METAL_PRAGMA_UNROLLfor (ushort i = 0; i < TILE_ROWS; i++) {+ const ushort row_offset = i * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW;METAL_PRAGMA_UNROLLfor (ushort j = 0; j < TILE_COLS; j++) {- thread auto& block_data = c_fragment.multiply_accumulate_at(i, j);+ const ushort col_offset = j * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN;METAL_PRAGMA_UNROLL- for (ushort k = 0; k < decltype(c_fragment)::ELEMENTS_PER_MULTIPLY_ACCUMULATE; k++) {- const ushort row_offset = i * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW;- const ushort col_offset = j * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN;- DT c_value = c_pointer[row_offset * leading_dimension_c + col_offset * column_stride_c + k];- block_data[k] = epilogue_operation.apply(block_data[k], static_cast<AccumulatorType>(c_value));+ for (ushort k = 0; k < epm; k++) {+ fn(row_offset, col_offset, k, data[(i * TILE_COLS + j) * epm + k]);}}}}+ template <typename EpilogueOp>+ METAL_FUNC void apply_epilogue(const device DT* C, const int ld_c, const int cstride_c, thread const EpilogueOp& op) {+ const device DT* c_ptr = C + simdgroup_row_offset * ld_c + simdgroup_col_offset * cstride_c;+ for_each_output([&](ushort row_offset, ushort col_offset, ushort k, thread AccumulatorType& v) {+ v = op.apply(v, static_cast<AccumulatorType>(c_ptr[row_offset * ld_c + col_offset * cstride_c + k]));+ });+ }+template <typename EpilogueOp>METAL_FUNC void apply_epilogue_safe(const device DT* C,- const int leading_dimension_c,- const int column_stride_c,+ const int ld_c,+ const int cstride_c,short2 tile_dimensions,- thread const EpilogueOp& epilogue_operation+ thread const EpilogueOp& op) {- const device DT* c_pointer =- C + simdgroup_row_offset * leading_dimension_c + simdgroup_col_offset * column_stride_c;+ const device DT* c_ptr = C + simdgroup_row_offset * ld_c + simdgroup_col_offset * cstride_c;tile_dimensions -= short2(simdgroup_col_offset, simdgroup_row_offset);-- METAL_PRAGMA_UNROLL- for (ushort i = 0; i < TILE_ROWS; i++) {- METAL_PRAGMA_UNROLL- for (ushort j = 0; j < TILE_COLS; j++) {- thread auto& block_data = c_fragment.multiply_accumulate_at(i, j);- const ushort row_offset = i * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW;- const ushort col_offset = j * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN;- METAL_PRAGMA_UNROLL- for (ushort k = 0; k < decltype(c_fragment)::ELEMENTS_PER_MULTIPLY_ACCUMULATE; k++) {- if (row_offset < tile_dimensions.y && col_offset + k < tile_dimensions.x) {- DT c_value = c_pointer[row_offset * leading_dimension_c + col_offset * column_stride_c + k];- block_data[k] = epilogue_operation.apply(block_data[k], static_cast<AccumulatorType>(c_value));- }- }- }- }+ for_each_output([&](ushort row_offset, ushort col_offset, ushort k, thread AccumulatorType& v) {+ if (row_offset < tile_dimensions.y && col_offset + k < tile_dimensions.x)+ v = op.apply(v, static_cast<AccumulatorType>(c_ptr[row_offset * ld_c + col_offset * cstride_c + k]));+ });}METAL_FUNC void apply_bias(const device BT* bias) {- const device BT* bias_pointer = bias + simdgroup_col_offset;-- METAL_PRAGMA_UNROLL- for (ushort i = 0; i < TILE_ROWS; i++) {- METAL_PRAGMA_UNROLL- for (ushort j = 0; j < TILE_COLS; j++) {- thread auto& block_data = c_fragment.multiply_accumulate_at(i, j);- const ushort col_offset = j * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN;- METAL_PRAGMA_UNROLL- for (ushort k = 0; k < decltype(c_fragment)::ELEMENTS_PER_MULTIPLY_ACCUMULATE; k++) {- block_data[k] += static_cast<AccumulatorType>(bias_pointer[col_offset + k]);- }- }- }+ const device BT* bias_ptr = bias + simdgroup_col_offset;+ for_each_output([&](ushort, ushort col_offset, ushort k, thread AccumulatorType& v) {+ v += static_cast<AccumulatorType>(bias_ptr[col_offset + k]);+ });}METAL_FUNC void apply_bias_safe(const device BT* bias, short2 tile_dimensions) {- const device BT* bias_pointer = bias + simdgroup_col_offset;+ const device BT* bias_ptr = bias + simdgroup_col_offset;tile_dimensions -= short2(simdgroup_col_offset, simdgroup_row_offset);-- METAL_PRAGMA_UNROLL- for (ushort i = 0; i < TILE_ROWS; i++) {- METAL_PRAGMA_UNROLL- for (ushort j = 0; j < TILE_COLS; j++) {- thread auto& block_data = c_fragment.multiply_accumulate_at(i, j);- const ushort row_offset = i * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW;- const ushort col_offset = j * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN;- METAL_PRAGMA_UNROLL- for (ushort k = 0; k < decltype(c_fragment)::ELEMENTS_PER_MULTIPLY_ACCUMULATE; k++) {- if (row_offset < tile_dimensions.y && col_offset + k < tile_dimensions.x) {- block_data[k] += static_cast<AccumulatorType>(bias_pointer[col_offset + k]);- }- }- }- }+ for_each_output([&](ushort row_offset, ushort col_offset, ushort k, thread AccumulatorType& v) {+ if (row_offset < tile_dimensions.y && col_offset + k < tile_dimensions.x)+ v += static_cast<AccumulatorType>(bias_ptr[col_offset + k]);+ });}};diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.hindex 17112a7fb..ed743d330 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h@@ -105,7 +105,7 @@ struct MxuMmaCore {thread Loader& loader_b,const thread ThreadContext& thread_context) {- AccumFragment accumulator(thread_context);+ AccumFragment accumulator;accumulator.clear();threadgroup BT* b_shared_simdgroup = b_shared + tile_col_offset * SHARED_STRIDE_B;@@ -123,20 +123,22 @@ struct MxuMmaCore {METAL_PRAGMA_NO_UNROLLfor (int inner_k = 0; inner_k < QUANT_BK; inner_k += SIMDGROUP_BLOCK_K) {- uzu::matmul::Fragment<AT, TILES_M, TILES_K, uzu::matmul::MxuFragmentOps> left_tile(thread_context);- uzu::matmul::Fragment<BT, TILES_N, TILES_K, uzu::matmul::MxuFragmentOps> right_tile(thread_context);+ uzu::matmul::Fragment<AT, TILES_M, TILES_K, uzu::matmul::MxuFragmentOps> left_tile;+ uzu::matmul::Fragment<BT, TILES_N, TILES_K, uzu::matmul::MxuFragmentOps> right_tile;const int left_offset = inner_k;- if constexpr (ALIGNED_M) {- left_tile.load(a_simdgroup + left_offset, leading_dimension_a);- } else {- left_tile- .load_safe(a_simdgroup + left_offset, leading_dimension_a, short2(SIMDGROUP_BLOCK_K, simdgroup_limit_m));+ auto left_src = uzu::matmul::fragment_source(a_simdgroup + left_offset, leading_dimension_a);+ if constexpr (!ALIGNED_M) {+ left_src = left_src.bounded(simdgroup_limit_m, SIMDGROUP_BLOCK_K);}+ left_tile.load_from(thread_context.simd_lane_id, left_src);- right_tile.load(b_shared_simdgroup + inner_k, int(SHARED_STRIDE_B));+ right_tile.load_from(+ thread_context.simd_lane_id,+ uzu::matmul::fragment_source(b_shared_simdgroup + inner_k, int(SHARED_STRIDE_B))+ );- uzu::matmul::MxuFragmentOps::template tile_matmul<false, true>(accumulator, left_tile, right_tile);+ uzu::matmul::MxuFragmentOps::template fragment_mma<false, true>(accumulator, left_tile, right_tile);}a_simdgroup += QUANT_BK;@@ -210,8 +212,6 @@ struct MxuMmaCore {const bool apply_bias = output_transform.contains(GemmDTransform::BIAS);const device BT* bias_simdgroup = output_bias + size_t(block_col) + size_t(tile_col_offset);- using FragType = uzu::matmul::Fragment<DT, TILES_M, TILES_N, uzu::matmul::MxuFragmentOps>;- const short2 thread_position = FragType::get_position(thread_context);auto dispatch_aligned_k = [&](auto body) {if constexpr (B_PROLOGUE == GemmBPrologueKind::FullPrecision) {@@ -325,62 +325,38 @@ struct MxuMmaCore {if (apply_scale) {const AccumulatorType scale = AccumulatorType(params->ab_scale);- METAL_PRAGMA_UNROLL- for (ushort i = 0; i < accumulator_tile.ELEMENTS_PER_TILE; i++) {- accumulator_tile.elements()[i] *= scale;- }+ accumulator_tile.map([&](auto value) { return value * scale; });}if (apply_accumulate) {- uzu::matmul::Fragment<DT, TILES_M, TILES_N, uzu::matmul::MxuFragmentOps> existing_output(- thread_context- );- if constexpr (aligned_m.value && aligned_n.value) {- existing_output.load(d_simdgroup, int(params->leading_dimension_d));- } else {- existing_output.load_safe(- d_simdgroup,- int(params->leading_dimension_d),- short2(simdgroup_limit_n, simdgroup_limit_m)- );- }- METAL_PRAGMA_UNROLL- for (ushort i = 0; i < accumulator_tile.ELEMENTS_PER_TILE; i++) {- accumulator_tile.elements()[i] += AccumulatorType(existing_output.elements()[i]);+ uzu::matmul::Fragment<DT, TILES_M, TILES_N, uzu::matmul::MxuFragmentOps> existing_output;+ auto output_src = uzu::matmul::fragment_source(d_simdgroup, int(params->leading_dimension_d));+ if constexpr (!(aligned_m.value && aligned_n.value)) {+ output_src = output_src.bounded(simdgroup_limit_m, simdgroup_limit_n);}+ existing_output.load_from(thread_context.simd_lane_id, output_src);+ thread DT* existing_data = existing_output.elements();+ accumulator_tile.map([&](auto value) { return value + AccumulatorType(*(existing_data++)); });}if (apply_bias) {- METAL_PRAGMA_UNROLL- for (ushort tile_row = 0; tile_row < TILES_M; ++tile_row) {- METAL_PRAGMA_UNROLL- for (ushort tile_col = 0; tile_col < TILES_N; ++tile_col) {- thread auto& frag = accumulator_tile.fragment_at(tile_row, tile_col);- const short col_base =- short(tile_col * uzu::matmul::MxuFragmentOps::FRAGMENT_COLS + thread_position.x);- METAL_PRAGMA_UNROLL- for (ushort i = 0; i < uzu::matmul::MxuFragmentOps::THREAD_ELEMENT_ROWS; ++i) {- METAL_PRAGMA_UNROLL- for (ushort j = 0; j < uzu::matmul::MxuFragmentOps::THREAD_ELEMENT_COLS; ++j) {- const ushort element_index = i * uzu::matmul::MxuFragmentOps::THREAD_ELEMENT_COLS + j;- const short col_local = short(col_base + j);- if constexpr (aligned_n.value) {- frag[element_index] += AccumulatorType(bias_simdgroup[col_local]);- } else {- if (col_local < simdgroup_limit_n) {- frag[element_index] += AccumulatorType(bias_simdgroup[col_local]);- }- }- }+ accumulator_tile.map_coords(thread_context.simd_lane_id, [&](short, short col, auto value) {+ if constexpr (aligned_n.value) {+ return value + AccumulatorType(bias_simdgroup[col]);+ } else {+ if (col < simdgroup_limit_n) {+ return value + AccumulatorType(bias_simdgroup[col]);}+ return value;}- }+ });}if constexpr (aligned_m.value && aligned_n.value) {- accumulator_tile.store(d_simdgroup, int(params->leading_dimension_d));+ accumulator_tile.store(thread_context.simd_lane_id, d_simdgroup, int(params->leading_dimension_d));} else {accumulator_tile.store_safe(+ thread_context.simd_lane_id,d_simdgroup,int(params->leading_dimension_d),short2(simdgroup_limit_n, simdgroup_limit_m)diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.hindex 30be122da..99aa4033f 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h@@ -125,7 +125,7 @@ struct SimdgroupMmaCore {}threadgroup_barrier(mem_flags::mem_threadgroup);- accumulator.multiply_accumulate(a_shared, b_shared);+ accumulator.matmul(a_shared, b_shared);loader_a.next();loader_b.next();@@ -142,7 +142,7 @@ struct SimdgroupMmaCore {loader_b.load_safe(last_tile_dimensions_b);threadgroup_barrier(mem_flags::mem_threadgroup);- accumulator.multiply_accumulate(a_shared, b_shared);+ accumulator.matmul(a_shared, b_shared);}}diff --git a/crates/backend-uzu/src/encodable_block/attention/gemm.rs b/crates/backend-uzu/src/encodable_block/attention/gemm.rsindex 958aa92e3..e81d4a4b8 100644--- a/crates/backend-uzu/src/encodable_block/attention/gemm.rs+++ b/crates/backend-uzu/src/encodable_block/attention/gemm.rs@@ -5,7 +5,7 @@ use std::{use crate::{backends::common::{- Allocation, AsBufferRangeRef, Backend, Buffer, Encoder, Kernels,+ Allocation, AsBufferRangeRef, Backend, Buffer, Context, Encoder, Kernels,gpu_types::{AttnParams, ring::RingParams},kernel::AttentionGemmKernel,},@@ -23,21 +23,17 @@ pub struct AttentionGemmArguments<'a, B: Backend, KVBuf: AsBufferRangeRef> {pub sinks: Option<&'a Allocation<B>>,pub num_heads: usize,pub num_groups: usize,- pub suffix_length: usize, // qL- pub sequence_length: usize, // kL (prefix + suffix)- pub segment_prefix_length: usize, // qL_off+ pub suffix_length: usize,+ pub sequence_length: usize,+ pub segment_prefix_length: usize,pub ring_params: Option<RingParams>,pub head_dim: usize,pub sliding_window_size: Option<usize>,pub is_causal: bool,pub scale: f32,- /// Element stride between kv-heads (groups) in the K buffer.pub k_head_stride: u64,- /// Element stride between tokens in the K buffer.pub k_seq_stride: u64,- /// Element stride between kv-heads (groups) in the V buffer.pub v_head_stride: u64,- /// Element stride between tokens in the V buffer.pub v_seq_stride: u64,}@@ -60,13 +56,22 @@ impl<B: Backend> AttentionGemmBlock<B> {encoder: &mut Encoder<B>,args: AttentionGemmArguments<B, KVBuf>,) -> Result<(), B::Error> {- let bk: usize = if args.head_dim < 128 {+ let head_dim = args.head_dim;+ let use_mxu = encoder.context().supports_mxu()+ && self.data_type != DataType::F32+ && head_dim <= 128+ && args.suffix_length >= 64;+ let bk = if use_mxu || head_dim < 128 {32} else {16};- let head_dim = args.head_dim;- let align_q = args.suffix_length.is_multiple_of(BQ);+ let bq = if use_mxu {+ 64+ } else {+ BQ+ };+ let align_q = args.suffix_length.is_multiple_of(bq);let align_k = args.sequence_length.is_multiple_of(bk);let is_kv_cache_ring = args.ring_params.is_some();let is_causal = args.is_causal;@@ -76,6 +81,7 @@ impl<B: Backend> AttentionGemmBlock<B> {let key = KernelKey {bk,head_dim,+ use_mxu,align_q,align_k,is_kv_cache_ring,@@ -94,6 +100,7 @@ impl<B: Backend> AttentionGemmBlock<B> {self.data_type,bk as u32,head_dim as u32,+ use_mxu,align_q,align_k,is_kv_cache_ring,@@ -106,15 +113,12 @@ impl<B: Backend> AttentionGemmBlock<B> {},};- // Params (all strides in elements)let q_head_stride = (args.suffix_length * head_dim) as u64;let q_seq_stride = head_dim as u64;-let o_head_stride = head_dim as u64;let o_seq_stride = (args.num_heads * head_dim) as u64;-let nk = args.sequence_length.div_ceil(bk);- let nq_aligned = args.suffix_length / BQ;+ let nq_aligned = args.suffix_length / bq;let nk_aligned = args.sequence_length / bk;let params = AttnParams {@@ -128,7 +132,7 @@ impl<B: Backend> AttentionGemmBlock<B> {k_len: args.sequence_length as u32,q_off: args.segment_prefix_length as u32,nq_aligned: nq_aligned as u32,- q_rem: (args.suffix_length - nq_aligned * BQ) as u32,+ q_rem: (args.suffix_length - nq_aligned * bq) as u32,nk: nk as u32,nk_aligned: nk_aligned as u32,k_rem: (args.sequence_length - nk_aligned * bk) as u32,@@ -157,6 +161,7 @@ impl<B: Backend> AttentionGemmBlock<B> {struct KernelKey {bk: usize,head_dim: usize,+ use_mxu: bool,align_q: bool,align_k: bool,is_kv_cache_ring: bool,diff --git a/crates/backend-uzu/unit/encodable_block/attention_gemm_test.rs b/crates/backend-uzu/unit/encodable_block/attention_gemm_test.rsindex fbb493c25..cfdd1eb01 100644--- a/crates/backend-uzu/unit/encodable_block/attention_gemm_test.rs+++ b/crates/backend-uzu/unit/encodable_block/attention_gemm_test.rs@@ -40,21 +40,18 @@ fn get_test_data<T: ArrayElement + Float>(head_dim: usize,do_causal: bool,) -> (Input<T>, Vec<T>) {- // queries: [num_heads, suffix_length, head_dim] (contiguous, row-major)let q_size = num_heads * suffix_length * head_dim;let mut queries = vec![T::zero(); q_size];for i in 0..q_size {queries[i] = T::from((i as f32 * 0.13 + 0.5).sin() * 0.5).unwrap();}- // keys: [num_kv_heads, sequence_length, head_dim]let k_size = num_kv_heads * sequence_length * head_dim;let mut keys = vec![T::zero(); k_size];for i in 0..k_size {keys[i] = T::from((i as f32 * 0.07 + 1.0).cos() * 0.5).unwrap();}- // values: [num_kv_heads, sequence_length, head_dim]let v_size = num_kv_heads * sequence_length * head_dim;let mut values = vec![T::zero(); v_size];for i in 0..v_size {@@ -151,31 +148,25 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(}fn test_basic<T: ArrayElement + Float + Debug + Display>() {- // Non-causal, single tokenlet (input, expected) = get_test_data::<T>(4, 4, 8, 1, 64, false);test_internal(&input, &expected);- // Non-causal, multiple tokenslet (input, expected) = get_test_data::<T>(4, 4, 8, 4, 64, false);test_internal(&input, &expected);}fn test_causal<T: ArrayElement + Float + Debug + Display>() {- // Causal, single token decodelet (input, expected) = get_test_data::<T>(4, 4, 16, 1, 64, true);test_internal(&input, &expected);- // Causal, multi-token prefilllet (input, expected) = get_test_data::<T>(4, 4, 8, 4, 64, true);test_internal(&input, &expected);}fn test_gqa<T: ArrayElement + Float + Debug + Display>() {- // GQA: 8 query heads, 2 kv headslet (input, expected) = get_test_data::<T>(8, 2, 8, 1, 64, false);test_internal(&input, &expected);- // GQA causallet (input, expected) = get_test_data::<T>(8, 2, 8, 4, 64, true);test_internal(&input, &expected);}@@ -186,16 +177,13 @@ fn test_head_dim<T: ArrayElement + Float + Debug + Display>(head_dim: usize) {}fn test_unaligned<T: ArrayElement + Float + Debug + Display>() {- // suffix_length not aligned to BQ=32let (input, expected) = get_test_data::<T>(4, 4, 40, 7, 64, true);test_internal(&input, &expected);- // sequence_length not aligned to BKlet (input, expected) = get_test_data::<T>(4, 4, 13, 4, 64, false);test_internal(&input, &expected);}-// Basic tests#[uzu_test]fn test_basic_f32() {test_basic::<f32>();@@ -211,7 +199,6 @@ fn test_basic_bf16() {test_basic::<bf16>();}-// Causal tests#[uzu_test]fn test_causal_f32() {test_causal::<f32>();@@ -227,7 +214,6 @@ fn test_causal_bf16() {test_causal::<bf16>();}-// GQA tests#[uzu_test]fn test_gqa_f32() {test_gqa::<f32>();@@ -243,7 +229,6 @@ fn test_gqa_bf16() {test_gqa::<bf16>();}-// Head dim 128#[uzu_test]fn test_head_dim_128_f32() {test_head_dim::<f32>(128);@@ -259,7 +244,6 @@ fn test_head_dim_128_bf16() {test_head_dim::<bf16>(128);}-// Unaligned tests#[uzu_test]fn test_unaligned_f32() {test_unaligned::<f32>();@@ -274,3 +258,11 @@ fn test_unaligned_f16() {fn test_unaligned_bf16() {test_unaligned::<bf16>();}++#[uzu_test]+fn test_prefill_mxu() {+ let (input, expected) = get_test_data::<f16>(4, 4, 128, 128, 64, true);+ test_internal(&input, &expected);+ let (input, expected) = get_test_data::<bf16>(8, 2, 100, 96, 128, true);+ test_internal(&input, &expected);+}