Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Refactor fragment (#558)

CCYeh45bfe5874e07
diff --git a/crates/backend-uzu/src/backends/common/context.rs b/crates/backend-uzu/src/backends/common/context.rs
index 102897d1b..f76e7678c 100644
--- a/crates/backend-uzu/src/backends/common/context.rs
+++ b/crates/backend-uzu/src/backends/common/context.rs
@@ -49,4 +49,8 @@ pub trait Context: Sized {
fn stop_capture(&self) -> Result<(), <Self::Backend as Backend>::Error>;
fn sparse_buffers_supported(&self) -> bool;
+
+ fn supports_mxu(&self) -> bool {
+ false
+ }
}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_gemm.rs b/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_gemm.rs
index 0f685b838..ccdecba2f 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_gemm.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_gemm.rs
@@ -11,7 +11,8 @@ use crate::{
#[variants(T, f32, f16, bf16)]
#[variants(BK, 16, 32)]
#[variants(BD, 64, 128, 256)]
-pub fn attention_gemm<T: ArrayElement + Float, const BK: u32, const BD: u32>(
+#[variants(USE_MXU, false)]
+pub fn attention_gemm<T: ArrayElement + Float, const BK: u32, const BD: u32, const USE_MXU: bool>(
q: *const T,
k: *const T,
v: *const T,
diff --git a/crates/backend-uzu/src/backends/metal/context.rs b/crates/backend-uzu/src/backends/metal/context.rs
index 210113317..c82022df5 100644
--- a/crates/backend-uzu/src/backends/metal/context.rs
+++ b/crates/backend-uzu/src/backends/metal/context.rs
@@ -262,4 +262,8 @@ impl Context for MetalContext {
fn sparse_buffers_supported(&self) -> bool {
self.device.supports_placement_sparse_resources()
}
+
+ fn supports_mxu(&self) -> bool {
+ self.supports_mxu()
+ }
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal
index 252d45687..426a66134 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal
@@ -1,8 +1,10 @@
#include <metal_stdlib>
#include "../common/dsl.h"
#include "../common/thread_context.h"
+#include "../matmul/common/fragment.h"
#include "../matmul/common/loader.h"
-#include "../matmul/common/simdgroup_fragment.h"
+#include "../matmul/common/mxu_fragment_ops.h"
+#include "../matmul/common/simdgroup_fragment_ops.h"
#include "../generated/ring.h"
#include "../generated/trie.h"
#include "../generated/attention.h"
@@ -21,30 +23,27 @@ struct TransformScale {
METAL_FUNC T apply(T x) const { return scale * x; }
};
-template <typename T>
-METAL_FUNC T row_reduce_max(T v) {
- // SimdgroupMultiplyAccumulate::get_lane_coordinates mapping groups lanes for
- // a row as: {lane, lane^1, lane^8, (lane^1)^8}. Reduce in two steps.
- v = metal::max(v, simd_shuffle_xor(v, 1));
- v = metal::max(v, simd_shuffle_xor(v, 8));
- return v;
-}
-
-template <typename T>
-METAL_FUNC T row_reduce_sum(T v) {
- v += simd_shuffle_xor(v, 1);
- v += simd_shuffle_xor(v, 8);
- return v;
-}
-
-#define BLOCK_QUERY_ROWS 32
-#define SIMDGROUPS_PER_ROW 4
-#define SIMDGROUPS_PER_COLUMN 1
+template <typename T, uint BK, uint BD, bool USE_MXU>
+struct AttentionGemmLayout {
+ using Ops = metal::conditional_t<USE_MXU, MxuFragmentOps, SimdgroupFragmentOps>;
+
+ METAL_CONST int FRAGMENT_ROWS = Ops::FRAGMENT_ROWS;
+ METAL_CONST int SIMDGROUPS_PER_THREADGROUP = 4;
+ METAL_CONST int BLOCK_QUERY_ROWS = SIMDGROUPS_PER_THREADGROUP * FRAGMENT_ROWS;
+ METAL_CONST int ROW_ALIGNMENT_BYTES = 16;
+ METAL_CONST int ROW_PADDING_ELEMENTS = ROW_ALIGNMENT_BYTES / int(sizeof(T));
+ METAL_CONST int Q_SMEM_SIZE = USE_MXU ? 1 : (BLOCK_QUERY_ROWS * (int(BD) + ROW_PADDING_ELEMENTS));
+ METAL_CONST int KV_SMEM_SIZE = USE_MXU ? 1 : (int(BK) * (int(BD) + ROW_PADDING_ELEMENTS));
+};
-template <typename T, uint BK, uint BD>
+template <typename T, uint BK, uint BD, bool USE_MXU>
VARIANTS(T, float, half, bfloat)
VARIANTS(BK, 16, 32)
VARIANTS(BD, 64, 128, 256)
+VARIANTS(USE_MXU, false, true)
+CONSTRAINT(!USE_MXU || BK == 32)
+CONSTRAINT(!USE_MXU || T != "float")
+CONSTRAINT(!USE_MXU || BD != 256)
PUBLIC KERNEL(AttentionGemm)(
const device T* q,
const device T* k,
@@ -64,19 +63,24 @@ PUBLIC KERNEL(AttentionGemm)(
const bool is_trie SPECIALIZE,
const bool is_sliding_window SPECIALIZE,
const bool has_sinks SPECIALIZE,
- threadgroup T q_smem[BLOCK_QUERY_ROWS * (BD + 16 / sizeof(T))],
- threadgroup T kv_smem[BK * (BD + 16 / sizeof(T))],
+ threadgroup T q_smem[AttentionGemmLayout<T, BK, BD, USE_MXU>::Q_SMEM_SIZE],
+ threadgroup T kv_smem[AttentionGemmLayout<T, BK, BD, USE_MXU>::KV_SMEM_SIZE],
const ThreadContext thread_context,
- const uint q_tile_idx GROUPS(suffix_length.div_ceil(BLOCK_QUERY_ROWS)),
+ const uint q_tile_idx GROUPS(suffix_length.div_ceil(if USE_MXU { 64 } else { 32 })),
const uint head_idx GROUPS(num_heads),
const uint batch_idx GROUPS(1),
const uint lid THREADS(128)
) {
- // -------------------------------------------------------------------------
- // Pointer setup (all strides are in elements)
- // q_tile_idx: query tile index (BLOCK_QUERY_ROWS rows)
- // head_idx: query head index
- // batch_idx: batch index (currently 1 in uzu, but kept for completeness)
+ using AccumType = float;
+ using Layout = AttentionGemmLayout<T, BK, BD, USE_MXU>;
+ using Ops = typename Layout::Ops;
+ constexpr short FRAGMENT_ROWS = Layout::FRAGMENT_ROWS;
+ constexpr int BLOCK_QUERY_ROWS = Layout::BLOCK_QUERY_ROWS;
+ constexpr int ROW_PADDING_ELEMENTS = Layout::ROW_PADDING_ELEMENTS;
+ constexpr int SIMDGROUPS_PER_THREADGROUP = Layout::SIMDGROUPS_PER_THREADGROUP;
+ constexpr bool QUERY_IS_RESIDENT = USE_MXU;
+
+ using InputType = metal::conditional_t<USE_MXU, T, AccumType>;
q += batch_idx * params.q_strides[0] + head_idx * params.q_strides[1] +
q_tile_idx * int64_t(BLOCK_QUERY_ROWS) * params.q_strides[2];
@@ -92,329 +96,249 @@ PUBLIC KERNEL(AttentionGemm)(
trie += batch_idx * suffix_length;
}
- // -------------------------------------------------------------------------
- // Threadgroup memory
- constexpr short query_padding = 16 / sizeof(T);
- constexpr short key_padding = 16 / sizeof(T);
- constexpr short value_padding = 16 / sizeof(T);
-
- constexpr short query_leading_dimension = BD + query_padding;
- constexpr short key_leading_dimension = BD + key_padding;
- constexpr short value_leading_dimension = BD + value_padding;
-
- threadgroup T* query_shared = q_smem;
- threadgroup T* key_shared = kv_smem;
- threadgroup T* value_shared = kv_smem;
-
- //
- // -------------------------------------------------------------------------
- // Block loaders
- using QueryLoader = ThreadgroupLoader<
- T,
- BLOCK_QUERY_ROWS,
- BD,
- query_leading_dimension,
- 1,
- SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * 32>;
-
- using KeyLoader =
- ThreadgroupLoader<T, BK, BD, key_leading_dimension, 0, SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * 32>;
-
- using ValueLoader =
- ThreadgroupLoader<T, BK, BD, value_leading_dimension, 0, SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * 32>;
-
const int query_source_stride = int(params.q_strides[2]);
const int key_source_stride = int(params.k_strides[2]);
const int value_source_stride = int(params.v_strides[2]);
- thread QueryLoader query_loader(q, query_source_stride, query_shared, thread_context);
- thread KeyLoader key_loader(k, key_source_stride, key_shared, thread_context);
- thread ValueLoader value_loader(v, value_source_stride, value_shared, thread_context);
-
- TransformScale<T> ts(static_cast<T>(params.scale * M_LOG2E_F));
-
- // -------------------------------------------------------------------------
- // MMA tiles
- constexpr short SIMDGROUP_BLOCK_SIZE = 8;
- using AccumType = float;
- using SimdgroupMultiplyAccumulateType =
- SimdgroupMultiplyAccumulate<AccumType, SIMDGROUP_BLOCK_SIZE, SIMDGROUP_BLOCK_SIZE>;
-
- constexpr int SIMDGROUPS_PER_THREADGROUP = SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN;
- static_assert(
- BLOCK_QUERY_ROWS >= (SIMDGROUPS_PER_THREADGROUP * SIMDGROUP_BLOCK_SIZE) &&
- BLOCK_QUERY_ROWS % (SIMDGROUPS_PER_THREADGROUP * SIMDGROUP_BLOCK_SIZE) == 0,
- "Each simdgroup must host at least 1 simdgroup matrix along Q sequence."
- );
-
- // Q sequence multiply-accumulate blocks per simdgroup (QUERY_GRID_ROWS == 1
- // for the 32-row block layout)
- constexpr int QUERY_GRID_ROWS = BLOCK_QUERY_ROWS / (SIMDGROUPS_PER_THREADGROUP * SIMDGROUP_BLOCK_SIZE);
- constexpr int KEY_GRID_COLS = BK / SIMDGROUP_BLOCK_SIZE;
- constexpr int HEAD_DIM_GRID_COLS = BD / SIMDGROUP_BLOCK_SIZE;
-
- static_assert(QUERY_GRID_ROWS == 1, "Expected QUERY_GRID_ROWS == 1");
-
- SimdgroupFragment<AccumType, QUERY_GRID_ROWS, 1, SimdgroupMultiplyAccumulateType> query_fragment;
- SimdgroupFragment<AccumType, 1, KEY_GRID_COLS, SimdgroupMultiplyAccumulateType> key_fragment;
- SimdgroupFragment<AccumType, QUERY_GRID_ROWS, KEY_GRID_COLS, SimdgroupMultiplyAccumulateType> score_fragment;
- SimdgroupFragment<AccumType, 1, 1, SimdgroupMultiplyAccumulateType> value_fragment;
- SimdgroupFragment<AccumType, QUERY_GRID_ROWS, HEAD_DIM_GRID_COLS, SimdgroupMultiplyAccumulateType> output_fragment;
-
- output_fragment.clear();
-
- // -------------------------------------------------------------------------
- // Lane coordinates and pointer offsets
- const short2 lane_coordinates = SimdgroupMultiplyAccumulateType::get_lane_coordinates(thread_context.simd_lane_id);
- const short lane_row = lane_coordinates.y;
- const short lane_col = lane_coordinates.x;
+ constexpr int QUERY_ROW_FRAGMENTS = 1;
+ constexpr int KEY_COL_FRAGMENTS = BK / FRAGMENT_ROWS;
+ constexpr int HEAD_DIM_FRAGMENTS = BD / FRAGMENT_ROWS;
+
+ static_assert(!USE_MXU || KEY_COL_FRAGMENTS % 2 == 0, "MXU QK needs even N (KEY_COL_FRAGMENTS)");
+
+ constexpr int QUERY_FRAGMENT_COUNT = QUERY_IS_RESIDENT ? HEAD_DIM_FRAGMENTS : 1;
+ using QueryFragment = OperandFragment<InputType, QUERY_ROW_FRAGMENTS, 1, Ops>;
+ using KeyFragment = OperandFragment<InputType, 1, KEY_COL_FRAGMENTS, Ops, ReadTranspose>;
+ using ScoreFragment = Fragment<AccumType, QUERY_ROW_FRAGMENTS, KEY_COL_FRAGMENTS, Ops>;
+ QueryFragment query_frags[QUERY_FRAGMENT_COUNT];
+ ScoreFragment score_fragment;
+ constexpr int VALUE_COL_FRAGMENTS = USE_MXU ? 2 : HEAD_DIM_FRAGMENTS;
+ static_assert(HEAD_DIM_FRAGMENTS % VALUE_COL_FRAGMENTS == 0, "head-dim must split into VALUE_COL_FRAGMENTS frags");
+ static_assert(!USE_MXU || VALUE_COL_FRAGMENTS % 2 == 0, "MXU PV needs even N (VALUE_COL_FRAGMENTS)");
+ constexpr int OUTPUT_CHUNKS = HEAD_DIM_FRAGMENTS / VALUE_COL_FRAGMENTS;
+ constexpr short OUTPUT_CHUNK_COLS = VALUE_COL_FRAGMENTS * FRAGMENT_ROWS;
+ using ValueFragment = OperandFragment<InputType, KEY_COL_FRAGMENTS, VALUE_COL_FRAGMENTS, Ops>;
+ using OutputFragment = Fragment<AccumType, QUERY_ROW_FRAGMENTS, VALUE_COL_FRAGMENTS, Ops>;
+ OutputFragment output_chunks[OUTPUT_CHUNKS];
- const short simdgroup_row_base = SIMDGROUP_BLOCK_SIZE * QUERY_GRID_ROWS * short(thread_context.simdgroup_index);
-
- const short query_shared_offset = (simdgroup_row_base + lane_row) * query_leading_dimension + lane_col;
- constexpr short query_tile_stride = SIMDGROUP_BLOCK_SIZE;
-
- const short key_shared_offset = lane_col * key_leading_dimension + lane_row;
- constexpr short key_tile_stride = SIMDGROUP_BLOCK_SIZE;
+ METAL_PRAGMA_UNROLL
+ for (int c = 0; c < OUTPUT_CHUNKS; ++c) {
+ output_chunks[c].clear();
+ }
- const short value_shared_offset = lane_row * value_leading_dimension + lane_col;
+ const short simdgroup_row_base = FRAGMENT_ROWS * QUERY_ROW_FRAGMENTS * short(thread_context.simdgroup_index);
+ constexpr short head_dim_fragment_stride = FRAGMENT_ROWS;
- // -------------------------------------------------------------------------
- // Load Q block once (and apply scaling)
- threadgroup_barrier(mem_flags::mem_threadgroup);
+ const bool ragged_q = (!align_q && int(q_tile_idx) == params.nq_aligned);
+ constexpr short query_leading_dimension = BD + ROW_PADDING_ELEMENTS;
+ threadgroup T* query_shared = q_smem;
+ const short query_shared_offset = simdgroup_row_base * query_leading_dimension;
- if (!align_q && int(q_tile_idx) == params.nq_aligned) {
- query_loader.load_safe(short2(BD, params.q_rem));
+ if constexpr (QUERY_IS_RESIDENT) {
+ auto q_src = fragment_source(q + int64_t(simdgroup_row_base) * query_source_stride, query_source_stride);
+ if (ragged_q) {
+ q_src = q_src.bounded(params.q_rem - simdgroup_row_base, FRAGMENT_ROWS);
+ }
+ const InputType query_scale = static_cast<InputType>(params.scale * M_LOG2E_F);
+ METAL_PRAGMA_UNROLL
+ for (short dd = 0; dd < HEAD_DIM_FRAGMENTS; dd++) {
+ query_frags[dd].load_from(thread_context.simd_lane_id, q_src.advanced(dd * head_dim_fragment_stride));
+ query_frags[dd].map_coords(thread_context.simd_lane_id, [&](short, short, InputType v) {
+ return v * query_scale;
+ });
+ }
} else {
- query_loader.load_unsafe();
+ using QueryLoader = ThreadgroupLoader<
+ T,
+ BLOCK_QUERY_ROWS,
+ BD,
+ query_leading_dimension,
+ 1,
+ SIMDGROUPS_PER_THREADGROUP * METAL_SIMD_SIZE>;
+ thread QueryLoader query_loader(q, query_source_stride, query_shared, thread_context);
+ if (ragged_q) {
+ query_loader.load_safe(short2(BD, params.q_rem));
+ } else {
+ query_loader.load_unsafe();
+ }
+ query_loader.apply_inplace_op(TransformScale<T>(static_cast<T>(params.scale * M_LOG2E_F)));
+ threadgroup_barrier(mem_flags::mem_threadgroup);
}
- query_loader.apply_inplace_op(ts);
-
- // -------------------------------------------------------------------------
- // Streaming softmax state for this row (shared across lanes in a row)
- const AccumType neg_inf = static_cast<AccumType>(-1e9f) * M_LOG2E_F;
- AccumType max_score = -INFINITY;
- AccumType sum_score = AccumType(0);
- if (has_sinks) {
- max_score = M_LOG2E_F * static_cast<AccumType>(sinks[head_idx]);
- sum_score = AccumType(1);
+ const AccumType masked_score = static_cast<AccumType>(-1e9f) * M_LOG2E_F;
+ constexpr int ROWS_PER_LANE = QUERY_ROW_FRAGMENTS * Ops::THREAD_ELEMENT_ROWS;
+ const AccumType init_max = has_sinks ? AccumType(M_LOG2E_F * static_cast<AccumType>(sinks[head_idx])) : -INFINITY;
+ const AccumType init_sum = has_sinks ? AccumType(1) : AccumType(0);
+ AccumType max_score[ROWS_PER_LANE];
+ AccumType sum_score[ROWS_PER_LANE];
+ METAL_PRAGMA_UNROLL
+ for (int r = 0; r < ROWS_PER_LANE; ++r) {
+ max_score[r] = init_max;
+ sum_score[r] = init_sum;
}
- // Determine K block loop limit (causal can early-stop)
uint32_t kb_lim = params.nk;
if (is_causal) {
const int q_max = (int(q_tile_idx) + 1) * BLOCK_QUERY_ROWS + params.q_off;
- kb_lim = (q_max + BK - 1) / BK;
- kb_lim = min(params.nk, kb_lim);
+ kb_lim = min(params.nk, uint32_t((q_max + BK - 1) / BK));
}
- const int q_rel = int(q_tile_idx) * BLOCK_QUERY_ROWS + int(simdgroup_row_base) + int(lane_row); // [0, q_len)
-
+ const int q_base = int(q_tile_idx) * BLOCK_QUERY_ROWS + int(simdgroup_row_base);
const int prefix_length = params.q_off;
-
const int suffix_position = is_kv_cache_ring ? int(ring_params.ring_length) : prefix_length;
+ using KVBlockSource = BlockSource<
+ T,
+ BK,
+ BD,
+ BD + ROW_PADDING_ELEMENTS,
+ SIMDGROUPS_PER_THREADGROUP * METAL_SIMD_SIZE,
+ typename Ops::BlockStorage>;
+ threadgroup T* kv_shared = kv_smem;
- const int query_position =
- (is_trie && q_rel < params.q_len) ? suffix_position + int(trie[q_rel].height) : suffix_position + q_rel;
-
- // Loop over KV blocks
for (int kb = 0; kb < kb_lim; kb++) {
- // Load K block
- threadgroup_barrier(mem_flags::mem_threadgroup);
- if (!align_k && kb == params.nk_aligned) {
- key_loader.load_safe(short2(BD, params.k_rem));
- } else {
- key_loader.load_unsafe();
- }
-
- // Compute S = Q @ K^T for this block
+ const bool tail_k = (!align_k && kb == params.nk_aligned);
+ const short valid_k_rows = tail_k ? short(params.k_rem) : short(BK);
+ const device T* k_block = k + int64_t(kb) * int(BK) * key_source_stride;
score_fragment.clear();
- threadgroup_barrier(mem_flags::mem_threadgroup);
+ KVBlockSource key_source{
+ k_block,
+ key_source_stride,
+ kv_shared,
+ valid_k_rows,
+ thread_context,
+ };
+ key_source.prepare();
+ auto q_shared_src = fragment_source(query_shared + query_shared_offset, query_leading_dimension);
METAL_PRAGMA_UNROLL
- for (short dd = 0; dd < HEAD_DIM_GRID_COLS; dd++) {
- simdgroup_barrier(mem_flags::mem_none);
-
- query_fragment.template load<T, 1, 1, query_leading_dimension, 1>(
- &query_shared[query_shared_offset + dd * query_tile_stride]
- );
- key_fragment.template load<T, 1, 1, 1, key_leading_dimension>(
- &key_shared[key_shared_offset + dd * key_tile_stride]
- );
+ for (short dd = 0; dd < HEAD_DIM_FRAGMENTS; dd++) {
+ if constexpr (!QUERY_IS_RESIDENT) {
+ query_frags[0].load_from(thread_context.simd_lane_id, q_shared_src.advanced(dd * head_dim_fragment_stride));
+ }
+ const short q_idx = QUERY_IS_RESIDENT ? dd : 0;
+ KeyFragment key_fragment;
+ key_source.load(key_fragment, dd * head_dim_fragment_stride);
simdgroup_barrier(mem_flags::mem_none);
- tile_multiply_accumulate(score_fragment, query_fragment, key_fragment, score_fragment);
+ fragment_mma(score_fragment, query_frags[q_idx], key_fragment);
}
- // Mask out tail keys for the last (unaligned) K block
- if (!align_k && kb == params.nk_aligned) {
- const int k_rem = params.k_rem;
- METAL_PRAGMA_UNROLL
- for (short j = 0; j < KEY_GRID_COLS; j++) {
- thread auto& frag = score_fragment.multiply_accumulate_at(0, j);
- const int col0 = int(lane_col) + int(j) * SIMDGROUP_BLOCK_SIZE;
- if (col0 >= k_rem) {
- frag[0] = neg_inf;
- }
- if ((col0 + 1) >= k_rem) {
- frag[1] = neg_inf;
- }
- }
- }
-
- // Unified masking: causal, trie, ring, sliding window
- if (q_rel < params.q_len) {
- METAL_PRAGMA_UNROLL
- for (short j = 0; j < KEY_GRID_COLS; j++) {
- thread auto& frag = score_fragment.multiply_accumulate_at(0, j);
- const int col_base = kb * BK + int(lane_col) + int(j) * SIMDGROUP_BLOCK_SIZE;
-
- const int k0 = col_base;
- const int k1 = col_base + 1;
-
- if (k0 < params.k_len && !should_use_key(
- ring_params,
- trie,
- sliding_window_size,
- q_rel,
- prefix_length,
- suffix_position,
- query_position,
- k0,
- is_kv_cache_ring,
- is_causal,
- is_trie,
- is_sliding_window
- ))
- frag[0] = neg_inf;
-
- if (k1 < params.k_len && !should_use_key(
- ring_params,
- trie,
- sliding_window_size,
- q_rel,
- prefix_length,
- suffix_position,
- query_position,
- k1,
- is_kv_cache_ring,
- is_causal,
- is_trie,
- is_sliding_window
- ))
- frag[1] = neg_inf;
+ const int k_rem = params.k_rem;
+ if (!is_trie && !is_kv_cache_ring && !is_sliding_window) {
+ const bool diag = is_causal && ((int(kb) + 1) * int(BK) - 1 > prefix_length + q_base);
+ if (tail_k || diag) {
+ score_fragment.map_coords(thread_context.simd_lane_id, [&](short row, short col, AccumType v) {
+ if (tail_k && int(col) >= k_rem) {
+ return masked_score;
+ }
+ if (diag && (kb * int(BK) + int(col)) > prefix_length + q_base + int(row)) {
+ return masked_score;
+ }
+ return v;
+ });
}
- }
-
- // Load V block (overwriting K in shared memory)
- threadgroup_barrier(mem_flags::mem_threadgroup);
- if (!align_k && kb == params.nk_aligned) {
- value_loader.load_safe(short2(BD, params.k_rem));
} else {
- value_loader.load_unsafe();
+ score_fragment.map_coords(thread_context.simd_lane_id, [&](short row, short col, AccumType v) {
+ if (tail_k && int(col) >= k_rem) {
+ return masked_score;
+ }
+ const int q_rel = q_base + int(row);
+ if (q_rel >= int(params.q_len)) {
+ return v;
+ }
+ const int key = kb * BK + int(col);
+ if (key >= int(params.k_len)) {
+ return v;
+ }
+ const int query_position = is_trie ? suffix_position + int(trie[q_rel].height) : suffix_position + q_rel;
+ if (!should_use_key(
+ ring_params,
+ trie,
+ sliding_window_size,
+ q_rel,
+ prefix_length,
+ suffix_position,
+ query_position,
+ key,
+ is_kv_cache_ring,
+ is_causal,
+ is_trie,
+ is_sliding_window
+ )) {
+ return masked_score;
+ }
+ return v;
+ });
}
- // -----------------------------------------------------------------------
- // Streaming softmax update for this block
+ AccumType block_max[ROWS_PER_LANE];
+ score_fragment.row_reduce(block_max, -INFINITY, [](AccumType a, AccumType b) { return metal::max(a, b); });
- // Row max for this block
- AccumType block_max_local = -INFINITY;
+ AccumType new_max[ROWS_PER_LANE];
+ AccumType factor[ROWS_PER_LANE];
METAL_PRAGMA_UNROLL
- for (short j = 0; j < KEY_GRID_COLS; j++) {
- const thread auto& frag = score_fragment.multiply_accumulate_at(0, j);
- block_max_local = metal::max(block_max_local, frag[0]);
- block_max_local = metal::max(block_max_local, frag[1]);
+ for (int r = 0; r < ROWS_PER_LANE; ++r) {
+ new_max[r] = metal::max(max_score[r], block_max[r]);
+ factor[r] = fast::exp2(max_score[r] - new_max[r]);
+ max_score[r] = new_max[r];
+ sum_score[r] *= factor[r];
}
- const AccumType block_max = row_reduce_max(block_max_local);
-
- const AccumType new_max = metal::max(max_score, block_max);
- const AccumType factor = fast::exp2(max_score - new_max);
- max_score = new_max;
- // Rescale running sum
- sum_score *= factor;
+ score_fragment.map_rows(new_max, [](AccumType v, AccumType m) { return fast::exp2(v - m); });
- // exp2(S - new_max) and row sum
- AccumType block_sum_local = AccumType(0);
+ AccumType block_sum[ROWS_PER_LANE];
+ score_fragment.row_reduce(block_sum, AccumType(0), [](AccumType a, AccumType b) { return a + b; });
METAL_PRAGMA_UNROLL
- for (short j = 0; j < KEY_GRID_COLS; j++) {
- thread auto& frag = score_fragment.multiply_accumulate_at(0, j);
- frag[0] = fast::exp2(frag[0] - new_max);
- frag[1] = fast::exp2(frag[1] - new_max);
- block_sum_local += frag[0] + frag[1];
+ for (int r = 0; r < ROWS_PER_LANE; ++r) {
+ sum_score[r] += block_sum[r];
}
- const AccumType block_sum = row_reduce_sum(block_sum_local);
- sum_score += block_sum;
- // Rescale output accumulator
METAL_PRAGMA_UNROLL
- for (short id = 0; id < HEAD_DIM_GRID_COLS; id++) {
- thread auto& frag = output_fragment.multiply_accumulate_at(0, id);
- frag[0] *= factor;
- frag[1] *= factor;
+ for (int c = 0; c < OUTPUT_CHUNKS; ++c) {
+ output_chunks[c].map_rows(factor, [](AccumType v, AccumType f) { return v * f; });
}
- // Accumulate output: output_fragment += score_fragment * Vblock
- threadgroup_barrier(mem_flags::mem_threadgroup);
+ const device T* v_block = v + int64_t(kb) * int(BK) * value_source_stride;
+ KVBlockSource value_source{
+ v_block,
+ value_source_stride,
+ kv_shared,
+ valid_k_rows,
+ thread_context,
+ };
+ value_source.prepare();
METAL_PRAGMA_UNROLL
- for (short id = 0; id < HEAD_DIM_GRID_COLS; id++) {
- METAL_PRAGMA_UNROLL
- for (short ik = 0; ik < KEY_GRID_COLS; ik++) {
- if constexpr (BD == 128) {
- simdgroup_barrier(mem_flags::mem_none);
- }
-
- const short kk = ik * SIMDGROUP_BLOCK_SIZE;
- const short dd = id * SIMDGROUP_BLOCK_SIZE;
-
- value_fragment.template load<T, 1, 1, value_leading_dimension, 1>(
- &value_shared[value_shared_offset + kk * value_leading_dimension + dd]
- );
-
- if constexpr (BD == 128) {
- simdgroup_barrier(mem_flags::mem_none);
- }
-
- SimdgroupMultiplyAccumulateType::multiply_accumulate(
- output_fragment.multiply_accumulate_at(0, id),
- score_fragment.multiply_accumulate_at(0, ik),
- value_fragment.multiply_accumulate_at(0, 0),
- output_fragment.multiply_accumulate_at(0, id)
- );
- }
+ for (int c = 0; c < OUTPUT_CHUNKS; ++c) {
+ ValueFragment value_chunk;
+ value_source.load(value_chunk, c * OUTPUT_CHUNK_COLS);
+ fragment_mma(output_chunks[c], score_fragment, value_chunk);
}
-
- // Prepare for next iteration
- key_loader.next();
- value_loader.next();
}
- // -------------------------------------------------------------------------
- // Normalize output by sum_score (avoid div-by-zero for masked-out rows)
- const AccumType inv_sum = AccumType(1) / sum_score;
+ AccumType inv_sum[ROWS_PER_LANE];
METAL_PRAGMA_UNROLL
- for (short id = 0; id < HEAD_DIM_GRID_COLS; id++) {
- thread auto& frag = output_fragment.multiply_accumulate_at(0, id);
- frag[0] *= inv_sum;
- frag[1] *= inv_sum;
+ for (int r = 0; r < ROWS_PER_LANE; ++r) {
+ inv_sum[r] = AccumType(1) / sum_score[r];
+ }
+ METAL_PRAGMA_UNROLL
+ for (int c = 0; c < OUTPUT_CHUNKS; ++c) {
+ output_chunks[c].map_rows(inv_sum, [](AccumType v, AccumType s) { return v * s; });
}
- threadgroup_barrier(mem_flags::mem_none);
-
- // Store results (O is row-major with row-stride params.o_strides[2])
- o += int64_t(simdgroup_row_base + lane_row) * params.o_strides[2] + int64_t(lane_col);
-
- if (!align_q && int(q_tile_idx) == params.nq_aligned) {
- const short2 dst_tile_dims = short2(BD - lane_col, params.q_rem - (simdgroup_row_base + lane_row));
+ o += int64_t(simdgroup_row_base) * params.o_strides[2];
- if (dst_tile_dims.x <= 0 || dst_tile_dims.y <= 0) {
- return;
+ if (ragged_q && params.q_rem <= int(simdgroup_row_base)) {
+ return;
+ }
+ METAL_PRAGMA_UNROLL
+ for (int c = 0; c < OUTPUT_CHUNKS; ++c) {
+ device T* o_chunk = o + c * OUTPUT_CHUNK_COLS;
+ if (ragged_q) {
+ const short2 dst_tile_dims = short2(OUTPUT_CHUNK_COLS, params.q_rem - simdgroup_row_base);
+ output_chunks[c].store_safe(thread_context.simd_lane_id, o_chunk, int(params.o_strides[2]), dst_tile_dims);
+ } else {
+ output_chunks[c].store(thread_context.simd_lane_id, o_chunk, int(params.o_strides[2]));
}
-
- output_fragment.template store_safe<T, 1, 1>(o, int(params.o_strides[2]), dst_tile_dims);
- } else {
- output_fragment.template store<T, 1, 1>(o, int(params.o_strides[2]));
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h
index 6c3f08682..a12b2a5cb 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h
@@ -11,13 +11,45 @@ using namespace metal;
namespace uzu {
namespace matmul {
-///////////////////////////////////////////////////////////////////////////////
-// Fragment - a thread-private tile of values arranged as TILE_ROWS x TILE_COLS
-// sub-tiles. Ops provides the sub-tile shape constants and the MMA primitive;
-// Fragment itself owns lane positioning and device<->register transfer.
-///////////////////////////////////////////////////////////////////////////////
+template <typename Ptr>
+struct FragmentSource {
+ Ptr base;
+ int row_stride;
+ int col_stride;
+ short row_bound;
+ short col_bound;
+ bool ragged;
-template <typename T, ushort TILE_ROWS_, ushort TILE_COLS_, class Ops>
+ METAL_FUNC FragmentSource advanced(int elements) const thread {
+ FragmentSource out = *this;
+ out.base += elements;
+ return out;
+ }
+
+ METAL_FUNC FragmentSource bounded(short rows, short cols) const thread {
+ FragmentSource out = *this;
+ out.row_bound = rows;
+ out.col_bound = cols;
+ out.ragged = true;
+ return out;
+ }
+};
+
+template <typename Ptr>
+METAL_FUNC FragmentSource<Ptr> fragment_source(Ptr base, int row_stride, int col_stride = 1) {
+ return FragmentSource<Ptr>{base, row_stride, col_stride, 0, 0, false};
+}
+
+struct ReadDirect {};
+struct ReadTranspose {};
+
+template <
+ typename T,
+ ushort ROW_FRAGMENTS_,
+ ushort COL_FRAGMENTS_,
+ class Ops,
+ class ReadPolicy = ReadDirect,
+ bool MMA_TRANSPOSE_ = false>
struct Fragment {
using FragmentOpsType = Ops;
using ElementType = T;
@@ -38,33 +70,27 @@ struct Fragment {
"(elements() relies on it)"
);
- METAL_CONST ushort TILE_ROWS = TILE_ROWS_;
- METAL_CONST ushort TILE_COLS = TILE_COLS_;
+ METAL_CONST ushort ROW_FRAGMENTS = ROW_FRAGMENTS_;
+ METAL_CONST ushort COL_FRAGMENTS = COL_FRAGMENTS_;
METAL_CONST ushort FRAGMENT_ROWS = ushort(Ops::FRAGMENT_ROWS);
METAL_CONST ushort FRAGMENT_COLS = ushort(Ops::FRAGMENT_COLS);
- METAL_CONST ushort NUM_FRAGS = TILE_ROWS * TILE_COLS;
- METAL_CONST ushort ELEMENTS_PER_TILE = NUM_FRAGS * Ops::ELEMENTS_PER_THREAD;
+ METAL_CONST ushort NUM_FRAGS = ROW_FRAGMENTS * COL_FRAGMENTS;
+ METAL_CONST ushort ELEMENTS_PER_FRAGMENT = NUM_FRAGS * Ops::ELEMENTS_PER_THREAD;
+ METAL_CONST ushort ROW_REDUCE_LANE_XOR_0 = 1;
+ METAL_CONST ushort ROW_REDUCE_LANE_XOR_1 = 8;
+ METAL_CONST bool MMA_TRANSPOSE = MMA_TRANSPOSE_;
ThreadVectorType fragment_data[NUM_FRAGS];
- ThreadContext thread_context;
- METAL_FUNC Fragment(const thread ThreadContext& thread_context) thread : thread_context(thread_context) {}
-
- // Lane origin (row, col) within a single FRAGMENT_ROWS x FRAGMENT_COLS
- // sub-tile. The mapping depends only on Ops::THREAD_ELEMENT_COLS, so it is
- // identical for every Fragment sharing that Ops type.
- METAL_FUNC static constexpr short2 get_position(const thread ThreadContext& thread_context) {
- const ushort simd_lane_id = ushort(thread_context.simd_lane_id);
+ METAL_FUNC static constexpr short2 get_position(const ushort simd_lane_id) {
const short quad = simd_lane_id / 4;
const short row = (quad & 4) + (simd_lane_id / 2) % 4;
const short col = ((quad & 2) + simd_lane_id % 2) * Ops::THREAD_ELEMENT_COLS;
return short2{col, row};
}
- METAL_FUNC short2 get_position() const thread { return get_position(thread_context); }
-
METAL_FUNC constexpr void clear() {
METAL_PRAGMA_UNROLL
for (ushort index = 0; index < NUM_FRAGS; ++index) {
@@ -73,7 +99,7 @@ struct Fragment {
}
METAL_FUNC constexpr thread ThreadVectorType& fragment_at(const ushort row_index, const ushort col_index) {
- return fragment_data[row_index * TILE_COLS + col_index];
+ return fragment_data[row_index * COL_FRAGMENTS + col_index];
}
template <bool transpose>
@@ -91,81 +117,161 @@ struct Fragment {
METAL_FUNC thread ElementType* elements() { return reinterpret_cast<thread ElementType*>(fragment_data); }
- // Unsafe load: copy a (TILE_ROWS * FRAGMENT_ROWS) x (TILE_COLS *
- // FRAGMENT_COLS) block from device memory into fragment registers. ColStride
- // defaults to the compile-time constant 1 so that the row-major fast path
- // triggers when the caller omits it. Tile strides control the gap between
- // sub-tiles.
- template <
- class Ptr,
- class RowStride,
- class ColStride = Int<1>,
- class TileRowStride = Int<1>,
- class TileColStride = Int<1>>
- METAL_FUNC void load(
- Ptr source,
- RowStride row_stride,
- ColStride col_stride = {},
- TileRowStride tile_row_stride = {},
- TileColStride tile_col_stride = {}
- ) thread {
- transfer<LOAD, UNSAFE>(source, row_stride, col_stride, Int<0>{}, Int<0>{}, tile_row_stride, tile_col_stride);
+ template <class Fn>
+ METAL_FUNC void map(Fn fn) thread {
+ thread ElementType* data = elements();
+ METAL_PRAGMA_UNROLL
+ for (ushort i = 0; i < ELEMENTS_PER_FRAGMENT; i++) {
+ data[i] = ElementType(fn(data[i]));
+ }
}
- // Safe load: col_stride is implicitly 1; out-of-bounds elements become T(0).
- template <class Ptr, class TileRowStride = Int<1>, class TileColStride = Int<1>>
- METAL_FUNC void load_safe(
- Ptr source,
- const int leading_dimension,
- const short2 tile_dimensions,
- TileRowStride tile_row_stride = {},
- TileColStride tile_col_stride = {}
- ) thread {
- transfer<LOAD, SAFE>(
- source,
- leading_dimension,
- Int<1>{},
- tile_dimensions.y,
- tile_dimensions.x,
- tile_row_stride,
- tile_col_stride
- );
+ template <class Fn>
+ METAL_FUNC void map_coords(const ushort simd_lane_id, Fn fn) thread {
+ const short2 position = get_position(simd_lane_id);
+ thread ElementType* data = elements();
+ for_each_fragment([&](auto fragment_row, auto fragment_col) {
+ const short row_base = position.y + short(fragment_row.value) * FRAGMENT_ROWS;
+ const short col_base = position.x + short(fragment_col.value) * FRAGMENT_COLS;
+ const ushort frag_base = (fragment_row.value * COL_FRAGMENTS + fragment_col.value) * Ops::ELEMENTS_PER_THREAD;
+ METAL_PRAGMA_UNROLL
+ for (ushort i = 0; i < Ops::THREAD_ELEMENT_ROWS; ++i) {
+ METAL_PRAGMA_UNROLL
+ for (ushort j = 0; j < Ops::THREAD_ELEMENT_COLS; ++j) {
+ const short row = row_base + short(i) * Ops::THREAD_ELEMENT_ROW_STRIDE;
+ const short col = col_base + short(j);
+ thread ElementType& value = data[frag_base + i * Ops::THREAD_ELEMENT_COLS + j];
+ value = ElementType(fn(row, col, value));
+ }
+ }
+ });
+ }
+
+ // Row lanes differ in bits {0,3} for both simdgroup and MXU layouts.
+ template <typename Acc, class Fn>
+ METAL_FUNC void row_reduce(thread Acc* out, const Acc identity, Fn op) thread {
+ thread ElementType* data = elements();
+ METAL_PRAGMA_UNROLL
+ for (ushort tr = 0; tr < ROW_FRAGMENTS; ++tr) {
+ METAL_PRAGMA_UNROLL
+ for (ushort i = 0; i < Ops::THREAD_ELEMENT_ROWS; ++i) {
+ Acc acc = identity;
+ METAL_PRAGMA_UNROLL
+ for (ushort tc = 0; tc < COL_FRAGMENTS; ++tc) {
+ const ushort frag_base = (tr * COL_FRAGMENTS + tc) * Ops::ELEMENTS_PER_THREAD;
+ METAL_PRAGMA_UNROLL
+ for (ushort j = 0; j < Ops::THREAD_ELEMENT_COLS; ++j) {
+ acc = op(acc, Acc(data[frag_base + i * Ops::THREAD_ELEMENT_COLS + j]));
+ }
+ }
+ acc = op(acc, simd_shuffle_xor(acc, ROW_REDUCE_LANE_XOR_0));
+ acc = op(acc, simd_shuffle_xor(acc, ROW_REDUCE_LANE_XOR_1));
+ out[tr * Ops::THREAD_ELEMENT_ROWS + i] = acc;
+ }
+ }
+ }
+
+ template <typename Acc, class Fn>
+ METAL_FUNC void map_rows(const thread Acc* row_vals, Fn fn) thread {
+ thread ElementType* data = elements();
+ METAL_PRAGMA_UNROLL
+ for (ushort tr = 0; tr < ROW_FRAGMENTS; ++tr) {
+ METAL_PRAGMA_UNROLL
+ for (ushort i = 0; i < Ops::THREAD_ELEMENT_ROWS; ++i) {
+ const Acc rv = row_vals[tr * Ops::THREAD_ELEMENT_ROWS + i];
+ METAL_PRAGMA_UNROLL
+ for (ushort tc = 0; tc < COL_FRAGMENTS; ++tc) {
+ const ushort frag_base = (tr * COL_FRAGMENTS + tc) * Ops::ELEMENTS_PER_THREAD;
+ METAL_PRAGMA_UNROLL
+ for (ushort j = 0; j < Ops::THREAD_ELEMENT_COLS; ++j) {
+ const ushort e = frag_base + i * Ops::THREAD_ELEMENT_COLS + j;
+ data[e] = ElementType(fn(Acc(data[e]), rv));
+ }
+ }
+ }
+ }
+ }
+
+ template <class Ptr>
+ METAL_FUNC void load_from(const ushort simd_lane_id, FragmentSource<Ptr> src) thread {
+ if constexpr (metal::is_same_v<ReadPolicy, ReadTranspose> && Ops::READ_TRANSPOSE_SWAPS_SOURCE_STRIDES) {
+ const int row_stride = src.row_stride;
+ src.row_stride = src.col_stride;
+ src.col_stride = row_stride;
+ const short row_bound = src.row_bound;
+ src.row_bound = src.col_bound;
+ src.col_bound = row_bound;
+ }
+
+ if (src.ragged) {
+ transfer<LOAD, SAFE>(
+ simd_lane_id,
+ src.base,
+ src.row_stride,
+ src.col_stride,
+ src.row_bound,
+ src.col_bound,
+ Int<1>{},
+ Int<1>{}
+ );
+ } else {
+ transfer<LOAD, UNSAFE>(
+ simd_lane_id,
+ src.base,
+ src.row_stride,
+ src.col_stride,
+ Int<0>{},
+ Int<0>{},
+ Int<1>{},
+ Int<1>{}
+ );
+ }
}
- // Unsafe store: mirror of load. Same defaults apply.
template <
class Ptr,
class RowStride,
class ColStride = Int<1>,
- class TileRowStride = Int<1>,
- class TileColStride = Int<1>>
+ class FragmentRowStride = Int<1>,
+ class FragmentColStride = Int<1>>
METAL_FUNC void store(
+ const ushort simd_lane_id,
Ptr destination,
RowStride row_stride,
ColStride col_stride = {},
- TileRowStride tile_row_stride = {},
- TileColStride tile_col_stride = {}
+ FragmentRowStride fragment_row_stride = {},
+ FragmentColStride fragment_col_stride = {}
) thread {
- transfer<STORE, UNSAFE>(destination, row_stride, col_stride, Int<0>{}, Int<0>{}, tile_row_stride, tile_col_stride);
+ transfer<STORE, UNSAFE>(
+ simd_lane_id,
+ destination,
+ row_stride,
+ col_stride,
+ Int<0>{},
+ Int<0>{},
+ fragment_row_stride,
+ fragment_col_stride
+ );
}
- // Safe store: col_stride is implicitly 1; out-of-bounds elements are skipped.
- template <class Ptr, class TileRowStride = Int<1>, class TileColStride = Int<1>>
+ template <class Ptr, class FragmentRowStride = Int<1>, class FragmentColStride = Int<1>>
METAL_FUNC void store_safe(
+ const ushort simd_lane_id,
Ptr destination,
const int leading_dimension,
const short2 tile_dimensions,
- TileRowStride tile_row_stride = {},
- TileColStride tile_col_stride = {}
+ FragmentRowStride fragment_row_stride = {},
+ FragmentColStride fragment_col_stride = {}
) thread {
transfer<STORE, SAFE>(
+ simd_lane_id,
destination,
leading_dimension,
Int<1>{},
tile_dimensions.y,
tile_dimensions.x,
- tile_row_stride,
- tile_col_stride
+ fragment_row_stride,
+ fragment_col_stride
);
}
@@ -177,20 +283,11 @@ private:
template <class Fn>
METAL_FUNC void for_each_fragment(Fn fn) thread {
- const_for_loop<0, TILE_ROWS, 1>([&](auto row_index) {
- const_for_loop<0, TILE_COLS, 1>([&](auto col_index) { fn(row_index, col_index); });
+ const_for_loop<0, ROW_FRAGMENTS, 1>([&](auto row_index) {
+ const_for_loop<0, COL_FRAGMENTS, 1>([&](auto col_index) { fn(row_index, col_index); });
});
}
- // Unified memory transfer between device memory and fragment registers.
- //
- // IS_LOAD - true to read into fragment_data, false to write back.
- // IS_SAFE - true to bounds-check each element against (row_limit,
- // col_limit);
- // on load, out-of-bounds elements become T(0); on store, skipped.
- //
- // When ColStride is a compile-time 1, the column-stride multiplication is
- // elided to help the compiler collapse strength-reduced addressing.
template <
bool IS_LOAD,
bool IS_SAFE,
@@ -199,29 +296,30 @@ private:
class ColStride,
class RowLimit,
class ColLimit,
- class TileRowStride,
- class TileColStride>
+ class FragmentRowStride,
+ class FragmentColStride>
METAL_FUNC void transfer(
+ const ushort simd_lane_id,
Ptr ptr,
RowStride row_stride,
ColStride col_stride,
RowLimit row_limit,
ColLimit col_limit,
- TileRowStride tile_row_stride,
- TileColStride tile_col_stride
+ FragmentRowStride fragment_row_stride,
+ FragmentColStride fragment_col_stride
) thread {
using U = PointerElementType<Ptr>;
constexpr bool col_stride_is_one = metal::is_same_v<ColStride, Int<1>>;
- const short2 position = get_position();
+ const short2 position = get_position(simd_lane_id);
ptr += position.y * row_stride + position.x * col_stride;
const auto local_row_limit = row_limit - position.y;
const auto local_col_limit = col_limit - position.x;
- for_each_fragment([&](auto tile_row, auto tile_col) {
- thread auto& frag = fragment_at(tile_row.value, tile_col.value);
- const auto row_base = tile_row.value * FRAGMENT_ROWS * tile_row_stride;
- const auto col_base = tile_col.value * FRAGMENT_COLS * tile_col_stride;
+ for_each_fragment([&](auto fragment_row, auto fragment_col) {
+ thread auto& frag = fragment_at(fragment_row.value, fragment_col.value);
+ const auto row_base = fragment_row.value * FRAGMENT_ROWS * fragment_row_stride;
+ const auto col_base = fragment_col.value * FRAGMENT_COLS * fragment_col_stride;
METAL_PRAGMA_UNROLL
for (ushort i = 0; i < Ops::THREAD_ELEMENT_ROWS; i++) {
@@ -254,5 +352,34 @@ private:
}
};
+template <class Ops, class Read>
+struct OperandFragmentTraits {
+ METAL_CONST bool READ_TRANSPOSE = metal::is_same_v<Read, ReadTranspose>;
+ METAL_CONST bool MMA_TRANSPOSE = READ_TRANSPOSE && !Ops::READ_TRANSPOSE_SWAPS_SOURCE_STRIDES;
+};
+
+template <typename T, ushort ROW_FRAGMENTS, ushort COL_FRAGMENTS, class Ops, class Read = ReadDirect>
+using OperandFragment = Fragment<
+ T,
+ OperandFragmentTraits<Ops, Read>::MMA_TRANSPOSE ? COL_FRAGMENTS : ROW_FRAGMENTS,
+ OperandFragmentTraits<Ops, Read>::MMA_TRANSPOSE ? ROW_FRAGMENTS : COL_FRAGMENTS,
+ Ops,
+ Read,
+ OperandFragmentTraits<Ops, Read>::MMA_TRANSPOSE>;
+
+template <class OutputFragment, class LeftFragment, class RightFragment>
+METAL_FUNC void fragment_mma(thread OutputFragment& output, thread LeftFragment& left, thread RightFragment& right) {
+ static_assert(
+ metal::is_same_v<typename OutputFragment::FragmentOpsType, typename LeftFragment::FragmentOpsType> &&
+ metal::is_same_v<typename OutputFragment::FragmentOpsType, typename RightFragment::FragmentOpsType>,
+ "fragment_mma requires output, left, and right fragments to use the same FragmentOps"
+ );
+ OutputFragment::FragmentOpsType::template fragment_mma<LeftFragment::MMA_TRANSPOSE, RightFragment::MMA_TRANSPOSE>(
+ output,
+ left,
+ right
+ );
+}
+
} // namespace matmul
} // namespace uzu
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/loader.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/loader.h
index abf7c3c6a..81eec16f0 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/loader.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/loader.h
@@ -2,16 +2,13 @@
#include "../../common/defines.h"
#include "../../common/thread_context.h"
+#include "fragment.h"
using namespace metal;
namespace uzu {
namespace matmul {
-///////////////////////////////////////////////////////////////////////////////
-// Threadgroup Loader - loads tiles from device memory to threadgroup memory
-///////////////////////////////////////////////////////////////////////////////
-
template <
typename T,
ushort THREADGROUP_TILE_ROWS,
@@ -117,5 +114,73 @@ struct ThreadgroupLoader {
METAL_FUNC void next() { source += tile_stride; }
};
+struct DeviceBlockStorage {};
+struct ThreadgroupBlockStorage {};
+
+template <typename T, ushort ROWS, ushort COLS, ushort SHARED_LEADING_DIMENSION, ushort THREADGROUP_SIZE, class Storage>
+struct BlockSource;
+
+template <typename T, ushort ROWS, ushort COLS, ushort SHARED_LEADING_DIMENSION, ushort THREADGROUP_SIZE>
+struct BlockSource<T, ROWS, COLS, SHARED_LEADING_DIMENSION, THREADGROUP_SIZE, DeviceBlockStorage> {
+ FragmentSource<const device T*> source;
+ ushort simd_lane_id;
+
+ METAL_FUNC BlockSource(
+ const device T* source_,
+ const int source_leading_dimension_,
+ threadgroup T*,
+ const short valid_rows_,
+ const thread ThreadContext& thread_context_
+ )
+ : source(fragment_source(source_, source_leading_dimension_)), simd_lane_id(thread_context_.simd_lane_id) {
+ if (valid_rows_ < short(ROWS)) {
+ source = source.bounded(valid_rows_, COLS);
+ }
+ }
+
+ METAL_FUNC void prepare() const thread {}
+
+ template <class FragmentT>
+ METAL_FUNC void load(thread FragmentT& frag, const int offset) const thread {
+ frag.load_from(simd_lane_id, source.advanced(offset));
+ }
+};
+
+template <typename T, ushort ROWS, ushort COLS, ushort SHARED_LEADING_DIMENSION, ushort THREADGROUP_SIZE>
+struct BlockSource<T, ROWS, COLS, SHARED_LEADING_DIMENSION, THREADGROUP_SIZE, ThreadgroupBlockStorage> {
+ const device T* source;
+ int source_leading_dimension;
+ short valid_rows;
+ ThreadContext thread_context;
+ FragmentSource<threadgroup T*> shared_source;
+
+ METAL_FUNC BlockSource(
+ const device T* source_,
+ const int source_leading_dimension_,
+ threadgroup T* shared_,
+ const short valid_rows_,
+ const thread ThreadContext& thread_context_
+ )
+ : source(source_), source_leading_dimension(source_leading_dimension_), valid_rows(valid_rows_),
+ thread_context(thread_context_), shared_source(fragment_source(shared_, SHARED_LEADING_DIMENSION)) {}
+
+ METAL_FUNC void prepare() const thread {
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ ThreadgroupLoader<T, ROWS, COLS, SHARED_LEADING_DIMENSION, 0, THREADGROUP_SIZE>
+ loader(source, source_leading_dimension, shared_source.base, thread_context);
+ if (valid_rows < short(ROWS)) {
+ loader.load_safe(short2(COLS, valid_rows));
+ } else {
+ loader.load_unsafe();
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+
+ template <class FragmentT>
+ METAL_FUNC void load(thread FragmentT& frag, const int offset) const thread {
+ frag.load_from(thread_context.simd_lane_id, shared_source.advanced(offset));
+ }
+};
+
} // namespace matmul
} // namespace uzu
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment_ops.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment_ops.h
index cce0638d8..99fbadad9 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment_ops.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment_ops.h
@@ -9,6 +9,7 @@
using namespace uzu;
#include "defines.h"
+#include "loader.h"
#include <MetalPerformancePrimitives/MetalPerformancePrimitives.h>
@@ -17,16 +18,21 @@ using namespace metal;
namespace uzu {
namespace matmul {
+METAL_CONST ushort MXU_MMA_ROWS = 16;
+METAL_CONST ushort MXU_MMA_COLS = 16;
+
struct MxuFragmentOps {
- METAL_CONST ushort FRAGMENT_ROWS = 16;
- METAL_CONST ushort FRAGMENT_COLS = 16;
+ METAL_CONST ushort FRAGMENT_ROWS = MXU_MMA_ROWS;
+ METAL_CONST ushort FRAGMENT_COLS = MXU_MMA_COLS;
+ METAL_CONST bool READ_TRANSPOSE_SWAPS_SOURCE_STRIDES = false;
+ using BlockStorage = DeviceBlockStorage;
METAL_CONST ushort ELEMENTS_PER_THREAD = (FRAGMENT_ROWS * FRAGMENT_COLS) / METAL_SIMD_SIZE;
METAL_CONST ushort THREAD_ELEMENT_ROWS = 2;
METAL_CONST ushort THREAD_ELEMENT_COLS = 4;
- METAL_CONST ushort THREAD_ELEMENT_ROW_STRIDE = 8;
+ METAL_CONST ushort THREAD_ELEMENT_ROW_STRIDE = FRAGMENT_ROWS / THREAD_ELEMENT_ROWS;
static_assert(
THREAD_ELEMENT_ROWS * THREAD_ELEMENT_COLS == ELEMENTS_PER_THREAD,
@@ -36,10 +42,7 @@ struct MxuFragmentOps {
template <typename U>
using ThreadVector = typename metal::vec<U, ELEMENTS_PER_THREAD>;
- // Runs a 16x32 MPP matmul accumulating into (output_0 | output_1). The caller
- // supplies a `marshal_inputs` callable that copies its fragments into the
- // cooperative left/right tensors; the descriptor and output marshalling are
- // shared between all call shapes.
+ // MPP has no valid 16x16x16 op; fragment_mma pairs fragments into 16x32.
template <typename CType, typename AType, typename BType, bool transpose_a, bool transpose_b, typename MarshalInputs>
METAL_FUNC static void mma_impl(
thread ThreadVector<CType>& output_0,
@@ -82,7 +85,6 @@ struct MxuFragmentOps {
}
}
- // N-paired: output is two columns, left is one fragment, right is two.
template <typename CType, typename AType, typename BType, bool transpose_a = false, bool transpose_b = false>
METAL_FUNC static void mma(
thread ThreadVector<CType>& output_col_0,
@@ -107,7 +109,6 @@ struct MxuFragmentOps {
);
}
- // M-paired: output is two rows, left is two fragments, right is one.
template <typename CType, typename AType, typename BType, bool transpose_a = false, bool transpose_b = false>
METAL_FUNC static void mma(
thread ThreadVector<CType>& output_row_0,
@@ -132,33 +133,42 @@ struct MxuFragmentOps {
);
}
- template <bool transpose_a, bool transpose_b, class OutputTile, class LeftTile, class RightTile>
- METAL_FUNC static void tile_matmul(thread OutputTile& output, thread LeftTile& left, thread RightTile& right) {
- constexpr ushort left_tile_m = transpose_a ? LeftTile::TILE_COLS : LeftTile::TILE_ROWS;
- constexpr ushort tile_m = OutputTile::TILE_ROWS;
- static_assert(left_tile_m == tile_m, "tile matmul: M dimensions do not match");
+ template <bool transpose_a, bool transpose_b, class OutputFragment, class LeftFragment, class RightFragment>
+ METAL_FUNC static void fragment_mma(
+ thread OutputFragment& output,
+ thread LeftFragment& left,
+ thread RightFragment& right
+ ) {
+ constexpr ushort left_rows = transpose_a ? LeftFragment::COL_FRAGMENTS : LeftFragment::ROW_FRAGMENTS;
+ constexpr ushort rows = OutputFragment::ROW_FRAGMENTS;
+ static_assert(left_rows == rows, "fragment matmul: M dimensions do not match");
+
+ constexpr ushort right_cols = transpose_b ? RightFragment::ROW_FRAGMENTS : RightFragment::COL_FRAGMENTS;
+ constexpr ushort cols = OutputFragment::COL_FRAGMENTS;
+ static_assert(right_cols == cols, "fragment matmul: N dimensions do not match");
- constexpr ushort right_tile_n = transpose_b ? RightTile::TILE_ROWS : RightTile::TILE_COLS;
- constexpr ushort tile_n = OutputTile::TILE_COLS;
- static_assert(right_tile_n == tile_n, "tile matmul: N dimensions do not match");
+ constexpr ushort left_depth = transpose_a ? LeftFragment::ROW_FRAGMENTS : LeftFragment::COL_FRAGMENTS;
+ constexpr ushort depth = transpose_b ? RightFragment::COL_FRAGMENTS : RightFragment::ROW_FRAGMENTS;
+ static_assert(left_depth == depth, "fragment matmul: K dimensions do not match");
- constexpr ushort left_tile_k = transpose_a ? LeftTile::TILE_ROWS : LeftTile::TILE_COLS;
- constexpr ushort tile_k = transpose_b ? RightTile::TILE_COLS : RightTile::TILE_ROWS;
- static_assert(left_tile_k == tile_k, "tile matmul: K dimensions do not match");
+ static_assert(
+ (cols % 2 == 0) || (cols == 1 && rows % 2 == 0),
+ "MXU fragment_mma requires even N, or N==1 with even M (MPP pairing)"
+ );
constexpr auto transpose_left = metal::bool_constant<transpose_a>{};
constexpr auto transpose_right = metal::bool_constant<transpose_b>{};
- if constexpr (tile_n == 1 && tile_m % 2 == 0) {
+ if constexpr (cols == 1 && rows % 2 == 0) {
METAL_PRAGMA_UNROLL
- for (ushort row = 0; row < tile_m; row += 2) {
+ for (ushort row = 0; row < rows; row += 2) {
METAL_PRAGMA_UNROLL
- for (ushort col = 0; col < tile_n; ++col) {
+ for (ushort col = 0; col < cols; ++col) {
METAL_PRAGMA_UNROLL
- for (ushort k = 0; k < tile_k; ++k) {
- mma<typename OutputTile::ElementType,
- typename LeftTile::ElementType,
- typename RightTile::ElementType,
+ for (ushort k = 0; k < depth; ++k) {
+ mma<typename OutputFragment::ElementType,
+ typename LeftFragment::ElementType,
+ typename RightFragment::ElementType,
transpose_a,
transpose_b>(
output.fragment_at(row, col),
@@ -172,16 +182,16 @@ struct MxuFragmentOps {
}
}
}
- } else if constexpr (tile_n % 2 == 0) {
+ } else if constexpr (cols % 2 == 0) {
METAL_PRAGMA_UNROLL
- for (ushort row = 0; row < tile_m; ++row) {
+ for (ushort row = 0; row < rows; ++row) {
METAL_PRAGMA_UNROLL
- for (ushort col = 0; col < tile_n; col += 2) {
+ for (ushort col = 0; col < cols; col += 2) {
METAL_PRAGMA_UNROLL
- for (ushort k = 0; k < tile_k; ++k) {
- mma<typename OutputTile::ElementType,
- typename LeftTile::ElementType,
- typename RightTile::ElementType,
+ for (ushort k = 0; k < depth; ++k) {
+ mma<typename OutputFragment::ElementType,
+ typename LeftFragment::ElementType,
+ typename RightFragment::ElementType,
transpose_a,
transpose_b>(
output.fragment_at(row, col),
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.h
index 4fd31c489..ad5c92331 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.h
@@ -43,7 +43,7 @@ METAL_FUNC auto gemm_loop(
constexpr ushort RIGHT_TILE_ROWS = transpose_b ? TILES_N : TILES_K;
constexpr ushort RIGHT_TILE_COLS = transpose_b ? TILES_K : TILES_N;
- Fragment<AccumulatorType, TILES_M, TILES_N, MxuFragmentOps> accumulator(thread_context);
+ Fragment<AccumulatorType, TILES_M, TILES_N, MxuFragmentOps> accumulator;
accumulator.clear();
METAL_PRAGMA_NO_UNROLL
@@ -52,31 +52,31 @@ METAL_FUNC auto gemm_loop(
METAL_PRAGMA_NO_UNROLL
for (int inner_k = 0; inner_k < BLOCK_K; inner_k += SIMDGROUP_BLOCK_K) {
- Fragment<AT, LEFT_TILE_ROWS, LEFT_TILE_COLS, MxuFragmentOps> left_tile(thread_context);
- Fragment<BT, RIGHT_TILE_ROWS, RIGHT_TILE_COLS, MxuFragmentOps> right_tile(thread_context);
+ Fragment<AT, LEFT_TILE_ROWS, LEFT_TILE_COLS, MxuFragmentOps> left_tile;
+ Fragment<BT, RIGHT_TILE_ROWS, RIGHT_TILE_COLS, MxuFragmentOps> right_tile;
volatile int mxu_iteration_fence;
const int left_offset = transpose_a ? inner_k * leading_dimension_a : inner_k;
const int right_offset = transpose_b ? inner_k : inner_k * leading_dimension_b;
- if constexpr (aligned_m) {
- left_tile.load(left_ptr + left_offset, leading_dimension_a);
- } else {
+ auto left_src = fragment_source(left_ptr + left_offset, leading_dimension_a);
+ if constexpr (!aligned_m) {
const short row_limit = transpose_a ? SIMDGROUP_BLOCK_K : simdgroup_limit_m;
const short col_limit = transpose_a ? simdgroup_limit_m : SIMDGROUP_BLOCK_K;
- left_tile.load_safe(left_ptr + left_offset, leading_dimension_a, short2(col_limit, row_limit));
+ left_src = left_src.bounded(row_limit, col_limit);
}
+ left_tile.load_from(thread_context.simd_lane_id, left_src);
- if constexpr (aligned_n) {
- right_tile.load(right_ptr + right_offset, leading_dimension_b);
- } else {
+ auto right_src = fragment_source(right_ptr + right_offset, leading_dimension_b);
+ if constexpr (!aligned_n) {
const short row_limit = transpose_b ? simdgroup_limit_n : SIMDGROUP_BLOCK_K;
const short col_limit = transpose_b ? SIMDGROUP_BLOCK_K : simdgroup_limit_n;
- right_tile.load_safe(right_ptr + right_offset, leading_dimension_b, short2(col_limit, row_limit));
+ right_src = right_src.bounded(row_limit, col_limit);
}
+ right_tile.load_from(thread_context.simd_lane_id, right_src);
- MxuFragmentOps::template tile_matmul<transpose_a, transpose_b>(accumulator, left_tile, right_tile);
+ MxuFragmentOps::template fragment_mma<transpose_a, transpose_b>(accumulator, left_tile, right_tile);
(void)mxu_iteration_fence;
}
@@ -92,8 +92,8 @@ METAL_FUNC auto gemm_loop(
METAL_PRAGMA_NO_UNROLL
for (int inner_k = 0; inner_k < remaining_k; inner_k += SIMDGROUP_BLOCK_K) {
- Fragment<AT, LEFT_TILE_ROWS, LEFT_TILE_COLS, MxuFragmentOps> left_tile(thread_context);
- Fragment<BT, RIGHT_TILE_ROWS, RIGHT_TILE_COLS, MxuFragmentOps> right_tile(thread_context);
+ Fragment<AT, LEFT_TILE_ROWS, LEFT_TILE_COLS, MxuFragmentOps> left_tile;
+ Fragment<BT, RIGHT_TILE_ROWS, RIGHT_TILE_COLS, MxuFragmentOps> right_tile;
const short safe_k = max(short(0), short(remaining_k - inner_k));
@@ -103,10 +103,16 @@ METAL_FUNC auto gemm_loop(
const int left_offset = transpose_a ? inner_k * leading_dimension_a : inner_k;
const int right_offset = transpose_b ? inner_k : inner_k * leading_dimension_b;
- left_tile.load_safe(left_ptr + left_offset, leading_dimension_a, left_limits);
- right_tile.load_safe(right_ptr + right_offset, leading_dimension_b, right_limits);
+ left_tile.load_from(
+ thread_context.simd_lane_id,
+ fragment_source(left_ptr + left_offset, leading_dimension_a).bounded(left_limits.y, left_limits.x)
+ );
+ right_tile.load_from(
+ thread_context.simd_lane_id,
+ fragment_source(right_ptr + right_offset, leading_dimension_b).bounded(right_limits.y, right_limits.x)
+ );
- MxuFragmentOps::template tile_matmul<transpose_a, transpose_b>(accumulator, left_tile, right_tile);
+ MxuFragmentOps::template fragment_mma<transpose_a, transpose_b>(accumulator, left_tile, right_tile);
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment.h
deleted file mode 100644
index cae407263..000000000
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment.h
+++ /dev/null
@@ -1,154 +0,0 @@
-#pragma once
-
-#include "simdgroup_multiply_accumulate.h"
-
-using namespace metal;
-
-namespace uzu {
-namespace matmul {
-
-///////////////////////////////////////////////////////////////////////////////
-// SimdgroupFragment - thread's portion of simdgroup tile data
-///////////////////////////////////////////////////////////////////////////////
-
-template <
- typename T,
- int GRID_ROWS_,
- int GRID_COLS_,
- class SimdgroupMultiplyAccumulateOps_ = SimdgroupMultiplyAccumulate<T, 8, 8>>
-struct SimdgroupFragment {
- using SimdgroupMultiplyAccumulateOpsType = SimdgroupMultiplyAccumulateOps_;
- using ElementType = T;
- METAL_CONST int MULTIPLY_ACCUMULATE_ROWS = SimdgroupMultiplyAccumulateOpsType::ROWS;
- METAL_CONST int MULTIPLY_ACCUMULATE_COLS = SimdgroupMultiplyAccumulateOpsType::COLS;
- METAL_CONST int ELEMENTS_PER_MULTIPLY_ACCUMULATE = SimdgroupMultiplyAccumulateOpsType::THREAD_ELEMENT_COUNT;
-
- METAL_CONST int GRID_ROWS = GRID_ROWS_;
- METAL_CONST int GRID_COLS = GRID_COLS_;
-
- METAL_CONST int TOTAL_ROWS = GRID_ROWS * MULTIPLY_ACCUMULATE_ROWS;
- METAL_CONST int TOTAL_COLS = GRID_COLS * MULTIPLY_ACCUMULATE_COLS;
-
- METAL_CONST int MULTIPLY_ACCUMULATE_COUNT = GRID_ROWS * GRID_COLS;
- METAL_CONST int ELEMENTS_PER_FRAGMENT = MULTIPLY_ACCUMULATE_COUNT * ELEMENTS_PER_MULTIPLY_ACCUMULATE;
-
- typedef typename SimdgroupMultiplyAccumulateOpsType::SimdgroupMatrixType SimdgroupMatrixType;
- typedef typename SimdgroupMultiplyAccumulateOpsType::ThreadDataType ThreadDataType;
-
- ThreadDataType multiply_accumulate_data[MULTIPLY_ACCUMULATE_COUNT] = {ThreadDataType(0)};
-
- METAL_FUNC SimdgroupFragment() thread {}
-
- METAL_FUNC constexpr void clear() {
- METAL_PRAGMA_UNROLL
- for (ushort i = 0; i < MULTIPLY_ACCUMULATE_COUNT; ++i) {
- multiply_accumulate_data[i] = ThreadDataType(0);
- }
- }
-
- METAL_FUNC constexpr thread ThreadDataType& multiply_accumulate_at(const ushort i, const ushort j) {
- return multiply_accumulate_data[i * GRID_COLS + j];
- }
-
- METAL_FUNC constexpr const thread ThreadDataType& multiply_accumulate_at(const ushort i, const ushort j) const {
- return multiply_accumulate_data[i * GRID_COLS + j];
- }
-
- METAL_FUNC thread ElementType* elements() { return reinterpret_cast<thread ElementType*>(multiply_accumulate_data); }
-
- METAL_FUNC const thread ElementType* elements() const {
- return reinterpret_cast<const thread ElementType*>(multiply_accumulate_data);
- }
-
- template <typename U, int SIMDGROUP_STRIDE_X, int SIMDGROUP_STRIDE_Y, int STRIDE_X, int STRIDE_Y>
- METAL_FUNC void load(const threadgroup U* source) {
- METAL_PRAGMA_UNROLL
- for (ushort i = 0; i < GRID_ROWS; ++i) {
- METAL_PRAGMA_UNROLL
- for (ushort j = 0; j < GRID_COLS; ++j) {
- SimdgroupMultiplyAccumulateOpsType::load(
- multiply_accumulate_at(i, j),
- &(source
- [(i * MULTIPLY_ACCUMULATE_ROWS) * SIMDGROUP_STRIDE_X * STRIDE_X +
- (j * MULTIPLY_ACCUMULATE_COLS) * SIMDGROUP_STRIDE_Y * STRIDE_Y]),
- STRIDE_X,
- STRIDE_Y
- );
- }
- }
- }
-
- template <typename U, int SIMDGROUP_STRIDE_X, int SIMDGROUP_STRIDE_Y>
- METAL_FUNC void store(device U* destination, const int leading_dimension) const {
- METAL_PRAGMA_UNROLL
- for (ushort i = 0; i < GRID_ROWS; ++i) {
- METAL_PRAGMA_UNROLL
- for (ushort j = 0; j < GRID_COLS; ++j) {
- SimdgroupMultiplyAccumulateOpsType::store(
- multiply_accumulate_at(i, j),
- &(destination
- [(i * MULTIPLY_ACCUMULATE_ROWS) * SIMDGROUP_STRIDE_X * leading_dimension +
- (j * MULTIPLY_ACCUMULATE_COLS) * SIMDGROUP_STRIDE_Y]),
- leading_dimension,
- 1
- );
- }
- }
- }
-
- template <typename U, int SIMDGROUP_STRIDE_X, int SIMDGROUP_STRIDE_Y>
- METAL_FUNC void store_safe(
- device U* destination,
- const int leading_dimension,
- const short2 destination_tile_dimensions
- ) const {
- METAL_PRAGMA_UNROLL
- for (int i = 0; i < GRID_ROWS; ++i) {
- METAL_PRAGMA_UNROLL
- for (int j = 0; j < GRID_COLS; ++j) {
- SimdgroupMultiplyAccumulateOpsType::store_safe(
- multiply_accumulate_at(i, j),
- destination,
- leading_dimension,
- 1,
- destination_tile_dimensions.y,
- destination_tile_dimensions.x,
- (i * MULTIPLY_ACCUMULATE_ROWS) * SIMDGROUP_STRIDE_X,
- (j * MULTIPLY_ACCUMULATE_COLS) * SIMDGROUP_STRIDE_Y
- );
- }
- }
- }
-};
-
-///////////////////////////////////////////////////////////////////////////////
-// Tile multiply-accumulate
-///////////////////////////////////////////////////////////////////////////////
-
-template <typename T, typename U, int M, int N, int K>
-METAL_FUNC void tile_multiply_accumulate(
- thread SimdgroupFragment<T, M, N>& D,
- thread SimdgroupFragment<U, M, K>& A,
- thread SimdgroupFragment<U, K, N>& B,
- thread SimdgroupFragment<T, M, N>& C
-) {
- METAL_PRAGMA_UNROLL
- for (ushort m = 0; m < M; ++m) {
- METAL_PRAGMA_UNROLL
- for (ushort n = 0; n < N; ++n) {
- const ushort column_serpentine = (m % 2) ? (N - 1 - n) : n;
- METAL_PRAGMA_UNROLL
- for (ushort k = 0; k < K; ++k) {
- SimdgroupFragment<T, M, N>::SimdgroupMultiplyAccumulateOpsType::multiply_accumulate(
- D.multiply_accumulate_at(m, column_serpentine),
- A.multiply_accumulate_at(m, k),
- B.multiply_accumulate_at(k, column_serpentine),
- C.multiply_accumulate_at(m, column_serpentine)
- );
- }
- }
- }
-}
-
-} // namespace matmul
-} // namespace uzu
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment_ops.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment_ops.h
new file mode 100644
index 000000000..b96d6c67f
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment_ops.h
@@ -0,0 +1,132 @@
+#pragma once
+
+#include "defines.h"
+#include "loader.h"
+#include "simdgroup_multiply_accumulate.h"
+
+using namespace metal;
+
+namespace uzu {
+namespace matmul {
+
+struct SimdgroupFragmentOps {
+ METAL_CONST ushort FRAGMENT_ROWS = SIMDGROUP_MMA_ROWS;
+ METAL_CONST ushort FRAGMENT_COLS = SIMDGROUP_MMA_COLS;
+ METAL_CONST bool READ_TRANSPOSE_SWAPS_SOURCE_STRIDES = true;
+ using BlockStorage = ThreadgroupBlockStorage;
+
+ METAL_CONST ushort ELEMENTS_PER_THREAD = (FRAGMENT_ROWS * FRAGMENT_COLS) / METAL_SIMD_SIZE;
+ METAL_CONST ushort THREAD_ELEMENT_ROWS = 1;
+ METAL_CONST ushort THREAD_ELEMENT_COLS = 2;
+ METAL_CONST ushort THREAD_ELEMENT_ROW_STRIDE = FRAGMENT_ROWS / THREAD_ELEMENT_ROWS;
+
+ template <typename U>
+ using ThreadVector = typename metal::vec<U, ELEMENTS_PER_THREAD>;
+
+ // Serpentine order matches the chain kernel's register reuse.
+ template <bool transpose_a, bool transpose_b, class OutputFragment, class LeftFragment, class RightFragment>
+ METAL_FUNC static void fragment_mma(
+ thread OutputFragment& output,
+ thread LeftFragment& left,
+ thread RightFragment& right
+ ) {
+ static_assert(!transpose_a && !transpose_b, "SimdgroupFragmentOps::fragment_mma: transpose not supported");
+ constexpr ushort rows = OutputFragment::ROW_FRAGMENTS;
+ constexpr ushort cols = OutputFragment::COL_FRAGMENTS;
+ constexpr ushort depth = LeftFragment::COL_FRAGMENTS;
+ static_assert(LeftFragment::ROW_FRAGMENTS == rows, "fragment matmul: M dimensions do not match");
+ static_assert(RightFragment::COL_FRAGMENTS == cols, "fragment matmul: N dimensions do not match");
+ static_assert(RightFragment::ROW_FRAGMENTS == depth, "fragment matmul: K dimensions do not match");
+ using Sg = SimdgroupMMA<typename OutputFragment::ElementType, FRAGMENT_ROWS, FRAGMENT_COLS>;
+
+ METAL_PRAGMA_UNROLL
+ for (ushort m = 0; m < rows; ++m) {
+ METAL_PRAGMA_UNROLL
+ for (ushort n = 0; n < cols; ++n) {
+ const ushort col = (m % 2) ? (cols - 1 - n) : n;
+ METAL_PRAGMA_UNROLL
+ for (ushort k = 0; k < depth; ++k) {
+ Sg::mma(
+ output.fragment_at(m, col),
+ left.fragment_at(m, k),
+ right.fragment_at(k, col),
+ output.fragment_at(m, col)
+ );
+ }
+ }
+ }
+ }
+};
+
+template <typename U, int SIMDGROUP_STRIDE_X, int SIMDGROUP_STRIDE_Y, int STRIDE_X, int STRIDE_Y, typename FragT>
+METAL_FUNC void fragment_load(thread FragT& frag, const threadgroup U* source) {
+ constexpr ushort fragment_rows = SimdgroupFragmentOps::FRAGMENT_ROWS;
+ constexpr ushort fragment_cols = SimdgroupFragmentOps::FRAGMENT_COLS;
+ using Sg = SimdgroupMMA<typename FragT::ElementType, fragment_rows, fragment_cols>;
+ METAL_PRAGMA_UNROLL
+ for (ushort i = 0; i < FragT::ROW_FRAGMENTS; ++i) {
+ METAL_PRAGMA_UNROLL
+ for (ushort j = 0; j < FragT::COL_FRAGMENTS; ++j) {
+ Sg::load(
+ frag.fragment_at(i, j),
+ &(source
+ [(i * fragment_rows) * SIMDGROUP_STRIDE_X * STRIDE_X +
+ (j * fragment_cols) * SIMDGROUP_STRIDE_Y * STRIDE_Y]),
+ STRIDE_X,
+ STRIDE_Y
+ );
+ }
+ }
+}
+
+template <typename U, int SIMDGROUP_STRIDE_X, int SIMDGROUP_STRIDE_Y, typename FragT>
+METAL_FUNC void fragment_store(thread FragT& frag, device U* destination, const int leading_dimension) {
+ constexpr ushort fragment_rows = SimdgroupFragmentOps::FRAGMENT_ROWS;
+ constexpr ushort fragment_cols = SimdgroupFragmentOps::FRAGMENT_COLS;
+ using Sg = SimdgroupMMA<typename FragT::ElementType, fragment_rows, fragment_cols>;
+ METAL_PRAGMA_UNROLL
+ for (ushort i = 0; i < FragT::ROW_FRAGMENTS; ++i) {
+ METAL_PRAGMA_UNROLL
+ for (ushort j = 0; j < FragT::COL_FRAGMENTS; ++j) {
+ Sg::store(
+ frag.fragment_at(i, j),
+ &(destination
+ [(i * fragment_rows) * SIMDGROUP_STRIDE_X * leading_dimension +
+ (j * fragment_cols) * SIMDGROUP_STRIDE_Y]),
+ leading_dimension,
+ 1
+ );
+ }
+ }
+}
+
+template <typename U, int SIMDGROUP_STRIDE_X, int SIMDGROUP_STRIDE_Y, typename FragT>
+METAL_FUNC void fragment_store_safe(
+ thread FragT& frag,
+ device U* destination,
+ const int leading_dimension,
+ const short2 destination_tile_dimensions
+) {
+ constexpr ushort fragment_rows = SimdgroupFragmentOps::FRAGMENT_ROWS;
+ constexpr ushort fragment_cols = SimdgroupFragmentOps::FRAGMENT_COLS;
+ using Sg = SimdgroupMMA<typename FragT::ElementType, fragment_rows, fragment_cols>;
+ METAL_PRAGMA_UNROLL
+ for (int i = 0; i < FragT::ROW_FRAGMENTS; ++i) {
+ METAL_PRAGMA_UNROLL
+ for (int j = 0; j < FragT::COL_FRAGMENTS; ++j) {
+ Sg::store_safe(
+ frag.fragment_at(i, j),
+ destination,
+ leading_dimension,
+ 1,
+ destination_tile_dimensions.y,
+ destination_tile_dimensions.x,
+ (i * fragment_rows) * SIMDGROUP_STRIDE_X,
+ (j * fragment_cols) * SIMDGROUP_STRIDE_Y
+ );
+ }
+ }
+}
+
+} // namespace matmul
+} // namespace uzu
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_multiply_accumulate.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_multiply_accumulate.h
index 939635b80..e82edcc49 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_multiply_accumulate.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_multiply_accumulate.h
@@ -7,27 +7,30 @@ using namespace metal;
namespace uzu {
namespace matmul {
+METAL_CONST int SIMDGROUP_MMA_ROWS = 8;
+METAL_CONST int SIMDGROUP_MMA_COLS = 8;
+
///////////////////////////////////////////////////////////////////////////////
-// SimdgroupMultiplyAccumulate - stateless traits/ops for 8x8 simdgroup_matrix
+// SimdgroupMMA - stateless traits/ops for 8x8 simdgroup_matrix
///////////////////////////////////////////////////////////////////////////////
template <typename T, int ROWS_, int COLS_>
-struct SimdgroupMultiplyAccumulate {
- static_assert(ROWS_ == 8, "Only 8 x 8 fragment matrices are currently supported");
- static_assert(COLS_ == 8, "Only 8 x 8 fragment matrices are currently supported");
+struct SimdgroupMMA {
+ static_assert(ROWS_ == SIMDGROUP_MMA_ROWS, "Only 8 x 8 fragment matrices are currently supported");
+ static_assert(COLS_ == SIMDGROUP_MMA_COLS, "Only 8 x 8 fragment matrices are currently supported");
};
template <typename T>
-struct SimdgroupMultiplyAccumulate<T, 8, 8> {
- METAL_CONST int ROWS = 8;
- METAL_CONST int COLS = 8;
+struct SimdgroupMMA<T, SIMDGROUP_MMA_ROWS, SIMDGROUP_MMA_COLS> {
+ METAL_CONST int ROWS = SIMDGROUP_MMA_ROWS;
+ METAL_CONST int COLS = SIMDGROUP_MMA_COLS;
METAL_CONST int THREAD_ELEMENT_COUNT = (ROWS * COLS) / METAL_SIMD_SIZE;
METAL_CONST int THREAD_ELEMENT_ROWS = 1;
METAL_CONST int THREAD_ELEMENT_COLS = 2;
static_assert(
THREAD_ELEMENT_ROWS * THREAD_ELEMENT_COLS == THREAD_ELEMENT_COUNT,
- "SimdgroupMultiplyAccumulate shape is not consistent with element count"
+ "SimdgroupMMA shape is not consistent with element count"
);
typedef metal::simdgroup_matrix<T, ROWS, COLS> SimdgroupMatrixType;
@@ -138,7 +141,7 @@ struct SimdgroupMultiplyAccumulate<T, 8, 8> {
}
}
- METAL_FUNC static constexpr void multiply_accumulate(
+ METAL_FUNC static constexpr void mma(
thread ThreadDataType& D,
thread ThreadDataType& A,
thread ThreadDataType& B,
@@ -153,12 +156,12 @@ struct SimdgroupMultiplyAccumulate<T, 8, 8> {
reinterpret_cast<thread ThreadDataType&>(B_mat.thread_elements()) = B;
reinterpret_cast<thread ThreadDataType&>(C_mat.thread_elements()) = C;
- multiply_accumulate(D_mat, A_mat, B_mat, C_mat);
+ mma(D_mat, A_mat, B_mat, C_mat);
D = reinterpret_cast<thread ThreadDataType&>(D_mat.thread_elements());
}
- METAL_FUNC static constexpr void multiply_accumulate(
+ METAL_FUNC static constexpr void mma(
thread SimdgroupMatrixType& D,
thread SimdgroupMatrixType& A,
thread SimdgroupMatrixType& B,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h
index 451bc066b..f59214c2d 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h
@@ -1,17 +1,14 @@
#pragma once
#include "../../common/thread_context.h"
-#include "simdgroup_fragment.h"
+#include "fragment.h"
+#include "simdgroup_fragment_ops.h"
using namespace metal;
namespace uzu {
namespace matmul {
-///////////////////////////////////////////////////////////////////////////////
-// ThreadgroupTile - manages the GEMM computation for a threadgroup
-///////////////////////////////////////////////////////////////////////////////
-
template <
typename AT,
typename BT,
@@ -29,8 +26,7 @@ template <
typename Epilogue = TransformNone<DT, AccumulatorType>>
struct ThreadgroupTile {
METAL_CONST ushort SIMDGROUP_BLOCK_SIZE = 8;
- using SimdgroupMultiplyAccumulateType =
- SimdgroupMultiplyAccumulate<AccumulatorType, SIMDGROUP_BLOCK_SIZE, SIMDGROUP_BLOCK_SIZE>;
+ using SimdgroupMMAType = SimdgroupMMA<AccumulatorType, SIMDGROUP_BLOCK_SIZE, SIMDGROUP_BLOCK_SIZE>;
METAL_CONST ushort TILE_ROW_STRIDE = SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW;
METAL_CONST ushort TILE_COL_STRIDE = SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN;
@@ -47,9 +43,9 @@ struct ThreadgroupTile {
METAL_CONST ushort TILE_STRIDE_A = SIMDGROUP_BLOCK_SIZE * A_STRIDE_INNER;
METAL_CONST ushort TILE_STRIDE_B = SIMDGROUP_BLOCK_SIZE * B_STRIDE_INNER;
- SimdgroupFragment<AccumulatorType, TILE_ROWS, 1, SimdgroupMultiplyAccumulateType> a_fragment;
- SimdgroupFragment<AccumulatorType, 1, TILE_COLS, SimdgroupMultiplyAccumulateType> b_fragment;
- SimdgroupFragment<AccumulatorType, TILE_ROWS, TILE_COLS, SimdgroupMultiplyAccumulateType> c_fragment;
+ Fragment<AccumulatorType, TILE_ROWS, 1, SimdgroupFragmentOps> a_fragment;
+ Fragment<AccumulatorType, 1, TILE_COLS, SimdgroupFragmentOps> b_fragment;
+ Fragment<AccumulatorType, TILE_ROWS, TILE_COLS, SimdgroupFragmentOps> c_fragment;
ushort simdgroup_row_offset;
ushort simdgroup_col_offset;
@@ -61,8 +57,7 @@ struct ThreadgroupTile {
ushort tile_row_base = SIMDGROUP_BLOCK_SIZE * (thread_context.simdgroup_index / SIMDGROUPS_PER_COLUMN);
ushort tile_col_base = SIMDGROUP_BLOCK_SIZE * (thread_context.simdgroup_index % SIMDGROUPS_PER_COLUMN);
- const ushort2 simdgroup_coordinates =
- ushort2(SimdgroupMultiplyAccumulateType::get_lane_coordinates(thread_context.simd_lane_id));
+ const ushort2 simdgroup_coordinates = ushort2(SimdgroupMMAType::get_lane_coordinates(thread_context.simd_lane_id));
simdgroup_row_offset = simdgroup_coordinates.y;
simdgroup_col_offset = simdgroup_coordinates.x;
@@ -73,17 +68,17 @@ struct ThreadgroupTile {
simdgroup_col_offset += tile_col_base;
}
- METAL_FUNC void multiply_accumulate(const threadgroup AT* a_shared, const threadgroup BT* b_shared) {
+ METAL_FUNC void matmul(const threadgroup AT* a_shared, const threadgroup BT* b_shared) {
a_shared += a_shared_offset;
b_shared += b_shared_offset;
METAL_PRAGMA_UNROLL
for (ushort k_block_index = 0; k_block_index < BLOCK_DEPTH; k_block_index += SIMDGROUP_BLOCK_SIZE) {
- a_fragment.template load<AT, SIMDGROUPS_PER_ROW, 1, A_STRIDE_ROW, A_STRIDE_INNER>(a_shared);
+ fragment_load<AT, SIMDGROUPS_PER_ROW, 1, A_STRIDE_ROW, A_STRIDE_INNER>(a_fragment, a_shared);
- b_fragment.template load<BT, 1, SIMDGROUPS_PER_COLUMN, B_STRIDE_INNER, B_STRIDE_COL>(b_shared);
+ fragment_load<BT, 1, SIMDGROUPS_PER_COLUMN, B_STRIDE_INNER, B_STRIDE_COL>(b_fragment, b_shared);
- tile_multiply_accumulate(c_fragment, a_fragment, b_fragment, c_fragment);
+ fragment_mma(c_fragment, a_fragment, b_fragment);
a_shared += TILE_STRIDE_A;
b_shared += TILE_STRIDE_B;
@@ -91,21 +86,15 @@ struct ThreadgroupTile {
}
METAL_FUNC void store_result(device DT* D, const int leading_dimension_d) {
- METAL_PRAGMA_UNROLL
- for (ushort i = 0; i < decltype(c_fragment)::ELEMENTS_PER_FRAGMENT; i++) {
- c_fragment.elements()[i] = Epilogue::apply(c_fragment.elements()[i]);
- }
+ c_fragment.map([](AccumulatorType value) { return Epilogue::apply(value); });
D += simdgroup_row_offset * leading_dimension_d + simdgroup_col_offset;
- c_fragment.template store<DT, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN>(D, leading_dimension_d);
+ fragment_store<DT, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN>(c_fragment, D, leading_dimension_d);
}
METAL_FUNC void store_result_safe(device DT* D, const int leading_dimension_d, short2 destination_tile_dimensions) {
- METAL_PRAGMA_UNROLL
- for (ushort i = 0; i < decltype(c_fragment)::ELEMENTS_PER_FRAGMENT; i++) {
- c_fragment.elements()[i] = Epilogue::apply(c_fragment.elements()[i]);
- }
+ c_fragment.map([](AccumulatorType value) { return Epilogue::apply(value); });
D += simdgroup_row_offset * leading_dimension_d + simdgroup_col_offset;
destination_tile_dimensions -= short2(simdgroup_col_offset, simdgroup_row_offset);
@@ -113,105 +102,70 @@ struct ThreadgroupTile {
if (destination_tile_dimensions.x <= 0 || destination_tile_dimensions.y <= 0)
return;
- c_fragment.template store_safe<DT, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN>(
+ fragment_store_safe<DT, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN>(
+ c_fragment,
D,
leading_dimension_d,
destination_tile_dimensions
);
}
- template <typename EpilogueOp>
- METAL_FUNC void apply_epilogue(
- const device DT* C,
- const int leading_dimension_c,
- const int column_stride_c,
- thread const EpilogueOp& epilogue_operation
- ) {
- const device DT* c_pointer =
- C + simdgroup_row_offset * leading_dimension_c + simdgroup_col_offset * column_stride_c;
-
+ template <class Fn>
+ METAL_FUNC void for_each_output(Fn fn) {
+ thread AccumulatorType* data = c_fragment.elements();
+ constexpr ushort epm = SimdgroupFragmentOps::ELEMENTS_PER_THREAD;
METAL_PRAGMA_UNROLL
for (ushort i = 0; i < TILE_ROWS; i++) {
+ const ushort row_offset = i * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW;
METAL_PRAGMA_UNROLL
for (ushort j = 0; j < TILE_COLS; j++) {
- thread auto& block_data = c_fragment.multiply_accumulate_at(i, j);
+ const ushort col_offset = j * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN;
METAL_PRAGMA_UNROLL
- for (ushort k = 0; k < decltype(c_fragment)::ELEMENTS_PER_MULTIPLY_ACCUMULATE; k++) {
- const ushort row_offset = i * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW;
- const ushort col_offset = j * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN;
- DT c_value = c_pointer[row_offset * leading_dimension_c + col_offset * column_stride_c + k];
- block_data[k] = epilogue_operation.apply(block_data[k], static_cast<AccumulatorType>(c_value));
+ for (ushort k = 0; k < epm; k++) {
+ fn(row_offset, col_offset, k, data[(i * TILE_COLS + j) * epm + k]);
}
}
}
}
+ template <typename EpilogueOp>
+ METAL_FUNC void apply_epilogue(const device DT* C, const int ld_c, const int cstride_c, thread const EpilogueOp& op) {
+ const device DT* c_ptr = C + simdgroup_row_offset * ld_c + simdgroup_col_offset * cstride_c;
+ for_each_output([&](ushort row_offset, ushort col_offset, ushort k, thread AccumulatorType& v) {
+ v = op.apply(v, static_cast<AccumulatorType>(c_ptr[row_offset * ld_c + col_offset * cstride_c + k]));
+ });
+ }
+
template <typename EpilogueOp>
METAL_FUNC void apply_epilogue_safe(
const device DT* C,
- const int leading_dimension_c,
- const int column_stride_c,
+ const int ld_c,
+ const int cstride_c,
short2 tile_dimensions,
- thread const EpilogueOp& epilogue_operation
+ thread const EpilogueOp& op
) {
- const device DT* c_pointer =
- C + simdgroup_row_offset * leading_dimension_c + simdgroup_col_offset * column_stride_c;
+ const device DT* c_ptr = C + simdgroup_row_offset * ld_c + simdgroup_col_offset * cstride_c;
tile_dimensions -= short2(simdgroup_col_offset, simdgroup_row_offset);
-
- METAL_PRAGMA_UNROLL
- for (ushort i = 0; i < TILE_ROWS; i++) {
- METAL_PRAGMA_UNROLL
- for (ushort j = 0; j < TILE_COLS; j++) {
- thread auto& block_data = c_fragment.multiply_accumulate_at(i, j);
- const ushort row_offset = i * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW;
- const ushort col_offset = j * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN;
- METAL_PRAGMA_UNROLL
- for (ushort k = 0; k < decltype(c_fragment)::ELEMENTS_PER_MULTIPLY_ACCUMULATE; k++) {
- if (row_offset < tile_dimensions.y && col_offset + k < tile_dimensions.x) {
- DT c_value = c_pointer[row_offset * leading_dimension_c + col_offset * column_stride_c + k];
- block_data[k] = epilogue_operation.apply(block_data[k], static_cast<AccumulatorType>(c_value));
- }
- }
- }
- }
+ for_each_output([&](ushort row_offset, ushort col_offset, ushort k, thread AccumulatorType& v) {
+ if (row_offset < tile_dimensions.y && col_offset + k < tile_dimensions.x)
+ v = op.apply(v, static_cast<AccumulatorType>(c_ptr[row_offset * ld_c + col_offset * cstride_c + k]));
+ });
}
METAL_FUNC void apply_bias(const device BT* bias) {
- const device BT* bias_pointer = bias + simdgroup_col_offset;
-
- METAL_PRAGMA_UNROLL
- for (ushort i = 0; i < TILE_ROWS; i++) {
- METAL_PRAGMA_UNROLL
- for (ushort j = 0; j < TILE_COLS; j++) {
- thread auto& block_data = c_fragment.multiply_accumulate_at(i, j);
- const ushort col_offset = j * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN;
- METAL_PRAGMA_UNROLL
- for (ushort k = 0; k < decltype(c_fragment)::ELEMENTS_PER_MULTIPLY_ACCUMULATE; k++) {
- block_data[k] += static_cast<AccumulatorType>(bias_pointer[col_offset + k]);
- }
- }
- }
+ const device BT* bias_ptr = bias + simdgroup_col_offset;
+ for_each_output([&](ushort, ushort col_offset, ushort k, thread AccumulatorType& v) {
+ v += static_cast<AccumulatorType>(bias_ptr[col_offset + k]);
+ });
}
METAL_FUNC void apply_bias_safe(const device BT* bias, short2 tile_dimensions) {
- const device BT* bias_pointer = bias + simdgroup_col_offset;
+ const device BT* bias_ptr = bias + simdgroup_col_offset;
tile_dimensions -= short2(simdgroup_col_offset, simdgroup_row_offset);
-
- METAL_PRAGMA_UNROLL
- for (ushort i = 0; i < TILE_ROWS; i++) {
- METAL_PRAGMA_UNROLL
- for (ushort j = 0; j < TILE_COLS; j++) {
- thread auto& block_data = c_fragment.multiply_accumulate_at(i, j);
- const ushort row_offset = i * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW;
- const ushort col_offset = j * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN;
- METAL_PRAGMA_UNROLL
- for (ushort k = 0; k < decltype(c_fragment)::ELEMENTS_PER_MULTIPLY_ACCUMULATE; k++) {
- if (row_offset < tile_dimensions.y && col_offset + k < tile_dimensions.x) {
- block_data[k] += static_cast<AccumulatorType>(bias_pointer[col_offset + k]);
- }
- }
- }
- }
+ for_each_output([&](ushort row_offset, ushort col_offset, ushort k, thread AccumulatorType& v) {
+ if (row_offset < tile_dimensions.y && col_offset + k < tile_dimensions.x)
+ v += static_cast<AccumulatorType>(bias_ptr[col_offset + k]);
+ });
}
};
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
index 17112a7fb..ed743d330 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
@@ -105,7 +105,7 @@ struct MxuMmaCore {
thread Loader& loader_b,
const thread ThreadContext& thread_context
) {
- AccumFragment accumulator(thread_context);
+ AccumFragment accumulator;
accumulator.clear();
threadgroup BT* b_shared_simdgroup = b_shared + tile_col_offset * SHARED_STRIDE_B;
@@ -123,20 +123,22 @@ struct MxuMmaCore {
METAL_PRAGMA_NO_UNROLL
for (int inner_k = 0; inner_k < QUANT_BK; inner_k += SIMDGROUP_BLOCK_K) {
- uzu::matmul::Fragment<AT, TILES_M, TILES_K, uzu::matmul::MxuFragmentOps> left_tile(thread_context);
- uzu::matmul::Fragment<BT, TILES_N, TILES_K, uzu::matmul::MxuFragmentOps> right_tile(thread_context);
+ uzu::matmul::Fragment<AT, TILES_M, TILES_K, uzu::matmul::MxuFragmentOps> left_tile;
+ uzu::matmul::Fragment<BT, TILES_N, TILES_K, uzu::matmul::MxuFragmentOps> right_tile;
const int left_offset = inner_k;
- if constexpr (ALIGNED_M) {
- left_tile.load(a_simdgroup + left_offset, leading_dimension_a);
- } else {
- left_tile
- .load_safe(a_simdgroup + left_offset, leading_dimension_a, short2(SIMDGROUP_BLOCK_K, simdgroup_limit_m));
+ auto left_src = uzu::matmul::fragment_source(a_simdgroup + left_offset, leading_dimension_a);
+ if constexpr (!ALIGNED_M) {
+ left_src = left_src.bounded(simdgroup_limit_m, SIMDGROUP_BLOCK_K);
}
+ left_tile.load_from(thread_context.simd_lane_id, left_src);
- right_tile.load(b_shared_simdgroup + inner_k, int(SHARED_STRIDE_B));
+ right_tile.load_from(
+ thread_context.simd_lane_id,
+ uzu::matmul::fragment_source(b_shared_simdgroup + inner_k, int(SHARED_STRIDE_B))
+ );
- uzu::matmul::MxuFragmentOps::template tile_matmul<false, true>(accumulator, left_tile, right_tile);
+ uzu::matmul::MxuFragmentOps::template fragment_mma<false, true>(accumulator, left_tile, right_tile);
}
a_simdgroup += QUANT_BK;
@@ -210,8 +212,6 @@ struct MxuMmaCore {
const bool apply_bias = output_transform.contains(GemmDTransform::BIAS);
const device BT* bias_simdgroup = output_bias + size_t(block_col) + size_t(tile_col_offset);
- using FragType = uzu::matmul::Fragment<DT, TILES_M, TILES_N, uzu::matmul::MxuFragmentOps>;
- const short2 thread_position = FragType::get_position(thread_context);
auto dispatch_aligned_k = [&](auto body) {
if constexpr (B_PROLOGUE == GemmBPrologueKind::FullPrecision) {
@@ -325,62 +325,38 @@ struct MxuMmaCore {
if (apply_scale) {
const AccumulatorType scale = AccumulatorType(params->ab_scale);
- METAL_PRAGMA_UNROLL
- for (ushort i = 0; i < accumulator_tile.ELEMENTS_PER_TILE; i++) {
- accumulator_tile.elements()[i] *= scale;
- }
+ accumulator_tile.map([&](auto value) { return value * scale; });
}
if (apply_accumulate) {
- uzu::matmul::Fragment<DT, TILES_M, TILES_N, uzu::matmul::MxuFragmentOps> existing_output(
- thread_context
- );
- if constexpr (aligned_m.value && aligned_n.value) {
- existing_output.load(d_simdgroup, int(params->leading_dimension_d));
- } else {
- existing_output.load_safe(
- d_simdgroup,
- int(params->leading_dimension_d),
- short2(simdgroup_limit_n, simdgroup_limit_m)
- );
- }
- METAL_PRAGMA_UNROLL
- for (ushort i = 0; i < accumulator_tile.ELEMENTS_PER_TILE; i++) {
- accumulator_tile.elements()[i] += AccumulatorType(existing_output.elements()[i]);
+ uzu::matmul::Fragment<DT, TILES_M, TILES_N, uzu::matmul::MxuFragmentOps> existing_output;
+ auto output_src = uzu::matmul::fragment_source(d_simdgroup, int(params->leading_dimension_d));
+ if constexpr (!(aligned_m.value && aligned_n.value)) {
+ output_src = output_src.bounded(simdgroup_limit_m, simdgroup_limit_n);
}
+ existing_output.load_from(thread_context.simd_lane_id, output_src);
+ thread DT* existing_data = existing_output.elements();
+ accumulator_tile.map([&](auto value) { return value + AccumulatorType(*(existing_data++)); });
}
if (apply_bias) {
- METAL_PRAGMA_UNROLL
- for (ushort tile_row = 0; tile_row < TILES_M; ++tile_row) {
- METAL_PRAGMA_UNROLL
- for (ushort tile_col = 0; tile_col < TILES_N; ++tile_col) {
- thread auto& frag = accumulator_tile.fragment_at(tile_row, tile_col);
- const short col_base =
- short(tile_col * uzu::matmul::MxuFragmentOps::FRAGMENT_COLS + thread_position.x);
- METAL_PRAGMA_UNROLL
- for (ushort i = 0; i < uzu::matmul::MxuFragmentOps::THREAD_ELEMENT_ROWS; ++i) {
- METAL_PRAGMA_UNROLL
- for (ushort j = 0; j < uzu::matmul::MxuFragmentOps::THREAD_ELEMENT_COLS; ++j) {
- const ushort element_index = i * uzu::matmul::MxuFragmentOps::THREAD_ELEMENT_COLS + j;
- const short col_local = short(col_base + j);
- if constexpr (aligned_n.value) {
- frag[element_index] += AccumulatorType(bias_simdgroup[col_local]);
- } else {
- if (col_local < simdgroup_limit_n) {
- frag[element_index] += AccumulatorType(bias_simdgroup[col_local]);
- }
- }
- }
+ accumulator_tile.map_coords(thread_context.simd_lane_id, [&](short, short col, auto value) {
+ if constexpr (aligned_n.value) {
+ return value + AccumulatorType(bias_simdgroup[col]);
+ } else {
+ if (col < simdgroup_limit_n) {
+ return value + AccumulatorType(bias_simdgroup[col]);
}
+ return value;
}
- }
+ });
}
if constexpr (aligned_m.value && aligned_n.value) {
- accumulator_tile.store(d_simdgroup, int(params->leading_dimension_d));
+ accumulator_tile.store(thread_context.simd_lane_id, d_simdgroup, int(params->leading_dimension_d));
} else {
accumulator_tile.store_safe(
+ thread_context.simd_lane_id,
d_simdgroup,
int(params->leading_dimension_d),
short2(simdgroup_limit_n, simdgroup_limit_m)
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h
index 30be122da..99aa4033f 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h
@@ -125,7 +125,7 @@ struct SimdgroupMmaCore {
}
threadgroup_barrier(mem_flags::mem_threadgroup);
- accumulator.multiply_accumulate(a_shared, b_shared);
+ accumulator.matmul(a_shared, b_shared);
loader_a.next();
loader_b.next();
@@ -142,7 +142,7 @@ struct SimdgroupMmaCore {
loader_b.load_safe(last_tile_dimensions_b);
threadgroup_barrier(mem_flags::mem_threadgroup);
- accumulator.multiply_accumulate(a_shared, b_shared);
+ accumulator.matmul(a_shared, b_shared);
}
}
diff --git a/crates/backend-uzu/src/encodable_block/attention/gemm.rs b/crates/backend-uzu/src/encodable_block/attention/gemm.rs
index 958aa92e3..e81d4a4b8 100644
--- a/crates/backend-uzu/src/encodable_block/attention/gemm.rs
+++ b/crates/backend-uzu/src/encodable_block/attention/gemm.rs
@@ -5,7 +5,7 @@ use std::{
use crate::{
backends::common::{
- Allocation, AsBufferRangeRef, Backend, Buffer, Encoder, Kernels,
+ Allocation, AsBufferRangeRef, Backend, Buffer, Context, Encoder, Kernels,
gpu_types::{AttnParams, ring::RingParams},
kernel::AttentionGemmKernel,
},
@@ -23,21 +23,17 @@ pub struct AttentionGemmArguments<'a, B: Backend, KVBuf: AsBufferRangeRef> {
pub sinks: Option<&'a Allocation<B>>,
pub num_heads: usize,
pub num_groups: usize,
- pub suffix_length: usize, // qL
- pub sequence_length: usize, // kL (prefix + suffix)
- pub segment_prefix_length: usize, // qL_off
+ pub suffix_length: usize,
+ pub sequence_length: usize,
+ pub segment_prefix_length: usize,
pub ring_params: Option<RingParams>,
pub head_dim: usize,
pub sliding_window_size: Option<usize>,
pub is_causal: bool,
pub scale: f32,
- /// Element stride between kv-heads (groups) in the K buffer.
pub k_head_stride: u64,
- /// Element stride between tokens in the K buffer.
pub k_seq_stride: u64,
- /// Element stride between kv-heads (groups) in the V buffer.
pub v_head_stride: u64,
- /// Element stride between tokens in the V buffer.
pub v_seq_stride: u64,
}
@@ -60,13 +56,22 @@ impl<B: Backend> AttentionGemmBlock<B> {
encoder: &mut Encoder<B>,
args: AttentionGemmArguments<B, KVBuf>,
) -> Result<(), B::Error> {
- let bk: usize = if args.head_dim < 128 {
+ let head_dim = args.head_dim;
+ let use_mxu = encoder.context().supports_mxu()
+ && self.data_type != DataType::F32
+ && head_dim <= 128
+ && args.suffix_length >= 64;
+ let bk = if use_mxu || head_dim < 128 {
32
} else {
16
};
- let head_dim = args.head_dim;
- let align_q = args.suffix_length.is_multiple_of(BQ);
+ let bq = if use_mxu {
+ 64
+ } else {
+ BQ
+ };
+ let align_q = args.suffix_length.is_multiple_of(bq);
let align_k = args.sequence_length.is_multiple_of(bk);
let is_kv_cache_ring = args.ring_params.is_some();
let is_causal = args.is_causal;
@@ -76,6 +81,7 @@ impl<B: Backend> AttentionGemmBlock<B> {
let key = KernelKey {
bk,
head_dim,
+ use_mxu,
align_q,
align_k,
is_kv_cache_ring,
@@ -94,6 +100,7 @@ impl<B: Backend> AttentionGemmBlock<B> {
self.data_type,
bk as u32,
head_dim as u32,
+ use_mxu,
align_q,
align_k,
is_kv_cache_ring,
@@ -106,15 +113,12 @@ impl<B: Backend> AttentionGemmBlock<B> {
},
};
- // Params (all strides in elements)
let q_head_stride = (args.suffix_length * head_dim) as u64;
let q_seq_stride = head_dim as u64;
-
let o_head_stride = head_dim as u64;
let o_seq_stride = (args.num_heads * head_dim) as u64;
-
let nk = args.sequence_length.div_ceil(bk);
- let nq_aligned = args.suffix_length / BQ;
+ let nq_aligned = args.suffix_length / bq;
let nk_aligned = args.sequence_length / bk;
let params = AttnParams {
@@ -128,7 +132,7 @@ impl<B: Backend> AttentionGemmBlock<B> {
k_len: args.sequence_length as u32,
q_off: args.segment_prefix_length as u32,
nq_aligned: nq_aligned as u32,
- q_rem: (args.suffix_length - nq_aligned * BQ) as u32,
+ q_rem: (args.suffix_length - nq_aligned * bq) as u32,
nk: nk as u32,
nk_aligned: nk_aligned as u32,
k_rem: (args.sequence_length - nk_aligned * bk) as u32,
@@ -157,6 +161,7 @@ impl<B: Backend> AttentionGemmBlock<B> {
struct KernelKey {
bk: usize,
head_dim: usize,
+ use_mxu: bool,
align_q: bool,
align_k: bool,
is_kv_cache_ring: bool,
diff --git a/crates/backend-uzu/unit/encodable_block/attention_gemm_test.rs b/crates/backend-uzu/unit/encodable_block/attention_gemm_test.rs
index fbb493c25..cfdd1eb01 100644
--- a/crates/backend-uzu/unit/encodable_block/attention_gemm_test.rs
+++ b/crates/backend-uzu/unit/encodable_block/attention_gemm_test.rs
@@ -40,21 +40,18 @@ fn get_test_data<T: ArrayElement + Float>(
head_dim: usize,
do_causal: bool,
) -> (Input<T>, Vec<T>) {
- // queries: [num_heads, suffix_length, head_dim] (contiguous, row-major)
let q_size = num_heads * suffix_length * head_dim;
let mut queries = vec![T::zero(); q_size];
for i in 0..q_size {
queries[i] = T::from((i as f32 * 0.13 + 0.5).sin() * 0.5).unwrap();
}
- // keys: [num_kv_heads, sequence_length, head_dim]
let k_size = num_kv_heads * sequence_length * head_dim;
let mut keys = vec![T::zero(); k_size];
for i in 0..k_size {
keys[i] = T::from((i as f32 * 0.07 + 1.0).cos() * 0.5).unwrap();
}
- // values: [num_kv_heads, sequence_length, head_dim]
let v_size = num_kv_heads * sequence_length * head_dim;
let mut values = vec![T::zero(); v_size];
for i in 0..v_size {
@@ -151,31 +148,25 @@ fn test_internal<T: ArrayElement + Float + Debug + Display>(
}
fn test_basic<T: ArrayElement + Float + Debug + Display>() {
- // Non-causal, single token
let (input, expected) = get_test_data::<T>(4, 4, 8, 1, 64, false);
test_internal(&input, &expected);
- // Non-causal, multiple tokens
let (input, expected) = get_test_data::<T>(4, 4, 8, 4, 64, false);
test_internal(&input, &expected);
}
fn test_causal<T: ArrayElement + Float + Debug + Display>() {
- // Causal, single token decode
let (input, expected) = get_test_data::<T>(4, 4, 16, 1, 64, true);
test_internal(&input, &expected);
- // Causal, multi-token prefill
let (input, expected) = get_test_data::<T>(4, 4, 8, 4, 64, true);
test_internal(&input, &expected);
}
fn test_gqa<T: ArrayElement + Float + Debug + Display>() {
- // GQA: 8 query heads, 2 kv heads
let (input, expected) = get_test_data::<T>(8, 2, 8, 1, 64, false);
test_internal(&input, &expected);
- // GQA causal
let (input, expected) = get_test_data::<T>(8, 2, 8, 4, 64, true);
test_internal(&input, &expected);
}
@@ -186,16 +177,13 @@ fn test_head_dim<T: ArrayElement + Float + Debug + Display>(head_dim: usize) {
}
fn test_unaligned<T: ArrayElement + Float + Debug + Display>() {
- // suffix_length not aligned to BQ=32
let (input, expected) = get_test_data::<T>(4, 4, 40, 7, 64, true);
test_internal(&input, &expected);
- // sequence_length not aligned to BK
let (input, expected) = get_test_data::<T>(4, 4, 13, 4, 64, false);
test_internal(&input, &expected);
}
-// Basic tests
#[uzu_test]
fn test_basic_f32() {
test_basic::<f32>();
@@ -211,7 +199,6 @@ fn test_basic_bf16() {
test_basic::<bf16>();
}
-// Causal tests
#[uzu_test]
fn test_causal_f32() {
test_causal::<f32>();
@@ -227,7 +214,6 @@ fn test_causal_bf16() {
test_causal::<bf16>();
}
-// GQA tests
#[uzu_test]
fn test_gqa_f32() {
test_gqa::<f32>();
@@ -243,7 +229,6 @@ fn test_gqa_bf16() {
test_gqa::<bf16>();
}
-// Head dim 128
#[uzu_test]
fn test_head_dim_128_f32() {
test_head_dim::<f32>(128);
@@ -259,7 +244,6 @@ fn test_head_dim_128_bf16() {
test_head_dim::<bf16>(128);
}
-// Unaligned tests
#[uzu_test]
fn test_unaligned_f32() {
test_unaligned::<f32>();
@@ -274,3 +258,11 @@ fn test_unaligned_f16() {
fn test_unaligned_bf16() {
test_unaligned::<bf16>();
}
+
+#[uzu_test]
+fn test_prefill_mxu() {
+ let (input, expected) = get_test_data::<f16>(4, 4, 128, 128, 64, true);
+ test_internal(&input, &expected);
+ let (input, expected) = get_test_data::<bf16>(8, 2, 100, 96, 128, true);
+ test_internal(&input, &expected);
+}