A high-performance inference engine for AI models
Support strict mxu and matmul without accumulation (#564)
CCYeh3a4ff207c453
diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metalindex 426a66134..7a2e7afc0 100644--- a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal@@ -25,7 +25,7 @@ struct TransformScale {template <typename T, uint BK, uint BD, bool USE_MXU>struct AttentionGemmLayout {- using Ops = metal::conditional_t<USE_MXU, MxuFragmentOps, SimdgroupFragmentOps>;+ using Ops = metal::conditional_t<USE_MXU, MxuFragmentOps<>, SimdgroupFragmentOps>;METAL_CONST int FRAGMENT_ROWS = Ops::FRAGMENT_ROWS;METAL_CONST int SIMDGROUPS_PER_THREADGROUP = 4;diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.hindex a12b2a5cb..1f40431e4 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h@@ -84,12 +84,7 @@ struct Fragment {ThreadVectorType fragment_data[NUM_FRAGS];- METAL_FUNC static constexpr short2 get_position(const ushort simd_lane_id) {- const short quad = simd_lane_id / 4;- const short row = (quad & 4) + (simd_lane_id / 2) % 4;- const short col = ((quad & 2) + simd_lane_id % 2) * Ops::THREAD_ELEMENT_COLS;- return short2{col, row};- }+ METAL_FUNC static constexpr short2 get_position(const ushort simd_lane_id) { return Ops::get_position(simd_lane_id); }METAL_FUNC constexpr void clear() {METAL_PRAGMA_UNROLL@@ -128,21 +123,19 @@ struct Fragment {template <class Fn>METAL_FUNC void map_coords(const ushort simd_lane_id, Fn fn) thread {- const short2 position = get_position(simd_lane_id);thread ElementType* data = elements();+ const short2 position = get_position(simd_lane_id);for_each_fragment([&](auto fragment_row, auto fragment_col) {+ const ushort frag_base = (fragment_row.value * COL_FRAGMENTS + fragment_col.value) * Ops::ELEMENTS_PER_THREAD;const short row_base = position.y + short(fragment_row.value) * FRAGMENT_ROWS;const short col_base = position.x + short(fragment_col.value) * FRAGMENT_COLS;- const ushort frag_base = (fragment_row.value * COL_FRAGMENTS + fragment_col.value) * Ops::ELEMENTS_PER_THREAD;METAL_PRAGMA_UNROLL- for (ushort i = 0; i < Ops::THREAD_ELEMENT_ROWS; ++i) {- METAL_PRAGMA_UNROLL- for (ushort j = 0; j < Ops::THREAD_ELEMENT_COLS; ++j) {- const short row = row_base + short(i) * Ops::THREAD_ELEMENT_ROW_STRIDE;- const short col = col_base + short(j);- thread ElementType& value = data[frag_base + i * Ops::THREAD_ELEMENT_COLS + j];- value = ElementType(fn(row, col, value));- }+ for (ushort element_index = 0; element_index < Ops::ELEMENTS_PER_THREAD; ++element_index) {+ const short2 element_offset = Ops::get_element_offset(element_index);+ const short row = row_base + element_offset.y;+ const short col = col_base + element_offset.x;+ thread ElementType& value = data[frag_base + element_index];+ value = ElementType(fn(row, col, value));}});}@@ -310,7 +303,6 @@ private:) thread {using U = PointerElementType<Ptr>;constexpr bool col_stride_is_one = metal::is_same_v<ColStride, Int<1>>;-const short2 position = get_position(simd_lane_id);ptr += position.y * row_stride + position.x * col_stride;const auto local_row_limit = row_limit - position.y;@@ -322,29 +314,26 @@ private:const auto col_base = fragment_col.value * FRAGMENT_COLS * fragment_col_stride;METAL_PRAGMA_UNROLL- for (ushort i = 0; i < Ops::THREAD_ELEMENT_ROWS; i++) {- const auto row = row_base + i * Ops::THREAD_ELEMENT_ROW_STRIDE;- METAL_PRAGMA_UNROLL- for (ushort j = 0; j < Ops::THREAD_ELEMENT_COLS; j++) {- const ushort element_index = i * Ops::THREAD_ELEMENT_COLS + j;- const auto col = col_base + j;- const auto offset = col_stride_is_one ? (row * row_stride + col) : (row * row_stride + col * col_stride);-- if constexpr (IS_LOAD) {- if constexpr (IS_SAFE) {- const bool in_bounds = (row < local_row_limit) && (col < local_col_limit);- frag[element_index] = in_bounds ? static_cast<T>(ptr[offset]) : T(0);- } else {- frag[element_index] = static_cast<T>(ptr[offset]);- }+ for (ushort element_index = 0; element_index < Ops::ELEMENTS_PER_THREAD; element_index++) {+ const short2 element_offset = Ops::get_element_offset(element_index);+ const auto row = row_base + element_offset.y;+ const auto col = col_base + element_offset.x;+ const auto offset = col_stride_is_one ? (row * row_stride + col) : (row * row_stride + col * col_stride);++ if constexpr (IS_LOAD) {+ if constexpr (IS_SAFE) {+ const bool in_bounds = (row < local_row_limit) && (col < local_col_limit);+ frag[element_index] = in_bounds ? static_cast<T>(ptr[offset]) : T(0);} else {- if constexpr (IS_SAFE) {- if ((row < local_row_limit) && (col < local_col_limit)) {- ptr[offset] = static_cast<U>(frag[element_index]);- }- } else {+ frag[element_index] = static_cast<T>(ptr[offset]);+ }+ } else {+ if constexpr (IS_SAFE) {+ if ((row < local_row_limit) && (col < local_col_limit)) {ptr[offset] = static_cast<U>(frag[element_index]);}+ } else {+ ptr[offset] = static_cast<U>(frag[element_index]);}}}@@ -381,5 +370,19 @@ METAL_FUNC void fragment_mma(thread OutputFragment& output, thread LeftFragment&);}+template <class OutputFragment, class LeftFragment, class RightFragment>+METAL_FUNC void fragment_mm(thread OutputFragment& output, thread LeftFragment& left, thread RightFragment& right) {+ static_assert(+ metal::is_same_v<typename OutputFragment::FragmentOpsType, typename LeftFragment::FragmentOpsType> &&+ metal::is_same_v<typename OutputFragment::FragmentOpsType, typename RightFragment::FragmentOpsType>,+ "fragment_mm requires output, left, and right fragments to use the same FragmentOps"+ );+ OutputFragment::FragmentOpsType::template fragment_mm<LeftFragment::MMA_TRANSPOSE, RightFragment::MMA_TRANSPOSE>(+ output,+ left,+ right+ );+}+} // namespace matmul} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment_ops.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment_ops.hindex 99fbadad9..0e86f32c4 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment_ops.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment_ops.h@@ -21,6 +21,8 @@ namespace matmul {METAL_CONST ushort MXU_MMA_ROWS = 16;METAL_CONST ushort MXU_MMA_COLS = 16;+// RELAXED=false uses the strict MPP layout; it currently performs about the same as simdgroup.+template <bool RELAXED = true>struct MxuFragmentOps {METAL_CONST ushort FRAGMENT_ROWS = MXU_MMA_ROWS;METAL_CONST ushort FRAGMENT_COLS = MXU_MMA_COLS;@@ -42,8 +44,41 @@ struct MxuFragmentOps {template <typename U>using ThreadVector = typename metal::vec<U, ELEMENTS_PER_THREAD>;+ METAL_FUNC static constexpr short2 get_position(ushort simd_lane_id) {+ if constexpr (RELAXED) {+ const short quad = simd_lane_id / 4;+ const short row = (quad & 4) + (simd_lane_id / 2) % 4;+ const short col = ((quad & 2) + simd_lane_id % 2) * THREAD_ELEMENT_COLS;+ return short2{col, row};+ } else {+ const short col = short((simd_lane_id & 1) * 2 + ((simd_lane_id >> 3) & 1) * 4);+ const short row = short(((simd_lane_id >> 1) & 3) + ((simd_lane_id >> 4) & 1) * 4);+ return short2{col, row};+ }+ }++ METAL_FUNC static constexpr short2 get_element_offset(ushort element_index) {+ if constexpr (RELAXED) {+ const short row = short((element_index / THREAD_ELEMENT_COLS) * THREAD_ELEMENT_ROW_STRIDE);+ const short col = short(element_index % THREAD_ELEMENT_COLS);+ return short2{col, row};+ } else {+ const short row = short((element_index / 4) * 8);+ const ushort col_slot = element_index & 3;+ const short col = short((col_slot & 1) + (col_slot / 2) * 8);+ return short2{col, row};+ }+ }+// MPP has no valid 16x16x16 op; fragment_mma pairs fragments into 16x32.- template <typename CType, typename AType, typename BType, bool transpose_a, bool transpose_b, typename MarshalInputs>+ template <+ bool ACCUMULATE,+ typename CType,+ typename AType,+ typename BType,+ bool transpose_a,+ bool transpose_b,+ typename MarshalInputs>METAL_FUNC static void mma_impl(thread ThreadVector<CType>& output_0,thread ThreadVector<CType>& output_1,@@ -55,8 +90,9 @@ struct MxuFragmentOps {FRAGMENT_COLS,transpose_a,transpose_b,- true,- mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate+ RELAXED,+ ACCUMULATE ? mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate+ : mpp::tensor_ops::matmul2d_descriptor::mode::multiply);mpp::tensor_ops::matmul2d<descriptor, metal::execution_simdgroup> matmul_op;@@ -70,23 +106,51 @@ struct MxuFragmentOps {marshal_inputs(cooperative_left, cooperative_right);- METAL_PRAGMA_UNROLL- for (ushort i = 0; i < ELEMENTS_PER_THREAD; i++) {- cooperative_output[i] = output_0[i];- cooperative_output[ELEMENTS_PER_THREAD + i] = output_1[i];+ if constexpr (ACCUMULATE) {+ if constexpr (RELAXED) {+ METAL_PRAGMA_UNROLL+ for (ushort i = 0; i < ELEMENTS_PER_THREAD; i++) {+ cooperative_output[i] = output_0[i];+ cooperative_output[ELEMENTS_PER_THREAD + i] = output_1[i];+ }+ } else {+ METAL_PRAGMA_UNROLL+ for (ushort i = 0; i < 4; i++) {+ cooperative_output[i] = output_0[i];+ cooperative_output[4 + i] = output_1[i];+ cooperative_output[8 + i] = output_0[4 + i];+ cooperative_output[12 + i] = output_1[4 + i];+ }+ }}matmul_op.run(cooperative_left, cooperative_right, cooperative_output);- METAL_PRAGMA_UNROLL- for (ushort i = 0; i < ELEMENTS_PER_THREAD; i++) {- output_0[i] = cooperative_output[i];- output_1[i] = cooperative_output[ELEMENTS_PER_THREAD + i];+ if constexpr (RELAXED) {+ METAL_PRAGMA_UNROLL+ for (ushort i = 0; i < ELEMENTS_PER_THREAD; i++) {+ output_0[i] = cooperative_output[i];+ output_1[i] = cooperative_output[ELEMENTS_PER_THREAD + i];+ }+ } else {+ METAL_PRAGMA_UNROLL+ for (ushort i = 0; i < 4; i++) {+ output_0[i] = cooperative_output[i];+ output_1[i] = cooperative_output[4 + i];+ output_0[4 + i] = cooperative_output[8 + i];+ output_1[4 + i] = cooperative_output[12 + i];+ }}}- template <typename CType, typename AType, typename BType, bool transpose_a = false, bool transpose_b = false>- METAL_FUNC static void mma(+ template <+ bool ACCUMULATE,+ typename CType,+ typename AType,+ typename BType,+ bool transpose_a = false,+ bool transpose_b = false>+ METAL_FUNC static void matmul(thread ThreadVector<CType>& output_col_0,thread ThreadVector<CType>& output_col_1,const thread ThreadVector<AType>& left,@@ -95,22 +159,41 @@ struct MxuFragmentOps {const thread ThreadVector<BType>& right_col_1,metal::bool_constant<transpose_b>) {- mma_impl<CType, AType, BType, transpose_a, transpose_b>(+ mma_impl<ACCUMULATE, CType, AType, BType, transpose_a, transpose_b>(output_col_0,output_col_1,[&](thread auto& cooperative_left, thread auto& cooperative_right) {METAL_PRAGMA_UNROLLfor (ushort i = 0; i < ELEMENTS_PER_THREAD; i++) {cooperative_left[i] = left[i];- cooperative_right[i] = right_col_0[i];- cooperative_right[ELEMENTS_PER_THREAD + i] = right_col_1[i];+ }+ if constexpr (RELAXED) {+ METAL_PRAGMA_UNROLL+ for (ushort i = 0; i < ELEMENTS_PER_THREAD; i++) {+ cooperative_right[i] = right_col_0[i];+ cooperative_right[ELEMENTS_PER_THREAD + i] = right_col_1[i];+ }+ } else {+ METAL_PRAGMA_UNROLL+ for (ushort i = 0; i < 4; i++) {+ cooperative_right[i] = right_col_0[i];+ cooperative_right[4 + i] = right_col_1[i];+ cooperative_right[8 + i] = right_col_0[4 + i];+ cooperative_right[12 + i] = right_col_1[4 + i];+ }}});}- template <typename CType, typename AType, typename BType, bool transpose_a = false, bool transpose_b = false>- METAL_FUNC static void mma(+ template <+ bool ACCUMULATE,+ typename CType,+ typename AType,+ typename BType,+ bool transpose_a = false,+ bool transpose_b = false>+ METAL_FUNC static void matmul(thread ThreadVector<CType>& output_row_0,thread ThreadVector<CType>& output_row_1,const thread ThreadVector<AType>& left_row_0,@@ -119,7 +202,8 @@ struct MxuFragmentOps {const thread ThreadVector<BType>& right,metal::bool_constant<transpose_b>) {- mma_impl<CType, AType, BType, transpose_a, transpose_b>(+ static_assert(RELAXED, "strict MXU row-pairing is not implemented");+ mma_impl<ACCUMULATE, CType, AType, BType, transpose_a, transpose_b>(output_row_0,output_row_1,[&](thread auto& cooperative_left, thread auto& cooperative_right) {@@ -133,8 +217,14 @@ struct MxuFragmentOps {);}- template <bool transpose_a, bool transpose_b, class OutputFragment, class LeftFragment, class RightFragment>- METAL_FUNC static void fragment_mma(+ template <+ bool ACCUMULATE,+ bool transpose_a,+ bool transpose_b,+ class OutputFragment,+ class LeftFragment,+ class RightFragment>+ METAL_FUNC static void fragment_matmul(thread OutputFragment& output,thread LeftFragment& left,thread RightFragment& right@@ -164,9 +254,28 @@ struct MxuFragmentOps {for (ushort row = 0; row < rows; row += 2) {METAL_PRAGMA_UNROLLfor (ushort col = 0; col < cols; ++col) {+ if constexpr (!ACCUMULATE) {+ matmul<+ false,+ typename OutputFragment::ElementType,+ typename LeftFragment::ElementType,+ typename RightFragment::ElementType,+ transpose_a,+ transpose_b>(+ output.fragment_at(row, col),+ output.fragment_at(row + 1, col),+ left.fragment_at(row, 0, transpose_left),+ left.fragment_at(row + 1, 0, transpose_left),+ transpose_left,+ right.fragment_at(0, col, transpose_right),+ transpose_right+ );+ }METAL_PRAGMA_UNROLL- for (ushort k = 0; k < depth; ++k) {- mma<typename OutputFragment::ElementType,+ for (ushort k = ACCUMULATE ? 0 : 1; k < depth; ++k) {+ matmul<+ true,+ typename OutputFragment::ElementType,typename LeftFragment::ElementType,typename RightFragment::ElementType,transpose_a,@@ -187,9 +296,28 @@ struct MxuFragmentOps {for (ushort row = 0; row < rows; ++row) {METAL_PRAGMA_UNROLLfor (ushort col = 0; col < cols; col += 2) {+ if constexpr (!ACCUMULATE) {+ matmul<+ false,+ typename OutputFragment::ElementType,+ typename LeftFragment::ElementType,+ typename RightFragment::ElementType,+ transpose_a,+ transpose_b>(+ output.fragment_at(row, col),+ output.fragment_at(row, col + 1),+ left.fragment_at(row, 0, transpose_left),+ transpose_left,+ right.fragment_at(0, col, transpose_right),+ right.fragment_at(0, col + 1, transpose_right),+ transpose_right+ );+ }METAL_PRAGMA_UNROLL- for (ushort k = 0; k < depth; ++k) {- mma<typename OutputFragment::ElementType,+ for (ushort k = ACCUMULATE ? 0 : 1; k < depth; ++k) {+ matmul<+ true,+ typename OutputFragment::ElementType,typename LeftFragment::ElementType,typename RightFragment::ElementType,transpose_a,@@ -207,7 +335,28 @@ struct MxuFragmentOps {}}}++ template <bool transpose_a, bool transpose_b, class OutputFragment, class LeftFragment, class RightFragment>+ METAL_FUNC static void fragment_mma(+ thread OutputFragment& output,+ thread LeftFragment& left,+ thread RightFragment& right+ ) {+ fragment_matmul<true, transpose_a, transpose_b>(output, left, right);+ }++ template <bool transpose_a, bool transpose_b, class OutputFragment, class LeftFragment, class RightFragment>+ METAL_FUNC static void fragment_mm(+ thread OutputFragment& output,+ thread LeftFragment& left,+ thread RightFragment& right+ ) {+ // MXU relaxed multiply is slightly faster than multiply_accumulate for pure matmul.+ fragment_matmul<false, transpose_a, transpose_b>(output, left, right);+ }};+using MxuStrictFragmentOps = MxuFragmentOps<false>;+} // namespace matmul} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.hindex ad5c92331..24fe2ba3b 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.h@@ -33,9 +33,9 @@ METAL_FUNC auto gemm_loop(const short simdgroup_limit_n,const thread ThreadContext& thread_context) {- constexpr ushort TILES_M = SIMDGROUP_BLOCK_M / MxuFragmentOps::FRAGMENT_ROWS;- constexpr ushort TILES_N = SIMDGROUP_BLOCK_N / MxuFragmentOps::FRAGMENT_COLS;- constexpr ushort TILES_K = SIMDGROUP_BLOCK_K / MxuFragmentOps::FRAGMENT_ROWS;+ constexpr ushort TILES_M = SIMDGROUP_BLOCK_M / MxuFragmentOps<>::FRAGMENT_ROWS;+ constexpr ushort TILES_N = SIMDGROUP_BLOCK_N / MxuFragmentOps<>::FRAGMENT_COLS;+ constexpr ushort TILES_K = SIMDGROUP_BLOCK_K / MxuFragmentOps<>::FRAGMENT_ROWS;constexpr ushort LEFT_TILE_ROWS = transpose_a ? TILES_K : TILES_M;constexpr ushort LEFT_TILE_COLS = transpose_a ? TILES_M : TILES_K;@@ -43,7 +43,7 @@ METAL_FUNC auto gemm_loop(constexpr ushort RIGHT_TILE_ROWS = transpose_b ? TILES_N : TILES_K;constexpr ushort RIGHT_TILE_COLS = transpose_b ? TILES_K : TILES_N;- Fragment<AccumulatorType, TILES_M, TILES_N, MxuFragmentOps> accumulator;+ Fragment<AccumulatorType, TILES_M, TILES_N, MxuFragmentOps<>> accumulator;accumulator.clear();METAL_PRAGMA_NO_UNROLL@@ -52,8 +52,8 @@ METAL_FUNC auto gemm_loop(METAL_PRAGMA_NO_UNROLLfor (int inner_k = 0; inner_k < BLOCK_K; inner_k += SIMDGROUP_BLOCK_K) {- Fragment<AT, LEFT_TILE_ROWS, LEFT_TILE_COLS, MxuFragmentOps> left_tile;- Fragment<BT, RIGHT_TILE_ROWS, RIGHT_TILE_COLS, MxuFragmentOps> right_tile;+ Fragment<AT, LEFT_TILE_ROWS, LEFT_TILE_COLS, MxuFragmentOps<>> left_tile;+ Fragment<BT, RIGHT_TILE_ROWS, RIGHT_TILE_COLS, MxuFragmentOps<>> right_tile;volatile int mxu_iteration_fence;@@ -76,7 +76,7 @@ METAL_FUNC auto gemm_loop(}right_tile.load_from(thread_context.simd_lane_id, right_src);- MxuFragmentOps::template fragment_mma<transpose_a, transpose_b>(accumulator, left_tile, right_tile);+ MxuFragmentOps<>::template fragment_mma<transpose_a, transpose_b>(accumulator, left_tile, right_tile);(void)mxu_iteration_fence;}@@ -92,8 +92,8 @@ METAL_FUNC auto gemm_loop(METAL_PRAGMA_NO_UNROLLfor (int inner_k = 0; inner_k < remaining_k; inner_k += SIMDGROUP_BLOCK_K) {- Fragment<AT, LEFT_TILE_ROWS, LEFT_TILE_COLS, MxuFragmentOps> left_tile;- Fragment<BT, RIGHT_TILE_ROWS, RIGHT_TILE_COLS, MxuFragmentOps> right_tile;+ Fragment<AT, LEFT_TILE_ROWS, LEFT_TILE_COLS, MxuFragmentOps<>> left_tile;+ Fragment<BT, RIGHT_TILE_ROWS, RIGHT_TILE_COLS, MxuFragmentOps<>> right_tile;const short safe_k = max(short(0), short(remaining_k - inner_k));@@ -112,7 +112,7 @@ METAL_FUNC auto gemm_loop(fragment_source(right_ptr + right_offset, leading_dimension_b).bounded(right_limits.y, right_limits.x));- MxuFragmentOps::template fragment_mma<transpose_a, transpose_b>(accumulator, left_tile, right_tile);+ MxuFragmentOps<>::template fragment_mma<transpose_a, transpose_b>(accumulator, left_tile, right_tile);}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment_ops.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment_ops.hindex b96d6c67f..f5342cd18 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment_ops.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment_ops.h@@ -23,7 +23,17 @@ struct SimdgroupFragmentOps {template <typename U>using ThreadVector = typename metal::vec<U, ELEMENTS_PER_THREAD>;- // Serpentine order matches the chain kernel's register reuse.+ METAL_FUNC static constexpr short2 get_position(ushort simd_lane_id) {+ const short quad = simd_lane_id / 4;+ const short row = (quad & 4) + (simd_lane_id / 2) % 4;+ const short col = ((quad & 2) + simd_lane_id % 2) * THREAD_ELEMENT_COLS;+ return short2{col, row};+ }++ METAL_FUNC static constexpr short2 get_element_offset(ushort element_index) {+ return short2{short(element_index), 0};+ }+template <bool transpose_a, bool transpose_b, class OutputFragment, class LeftFragment, class RightFragment>METAL_FUNC static void fragment_mma(thread OutputFragment& output,@@ -56,6 +66,17 @@ struct SimdgroupFragmentOps {}}}++ template <bool transpose_a, bool transpose_b, class OutputFragment, class LeftFragment, class RightFragment>+ METAL_FUNC static void fragment_mm(+ thread OutputFragment& output,+ thread LeftFragment& left,+ thread RightFragment& right+ ) {+ // simdgroup_multiply is slower here.+ output.clear();+ fragment_mma<transpose_a, transpose_b>(output, left, right);+ }};template <typename U, int SIMDGROUP_STRIDE_X, int SIMDGROUP_STRIDE_Y, int STRIDE_X, int STRIDE_Y, typename FragT>diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.hindex ed743d330..1dcb3d983 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h@@ -42,9 +42,9 @@ struct MxuMmaCore {THREADGROUP_BLOCK_K_FP % SIMDGROUP_BLOCK_K == 0,"FP THREADGROUP_BLOCK_K must be a multiple of SIMDGROUP_BLOCK_K");- METAL_CONST ushort TILES_M = SIMDGROUP_BLOCK_M / uzu::matmul::MxuFragmentOps::FRAGMENT_ROWS;- METAL_CONST ushort TILES_N = SIMDGROUP_BLOCK_N / uzu::matmul::MxuFragmentOps::FRAGMENT_COLS;- METAL_CONST ushort TILES_K = SIMDGROUP_BLOCK_K / uzu::matmul::MxuFragmentOps::FRAGMENT_ROWS;+ METAL_CONST ushort TILES_M = SIMDGROUP_BLOCK_M / uzu::matmul::MxuFragmentOps<>::FRAGMENT_ROWS;+ METAL_CONST ushort TILES_N = SIMDGROUP_BLOCK_N / uzu::matmul::MxuFragmentOps<>::FRAGMENT_COLS;+ METAL_CONST ushort TILES_K = SIMDGROUP_BLOCK_K / uzu::matmul::MxuFragmentOps<>::FRAGMENT_ROWS;METAL_CONST ushort QUANT_BK = (B_PROLOGUE == GemmBPrologueKind::FullPrecision) ? 0 : GROUP_SIZE;METAL_CONST ushort PADDING_B = 16 / sizeof(BT);@@ -90,7 +90,7 @@ struct MxuMmaCore {(BITS > 0) ? BITS : 4,true>;- using AccumFragment = uzu::matmul::Fragment<AccumulatorType, TILES_M, TILES_N, uzu::matmul::MxuFragmentOps>;+ using AccumFragment = uzu::matmul::Fragment<AccumulatorType, TILES_M, TILES_N, uzu::matmul::MxuFragmentOps<>>;template <bool ALIGNED_M, bool ALIGNED_N, typename Loader>static METAL_FUNC AccumFragment quant_k_loop(@@ -123,8 +123,8 @@ struct MxuMmaCore {METAL_PRAGMA_NO_UNROLLfor (int inner_k = 0; inner_k < QUANT_BK; inner_k += SIMDGROUP_BLOCK_K) {- uzu::matmul::Fragment<AT, TILES_M, TILES_K, uzu::matmul::MxuFragmentOps> left_tile;- uzu::matmul::Fragment<BT, TILES_N, TILES_K, uzu::matmul::MxuFragmentOps> right_tile;+ uzu::matmul::Fragment<AT, TILES_M, TILES_K, uzu::matmul::MxuFragmentOps<>> left_tile;+ uzu::matmul::Fragment<BT, TILES_N, TILES_K, uzu::matmul::MxuFragmentOps<>> right_tile;const int left_offset = inner_k;auto left_src = uzu::matmul::fragment_source(a_simdgroup + left_offset, leading_dimension_a);@@ -138,7 +138,7 @@ struct MxuMmaCore {uzu::matmul::fragment_source(b_shared_simdgroup + inner_k, int(SHARED_STRIDE_B)));- uzu::matmul::MxuFragmentOps::template fragment_mma<false, true>(accumulator, left_tile, right_tile);+ uzu::matmul::MxuFragmentOps<>::template fragment_mma<false, true>(accumulator, left_tile, right_tile);}a_simdgroup += QUANT_BK;@@ -329,7 +329,7 @@ struct MxuMmaCore {}if (apply_accumulate) {- uzu::matmul::Fragment<DT, TILES_M, TILES_N, uzu::matmul::MxuFragmentOps> existing_output;+ uzu::matmul::Fragment<DT, TILES_M, TILES_N, uzu::matmul::MxuFragmentOps<>> existing_output;auto output_src = uzu::matmul::fragment_source(d_simdgroup, int(params->leading_dimension_d));if constexpr (!(aligned_m.value && aligned_n.value)) {output_src = output_src.bounded(simdgroup_limit_m, simdgroup_limit_n);