Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Support strict mxu and matmul without accumulation (#564)

CCYeh3a4ff207c453
diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal
index 426a66134..7a2e7afc0 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal
@@ -25,7 +25,7 @@ struct TransformScale {
template <typename T, uint BK, uint BD, bool USE_MXU>
struct AttentionGemmLayout {
- using Ops = metal::conditional_t<USE_MXU, MxuFragmentOps, SimdgroupFragmentOps>;
+ using Ops = metal::conditional_t<USE_MXU, MxuFragmentOps<>, SimdgroupFragmentOps>;
METAL_CONST int FRAGMENT_ROWS = Ops::FRAGMENT_ROWS;
METAL_CONST int SIMDGROUPS_PER_THREADGROUP = 4;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h
index a12b2a5cb..1f40431e4 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h
@@ -84,12 +84,7 @@ struct Fragment {
ThreadVectorType fragment_data[NUM_FRAGS];
- METAL_FUNC static constexpr short2 get_position(const ushort simd_lane_id) {
- const short quad = simd_lane_id / 4;
- const short row = (quad & 4) + (simd_lane_id / 2) % 4;
- const short col = ((quad & 2) + simd_lane_id % 2) * Ops::THREAD_ELEMENT_COLS;
- return short2{col, row};
- }
+ METAL_FUNC static constexpr short2 get_position(const ushort simd_lane_id) { return Ops::get_position(simd_lane_id); }
METAL_FUNC constexpr void clear() {
METAL_PRAGMA_UNROLL
@@ -128,21 +123,19 @@ struct Fragment {
template <class Fn>
METAL_FUNC void map_coords(const ushort simd_lane_id, Fn fn) thread {
- const short2 position = get_position(simd_lane_id);
thread ElementType* data = elements();
+ const short2 position = get_position(simd_lane_id);
for_each_fragment([&](auto fragment_row, auto fragment_col) {
+ const ushort frag_base = (fragment_row.value * COL_FRAGMENTS + fragment_col.value) * Ops::ELEMENTS_PER_THREAD;
const short row_base = position.y + short(fragment_row.value) * FRAGMENT_ROWS;
const short col_base = position.x + short(fragment_col.value) * FRAGMENT_COLS;
- const ushort frag_base = (fragment_row.value * COL_FRAGMENTS + fragment_col.value) * Ops::ELEMENTS_PER_THREAD;
METAL_PRAGMA_UNROLL
- for (ushort i = 0; i < Ops::THREAD_ELEMENT_ROWS; ++i) {
- METAL_PRAGMA_UNROLL
- for (ushort j = 0; j < Ops::THREAD_ELEMENT_COLS; ++j) {
- const short row = row_base + short(i) * Ops::THREAD_ELEMENT_ROW_STRIDE;
- const short col = col_base + short(j);
- thread ElementType& value = data[frag_base + i * Ops::THREAD_ELEMENT_COLS + j];
- value = ElementType(fn(row, col, value));
- }
+ for (ushort element_index = 0; element_index < Ops::ELEMENTS_PER_THREAD; ++element_index) {
+ const short2 element_offset = Ops::get_element_offset(element_index);
+ const short row = row_base + element_offset.y;
+ const short col = col_base + element_offset.x;
+ thread ElementType& value = data[frag_base + element_index];
+ value = ElementType(fn(row, col, value));
}
});
}
@@ -310,7 +303,6 @@ private:
) thread {
using U = PointerElementType<Ptr>;
constexpr bool col_stride_is_one = metal::is_same_v<ColStride, Int<1>>;
-
const short2 position = get_position(simd_lane_id);
ptr += position.y * row_stride + position.x * col_stride;
const auto local_row_limit = row_limit - position.y;
@@ -322,29 +314,26 @@ private:
const auto col_base = fragment_col.value * FRAGMENT_COLS * fragment_col_stride;
METAL_PRAGMA_UNROLL
- for (ushort i = 0; i < Ops::THREAD_ELEMENT_ROWS; i++) {
- const auto row = row_base + i * Ops::THREAD_ELEMENT_ROW_STRIDE;
- METAL_PRAGMA_UNROLL
- for (ushort j = 0; j < Ops::THREAD_ELEMENT_COLS; j++) {
- const ushort element_index = i * Ops::THREAD_ELEMENT_COLS + j;
- const auto col = col_base + j;
- const auto offset = col_stride_is_one ? (row * row_stride + col) : (row * row_stride + col * col_stride);
-
- if constexpr (IS_LOAD) {
- if constexpr (IS_SAFE) {
- const bool in_bounds = (row < local_row_limit) && (col < local_col_limit);
- frag[element_index] = in_bounds ? static_cast<T>(ptr[offset]) : T(0);
- } else {
- frag[element_index] = static_cast<T>(ptr[offset]);
- }
+ for (ushort element_index = 0; element_index < Ops::ELEMENTS_PER_THREAD; element_index++) {
+ const short2 element_offset = Ops::get_element_offset(element_index);
+ const auto row = row_base + element_offset.y;
+ const auto col = col_base + element_offset.x;
+ const auto offset = col_stride_is_one ? (row * row_stride + col) : (row * row_stride + col * col_stride);
+
+ if constexpr (IS_LOAD) {
+ if constexpr (IS_SAFE) {
+ const bool in_bounds = (row < local_row_limit) && (col < local_col_limit);
+ frag[element_index] = in_bounds ? static_cast<T>(ptr[offset]) : T(0);
} else {
- if constexpr (IS_SAFE) {
- if ((row < local_row_limit) && (col < local_col_limit)) {
- ptr[offset] = static_cast<U>(frag[element_index]);
- }
- } else {
+ frag[element_index] = static_cast<T>(ptr[offset]);
+ }
+ } else {
+ if constexpr (IS_SAFE) {
+ if ((row < local_row_limit) && (col < local_col_limit)) {
ptr[offset] = static_cast<U>(frag[element_index]);
}
+ } else {
+ ptr[offset] = static_cast<U>(frag[element_index]);
}
}
}
@@ -381,5 +370,19 @@ METAL_FUNC void fragment_mma(thread OutputFragment& output, thread LeftFragment&
);
}
+template <class OutputFragment, class LeftFragment, class RightFragment>
+METAL_FUNC void fragment_mm(thread OutputFragment& output, thread LeftFragment& left, thread RightFragment& right) {
+ static_assert(
+ metal::is_same_v<typename OutputFragment::FragmentOpsType, typename LeftFragment::FragmentOpsType> &&
+ metal::is_same_v<typename OutputFragment::FragmentOpsType, typename RightFragment::FragmentOpsType>,
+ "fragment_mm requires output, left, and right fragments to use the same FragmentOps"
+ );
+ OutputFragment::FragmentOpsType::template fragment_mm<LeftFragment::MMA_TRANSPOSE, RightFragment::MMA_TRANSPOSE>(
+ output,
+ left,
+ right
+ );
+}
+
} // namespace matmul
} // namespace uzu
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment_ops.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment_ops.h
index 99fbadad9..0e86f32c4 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment_ops.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment_ops.h
@@ -21,6 +21,8 @@ namespace matmul {
METAL_CONST ushort MXU_MMA_ROWS = 16;
METAL_CONST ushort MXU_MMA_COLS = 16;
+// RELAXED=false uses the strict MPP layout; it currently performs about the same as simdgroup.
+template <bool RELAXED = true>
struct MxuFragmentOps {
METAL_CONST ushort FRAGMENT_ROWS = MXU_MMA_ROWS;
METAL_CONST ushort FRAGMENT_COLS = MXU_MMA_COLS;
@@ -42,8 +44,41 @@ struct MxuFragmentOps {
template <typename U>
using ThreadVector = typename metal::vec<U, ELEMENTS_PER_THREAD>;
+ METAL_FUNC static constexpr short2 get_position(ushort simd_lane_id) {
+ if constexpr (RELAXED) {
+ const short quad = simd_lane_id / 4;
+ const short row = (quad & 4) + (simd_lane_id / 2) % 4;
+ const short col = ((quad & 2) + simd_lane_id % 2) * THREAD_ELEMENT_COLS;
+ return short2{col, row};
+ } else {
+ const short col = short((simd_lane_id & 1) * 2 + ((simd_lane_id >> 3) & 1) * 4);
+ const short row = short(((simd_lane_id >> 1) & 3) + ((simd_lane_id >> 4) & 1) * 4);
+ return short2{col, row};
+ }
+ }
+
+ METAL_FUNC static constexpr short2 get_element_offset(ushort element_index) {
+ if constexpr (RELAXED) {
+ const short row = short((element_index / THREAD_ELEMENT_COLS) * THREAD_ELEMENT_ROW_STRIDE);
+ const short col = short(element_index % THREAD_ELEMENT_COLS);
+ return short2{col, row};
+ } else {
+ const short row = short((element_index / 4) * 8);
+ const ushort col_slot = element_index & 3;
+ const short col = short((col_slot & 1) + (col_slot / 2) * 8);
+ return short2{col, row};
+ }
+ }
+
// MPP has no valid 16x16x16 op; fragment_mma pairs fragments into 16x32.
- template <typename CType, typename AType, typename BType, bool transpose_a, bool transpose_b, typename MarshalInputs>
+ template <
+ bool ACCUMULATE,
+ typename CType,
+ typename AType,
+ typename BType,
+ bool transpose_a,
+ bool transpose_b,
+ typename MarshalInputs>
METAL_FUNC static void mma_impl(
thread ThreadVector<CType>& output_0,
thread ThreadVector<CType>& output_1,
@@ -55,8 +90,9 @@ struct MxuFragmentOps {
FRAGMENT_COLS,
transpose_a,
transpose_b,
- true,
- mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate
+ RELAXED,
+ ACCUMULATE ? mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate
+ : mpp::tensor_ops::matmul2d_descriptor::mode::multiply
);
mpp::tensor_ops::matmul2d<descriptor, metal::execution_simdgroup> matmul_op;
@@ -70,23 +106,51 @@ struct MxuFragmentOps {
marshal_inputs(cooperative_left, cooperative_right);
- METAL_PRAGMA_UNROLL
- for (ushort i = 0; i < ELEMENTS_PER_THREAD; i++) {
- cooperative_output[i] = output_0[i];
- cooperative_output[ELEMENTS_PER_THREAD + i] = output_1[i];
+ if constexpr (ACCUMULATE) {
+ if constexpr (RELAXED) {
+ METAL_PRAGMA_UNROLL
+ for (ushort i = 0; i < ELEMENTS_PER_THREAD; i++) {
+ cooperative_output[i] = output_0[i];
+ cooperative_output[ELEMENTS_PER_THREAD + i] = output_1[i];
+ }
+ } else {
+ METAL_PRAGMA_UNROLL
+ for (ushort i = 0; i < 4; i++) {
+ cooperative_output[i] = output_0[i];
+ cooperative_output[4 + i] = output_1[i];
+ cooperative_output[8 + i] = output_0[4 + i];
+ cooperative_output[12 + i] = output_1[4 + i];
+ }
+ }
}
matmul_op.run(cooperative_left, cooperative_right, cooperative_output);
- METAL_PRAGMA_UNROLL
- for (ushort i = 0; i < ELEMENTS_PER_THREAD; i++) {
- output_0[i] = cooperative_output[i];
- output_1[i] = cooperative_output[ELEMENTS_PER_THREAD + i];
+ if constexpr (RELAXED) {
+ METAL_PRAGMA_UNROLL
+ for (ushort i = 0; i < ELEMENTS_PER_THREAD; i++) {
+ output_0[i] = cooperative_output[i];
+ output_1[i] = cooperative_output[ELEMENTS_PER_THREAD + i];
+ }
+ } else {
+ METAL_PRAGMA_UNROLL
+ for (ushort i = 0; i < 4; i++) {
+ output_0[i] = cooperative_output[i];
+ output_1[i] = cooperative_output[4 + i];
+ output_0[4 + i] = cooperative_output[8 + i];
+ output_1[4 + i] = cooperative_output[12 + i];
+ }
}
}
- template <typename CType, typename AType, typename BType, bool transpose_a = false, bool transpose_b = false>
- METAL_FUNC static void mma(
+ template <
+ bool ACCUMULATE,
+ typename CType,
+ typename AType,
+ typename BType,
+ bool transpose_a = false,
+ bool transpose_b = false>
+ METAL_FUNC static void matmul(
thread ThreadVector<CType>& output_col_0,
thread ThreadVector<CType>& output_col_1,
const thread ThreadVector<AType>& left,
@@ -95,22 +159,41 @@ struct MxuFragmentOps {
const thread ThreadVector<BType>& right_col_1,
metal::bool_constant<transpose_b>
) {
- mma_impl<CType, AType, BType, transpose_a, transpose_b>(
+ mma_impl<ACCUMULATE, CType, AType, BType, transpose_a, transpose_b>(
output_col_0,
output_col_1,
[&](thread auto& cooperative_left, thread auto& cooperative_right) {
METAL_PRAGMA_UNROLL
for (ushort i = 0; i < ELEMENTS_PER_THREAD; i++) {
cooperative_left[i] = left[i];
- cooperative_right[i] = right_col_0[i];
- cooperative_right[ELEMENTS_PER_THREAD + i] = right_col_1[i];
+ }
+ if constexpr (RELAXED) {
+ METAL_PRAGMA_UNROLL
+ for (ushort i = 0; i < ELEMENTS_PER_THREAD; i++) {
+ cooperative_right[i] = right_col_0[i];
+ cooperative_right[ELEMENTS_PER_THREAD + i] = right_col_1[i];
+ }
+ } else {
+ METAL_PRAGMA_UNROLL
+ for (ushort i = 0; i < 4; i++) {
+ cooperative_right[i] = right_col_0[i];
+ cooperative_right[4 + i] = right_col_1[i];
+ cooperative_right[8 + i] = right_col_0[4 + i];
+ cooperative_right[12 + i] = right_col_1[4 + i];
+ }
}
}
);
}
- template <typename CType, typename AType, typename BType, bool transpose_a = false, bool transpose_b = false>
- METAL_FUNC static void mma(
+ template <
+ bool ACCUMULATE,
+ typename CType,
+ typename AType,
+ typename BType,
+ bool transpose_a = false,
+ bool transpose_b = false>
+ METAL_FUNC static void matmul(
thread ThreadVector<CType>& output_row_0,
thread ThreadVector<CType>& output_row_1,
const thread ThreadVector<AType>& left_row_0,
@@ -119,7 +202,8 @@ struct MxuFragmentOps {
const thread ThreadVector<BType>& right,
metal::bool_constant<transpose_b>
) {
- mma_impl<CType, AType, BType, transpose_a, transpose_b>(
+ static_assert(RELAXED, "strict MXU row-pairing is not implemented");
+ mma_impl<ACCUMULATE, CType, AType, BType, transpose_a, transpose_b>(
output_row_0,
output_row_1,
[&](thread auto& cooperative_left, thread auto& cooperative_right) {
@@ -133,8 +217,14 @@ struct MxuFragmentOps {
);
}
- template <bool transpose_a, bool transpose_b, class OutputFragment, class LeftFragment, class RightFragment>
- METAL_FUNC static void fragment_mma(
+ template <
+ bool ACCUMULATE,
+ bool transpose_a,
+ bool transpose_b,
+ class OutputFragment,
+ class LeftFragment,
+ class RightFragment>
+ METAL_FUNC static void fragment_matmul(
thread OutputFragment& output,
thread LeftFragment& left,
thread RightFragment& right
@@ -164,9 +254,28 @@ struct MxuFragmentOps {
for (ushort row = 0; row < rows; row += 2) {
METAL_PRAGMA_UNROLL
for (ushort col = 0; col < cols; ++col) {
+ if constexpr (!ACCUMULATE) {
+ matmul<
+ false,
+ typename OutputFragment::ElementType,
+ typename LeftFragment::ElementType,
+ typename RightFragment::ElementType,
+ transpose_a,
+ transpose_b>(
+ output.fragment_at(row, col),
+ output.fragment_at(row + 1, col),
+ left.fragment_at(row, 0, transpose_left),
+ left.fragment_at(row + 1, 0, transpose_left),
+ transpose_left,
+ right.fragment_at(0, col, transpose_right),
+ transpose_right
+ );
+ }
METAL_PRAGMA_UNROLL
- for (ushort k = 0; k < depth; ++k) {
- mma<typename OutputFragment::ElementType,
+ for (ushort k = ACCUMULATE ? 0 : 1; k < depth; ++k) {
+ matmul<
+ true,
+ typename OutputFragment::ElementType,
typename LeftFragment::ElementType,
typename RightFragment::ElementType,
transpose_a,
@@ -187,9 +296,28 @@ struct MxuFragmentOps {
for (ushort row = 0; row < rows; ++row) {
METAL_PRAGMA_UNROLL
for (ushort col = 0; col < cols; col += 2) {
+ if constexpr (!ACCUMULATE) {
+ matmul<
+ false,
+ typename OutputFragment::ElementType,
+ typename LeftFragment::ElementType,
+ typename RightFragment::ElementType,
+ transpose_a,
+ transpose_b>(
+ output.fragment_at(row, col),
+ output.fragment_at(row, col + 1),
+ left.fragment_at(row, 0, transpose_left),
+ transpose_left,
+ right.fragment_at(0, col, transpose_right),
+ right.fragment_at(0, col + 1, transpose_right),
+ transpose_right
+ );
+ }
METAL_PRAGMA_UNROLL
- for (ushort k = 0; k < depth; ++k) {
- mma<typename OutputFragment::ElementType,
+ for (ushort k = ACCUMULATE ? 0 : 1; k < depth; ++k) {
+ matmul<
+ true,
+ typename OutputFragment::ElementType,
typename LeftFragment::ElementType,
typename RightFragment::ElementType,
transpose_a,
@@ -207,7 +335,28 @@ struct MxuFragmentOps {
}
}
}
+
+ template <bool transpose_a, bool transpose_b, class OutputFragment, class LeftFragment, class RightFragment>
+ METAL_FUNC static void fragment_mma(
+ thread OutputFragment& output,
+ thread LeftFragment& left,
+ thread RightFragment& right
+ ) {
+ fragment_matmul<true, transpose_a, transpose_b>(output, left, right);
+ }
+
+ template <bool transpose_a, bool transpose_b, class OutputFragment, class LeftFragment, class RightFragment>
+ METAL_FUNC static void fragment_mm(
+ thread OutputFragment& output,
+ thread LeftFragment& left,
+ thread RightFragment& right
+ ) {
+ // MXU relaxed multiply is slightly faster than multiply_accumulate for pure matmul.
+ fragment_matmul<false, transpose_a, transpose_b>(output, left, right);
+ }
};
+using MxuStrictFragmentOps = MxuFragmentOps<false>;
+
} // namespace matmul
} // namespace uzu
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.h
index ad5c92331..24fe2ba3b 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.h
@@ -33,9 +33,9 @@ METAL_FUNC auto gemm_loop(
const short simdgroup_limit_n,
const thread ThreadContext& thread_context
) {
- constexpr ushort TILES_M = SIMDGROUP_BLOCK_M / MxuFragmentOps::FRAGMENT_ROWS;
- constexpr ushort TILES_N = SIMDGROUP_BLOCK_N / MxuFragmentOps::FRAGMENT_COLS;
- constexpr ushort TILES_K = SIMDGROUP_BLOCK_K / MxuFragmentOps::FRAGMENT_ROWS;
+ constexpr ushort TILES_M = SIMDGROUP_BLOCK_M / MxuFragmentOps<>::FRAGMENT_ROWS;
+ constexpr ushort TILES_N = SIMDGROUP_BLOCK_N / MxuFragmentOps<>::FRAGMENT_COLS;
+ constexpr ushort TILES_K = SIMDGROUP_BLOCK_K / MxuFragmentOps<>::FRAGMENT_ROWS;
constexpr ushort LEFT_TILE_ROWS = transpose_a ? TILES_K : TILES_M;
constexpr ushort LEFT_TILE_COLS = transpose_a ? TILES_M : TILES_K;
@@ -43,7 +43,7 @@ METAL_FUNC auto gemm_loop(
constexpr ushort RIGHT_TILE_ROWS = transpose_b ? TILES_N : TILES_K;
constexpr ushort RIGHT_TILE_COLS = transpose_b ? TILES_K : TILES_N;
- Fragment<AccumulatorType, TILES_M, TILES_N, MxuFragmentOps> accumulator;
+ Fragment<AccumulatorType, TILES_M, TILES_N, MxuFragmentOps<>> accumulator;
accumulator.clear();
METAL_PRAGMA_NO_UNROLL
@@ -52,8 +52,8 @@ METAL_FUNC auto gemm_loop(
METAL_PRAGMA_NO_UNROLL
for (int inner_k = 0; inner_k < BLOCK_K; inner_k += SIMDGROUP_BLOCK_K) {
- Fragment<AT, LEFT_TILE_ROWS, LEFT_TILE_COLS, MxuFragmentOps> left_tile;
- Fragment<BT, RIGHT_TILE_ROWS, RIGHT_TILE_COLS, MxuFragmentOps> right_tile;
+ Fragment<AT, LEFT_TILE_ROWS, LEFT_TILE_COLS, MxuFragmentOps<>> left_tile;
+ Fragment<BT, RIGHT_TILE_ROWS, RIGHT_TILE_COLS, MxuFragmentOps<>> right_tile;
volatile int mxu_iteration_fence;
@@ -76,7 +76,7 @@ METAL_FUNC auto gemm_loop(
}
right_tile.load_from(thread_context.simd_lane_id, right_src);
- MxuFragmentOps::template fragment_mma<transpose_a, transpose_b>(accumulator, left_tile, right_tile);
+ MxuFragmentOps<>::template fragment_mma<transpose_a, transpose_b>(accumulator, left_tile, right_tile);
(void)mxu_iteration_fence;
}
@@ -92,8 +92,8 @@ METAL_FUNC auto gemm_loop(
METAL_PRAGMA_NO_UNROLL
for (int inner_k = 0; inner_k < remaining_k; inner_k += SIMDGROUP_BLOCK_K) {
- Fragment<AT, LEFT_TILE_ROWS, LEFT_TILE_COLS, MxuFragmentOps> left_tile;
- Fragment<BT, RIGHT_TILE_ROWS, RIGHT_TILE_COLS, MxuFragmentOps> right_tile;
+ Fragment<AT, LEFT_TILE_ROWS, LEFT_TILE_COLS, MxuFragmentOps<>> left_tile;
+ Fragment<BT, RIGHT_TILE_ROWS, RIGHT_TILE_COLS, MxuFragmentOps<>> right_tile;
const short safe_k = max(short(0), short(remaining_k - inner_k));
@@ -112,7 +112,7 @@ METAL_FUNC auto gemm_loop(
fragment_source(right_ptr + right_offset, leading_dimension_b).bounded(right_limits.y, right_limits.x)
);
- MxuFragmentOps::template fragment_mma<transpose_a, transpose_b>(accumulator, left_tile, right_tile);
+ MxuFragmentOps<>::template fragment_mma<transpose_a, transpose_b>(accumulator, left_tile, right_tile);
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment_ops.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment_ops.h
index b96d6c67f..f5342cd18 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment_ops.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment_ops.h
@@ -23,7 +23,17 @@ struct SimdgroupFragmentOps {
template <typename U>
using ThreadVector = typename metal::vec<U, ELEMENTS_PER_THREAD>;
- // Serpentine order matches the chain kernel's register reuse.
+ METAL_FUNC static constexpr short2 get_position(ushort simd_lane_id) {
+ const short quad = simd_lane_id / 4;
+ const short row = (quad & 4) + (simd_lane_id / 2) % 4;
+ const short col = ((quad & 2) + simd_lane_id % 2) * THREAD_ELEMENT_COLS;
+ return short2{col, row};
+ }
+
+ METAL_FUNC static constexpr short2 get_element_offset(ushort element_index) {
+ return short2{short(element_index), 0};
+ }
+
template <bool transpose_a, bool transpose_b, class OutputFragment, class LeftFragment, class RightFragment>
METAL_FUNC static void fragment_mma(
thread OutputFragment& output,
@@ -56,6 +66,17 @@ struct SimdgroupFragmentOps {
}
}
}
+
+ template <bool transpose_a, bool transpose_b, class OutputFragment, class LeftFragment, class RightFragment>
+ METAL_FUNC static void fragment_mm(
+ thread OutputFragment& output,
+ thread LeftFragment& left,
+ thread RightFragment& right
+ ) {
+ // simdgroup_multiply is slower here.
+ output.clear();
+ fragment_mma<transpose_a, transpose_b>(output, left, right);
+ }
};
template <typename U, int SIMDGROUP_STRIDE_X, int SIMDGROUP_STRIDE_Y, int STRIDE_X, int STRIDE_Y, typename FragT>
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
index ed743d330..1dcb3d983 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
@@ -42,9 +42,9 @@ struct MxuMmaCore {
THREADGROUP_BLOCK_K_FP % SIMDGROUP_BLOCK_K == 0,
"FP THREADGROUP_BLOCK_K must be a multiple of SIMDGROUP_BLOCK_K"
);
- METAL_CONST ushort TILES_M = SIMDGROUP_BLOCK_M / uzu::matmul::MxuFragmentOps::FRAGMENT_ROWS;
- METAL_CONST ushort TILES_N = SIMDGROUP_BLOCK_N / uzu::matmul::MxuFragmentOps::FRAGMENT_COLS;
- METAL_CONST ushort TILES_K = SIMDGROUP_BLOCK_K / uzu::matmul::MxuFragmentOps::FRAGMENT_ROWS;
+ METAL_CONST ushort TILES_M = SIMDGROUP_BLOCK_M / uzu::matmul::MxuFragmentOps<>::FRAGMENT_ROWS;
+ METAL_CONST ushort TILES_N = SIMDGROUP_BLOCK_N / uzu::matmul::MxuFragmentOps<>::FRAGMENT_COLS;
+ METAL_CONST ushort TILES_K = SIMDGROUP_BLOCK_K / uzu::matmul::MxuFragmentOps<>::FRAGMENT_ROWS;
METAL_CONST ushort QUANT_BK = (B_PROLOGUE == GemmBPrologueKind::FullPrecision) ? 0 : GROUP_SIZE;
METAL_CONST ushort PADDING_B = 16 / sizeof(BT);
@@ -90,7 +90,7 @@ struct MxuMmaCore {
(BITS > 0) ? BITS : 4,
true>;
- using AccumFragment = uzu::matmul::Fragment<AccumulatorType, TILES_M, TILES_N, uzu::matmul::MxuFragmentOps>;
+ using AccumFragment = uzu::matmul::Fragment<AccumulatorType, TILES_M, TILES_N, uzu::matmul::MxuFragmentOps<>>;
template <bool ALIGNED_M, bool ALIGNED_N, typename Loader>
static METAL_FUNC AccumFragment quant_k_loop(
@@ -123,8 +123,8 @@ struct MxuMmaCore {
METAL_PRAGMA_NO_UNROLL
for (int inner_k = 0; inner_k < QUANT_BK; inner_k += SIMDGROUP_BLOCK_K) {
- uzu::matmul::Fragment<AT, TILES_M, TILES_K, uzu::matmul::MxuFragmentOps> left_tile;
- uzu::matmul::Fragment<BT, TILES_N, TILES_K, uzu::matmul::MxuFragmentOps> right_tile;
+ uzu::matmul::Fragment<AT, TILES_M, TILES_K, uzu::matmul::MxuFragmentOps<>> left_tile;
+ uzu::matmul::Fragment<BT, TILES_N, TILES_K, uzu::matmul::MxuFragmentOps<>> right_tile;
const int left_offset = inner_k;
auto left_src = uzu::matmul::fragment_source(a_simdgroup + left_offset, leading_dimension_a);
@@ -138,7 +138,7 @@ struct MxuMmaCore {
uzu::matmul::fragment_source(b_shared_simdgroup + inner_k, int(SHARED_STRIDE_B))
);
- uzu::matmul::MxuFragmentOps::template fragment_mma<false, true>(accumulator, left_tile, right_tile);
+ uzu::matmul::MxuFragmentOps<>::template fragment_mma<false, true>(accumulator, left_tile, right_tile);
}
a_simdgroup += QUANT_BK;
@@ -329,7 +329,7 @@ struct MxuMmaCore {
}
if (apply_accumulate) {
- uzu::matmul::Fragment<DT, TILES_M, TILES_N, uzu::matmul::MxuFragmentOps> existing_output;
+ uzu::matmul::Fragment<DT, TILES_M, TILES_N, uzu::matmul::MxuFragmentOps<>> existing_output;
auto output_src = uzu::matmul::fragment_source(d_simdgroup, int(params->leading_dimension_d));
if constexpr (!(aligned_m.value && aligned_n.value)) {
output_src = output_src.bounded(simdgroup_limit_m, simdgroup_limit_n);