Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

reuse and optimize reduce_k_slices (#1036)

CCYeh6141a641bdeb
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/fragment.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/fragment.h
index 0a939a0f2..0dd3809a3 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/fragment.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/fragment.h
@@ -370,24 +370,31 @@ struct Fragment {
if constexpr (K_SLICES == 1) {
return true;
} else {
+ static_assert(ELEMENTS_PER_FRAGMENT % 2 == 0, "K-slice partials are stored as 2- or 4-wide vectors");
+ constexpr ushort WIDTH = ELEMENTS_PER_FRAGMENT % 4 == 0 ? 4 : 2;
+ constexpr ushort VECTORS = ELEMENTS_PER_FRAGMENT / WIDTH;
+ using Vector = metal::vec<ElementType, WIDTH>;
const ushort k_slice = ushort(thread_context.simdgroup_index) / OUTPUT_GROUPS;
const ushort output_group = ushort(thread_context.simdgroup_index) % OUTPUT_GROUPS;
const ushort lane = ushort(thread_context.simd_lane_id);
- thread ElementType* values = elements();
- constexpr ushort ELEMENTS = ELEMENTS_PER_FRAGMENT;
- constexpr ushort GROUP_STRIDE = (K_SLICES - 1) * ELEMENTS * METAL_SIMD_SIZE;
+ threadgroup Vector* group_partials = reinterpret_cast<threadgroup Vector*>(partials) +
+ output_group * (K_SLICES - 1) * VECTORS * METAL_SIMD_SIZE + lane;
+ thread Vector* values = reinterpret_cast<thread Vector*>(elements());
if (k_slice != 0) {
- for (ushort e = 0; e < ELEMENTS; ++e) {
- partials[output_group * GROUP_STRIDE + ((k_slice - 1) * ELEMENTS + e) * METAL_SIMD_SIZE + lane] = values[e];
+ METAL_PRAGMA_UNROLL
+ for (ushort v = 0; v < VECTORS; ++v) {
+ group_partials[((k_slice - 1) * VECTORS + v) * METAL_SIMD_SIZE] = values[v];
}
}
threadgroup_barrier(mem_flags::mem_threadgroup);
if (k_slice != 0) {
return false;
}
+ METAL_PRAGMA_UNROLL
for (ushort slice = 1; slice < K_SLICES; ++slice) {
- for (ushort e = 0; e < ELEMENTS; ++e) {
- values[e] += partials[output_group * GROUP_STRIDE + ((slice - 1) * ELEMENTS + e) * METAL_SIMD_SIZE + lane];
+ METAL_PRAGMA_UNROLL
+ for (ushort v = 0; v < VECTORS; ++v) {
+ values[v] += group_partials[((slice - 1) * VECTORS + v) * METAL_SIMD_SIZE];
}
}
return true;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
index 7c415bd55..567890ed9 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
@@ -148,27 +148,11 @@ struct MxuMmaCore {
thread_context
);
- if constexpr (SIMDGROUPS_PER_K > 1) {
- threadgroup float* k_partials = reinterpret_cast<threadgroup float*>(b_shared);
- thread AccumulatorType* values = accumulator_tile.elements();
- const ushort lane = ushort(thread_context.simd_lane_id);
- if (k_slice != 0) {
- for (ushort element = 0; element < AccumFragment::ELEMENTS_PER_FRAGMENT; ++element) {
- k_partials
- [((k_slice - 1) * AccumFragment::ELEMENTS_PER_FRAGMENT + element) * METAL_SIMD_SIZE + lane] =
- values[element];
- }
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
- if (k_slice != 0) {
- return;
- }
- for (ushort slice = 1; slice < SIMDGROUPS_PER_K; ++slice) {
- for (ushort element = 0; element < AccumFragment::ELEMENTS_PER_FRAGMENT; ++element) {
- values[element] += k_partials
- [((slice - 1) * AccumFragment::ELEMENTS_PER_FRAGMENT + element) * METAL_SIMD_SIZE + lane];
- }
- }
+ if (!accumulator_tile.template reduce_k_slices<SIMDGROUPS_PER_K, 1>(
+ reinterpret_cast<threadgroup float*>(b_shared),
+ thread_context
+ )) {
+ return;
}
if (apply_scale) {