A high-performance inference engine for AI models
reuse and optimize reduce_k_slices (#1036)
CCYeh6141a641bdeb
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/fragment.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/fragment.hindex 0a939a0f2..0dd3809a3 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/common/fragment.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/common/fragment.h@@ -370,24 +370,31 @@ struct Fragment {if constexpr (K_SLICES == 1) {return true;} else {+ static_assert(ELEMENTS_PER_FRAGMENT % 2 == 0, "K-slice partials are stored as 2- or 4-wide vectors");+ constexpr ushort WIDTH = ELEMENTS_PER_FRAGMENT % 4 == 0 ? 4 : 2;+ constexpr ushort VECTORS = ELEMENTS_PER_FRAGMENT / WIDTH;+ using Vector = metal::vec<ElementType, WIDTH>;const ushort k_slice = ushort(thread_context.simdgroup_index) / OUTPUT_GROUPS;const ushort output_group = ushort(thread_context.simdgroup_index) % OUTPUT_GROUPS;const ushort lane = ushort(thread_context.simd_lane_id);- thread ElementType* values = elements();- constexpr ushort ELEMENTS = ELEMENTS_PER_FRAGMENT;- constexpr ushort GROUP_STRIDE = (K_SLICES - 1) * ELEMENTS * METAL_SIMD_SIZE;+ threadgroup Vector* group_partials = reinterpret_cast<threadgroup Vector*>(partials) ++ output_group * (K_SLICES - 1) * VECTORS * METAL_SIMD_SIZE + lane;+ thread Vector* values = reinterpret_cast<thread Vector*>(elements());if (k_slice != 0) {- for (ushort e = 0; e < ELEMENTS; ++e) {- partials[output_group * GROUP_STRIDE + ((k_slice - 1) * ELEMENTS + e) * METAL_SIMD_SIZE + lane] = values[e];+ METAL_PRAGMA_UNROLL+ for (ushort v = 0; v < VECTORS; ++v) {+ group_partials[((k_slice - 1) * VECTORS + v) * METAL_SIMD_SIZE] = values[v];}}threadgroup_barrier(mem_flags::mem_threadgroup);if (k_slice != 0) {return false;}+ METAL_PRAGMA_UNROLLfor (ushort slice = 1; slice < K_SLICES; ++slice) {- for (ushort e = 0; e < ELEMENTS; ++e) {- values[e] += partials[output_group * GROUP_STRIDE + ((slice - 1) * ELEMENTS + e) * METAL_SIMD_SIZE + lane];+ METAL_PRAGMA_UNROLL+ for (ushort v = 0; v < VECTORS; ++v) {+ values[v] += group_partials[((slice - 1) * VECTORS + v) * METAL_SIMD_SIZE];}}return true;diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.hindex 7c415bd55..567890ed9 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h@@ -148,27 +148,11 @@ struct MxuMmaCore {thread_context);- if constexpr (SIMDGROUPS_PER_K > 1) {- threadgroup float* k_partials = reinterpret_cast<threadgroup float*>(b_shared);- thread AccumulatorType* values = accumulator_tile.elements();- const ushort lane = ushort(thread_context.simd_lane_id);- if (k_slice != 0) {- for (ushort element = 0; element < AccumFragment::ELEMENTS_PER_FRAGMENT; ++element) {- k_partials- [((k_slice - 1) * AccumFragment::ELEMENTS_PER_FRAGMENT + element) * METAL_SIMD_SIZE + lane] =- values[element];- }- }- threadgroup_barrier(mem_flags::mem_threadgroup);- if (k_slice != 0) {- return;- }- for (ushort slice = 1; slice < SIMDGROUPS_PER_K; ++slice) {- for (ushort element = 0; element < AccumFragment::ELEMENTS_PER_FRAGMENT; ++element) {- values[element] += k_partials- [((slice - 1) * AccumFragment::ELEMENTS_PER_FRAGMENT + element) * METAL_SIMD_SIZE + lane];- }- }+ if (!accumulator_tile.template reduce_k_slices<SIMDGROUPS_PER_K, 1>(+ reinterpret_cast<threadgroup float*>(b_shared),+ thread_context+ )) {+ return;}if (apply_scale) {