A high-performance inference engine for AI models
Metal warnings (#622)
uuuvn55853f7bcfa0
diff --git a/crates/backend-uzu/build/metal/toolchain.rs b/crates/backend-uzu/build/metal/toolchain.rsindex 4d1f6dcf3..c9201c4a9 100644--- a/crates/backend-uzu/build/metal/toolchain.rs+++ b/crates/backend-uzu/build/metal/toolchain.rs@@ -115,14 +115,7 @@ impl MetalToolchain {_ => [OsString::from("-O2")].into(), // treat levels 2,3,s,z as O2 for metal};- let extra_options = Box::new([- OsString::from(format!("-m{}-version-min={}", sdk.os(), std.min_os())),- // NOTE: Previous build.rs script didn't forward metal compiler warnings to cargo warnings, the new one does.- // This is temporary to avoid warning spam without modifying unrelated things in the same pull request as introducing the dsl and converting kernels- OsString::from("-Wno-sign-compare"),- OsString::from("-Wno-macro-redefined"),- OsString::from("-Wno-unused-variable"),- ]);+ let extra_options = Box::new([OsString::from(format!("-m{}-version-min={}", sdk.os(), std.min_os()))]);let include_dirs = include_dir.into_iter().collect();diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metalindex d9d2bd890..77ce9233a 100644--- a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal@@ -27,13 +27,13 @@ template <typename T, uint BK, uint BD, bool USE_MXU>struct AttentionGemmLayout {using Ops = metal::conditional_t<USE_MXU, MxuFragmentOps<>, SimdgroupFragmentOps>;- METAL_CONST int FRAGMENT_ROWS = Ops::FRAGMENT_ROWS;- METAL_CONST int SIMDGROUPS_PER_THREADGROUP = 4;- METAL_CONST int BLOCK_QUERY_ROWS = SIMDGROUPS_PER_THREADGROUP * FRAGMENT_ROWS;- METAL_CONST int ROW_ALIGNMENT_BYTES = 16;- METAL_CONST int ROW_PADDING_ELEMENTS = ROW_ALIGNMENT_BYTES / int(sizeof(T));- METAL_CONST int Q_SMEM_SIZE = USE_MXU ? 1 : (BLOCK_QUERY_ROWS * (int(BD) + ROW_PADDING_ELEMENTS));- METAL_CONST int KV_SMEM_SIZE = USE_MXU ? 1 : (int(BK) * (int(BD) + ROW_PADDING_ELEMENTS));+ UZU_CONST int FRAGMENT_ROWS = Ops::FRAGMENT_ROWS;+ UZU_CONST int SIMDGROUPS_PER_THREADGROUP = 4;+ UZU_CONST int BLOCK_QUERY_ROWS = SIMDGROUPS_PER_THREADGROUP * FRAGMENT_ROWS;+ UZU_CONST int ROW_ALIGNMENT_BYTES = 16;+ UZU_CONST int ROW_PADDING_ELEMENTS = ROW_ALIGNMENT_BYTES / int(sizeof(T));+ UZU_CONST int Q_SMEM_SIZE = USE_MXU ? 1 : (BLOCK_QUERY_ROWS * (int(BD) + ROW_PADDING_ELEMENTS));+ UZU_CONST int KV_SMEM_SIZE = USE_MXU ? 1 : (int(BK) * (int(BD) + ROW_PADDING_ELEMENTS));};template <typename T, uint BK, uint BD, bool USE_MXU>@@ -129,7 +129,7 @@ KERNEL(AttentionGemm)(const short simdgroup_row_base = FRAGMENT_ROWS * QUERY_ROW_FRAGMENTS * short(thread_context.simdgroup_index);constexpr short head_dim_fragment_stride = FRAGMENT_ROWS;- const bool ragged_q = (!align_q && int(q_tile_idx) == params.nq_aligned);+ const bool ragged_q = (!align_q && q_tile_idx == params.nq_aligned);constexpr short query_leading_dimension = BD + ROW_PADDING_ELEMENTS;threadgroup T* query_shared = q_smem;const short query_shared_offset = simdgroup_row_base * query_leading_dimension;@@ -177,10 +177,10 @@ KERNEL(AttentionGemm)(sum_score[r] = init_sum;}- uint32_t kb_lim = params.nk;+ int kb_lim = int(params.nk);if (is_causal) {const int q_max = (int(q_tile_idx) + 1) * BLOCK_QUERY_ROWS + params.q_off;- kb_lim = min(params.nk, uint32_t((q_max + BK - 1) / BK));+ kb_lim = min(int(params.nk), (q_max + int(BK) - 1) / int(BK));}const int q_base = int(q_tile_idx) * BLOCK_QUERY_ROWS + int(simdgroup_row_base);@@ -196,7 +196,7 @@ KERNEL(AttentionGemm)(threadgroup T* kv_shared = kv_smem;for (int kb = 0; kb < kb_lim; kb++) {- const bool tail_k = (!align_k && kb == params.nk_aligned);+ const bool tail_k = (!align_k && kb == int(params.nk_aligned));const short valid_k_rows = tail_k ? short(params.k_rem) : short(BK);const device T* k_block = k + int64_t(kb) * int(BK) * key_source_stride;score_fragment.clear();@@ -328,7 +328,7 @@ KERNEL(AttentionGemm)(o += int64_t(simdgroup_row_base) * params.o_strides[2];- if (ragged_q && params.q_rem <= int(simdgroup_row_base)) {+ if (ragged_q && int(params.q_rem) <= int(simdgroup_row_base)) {return;}METAL_PRAGMA_UNROLLdiff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_single_pass.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_single_pass.metalindex 4fcf79338..923a9a135 100644--- a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_single_pass.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_single_pass.metal@@ -79,10 +79,10 @@ PUBLIC KERNEL(AttentionSinglePass)(out += o_offset * value_dim + thread_context.simdgroup_index * value_elements_per_thread;// Read the query and 0 the output accumulator- for (int i = 0; i < qk_elements_per_thread; i++) {+ for (uint i = 0; i < qk_elements_per_thread; i++) {q[i] = static_cast<U>(scale) * queries[i];}- for (int i = 0; i < value_elements_per_thread; i++) {+ for (uint i = 0; i < value_elements_per_thread; i++) {o[i] = 0;}@@ -132,7 +132,7 @@ PUBLIC KERNEL(AttentionSinglePass)(sum_exp_score = sum_exp_score * factor + exp_score;// Update the output accumulator- for (int j = 0; j < value_elements_per_thread; j++) {+ for (uint j = 0; j < value_elements_per_thread; j++) {o[j] = o[j] * factor + exp_score * values[j];}}@@ -154,7 +154,7 @@ PUBLIC KERNEL(AttentionSinglePass)(sum_exp_score = simd_sum(shared_sum_exp_scores[thread_context.simd_lane_id] * factor);// Now we need to aggregate all the outputs- for (int i = 0; i < value_elements_per_thread; i++) {+ for (uint i = 0; i < value_elements_per_thread; i++) {shared_outputs[thread_context.simd_lane_id * HEAD_BLOCK_SIZE + thread_context.simdgroup_index] = o[i];threadgroup_barrier(mem_flags::mem_threadgroup);o[i] = simd_sum(@@ -166,7 +166,7 @@ PUBLIC KERNEL(AttentionSinglePass)(// And write the outputif (thread_context.simd_lane_id == 0) {- for (int i = 0; i < value_elements_per_thread; i++) {+ for (uint i = 0; i < value_elements_per_thread; i++) {out[i] = static_cast<T>(o[i]);}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/common/defines.h b/crates/backend-uzu/src/backends/metal/kernel/common/defines.hindex 9d3bacc4a..ad435b673 100644--- a/crates/backend-uzu/src/backends/metal/kernel/common/defines.h+++ b/crates/backend-uzu/src/backends/metal/kernel/common/defines.h@@ -1,6 +1,6 @@#pragma once-#define METAL_CONST static constant constexpr+#define UZU_CONST static constant constexpr#define METAL_PRAGMA_UNROLL _Pragma("clang loop unroll(full)")#define METAL_PRAGMA_NO_UNROLL _Pragma("clang loop unroll(disable)")#define METAL_SIMD_SIZE 32diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.hindex e553d8ddc..5945254fe 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h@@ -70,17 +70,17 @@ struct Fragment {"(elements() relies on it)");- METAL_CONST ushort ROW_FRAGMENTS = ROW_FRAGMENTS_;- METAL_CONST ushort COL_FRAGMENTS = COL_FRAGMENTS_;+ UZU_CONST ushort ROW_FRAGMENTS = ROW_FRAGMENTS_;+ UZU_CONST ushort COL_FRAGMENTS = COL_FRAGMENTS_;- METAL_CONST ushort FRAGMENT_ROWS = Ops::FRAGMENT_ROWS;- METAL_CONST ushort FRAGMENT_COLS = Ops::FRAGMENT_COLS;+ UZU_CONST ushort FRAGMENT_ROWS = Ops::FRAGMENT_ROWS;+ UZU_CONST ushort FRAGMENT_COLS = Ops::FRAGMENT_COLS;- METAL_CONST ushort NUM_FRAGS = ROW_FRAGMENTS * COL_FRAGMENTS;- METAL_CONST ushort ELEMENTS_PER_FRAGMENT = NUM_FRAGS * Ops::ELEMENTS_PER_THREAD;- METAL_CONST ushort ROW_REDUCE_LANE_XOR_0 = 1;- METAL_CONST ushort ROW_REDUCE_LANE_XOR_1 = 8;- METAL_CONST bool MMA_TRANSPOSE = MMA_TRANSPOSE_;+ UZU_CONST ushort NUM_FRAGS = ROW_FRAGMENTS * COL_FRAGMENTS;+ UZU_CONST ushort ELEMENTS_PER_FRAGMENT = NUM_FRAGS * Ops::ELEMENTS_PER_THREAD;+ UZU_CONST ushort ROW_REDUCE_LANE_XOR_0 = 1;+ UZU_CONST ushort ROW_REDUCE_LANE_XOR_1 = 8;+ UZU_CONST bool MMA_TRANSPOSE = MMA_TRANSPOSE_;ThreadVectorType fragment_data[NUM_FRAGS];@@ -326,10 +326,10 @@ struct Fragment {}private:- METAL_CONST bool LOAD = true;- METAL_CONST bool STORE = false;- METAL_CONST bool SAFE = true;- METAL_CONST bool UNSAFE = false;+ UZU_CONST bool LOAD = true;+ UZU_CONST bool STORE = false;+ UZU_CONST bool SAFE = true;+ UZU_CONST bool UNSAFE = false;template <class Fn>METAL_FUNC static void for_each_fragment(Fn fn) {@@ -400,8 +400,8 @@ private:template <class Ops, class Read>struct OperandFragmentTraits {- METAL_CONST bool READ_TRANSPOSE = metal::is_same_v<Read, ReadTranspose>;- METAL_CONST bool MMA_TRANSPOSE = READ_TRANSPOSE && !Ops::READ_TRANSPOSE_SWAPS_SOURCE_STRIDES;+ UZU_CONST bool READ_TRANSPOSE = metal::is_same_v<Read, ReadTranspose>;+ UZU_CONST bool MMA_TRANSPOSE = READ_TRANSPOSE && !Ops::READ_TRANSPOSE_SWAPS_SOURCE_STRIDES;};template <typename T, ushort ROW_FRAGMENTS, ushort COL_FRAGMENTS, class Ops, class Read = ReadDirect>diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment_ops.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment_ops.hindex 0e86f32c4..6f571fe6e 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment_ops.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment_ops.h@@ -18,23 +18,23 @@ using namespace metal;namespace uzu {namespace matmul {-METAL_CONST ushort MXU_MMA_ROWS = 16;-METAL_CONST ushort MXU_MMA_COLS = 16;+UZU_CONST ushort MXU_MMA_ROWS = 16;+UZU_CONST ushort MXU_MMA_COLS = 16;// RELAXED=false uses the strict MPP layout; it currently performs about the same as simdgroup.template <bool RELAXED = true>struct MxuFragmentOps {- METAL_CONST ushort FRAGMENT_ROWS = MXU_MMA_ROWS;- METAL_CONST ushort FRAGMENT_COLS = MXU_MMA_COLS;- METAL_CONST bool READ_TRANSPOSE_SWAPS_SOURCE_STRIDES = false;+ UZU_CONST ushort FRAGMENT_ROWS = MXU_MMA_ROWS;+ UZU_CONST ushort FRAGMENT_COLS = MXU_MMA_COLS;+ UZU_CONST bool READ_TRANSPOSE_SWAPS_SOURCE_STRIDES = false;using BlockStorage = DeviceBlockStorage;- METAL_CONST ushort ELEMENTS_PER_THREAD = (FRAGMENT_ROWS * FRAGMENT_COLS) / METAL_SIMD_SIZE;+ UZU_CONST ushort ELEMENTS_PER_THREAD = (FRAGMENT_ROWS * FRAGMENT_COLS) / METAL_SIMD_SIZE;- METAL_CONST ushort THREAD_ELEMENT_ROWS = 2;- METAL_CONST ushort THREAD_ELEMENT_COLS = 4;+ UZU_CONST ushort THREAD_ELEMENT_ROWS = 2;+ UZU_CONST ushort THREAD_ELEMENT_COLS = 4;- METAL_CONST ushort THREAD_ELEMENT_ROW_STRIDE = FRAGMENT_ROWS / THREAD_ELEMENT_ROWS;+ UZU_CONST ushort THREAD_ELEMENT_ROW_STRIDE = FRAGMENT_ROWS / THREAD_ELEMENT_ROWS;static_assert(THREAD_ELEMENT_ROWS * THREAD_ELEMENT_COLS == ELEMENTS_PER_THREAD,diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment_ops.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment_ops.hindex 65b94dc47..27a6fed7d 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment_ops.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment_ops.h@@ -10,15 +10,15 @@ namespace uzu {namespace matmul {struct SimdgroupFragmentOps {- METAL_CONST ushort FRAGMENT_ROWS = SIMDGROUP_MMA_ROWS;- METAL_CONST ushort FRAGMENT_COLS = SIMDGROUP_MMA_COLS;- METAL_CONST bool READ_TRANSPOSE_SWAPS_SOURCE_STRIDES = true;+ UZU_CONST ushort FRAGMENT_ROWS = SIMDGROUP_MMA_ROWS;+ UZU_CONST ushort FRAGMENT_COLS = SIMDGROUP_MMA_COLS;+ UZU_CONST bool READ_TRANSPOSE_SWAPS_SOURCE_STRIDES = true;using BlockStorage = ThreadgroupBlockStorage;- METAL_CONST ushort ELEMENTS_PER_THREAD = (FRAGMENT_ROWS * FRAGMENT_COLS) / METAL_SIMD_SIZE;- METAL_CONST ushort THREAD_ELEMENT_ROWS = 1;- METAL_CONST ushort THREAD_ELEMENT_COLS = 2;- METAL_CONST ushort THREAD_ELEMENT_ROW_STRIDE = FRAGMENT_ROWS / THREAD_ELEMENT_ROWS;+ UZU_CONST ushort ELEMENTS_PER_THREAD = (FRAGMENT_ROWS * FRAGMENT_COLS) / METAL_SIMD_SIZE;+ UZU_CONST ushort THREAD_ELEMENT_ROWS = 1;+ UZU_CONST ushort THREAD_ELEMENT_COLS = 2;+ UZU_CONST ushort THREAD_ELEMENT_ROW_STRIDE = FRAGMENT_ROWS / THREAD_ELEMENT_ROWS;template <typename U>using ThreadVector = typename metal::vec<U, ELEMENTS_PER_THREAD>;diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_multiply_accumulate.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_multiply_accumulate.hindex 9051fa2dd..23a023f29 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_multiply_accumulate.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_multiply_accumulate.h@@ -7,8 +7,8 @@ using namespace metal;namespace uzu {namespace matmul {-METAL_CONST int SIMDGROUP_MMA_ROWS = 8;-METAL_CONST int SIMDGROUP_MMA_COLS = 8;+UZU_CONST int SIMDGROUP_MMA_ROWS = 8;+UZU_CONST int SIMDGROUP_MMA_COLS = 8;///////////////////////////////////////////////////////////////////////////////// SimdgroupMMA - stateless traits/ops for 8x8 simdgroup_matrix@@ -22,11 +22,11 @@ struct SimdgroupMMA {template <typename AccumulatorT, typename LeftT, typename RightT>struct SimdgroupMMA<AccumulatorT, SIMDGROUP_MMA_ROWS, SIMDGROUP_MMA_COLS, LeftT, RightT> {- METAL_CONST int ROWS = SIMDGROUP_MMA_ROWS;- METAL_CONST int COLS = SIMDGROUP_MMA_COLS;- METAL_CONST int THREAD_ELEMENT_COUNT = (ROWS * COLS) / METAL_SIMD_SIZE;- METAL_CONST int THREAD_ELEMENT_ROWS = 1;- METAL_CONST int THREAD_ELEMENT_COLS = 2;+ UZU_CONST int ROWS = SIMDGROUP_MMA_ROWS;+ UZU_CONST int COLS = SIMDGROUP_MMA_COLS;+ UZU_CONST int THREAD_ELEMENT_COUNT = (ROWS * COLS) / METAL_SIMD_SIZE;+ UZU_CONST int THREAD_ELEMENT_ROWS = 1;+ UZU_CONST int THREAD_ELEMENT_COLS = 2;static_assert(THREAD_ELEMENT_ROWS * THREAD_ELEMENT_COLS == THREAD_ELEMENT_COUNT,diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.hindex 4c7a97f21..8c688560a 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h@@ -25,20 +25,20 @@ template <typename AccumulatorType = float,typename Epilogue = TransformNone<DT, AccumulatorType>>struct ThreadgroupTile {- METAL_CONST ushort SIMDGROUP_BLOCK_SIZE = 8;+ UZU_CONST ushort SIMDGROUP_BLOCK_SIZE = 8;using SimdgroupMMAType = SimdgroupMMA<AccumulatorType, SIMDGROUP_BLOCK_SIZE, SIMDGROUP_BLOCK_SIZE>;- METAL_CONST ushort TILE_ROWS = BLOCK_ROWS / (SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW);- METAL_CONST ushort TILE_COLS = BLOCK_COLS / (SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN);+ UZU_CONST ushort TILE_ROWS = BLOCK_ROWS / (SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW);+ UZU_CONST ushort TILE_COLS = BLOCK_COLS / (SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN);- METAL_CONST ushort A_STRIDE_ROW = transpose_a ? 1 : THREADGROUP_LEADING_DIMENSION_A;- METAL_CONST ushort A_STRIDE_INNER = transpose_a ? THREADGROUP_LEADING_DIMENSION_A : 1;+ UZU_CONST ushort A_STRIDE_ROW = transpose_a ? 1 : THREADGROUP_LEADING_DIMENSION_A;+ UZU_CONST ushort A_STRIDE_INNER = transpose_a ? THREADGROUP_LEADING_DIMENSION_A : 1;- METAL_CONST ushort B_STRIDE_INNER = transpose_b ? 1 : THREADGROUP_LEADING_DIMENSION_B;- METAL_CONST ushort B_STRIDE_COL = transpose_b ? THREADGROUP_LEADING_DIMENSION_B : 1;+ UZU_CONST ushort B_STRIDE_INNER = transpose_b ? 1 : THREADGROUP_LEADING_DIMENSION_B;+ UZU_CONST ushort B_STRIDE_COL = transpose_b ? THREADGROUP_LEADING_DIMENSION_B : 1;- METAL_CONST ushort TILE_STRIDE_A = SIMDGROUP_BLOCK_SIZE * A_STRIDE_INNER;- METAL_CONST ushort TILE_STRIDE_B = SIMDGROUP_BLOCK_SIZE * B_STRIDE_INNER;+ UZU_CONST ushort TILE_STRIDE_A = SIMDGROUP_BLOCK_SIZE * A_STRIDE_INNER;+ UZU_CONST ushort TILE_STRIDE_B = SIMDGROUP_BLOCK_SIZE * B_STRIDE_INNER;Fragment<AT, TILE_ROWS, 1, SimdgroupFragmentOps> a_fragment;Fragment<BT, 1, TILE_COLS, SimdgroupFragmentOps> b_fragment;diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.hindex 1dcb3d983..0185c2141 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h@@ -30,26 +30,26 @@ template <int BITS = 0,int GROUP_SIZE = 0>struct MxuMmaCore {- METAL_CONST ushort THREADGROUP_BLOCK_M = gemm_tiling_block_m(GEMM_TILING);- METAL_CONST ushort THREADGROUP_BLOCK_N = gemm_tiling_block_n(GEMM_TILING);- METAL_CONST ushort SIMDGROUPS_PER_ROW = gemm_tiling_simdgroups_per_row(GEMM_TILING);- METAL_CONST ushort SIMDGROUPS_PER_COLUMN = gemm_tiling_simdgroups_per_column(GEMM_TILING);- METAL_CONST ushort SIMDGROUP_BLOCK_M = THREADGROUP_BLOCK_M / SIMDGROUPS_PER_ROW;- METAL_CONST ushort SIMDGROUP_BLOCK_N = THREADGROUP_BLOCK_N / SIMDGROUPS_PER_COLUMN;- METAL_CONST ushort SIMDGROUP_BLOCK_K = 32;- METAL_CONST ushort THREADGROUP_BLOCK_K_FP = gemm_tiling_block_k(GEMM_TILING);+ UZU_CONST ushort THREADGROUP_BLOCK_M = gemm_tiling_block_m(GEMM_TILING);+ UZU_CONST ushort THREADGROUP_BLOCK_N = gemm_tiling_block_n(GEMM_TILING);+ UZU_CONST ushort SIMDGROUPS_PER_ROW = gemm_tiling_simdgroups_per_row(GEMM_TILING);+ UZU_CONST ushort SIMDGROUPS_PER_COLUMN = gemm_tiling_simdgroups_per_column(GEMM_TILING);+ UZU_CONST ushort SIMDGROUP_BLOCK_M = THREADGROUP_BLOCK_M / SIMDGROUPS_PER_ROW;+ UZU_CONST ushort SIMDGROUP_BLOCK_N = THREADGROUP_BLOCK_N / SIMDGROUPS_PER_COLUMN;+ UZU_CONST ushort SIMDGROUP_BLOCK_K = 32;+ UZU_CONST ushort THREADGROUP_BLOCK_K_FP = gemm_tiling_block_k(GEMM_TILING);static_assert(THREADGROUP_BLOCK_K_FP % SIMDGROUP_BLOCK_K == 0,"FP THREADGROUP_BLOCK_K must be a multiple of SIMDGROUP_BLOCK_K");- METAL_CONST ushort TILES_M = SIMDGROUP_BLOCK_M / uzu::matmul::MxuFragmentOps<>::FRAGMENT_ROWS;- METAL_CONST ushort TILES_N = SIMDGROUP_BLOCK_N / uzu::matmul::MxuFragmentOps<>::FRAGMENT_COLS;- METAL_CONST ushort TILES_K = SIMDGROUP_BLOCK_K / uzu::matmul::MxuFragmentOps<>::FRAGMENT_ROWS;-- METAL_CONST ushort QUANT_BK = (B_PROLOGUE == GemmBPrologueKind::FullPrecision) ? 0 : GROUP_SIZE;- METAL_CONST ushort PADDING_B = 16 / sizeof(BT);- METAL_CONST ushort SHARED_STRIDE_B = (QUANT_BK > 0) ? (QUANT_BK + PADDING_B) : 1;- METAL_CONST ushort THREADGROUP_THREADS = SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * METAL_SIMD_SIZE;+ UZU_CONST ushort TILES_M = SIMDGROUP_BLOCK_M / uzu::matmul::MxuFragmentOps<>::FRAGMENT_ROWS;+ UZU_CONST ushort TILES_N = SIMDGROUP_BLOCK_N / uzu::matmul::MxuFragmentOps<>::FRAGMENT_COLS;+ UZU_CONST ushort TILES_K = SIMDGROUP_BLOCK_K / uzu::matmul::MxuFragmentOps<>::FRAGMENT_ROWS;++ UZU_CONST ushort QUANT_BK = (B_PROLOGUE == GemmBPrologueKind::FullPrecision) ? 0 : GROUP_SIZE;+ UZU_CONST ushort PADDING_B = 16 / sizeof(BT);+ UZU_CONST ushort SHARED_STRIDE_B = (QUANT_BK > 0) ? (QUANT_BK + PADDING_B) : 1;+ UZU_CONST ushort THREADGROUP_THREADS = SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * METAL_SIMD_SIZE;static_assert(B_PROLOGUE == GemmBPrologueKind::FullPrecision || QUANT_BK % SIMDGROUP_BLOCK_K == 0,"QUANT_BK must be a multiple of SIMDGROUP_BLOCK_K"diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.hindex bcdcc86a8..da377d270 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h@@ -26,14 +26,14 @@ struct QuantizedBlockLoaderScaleBias {static_assert(GROUP_SIZE % THREADGROUP_TILE_COLS == 0, "Group size should be divisible by columns");static_assert(BITS == 4 || BITS == 8, "Only int4 and int8 supported");- METAL_CONST short pack_factor = get_pack_factor<BITS, 8>();- METAL_CONST short bytes_per_pack = get_bytes_per_pack<BITS>();- METAL_CONST short THREADGROUP_TILE_COLS_PACKED = THREADGROUP_TILE_COLS / pack_factor;- METAL_CONST short READS_PER_THREAD = (THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS < THREADGROUP_SIZE)- ? 1- : (THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS) / THREADGROUP_SIZE;- METAL_CONST short GROUP_STEPS_PER_BLOCK = GROUP_SIZE / THREADGROUP_TILE_COLS;- METAL_CONST bool TILE_HAS_IDLE_THREADS = THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS < THREADGROUP_SIZE;+ UZU_CONST short pack_factor = get_pack_factor<BITS, 8>();+ UZU_CONST short bytes_per_pack = get_bytes_per_pack<BITS>();+ UZU_CONST short THREADGROUP_TILE_COLS_PACKED = THREADGROUP_TILE_COLS / pack_factor;+ UZU_CONST short READS_PER_THREAD = (THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS < THREADGROUP_SIZE)+ ? 1+ : (THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS) / THREADGROUP_SIZE;+ UZU_CONST short GROUP_STEPS_PER_BLOCK = GROUP_SIZE / THREADGROUP_TILE_COLS;+ UZU_CONST bool TILE_HAS_IDLE_THREADS = THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS < THREADGROUP_SIZE;const int src_leading_dim;const int tile_stride;diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.hindex 8a36d2549..330b30330 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h@@ -27,14 +27,14 @@ struct QuantizedBlockLoaderScaleZeroPoint {static_assert(GROUP_SIZE % THREADGROUP_TILE_COLS == 0, "Group size should be divisible by columns");static_assert(BITS == 4 || BITS == 8, "Only int4 and int8 supported");- METAL_CONST short pack_factor = get_pack_factor<BITS, 8>();- METAL_CONST short bytes_per_pack = get_bytes_per_pack<BITS>();- METAL_CONST short THREADGROUP_TILE_COLS_PACKED = THREADGROUP_TILE_COLS / pack_factor;- METAL_CONST short READS_PER_THREAD = (THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS < THREADGROUP_SIZE)- ? 1- : (THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS) / THREADGROUP_SIZE;- METAL_CONST short GROUP_STEPS_PER_BLOCK = GROUP_SIZE / THREADGROUP_TILE_COLS;- METAL_CONST bool TILE_HAS_IDLE_THREADS = THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS < THREADGROUP_SIZE;+ UZU_CONST short pack_factor = get_pack_factor<BITS, 8>();+ UZU_CONST short bytes_per_pack = get_bytes_per_pack<BITS>();+ UZU_CONST short THREADGROUP_TILE_COLS_PACKED = THREADGROUP_TILE_COLS / pack_factor;+ UZU_CONST short READS_PER_THREAD = (THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS < THREADGROUP_SIZE)+ ? 1+ : (THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS) / THREADGROUP_SIZE;+ UZU_CONST short GROUP_STEPS_PER_BLOCK = GROUP_SIZE / THREADGROUP_TILE_COLS;+ UZU_CONST bool TILE_HAS_IDLE_THREADS = THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS < THREADGROUP_SIZE;const int src_leading_dim;const int groups_per_row;diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.hindex 99aa4033f..bd1870521 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h@@ -30,16 +30,16 @@ template <int BITS = 0,int GROUP_SIZE = 0>struct SimdgroupMmaCore {- METAL_CONST int THREADGROUP_BLOCK_M = gemm_tiling_block_m(GEMM_TILING);- METAL_CONST int THREADGROUP_BLOCK_N = gemm_tiling_block_n(GEMM_TILING);- METAL_CONST int THREADGROUP_BLOCK_K = gemm_tiling_block_k(GEMM_TILING);- METAL_CONST int SIMDGROUPS_PER_ROW = gemm_tiling_simdgroups_per_row(GEMM_TILING);- METAL_CONST int SIMDGROUPS_PER_COLUMN = gemm_tiling_simdgroups_per_column(GEMM_TILING);- METAL_CONST ushort PADDING_A = 16 / sizeof(AT);- METAL_CONST ushort PADDING_B = 16 / sizeof(BT);- METAL_CONST ushort SHARED_STRIDE_A = THREADGROUP_BLOCK_K + PADDING_A;- METAL_CONST ushort SHARED_STRIDE_B = (TRANSPOSE_B ? THREADGROUP_BLOCK_K : THREADGROUP_BLOCK_N) + PADDING_B;- METAL_CONST ushort THREADGROUP_THREADS = SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * METAL_SIMD_SIZE;+ UZU_CONST int THREADGROUP_BLOCK_M = gemm_tiling_block_m(GEMM_TILING);+ UZU_CONST int THREADGROUP_BLOCK_N = gemm_tiling_block_n(GEMM_TILING);+ UZU_CONST int THREADGROUP_BLOCK_K = gemm_tiling_block_k(GEMM_TILING);+ UZU_CONST int SIMDGROUPS_PER_ROW = gemm_tiling_simdgroups_per_row(GEMM_TILING);+ UZU_CONST int SIMDGROUPS_PER_COLUMN = gemm_tiling_simdgroups_per_column(GEMM_TILING);+ UZU_CONST ushort PADDING_A = 16 / sizeof(AT);+ UZU_CONST ushort PADDING_B = 16 / sizeof(BT);+ UZU_CONST ushort SHARED_STRIDE_A = THREADGROUP_BLOCK_K + PADDING_A;+ UZU_CONST ushort SHARED_STRIDE_B = (TRANSPOSE_B ? THREADGROUP_BLOCK_K : THREADGROUP_BLOCK_N) + PADDING_B;+ UZU_CONST ushort THREADGROUP_THREADS = SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * METAL_SIMD_SIZE;using ALoader = uzu::matmul::ThreadgroupLoader<AT, THREADGROUP_BLOCK_M, THREADGROUP_BLOCK_K, SHARED_STRIDE_A, true, THREADGROUP_THREADS>;diff --git a/scripts/clang-format.sh b/scripts/clang-format.shindex 47df45edd..cd3246821 100755--- a/scripts/clang-format.sh+++ b/scripts/clang-format.sh@@ -45,7 +45,7 @@ if [[ "$MODE" == "check" ]]; then# Show diffs without writingFAILED=0for f in "${FILES[@]}"; do- if ! diff -u "$f" <(clang-format "$f"); then+ if ! diff -u "$f" <(/opt/homebrew/bin/clang-format "$f"); thenFAILED=1fidone