Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Metal warnings (#622)

uuuvn55853f7bcfa0
diff --git a/crates/backend-uzu/build/metal/toolchain.rs b/crates/backend-uzu/build/metal/toolchain.rs
index 4d1f6dcf3..c9201c4a9 100644
--- a/crates/backend-uzu/build/metal/toolchain.rs
+++ b/crates/backend-uzu/build/metal/toolchain.rs
@@ -115,14 +115,7 @@ impl MetalToolchain {
_ => [OsString::from("-O2")].into(), // treat levels 2,3,s,z as O2 for metal
};
- let extra_options = Box::new([
- OsString::from(format!("-m{}-version-min={}", sdk.os(), std.min_os())),
- // NOTE: Previous build.rs script didn't forward metal compiler warnings to cargo warnings, the new one does.
- // This is temporary to avoid warning spam without modifying unrelated things in the same pull request as introducing the dsl and converting kernels
- OsString::from("-Wno-sign-compare"),
- OsString::from("-Wno-macro-redefined"),
- OsString::from("-Wno-unused-variable"),
- ]);
+ let extra_options = Box::new([OsString::from(format!("-m{}-version-min={}", sdk.os(), std.min_os()))]);
let include_dirs = include_dir.into_iter().collect();
diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal
index d9d2bd890..77ce9233a 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal
@@ -27,13 +27,13 @@ template <typename T, uint BK, uint BD, bool USE_MXU>
struct AttentionGemmLayout {
using Ops = metal::conditional_t<USE_MXU, MxuFragmentOps<>, SimdgroupFragmentOps>;
- METAL_CONST int FRAGMENT_ROWS = Ops::FRAGMENT_ROWS;
- METAL_CONST int SIMDGROUPS_PER_THREADGROUP = 4;
- METAL_CONST int BLOCK_QUERY_ROWS = SIMDGROUPS_PER_THREADGROUP * FRAGMENT_ROWS;
- METAL_CONST int ROW_ALIGNMENT_BYTES = 16;
- METAL_CONST int ROW_PADDING_ELEMENTS = ROW_ALIGNMENT_BYTES / int(sizeof(T));
- METAL_CONST int Q_SMEM_SIZE = USE_MXU ? 1 : (BLOCK_QUERY_ROWS * (int(BD) + ROW_PADDING_ELEMENTS));
- METAL_CONST int KV_SMEM_SIZE = USE_MXU ? 1 : (int(BK) * (int(BD) + ROW_PADDING_ELEMENTS));
+ UZU_CONST int FRAGMENT_ROWS = Ops::FRAGMENT_ROWS;
+ UZU_CONST int SIMDGROUPS_PER_THREADGROUP = 4;
+ UZU_CONST int BLOCK_QUERY_ROWS = SIMDGROUPS_PER_THREADGROUP * FRAGMENT_ROWS;
+ UZU_CONST int ROW_ALIGNMENT_BYTES = 16;
+ UZU_CONST int ROW_PADDING_ELEMENTS = ROW_ALIGNMENT_BYTES / int(sizeof(T));
+ UZU_CONST int Q_SMEM_SIZE = USE_MXU ? 1 : (BLOCK_QUERY_ROWS * (int(BD) + ROW_PADDING_ELEMENTS));
+ UZU_CONST int KV_SMEM_SIZE = USE_MXU ? 1 : (int(BK) * (int(BD) + ROW_PADDING_ELEMENTS));
};
template <typename T, uint BK, uint BD, bool USE_MXU>
@@ -129,7 +129,7 @@ KERNEL(AttentionGemm)(
const short simdgroup_row_base = FRAGMENT_ROWS * QUERY_ROW_FRAGMENTS * short(thread_context.simdgroup_index);
constexpr short head_dim_fragment_stride = FRAGMENT_ROWS;
- const bool ragged_q = (!align_q && int(q_tile_idx) == params.nq_aligned);
+ const bool ragged_q = (!align_q && q_tile_idx == params.nq_aligned);
constexpr short query_leading_dimension = BD + ROW_PADDING_ELEMENTS;
threadgroup T* query_shared = q_smem;
const short query_shared_offset = simdgroup_row_base * query_leading_dimension;
@@ -177,10 +177,10 @@ KERNEL(AttentionGemm)(
sum_score[r] = init_sum;
}
- uint32_t kb_lim = params.nk;
+ int kb_lim = int(params.nk);
if (is_causal) {
const int q_max = (int(q_tile_idx) + 1) * BLOCK_QUERY_ROWS + params.q_off;
- kb_lim = min(params.nk, uint32_t((q_max + BK - 1) / BK));
+ kb_lim = min(int(params.nk), (q_max + int(BK) - 1) / int(BK));
}
const int q_base = int(q_tile_idx) * BLOCK_QUERY_ROWS + int(simdgroup_row_base);
@@ -196,7 +196,7 @@ KERNEL(AttentionGemm)(
threadgroup T* kv_shared = kv_smem;
for (int kb = 0; kb < kb_lim; kb++) {
- const bool tail_k = (!align_k && kb == params.nk_aligned);
+ const bool tail_k = (!align_k && kb == int(params.nk_aligned));
const short valid_k_rows = tail_k ? short(params.k_rem) : short(BK);
const device T* k_block = k + int64_t(kb) * int(BK) * key_source_stride;
score_fragment.clear();
@@ -328,7 +328,7 @@ KERNEL(AttentionGemm)(
o += int64_t(simdgroup_row_base) * params.o_strides[2];
- if (ragged_q && params.q_rem <= int(simdgroup_row_base)) {
+ if (ragged_q && int(params.q_rem) <= int(simdgroup_row_base)) {
return;
}
METAL_PRAGMA_UNROLL
diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_single_pass.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_single_pass.metal
index 4fcf79338..923a9a135 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_single_pass.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_single_pass.metal
@@ -79,10 +79,10 @@ PUBLIC KERNEL(AttentionSinglePass)(
out += o_offset * value_dim + thread_context.simdgroup_index * value_elements_per_thread;
// Read the query and 0 the output accumulator
- for (int i = 0; i < qk_elements_per_thread; i++) {
+ for (uint i = 0; i < qk_elements_per_thread; i++) {
q[i] = static_cast<U>(scale) * queries[i];
}
- for (int i = 0; i < value_elements_per_thread; i++) {
+ for (uint i = 0; i < value_elements_per_thread; i++) {
o[i] = 0;
}
@@ -132,7 +132,7 @@ PUBLIC KERNEL(AttentionSinglePass)(
sum_exp_score = sum_exp_score * factor + exp_score;
// Update the output accumulator
- for (int j = 0; j < value_elements_per_thread; j++) {
+ for (uint j = 0; j < value_elements_per_thread; j++) {
o[j] = o[j] * factor + exp_score * values[j];
}
}
@@ -154,7 +154,7 @@ PUBLIC KERNEL(AttentionSinglePass)(
sum_exp_score = simd_sum(shared_sum_exp_scores[thread_context.simd_lane_id] * factor);
// Now we need to aggregate all the outputs
- for (int i = 0; i < value_elements_per_thread; i++) {
+ for (uint i = 0; i < value_elements_per_thread; i++) {
shared_outputs[thread_context.simd_lane_id * HEAD_BLOCK_SIZE + thread_context.simdgroup_index] = o[i];
threadgroup_barrier(mem_flags::mem_threadgroup);
o[i] = simd_sum(
@@ -166,7 +166,7 @@ PUBLIC KERNEL(AttentionSinglePass)(
// And write the output
if (thread_context.simd_lane_id == 0) {
- for (int i = 0; i < value_elements_per_thread; i++) {
+ for (uint i = 0; i < value_elements_per_thread; i++) {
out[i] = static_cast<T>(o[i]);
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/common/defines.h b/crates/backend-uzu/src/backends/metal/kernel/common/defines.h
index 9d3bacc4a..ad435b673 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/common/defines.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/common/defines.h
@@ -1,6 +1,6 @@
#pragma once
-#define METAL_CONST static constant constexpr
+#define UZU_CONST static constant constexpr
#define METAL_PRAGMA_UNROLL _Pragma("clang loop unroll(full)")
#define METAL_PRAGMA_NO_UNROLL _Pragma("clang loop unroll(disable)")
#define METAL_SIMD_SIZE 32
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h
index e553d8ddc..5945254fe 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h
@@ -70,17 +70,17 @@ struct Fragment {
"(elements() relies on it)"
);
- METAL_CONST ushort ROW_FRAGMENTS = ROW_FRAGMENTS_;
- METAL_CONST ushort COL_FRAGMENTS = COL_FRAGMENTS_;
+ UZU_CONST ushort ROW_FRAGMENTS = ROW_FRAGMENTS_;
+ UZU_CONST ushort COL_FRAGMENTS = COL_FRAGMENTS_;
- METAL_CONST ushort FRAGMENT_ROWS = Ops::FRAGMENT_ROWS;
- METAL_CONST ushort FRAGMENT_COLS = Ops::FRAGMENT_COLS;
+ UZU_CONST ushort FRAGMENT_ROWS = Ops::FRAGMENT_ROWS;
+ UZU_CONST ushort FRAGMENT_COLS = Ops::FRAGMENT_COLS;
- METAL_CONST ushort NUM_FRAGS = ROW_FRAGMENTS * COL_FRAGMENTS;
- METAL_CONST ushort ELEMENTS_PER_FRAGMENT = NUM_FRAGS * Ops::ELEMENTS_PER_THREAD;
- METAL_CONST ushort ROW_REDUCE_LANE_XOR_0 = 1;
- METAL_CONST ushort ROW_REDUCE_LANE_XOR_1 = 8;
- METAL_CONST bool MMA_TRANSPOSE = MMA_TRANSPOSE_;
+ UZU_CONST ushort NUM_FRAGS = ROW_FRAGMENTS * COL_FRAGMENTS;
+ UZU_CONST ushort ELEMENTS_PER_FRAGMENT = NUM_FRAGS * Ops::ELEMENTS_PER_THREAD;
+ UZU_CONST ushort ROW_REDUCE_LANE_XOR_0 = 1;
+ UZU_CONST ushort ROW_REDUCE_LANE_XOR_1 = 8;
+ UZU_CONST bool MMA_TRANSPOSE = MMA_TRANSPOSE_;
ThreadVectorType fragment_data[NUM_FRAGS];
@@ -326,10 +326,10 @@ struct Fragment {
}
private:
- METAL_CONST bool LOAD = true;
- METAL_CONST bool STORE = false;
- METAL_CONST bool SAFE = true;
- METAL_CONST bool UNSAFE = false;
+ UZU_CONST bool LOAD = true;
+ UZU_CONST bool STORE = false;
+ UZU_CONST bool SAFE = true;
+ UZU_CONST bool UNSAFE = false;
template <class Fn>
METAL_FUNC static void for_each_fragment(Fn fn) {
@@ -400,8 +400,8 @@ private:
template <class Ops, class Read>
struct OperandFragmentTraits {
- METAL_CONST bool READ_TRANSPOSE = metal::is_same_v<Read, ReadTranspose>;
- METAL_CONST bool MMA_TRANSPOSE = READ_TRANSPOSE && !Ops::READ_TRANSPOSE_SWAPS_SOURCE_STRIDES;
+ UZU_CONST bool READ_TRANSPOSE = metal::is_same_v<Read, ReadTranspose>;
+ UZU_CONST bool MMA_TRANSPOSE = READ_TRANSPOSE && !Ops::READ_TRANSPOSE_SWAPS_SOURCE_STRIDES;
};
template <typename T, ushort ROW_FRAGMENTS, ushort COL_FRAGMENTS, class Ops, class Read = ReadDirect>
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment_ops.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment_ops.h
index 0e86f32c4..6f571fe6e 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment_ops.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment_ops.h
@@ -18,23 +18,23 @@ using namespace metal;
namespace uzu {
namespace matmul {
-METAL_CONST ushort MXU_MMA_ROWS = 16;
-METAL_CONST ushort MXU_MMA_COLS = 16;
+UZU_CONST ushort MXU_MMA_ROWS = 16;
+UZU_CONST ushort MXU_MMA_COLS = 16;
// RELAXED=false uses the strict MPP layout; it currently performs about the same as simdgroup.
template <bool RELAXED = true>
struct MxuFragmentOps {
- METAL_CONST ushort FRAGMENT_ROWS = MXU_MMA_ROWS;
- METAL_CONST ushort FRAGMENT_COLS = MXU_MMA_COLS;
- METAL_CONST bool READ_TRANSPOSE_SWAPS_SOURCE_STRIDES = false;
+ UZU_CONST ushort FRAGMENT_ROWS = MXU_MMA_ROWS;
+ UZU_CONST ushort FRAGMENT_COLS = MXU_MMA_COLS;
+ UZU_CONST bool READ_TRANSPOSE_SWAPS_SOURCE_STRIDES = false;
using BlockStorage = DeviceBlockStorage;
- METAL_CONST ushort ELEMENTS_PER_THREAD = (FRAGMENT_ROWS * FRAGMENT_COLS) / METAL_SIMD_SIZE;
+ UZU_CONST ushort ELEMENTS_PER_THREAD = (FRAGMENT_ROWS * FRAGMENT_COLS) / METAL_SIMD_SIZE;
- METAL_CONST ushort THREAD_ELEMENT_ROWS = 2;
- METAL_CONST ushort THREAD_ELEMENT_COLS = 4;
+ UZU_CONST ushort THREAD_ELEMENT_ROWS = 2;
+ UZU_CONST ushort THREAD_ELEMENT_COLS = 4;
- METAL_CONST ushort THREAD_ELEMENT_ROW_STRIDE = FRAGMENT_ROWS / THREAD_ELEMENT_ROWS;
+ UZU_CONST ushort THREAD_ELEMENT_ROW_STRIDE = FRAGMENT_ROWS / THREAD_ELEMENT_ROWS;
static_assert(
THREAD_ELEMENT_ROWS * THREAD_ELEMENT_COLS == ELEMENTS_PER_THREAD,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment_ops.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment_ops.h
index 65b94dc47..27a6fed7d 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment_ops.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment_ops.h
@@ -10,15 +10,15 @@ namespace uzu {
namespace matmul {
struct SimdgroupFragmentOps {
- METAL_CONST ushort FRAGMENT_ROWS = SIMDGROUP_MMA_ROWS;
- METAL_CONST ushort FRAGMENT_COLS = SIMDGROUP_MMA_COLS;
- METAL_CONST bool READ_TRANSPOSE_SWAPS_SOURCE_STRIDES = true;
+ UZU_CONST ushort FRAGMENT_ROWS = SIMDGROUP_MMA_ROWS;
+ UZU_CONST ushort FRAGMENT_COLS = SIMDGROUP_MMA_COLS;
+ UZU_CONST bool READ_TRANSPOSE_SWAPS_SOURCE_STRIDES = true;
using BlockStorage = ThreadgroupBlockStorage;
- METAL_CONST ushort ELEMENTS_PER_THREAD = (FRAGMENT_ROWS * FRAGMENT_COLS) / METAL_SIMD_SIZE;
- METAL_CONST ushort THREAD_ELEMENT_ROWS = 1;
- METAL_CONST ushort THREAD_ELEMENT_COLS = 2;
- METAL_CONST ushort THREAD_ELEMENT_ROW_STRIDE = FRAGMENT_ROWS / THREAD_ELEMENT_ROWS;
+ UZU_CONST ushort ELEMENTS_PER_THREAD = (FRAGMENT_ROWS * FRAGMENT_COLS) / METAL_SIMD_SIZE;
+ UZU_CONST ushort THREAD_ELEMENT_ROWS = 1;
+ UZU_CONST ushort THREAD_ELEMENT_COLS = 2;
+ UZU_CONST ushort THREAD_ELEMENT_ROW_STRIDE = FRAGMENT_ROWS / THREAD_ELEMENT_ROWS;
template <typename U>
using ThreadVector = typename metal::vec<U, ELEMENTS_PER_THREAD>;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_multiply_accumulate.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_multiply_accumulate.h
index 9051fa2dd..23a023f29 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_multiply_accumulate.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_multiply_accumulate.h
@@ -7,8 +7,8 @@ using namespace metal;
namespace uzu {
namespace matmul {
-METAL_CONST int SIMDGROUP_MMA_ROWS = 8;
-METAL_CONST int SIMDGROUP_MMA_COLS = 8;
+UZU_CONST int SIMDGROUP_MMA_ROWS = 8;
+UZU_CONST int SIMDGROUP_MMA_COLS = 8;
///////////////////////////////////////////////////////////////////////////////
// SimdgroupMMA - stateless traits/ops for 8x8 simdgroup_matrix
@@ -22,11 +22,11 @@ struct SimdgroupMMA {
template <typename AccumulatorT, typename LeftT, typename RightT>
struct SimdgroupMMA<AccumulatorT, SIMDGROUP_MMA_ROWS, SIMDGROUP_MMA_COLS, LeftT, RightT> {
- METAL_CONST int ROWS = SIMDGROUP_MMA_ROWS;
- METAL_CONST int COLS = SIMDGROUP_MMA_COLS;
- METAL_CONST int THREAD_ELEMENT_COUNT = (ROWS * COLS) / METAL_SIMD_SIZE;
- METAL_CONST int THREAD_ELEMENT_ROWS = 1;
- METAL_CONST int THREAD_ELEMENT_COLS = 2;
+ UZU_CONST int ROWS = SIMDGROUP_MMA_ROWS;
+ UZU_CONST int COLS = SIMDGROUP_MMA_COLS;
+ UZU_CONST int THREAD_ELEMENT_COUNT = (ROWS * COLS) / METAL_SIMD_SIZE;
+ UZU_CONST int THREAD_ELEMENT_ROWS = 1;
+ UZU_CONST int THREAD_ELEMENT_COLS = 2;
static_assert(
THREAD_ELEMENT_ROWS * THREAD_ELEMENT_COLS == THREAD_ELEMENT_COUNT,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h
index 4c7a97f21..8c688560a 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h
@@ -25,20 +25,20 @@ template <
typename AccumulatorType = float,
typename Epilogue = TransformNone<DT, AccumulatorType>>
struct ThreadgroupTile {
- METAL_CONST ushort SIMDGROUP_BLOCK_SIZE = 8;
+ UZU_CONST ushort SIMDGROUP_BLOCK_SIZE = 8;
using SimdgroupMMAType = SimdgroupMMA<AccumulatorType, SIMDGROUP_BLOCK_SIZE, SIMDGROUP_BLOCK_SIZE>;
- METAL_CONST ushort TILE_ROWS = BLOCK_ROWS / (SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW);
- METAL_CONST ushort TILE_COLS = BLOCK_COLS / (SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN);
+ UZU_CONST ushort TILE_ROWS = BLOCK_ROWS / (SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW);
+ UZU_CONST ushort TILE_COLS = BLOCK_COLS / (SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN);
- METAL_CONST ushort A_STRIDE_ROW = transpose_a ? 1 : THREADGROUP_LEADING_DIMENSION_A;
- METAL_CONST ushort A_STRIDE_INNER = transpose_a ? THREADGROUP_LEADING_DIMENSION_A : 1;
+ UZU_CONST ushort A_STRIDE_ROW = transpose_a ? 1 : THREADGROUP_LEADING_DIMENSION_A;
+ UZU_CONST ushort A_STRIDE_INNER = transpose_a ? THREADGROUP_LEADING_DIMENSION_A : 1;
- METAL_CONST ushort B_STRIDE_INNER = transpose_b ? 1 : THREADGROUP_LEADING_DIMENSION_B;
- METAL_CONST ushort B_STRIDE_COL = transpose_b ? THREADGROUP_LEADING_DIMENSION_B : 1;
+ UZU_CONST ushort B_STRIDE_INNER = transpose_b ? 1 : THREADGROUP_LEADING_DIMENSION_B;
+ UZU_CONST ushort B_STRIDE_COL = transpose_b ? THREADGROUP_LEADING_DIMENSION_B : 1;
- METAL_CONST ushort TILE_STRIDE_A = SIMDGROUP_BLOCK_SIZE * A_STRIDE_INNER;
- METAL_CONST ushort TILE_STRIDE_B = SIMDGROUP_BLOCK_SIZE * B_STRIDE_INNER;
+ UZU_CONST ushort TILE_STRIDE_A = SIMDGROUP_BLOCK_SIZE * A_STRIDE_INNER;
+ UZU_CONST ushort TILE_STRIDE_B = SIMDGROUP_BLOCK_SIZE * B_STRIDE_INNER;
Fragment<AT, TILE_ROWS, 1, SimdgroupFragmentOps> a_fragment;
Fragment<BT, 1, TILE_COLS, SimdgroupFragmentOps> b_fragment;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
index 1dcb3d983..0185c2141 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
@@ -30,26 +30,26 @@ template <
int BITS = 0,
int GROUP_SIZE = 0>
struct MxuMmaCore {
- METAL_CONST ushort THREADGROUP_BLOCK_M = gemm_tiling_block_m(GEMM_TILING);
- METAL_CONST ushort THREADGROUP_BLOCK_N = gemm_tiling_block_n(GEMM_TILING);
- METAL_CONST ushort SIMDGROUPS_PER_ROW = gemm_tiling_simdgroups_per_row(GEMM_TILING);
- METAL_CONST ushort SIMDGROUPS_PER_COLUMN = gemm_tiling_simdgroups_per_column(GEMM_TILING);
- METAL_CONST ushort SIMDGROUP_BLOCK_M = THREADGROUP_BLOCK_M / SIMDGROUPS_PER_ROW;
- METAL_CONST ushort SIMDGROUP_BLOCK_N = THREADGROUP_BLOCK_N / SIMDGROUPS_PER_COLUMN;
- METAL_CONST ushort SIMDGROUP_BLOCK_K = 32;
- METAL_CONST ushort THREADGROUP_BLOCK_K_FP = gemm_tiling_block_k(GEMM_TILING);
+ UZU_CONST ushort THREADGROUP_BLOCK_M = gemm_tiling_block_m(GEMM_TILING);
+ UZU_CONST ushort THREADGROUP_BLOCK_N = gemm_tiling_block_n(GEMM_TILING);
+ UZU_CONST ushort SIMDGROUPS_PER_ROW = gemm_tiling_simdgroups_per_row(GEMM_TILING);
+ UZU_CONST ushort SIMDGROUPS_PER_COLUMN = gemm_tiling_simdgroups_per_column(GEMM_TILING);
+ UZU_CONST ushort SIMDGROUP_BLOCK_M = THREADGROUP_BLOCK_M / SIMDGROUPS_PER_ROW;
+ UZU_CONST ushort SIMDGROUP_BLOCK_N = THREADGROUP_BLOCK_N / SIMDGROUPS_PER_COLUMN;
+ UZU_CONST ushort SIMDGROUP_BLOCK_K = 32;
+ UZU_CONST ushort THREADGROUP_BLOCK_K_FP = gemm_tiling_block_k(GEMM_TILING);
static_assert(
THREADGROUP_BLOCK_K_FP % SIMDGROUP_BLOCK_K == 0,
"FP THREADGROUP_BLOCK_K must be a multiple of SIMDGROUP_BLOCK_K"
);
- METAL_CONST ushort TILES_M = SIMDGROUP_BLOCK_M / uzu::matmul::MxuFragmentOps<>::FRAGMENT_ROWS;
- METAL_CONST ushort TILES_N = SIMDGROUP_BLOCK_N / uzu::matmul::MxuFragmentOps<>::FRAGMENT_COLS;
- METAL_CONST ushort TILES_K = SIMDGROUP_BLOCK_K / uzu::matmul::MxuFragmentOps<>::FRAGMENT_ROWS;
-
- METAL_CONST ushort QUANT_BK = (B_PROLOGUE == GemmBPrologueKind::FullPrecision) ? 0 : GROUP_SIZE;
- METAL_CONST ushort PADDING_B = 16 / sizeof(BT);
- METAL_CONST ushort SHARED_STRIDE_B = (QUANT_BK > 0) ? (QUANT_BK + PADDING_B) : 1;
- METAL_CONST ushort THREADGROUP_THREADS = SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * METAL_SIMD_SIZE;
+ UZU_CONST ushort TILES_M = SIMDGROUP_BLOCK_M / uzu::matmul::MxuFragmentOps<>::FRAGMENT_ROWS;
+ UZU_CONST ushort TILES_N = SIMDGROUP_BLOCK_N / uzu::matmul::MxuFragmentOps<>::FRAGMENT_COLS;
+ UZU_CONST ushort TILES_K = SIMDGROUP_BLOCK_K / uzu::matmul::MxuFragmentOps<>::FRAGMENT_ROWS;
+
+ UZU_CONST ushort QUANT_BK = (B_PROLOGUE == GemmBPrologueKind::FullPrecision) ? 0 : GROUP_SIZE;
+ UZU_CONST ushort PADDING_B = 16 / sizeof(BT);
+ UZU_CONST ushort SHARED_STRIDE_B = (QUANT_BK > 0) ? (QUANT_BK + PADDING_B) : 1;
+ UZU_CONST ushort THREADGROUP_THREADS = SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * METAL_SIMD_SIZE;
static_assert(
B_PROLOGUE == GemmBPrologueKind::FullPrecision || QUANT_BK % SIMDGROUP_BLOCK_K == 0,
"QUANT_BK must be a multiple of SIMDGROUP_BLOCK_K"
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h
index bcdcc86a8..da377d270 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h
@@ -26,14 +26,14 @@ struct QuantizedBlockLoaderScaleBias {
static_assert(GROUP_SIZE % THREADGROUP_TILE_COLS == 0, "Group size should be divisible by columns");
static_assert(BITS == 4 || BITS == 8, "Only int4 and int8 supported");
- METAL_CONST short pack_factor = get_pack_factor<BITS, 8>();
- METAL_CONST short bytes_per_pack = get_bytes_per_pack<BITS>();
- METAL_CONST short THREADGROUP_TILE_COLS_PACKED = THREADGROUP_TILE_COLS / pack_factor;
- METAL_CONST short READS_PER_THREAD = (THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS < THREADGROUP_SIZE)
- ? 1
- : (THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS) / THREADGROUP_SIZE;
- METAL_CONST short GROUP_STEPS_PER_BLOCK = GROUP_SIZE / THREADGROUP_TILE_COLS;
- METAL_CONST bool TILE_HAS_IDLE_THREADS = THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS < THREADGROUP_SIZE;
+ UZU_CONST short pack_factor = get_pack_factor<BITS, 8>();
+ UZU_CONST short bytes_per_pack = get_bytes_per_pack<BITS>();
+ UZU_CONST short THREADGROUP_TILE_COLS_PACKED = THREADGROUP_TILE_COLS / pack_factor;
+ UZU_CONST short READS_PER_THREAD = (THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS < THREADGROUP_SIZE)
+ ? 1
+ : (THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS) / THREADGROUP_SIZE;
+ UZU_CONST short GROUP_STEPS_PER_BLOCK = GROUP_SIZE / THREADGROUP_TILE_COLS;
+ UZU_CONST bool TILE_HAS_IDLE_THREADS = THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS < THREADGROUP_SIZE;
const int src_leading_dim;
const int tile_stride;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h
index 8a36d2549..330b30330 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h
@@ -27,14 +27,14 @@ struct QuantizedBlockLoaderScaleZeroPoint {
static_assert(GROUP_SIZE % THREADGROUP_TILE_COLS == 0, "Group size should be divisible by columns");
static_assert(BITS == 4 || BITS == 8, "Only int4 and int8 supported");
- METAL_CONST short pack_factor = get_pack_factor<BITS, 8>();
- METAL_CONST short bytes_per_pack = get_bytes_per_pack<BITS>();
- METAL_CONST short THREADGROUP_TILE_COLS_PACKED = THREADGROUP_TILE_COLS / pack_factor;
- METAL_CONST short READS_PER_THREAD = (THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS < THREADGROUP_SIZE)
- ? 1
- : (THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS) / THREADGROUP_SIZE;
- METAL_CONST short GROUP_STEPS_PER_BLOCK = GROUP_SIZE / THREADGROUP_TILE_COLS;
- METAL_CONST bool TILE_HAS_IDLE_THREADS = THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS < THREADGROUP_SIZE;
+ UZU_CONST short pack_factor = get_pack_factor<BITS, 8>();
+ UZU_CONST short bytes_per_pack = get_bytes_per_pack<BITS>();
+ UZU_CONST short THREADGROUP_TILE_COLS_PACKED = THREADGROUP_TILE_COLS / pack_factor;
+ UZU_CONST short READS_PER_THREAD = (THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS < THREADGROUP_SIZE)
+ ? 1
+ : (THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS) / THREADGROUP_SIZE;
+ UZU_CONST short GROUP_STEPS_PER_BLOCK = GROUP_SIZE / THREADGROUP_TILE_COLS;
+ UZU_CONST bool TILE_HAS_IDLE_THREADS = THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS < THREADGROUP_SIZE;
const int src_leading_dim;
const int groups_per_row;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h
index 99aa4033f..bd1870521 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h
@@ -30,16 +30,16 @@ template <
int BITS = 0,
int GROUP_SIZE = 0>
struct SimdgroupMmaCore {
- METAL_CONST int THREADGROUP_BLOCK_M = gemm_tiling_block_m(GEMM_TILING);
- METAL_CONST int THREADGROUP_BLOCK_N = gemm_tiling_block_n(GEMM_TILING);
- METAL_CONST int THREADGROUP_BLOCK_K = gemm_tiling_block_k(GEMM_TILING);
- METAL_CONST int SIMDGROUPS_PER_ROW = gemm_tiling_simdgroups_per_row(GEMM_TILING);
- METAL_CONST int SIMDGROUPS_PER_COLUMN = gemm_tiling_simdgroups_per_column(GEMM_TILING);
- METAL_CONST ushort PADDING_A = 16 / sizeof(AT);
- METAL_CONST ushort PADDING_B = 16 / sizeof(BT);
- METAL_CONST ushort SHARED_STRIDE_A = THREADGROUP_BLOCK_K + PADDING_A;
- METAL_CONST ushort SHARED_STRIDE_B = (TRANSPOSE_B ? THREADGROUP_BLOCK_K : THREADGROUP_BLOCK_N) + PADDING_B;
- METAL_CONST ushort THREADGROUP_THREADS = SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * METAL_SIMD_SIZE;
+ UZU_CONST int THREADGROUP_BLOCK_M = gemm_tiling_block_m(GEMM_TILING);
+ UZU_CONST int THREADGROUP_BLOCK_N = gemm_tiling_block_n(GEMM_TILING);
+ UZU_CONST int THREADGROUP_BLOCK_K = gemm_tiling_block_k(GEMM_TILING);
+ UZU_CONST int SIMDGROUPS_PER_ROW = gemm_tiling_simdgroups_per_row(GEMM_TILING);
+ UZU_CONST int SIMDGROUPS_PER_COLUMN = gemm_tiling_simdgroups_per_column(GEMM_TILING);
+ UZU_CONST ushort PADDING_A = 16 / sizeof(AT);
+ UZU_CONST ushort PADDING_B = 16 / sizeof(BT);
+ UZU_CONST ushort SHARED_STRIDE_A = THREADGROUP_BLOCK_K + PADDING_A;
+ UZU_CONST ushort SHARED_STRIDE_B = (TRANSPOSE_B ? THREADGROUP_BLOCK_K : THREADGROUP_BLOCK_N) + PADDING_B;
+ UZU_CONST ushort THREADGROUP_THREADS = SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * METAL_SIMD_SIZE;
using ALoader = uzu::matmul::
ThreadgroupLoader<AT, THREADGROUP_BLOCK_M, THREADGROUP_BLOCK_K, SHARED_STRIDE_A, true, THREADGROUP_THREADS>;
diff --git a/scripts/clang-format.sh b/scripts/clang-format.sh
index 47df45edd..cd3246821 100755
--- a/scripts/clang-format.sh
+++ b/scripts/clang-format.sh
@@ -45,7 +45,7 @@ if [[ "$MODE" == "check" ]]; then
# Show diffs without writing
FAILED=0
for f in "${FILES[@]}"; do
- if ! diff -u "$f" <(clang-format "$f"); then
+ if ! diff -u "$f" <(/opt/homebrew/bin/clang-format "$f"); then
FAILED=1
fi
done