A high-performance inference engine for AI models
Match rustfmt line length in clang-format (#513)
uuuvnf2fe36d77725
diff --git a/.clang-format b/.clang-formatindex ad70f0798..e3ab56814 100644--- a/.clang-format+++ b/.clang-format@@ -1,7 +1,7 @@BasedOnStyle: LLVMIndentWidth: 2ContinuationIndentWidth: 4-ColumnLimit: 80+ColumnLimit: 120# Keep return type with function nameAlwaysBreakAfterReturnType: Nonediff --git a/crates/backend-uzu/src/backends/metal/kernel/activation/activations.h b/crates/backend-uzu/src/backends/metal/kernel/activation/activations.hindex ede7a07cd..a369e4bd6 100644--- a/crates/backend-uzu/src/backends/metal/kernel/activation/activations.h+++ b/crates/backend-uzu/src/backends/metal/kernel/activation/activations.h@@ -23,8 +23,7 @@ inline T activate_gelu(T x) {constexpr float k0 = 0.044715f;constexpr float k1 = 0.7978845608f; // sqrt(2/pi)float xf = float(x);- float yf =- 0.5f * xf * (1.0f + metal::precise::tanh(k1 * (xf + k0 * xf * xf * xf)));+ float yf = 0.5f * xf * (1.0f + metal::precise::tanh(k1 * (xf + k0 * xf * xf * xf)));return T(yf);}diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_fallback.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_fallback.metalindex cccd17c89..14e2e12b2 100644--- a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_fallback.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_fallback.metal@@ -36,11 +36,8 @@ PUBLIC KERNEL(AttentionFallbackScatterScores)(const uint head_index = group_index * gqa_factor + head_in_group;const uint prefix_length = sequence_length - suffix_length;- const uint suffix_position =- is_kv_cache_ring ? uint(ring_params.ring_length) : prefix_length;- const uint query_position = is_trie- ? suffix_position + trie[query_index].height- : suffix_position + query_index;+ const uint suffix_position = is_kv_cache_ring ? uint(ring_params.ring_length) : prefix_length;+ const uint query_position = is_trie ? suffix_position + trie[query_index].height : suffix_position + query_index;const bool use_key = should_use_key(ring_params,@@ -57,10 +54,8 @@ PUBLIC KERNEL(AttentionFallbackScatterScores)(is_sliding_window);- scores- [(head_index * suffix_length + query_index) * sequence_length +- sequence_index] =- use_key ? group_scores[group_score_index] : T(-INFINITY);+ scores[(head_index * suffix_length + query_index) * sequence_length + sequence_index] =+ use_key ? group_scores[group_score_index] : T(-INFINITY);}template <typename T>@@ -81,6 +76,5 @@ PUBLIC KERNEL(AttentionFallbackScatterValues)(const uint query_index = row_index % suffix_length;const uint head_in_group = row_index / suffix_length;const uint head_index = group_index * gqa_factor + head_in_group;- out[(query_index * num_heads + head_index) * head_dim + dim_index] =- group_output[group_output_index];+ out[(query_index * num_heads + head_index) * head_dim + dim_index] = group_output[group_output_index];}diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metalindex 33d1ed248..252d45687 100644--- a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_gemm.metal@@ -82,10 +82,8 @@ PUBLIC KERNEL(AttentionGemm)(q_tile_idx * int64_t(BLOCK_QUERY_ROWS) * params.q_strides[2];const int kv_head_idx = int(head_idx) / params.gqa_factor;- k += batch_idx * params.k_strides[0] +- int64_t(kv_head_idx) * params.k_strides[1];- v += batch_idx * params.v_strides[0] +- int64_t(kv_head_idx) * params.v_strides[1];+ k += batch_idx * params.k_strides[0] + int64_t(kv_head_idx) * params.k_strides[1];+ v += batch_idx * params.v_strides[0] + int64_t(kv_head_idx) * params.v_strides[1];o += batch_idx * params.o_strides[0] + head_idx * params.o_strides[1] +q_tile_idx * int64_t(BLOCK_QUERY_ROWS) * params.o_strides[2];@@ -119,31 +117,19 @@ PUBLIC KERNEL(AttentionGemm)(1,SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * 32>;- using KeyLoader = ThreadgroupLoader<- T,- BK,- BD,- key_leading_dimension,- 0,- SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * 32>;+ using KeyLoader =+ ThreadgroupLoader<T, BK, BD, key_leading_dimension, 0, SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * 32>;- using ValueLoader = ThreadgroupLoader<- T,- BK,- BD,- value_leading_dimension,- 0,- SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * 32>;+ using ValueLoader =+ ThreadgroupLoader<T, BK, BD, value_leading_dimension, 0, SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * 32>;const int query_source_stride = int(params.q_strides[2]);const int key_source_stride = int(params.k_strides[2]);const int value_source_stride = int(params.v_strides[2]);- thread QueryLoader- query_loader(q, query_source_stride, query_shared, thread_context);+ thread QueryLoader query_loader(q, query_source_stride, query_shared, thread_context);thread KeyLoader key_loader(k, key_source_stride, key_shared, thread_context);- thread ValueLoader- value_loader(v, value_source_stride, value_shared, thread_context);+ thread ValueLoader value_loader(v, value_source_stride, value_shared, thread_context);TransformScale<T> ts(static_cast<T>(params.scale * M_LOG2E_F));@@ -151,80 +137,47 @@ PUBLIC KERNEL(AttentionGemm)(// MMA tilesconstexpr short SIMDGROUP_BLOCK_SIZE = 8;using AccumType = float;- using SimdgroupMultiplyAccumulateType = SimdgroupMultiplyAccumulate<- AccumType,- SIMDGROUP_BLOCK_SIZE,- SIMDGROUP_BLOCK_SIZE>;+ using SimdgroupMultiplyAccumulateType =+ SimdgroupMultiplyAccumulate<AccumType, SIMDGROUP_BLOCK_SIZE, SIMDGROUP_BLOCK_SIZE>;- constexpr int SIMDGROUPS_PER_THREADGROUP =- SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN;+ constexpr int SIMDGROUPS_PER_THREADGROUP = SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN;static_assert(BLOCK_QUERY_ROWS >= (SIMDGROUPS_PER_THREADGROUP * SIMDGROUP_BLOCK_SIZE) &&- BLOCK_QUERY_ROWS %- (SIMDGROUPS_PER_THREADGROUP * SIMDGROUP_BLOCK_SIZE) ==- 0,+ BLOCK_QUERY_ROWS % (SIMDGROUPS_PER_THREADGROUP * SIMDGROUP_BLOCK_SIZE) == 0,"Each simdgroup must host at least 1 simdgroup matrix along Q sequence.");// Q sequence multiply-accumulate blocks per simdgroup (QUERY_GRID_ROWS == 1// for the 32-row block layout)- constexpr int QUERY_GRID_ROWS =- BLOCK_QUERY_ROWS / (SIMDGROUPS_PER_THREADGROUP * SIMDGROUP_BLOCK_SIZE);+ constexpr int QUERY_GRID_ROWS = BLOCK_QUERY_ROWS / (SIMDGROUPS_PER_THREADGROUP * SIMDGROUP_BLOCK_SIZE);constexpr int KEY_GRID_COLS = BK / SIMDGROUP_BLOCK_SIZE;constexpr int HEAD_DIM_GRID_COLS = BD / SIMDGROUP_BLOCK_SIZE;static_assert(QUERY_GRID_ROWS == 1, "Expected QUERY_GRID_ROWS == 1");- SimdgroupFragment<- AccumType,- QUERY_GRID_ROWS,- 1,- SimdgroupMultiplyAccumulateType>- query_fragment;- SimdgroupFragment<- AccumType,- 1,- KEY_GRID_COLS,- SimdgroupMultiplyAccumulateType>- key_fragment;- SimdgroupFragment<- AccumType,- QUERY_GRID_ROWS,- KEY_GRID_COLS,- SimdgroupMultiplyAccumulateType>- score_fragment;- SimdgroupFragment<AccumType, 1, 1, SimdgroupMultiplyAccumulateType>- value_fragment;- SimdgroupFragment<- AccumType,- QUERY_GRID_ROWS,- HEAD_DIM_GRID_COLS,- SimdgroupMultiplyAccumulateType>- output_fragment;+ SimdgroupFragment<AccumType, QUERY_GRID_ROWS, 1, SimdgroupMultiplyAccumulateType> query_fragment;+ SimdgroupFragment<AccumType, 1, KEY_GRID_COLS, SimdgroupMultiplyAccumulateType> key_fragment;+ SimdgroupFragment<AccumType, QUERY_GRID_ROWS, KEY_GRID_COLS, SimdgroupMultiplyAccumulateType> score_fragment;+ SimdgroupFragment<AccumType, 1, 1, SimdgroupMultiplyAccumulateType> value_fragment;+ SimdgroupFragment<AccumType, QUERY_GRID_ROWS, HEAD_DIM_GRID_COLS, SimdgroupMultiplyAccumulateType> output_fragment;output_fragment.clear();// -------------------------------------------------------------------------// Lane coordinates and pointer offsets- const short2 lane_coordinates =- SimdgroupMultiplyAccumulateType::get_lane_coordinates(- thread_context.simd_lane_id- );+ const short2 lane_coordinates = SimdgroupMultiplyAccumulateType::get_lane_coordinates(thread_context.simd_lane_id);const short lane_row = lane_coordinates.y;const short lane_col = lane_coordinates.x;- const short simdgroup_row_base = SIMDGROUP_BLOCK_SIZE * QUERY_GRID_ROWS *- short(thread_context.simdgroup_index);+ const short simdgroup_row_base = SIMDGROUP_BLOCK_SIZE * QUERY_GRID_ROWS * short(thread_context.simdgroup_index);- const short query_shared_offset =- (simdgroup_row_base + lane_row) * query_leading_dimension + lane_col;+ const short query_shared_offset = (simdgroup_row_base + lane_row) * query_leading_dimension + lane_col;constexpr short query_tile_stride = SIMDGROUP_BLOCK_SIZE;const short key_shared_offset = lane_col * key_leading_dimension + lane_row;constexpr short key_tile_stride = SIMDGROUP_BLOCK_SIZE;- const short value_shared_offset =- lane_row * value_leading_dimension + lane_col;+ const short value_shared_offset = lane_row * value_leading_dimension + lane_col;// -------------------------------------------------------------------------// Load Q block once (and apply scaling)@@ -256,17 +209,14 @@ PUBLIC KERNEL(AttentionGemm)(kb_lim = min(params.nk, kb_lim);}- const int q_rel = int(q_tile_idx) * BLOCK_QUERY_ROWS +- int(simdgroup_row_base) + int(lane_row); // [0, q_len)+ const int q_rel = int(q_tile_idx) * BLOCK_QUERY_ROWS + int(simdgroup_row_base) + int(lane_row); // [0, q_len)const int prefix_length = params.q_off;- const int suffix_position =- is_kv_cache_ring ? int(ring_params.ring_length) : prefix_length;+ const int suffix_position = is_kv_cache_ring ? int(ring_params.ring_length) : prefix_length;- const int query_position = (is_trie && q_rel < params.q_len)- ? suffix_position + int(trie[q_rel].height)- : suffix_position + q_rel;+ const int query_position =+ (is_trie && q_rel < params.q_len) ? suffix_position + int(trie[q_rel].height) : suffix_position + q_rel;// Loop over KV blocksfor (int kb = 0; kb < kb_lim; kb++) {@@ -295,12 +245,7 @@ PUBLIC KERNEL(AttentionGemm)();simdgroup_barrier(mem_flags::mem_none);- tile_multiply_accumulate(- score_fragment,- query_fragment,- key_fragment,- score_fragment- );+ tile_multiply_accumulate(score_fragment, query_fragment, key_fragment, score_fragment);}// Mask out tail keys for the last (unaligned) K block@@ -324,8 +269,7 @@ PUBLIC KERNEL(AttentionGemm)(METAL_PRAGMA_UNROLLfor (short j = 0; j < KEY_GRID_COLS; j++) {thread auto& frag = score_fragment.multiply_accumulate_at(0, j);- const int col_base =- kb * BK + int(lane_col) + int(j) * SIMDGROUP_BLOCK_SIZE;+ const int col_base = kb * BK + int(lane_col) + int(j) * SIMDGROUP_BLOCK_SIZE;const int k0 = col_base;const int k1 = col_base + 1;@@ -426,8 +370,7 @@ PUBLIC KERNEL(AttentionGemm)(const short dd = id * SIMDGROUP_BLOCK_SIZE;value_fragment.template load<T, 1, 1, value_leading_dimension, 1>(- &value_shared- [value_shared_offset + kk * value_leading_dimension + dd]+ &value_shared[value_shared_offset + kk * value_leading_dimension + dd]);if constexpr (BD == 128) {@@ -461,22 +404,16 @@ PUBLIC KERNEL(AttentionGemm)(threadgroup_barrier(mem_flags::mem_none);// Store results (O is row-major with row-stride params.o_strides[2])- o += int64_t(simdgroup_row_base + lane_row) * params.o_strides[2] +- int64_t(lane_col);+ o += int64_t(simdgroup_row_base + lane_row) * params.o_strides[2] + int64_t(lane_col);if (!align_q && int(q_tile_idx) == params.nq_aligned) {- const short2 dst_tile_dims =- short2(BD - lane_col, params.q_rem - (simdgroup_row_base + lane_row));+ const short2 dst_tile_dims = short2(BD - lane_col, params.q_rem - (simdgroup_row_base + lane_row));if (dst_tile_dims.x <= 0 || dst_tile_dims.y <= 0) {return;}- output_fragment.template store_safe<T, 1, 1>(- o,- int(params.o_strides[2]),- dst_tile_dims- );+ output_fragment.template store_safe<T, 1, 1>(o, int(params.o_strides[2]), dst_tile_dims);} else {output_fragment.template store<T, 1, 1>(o, int(params.o_strides[2]));}diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_single_pass.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_single_pass.metalindex 221d1ed3d..4fcf79338 100644--- a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_single_pass.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_single_pass.metal@@ -62,28 +62,21 @@ PUBLIC KERNEL(AttentionSinglePass)(const uint kv_head_idx = head_idx / gqa_factor;const uint o_offset = q_seq_idx * num_heads + head_idx;- const uint q_offset = query_transposed ? num_heads * q_seq_idx + head_idx- : head_idx * suffix_length + q_seq_idx;+ const uint q_offset = query_transposed ? num_heads * q_seq_idx + head_idx : head_idx * suffix_length + q_seq_idx;const uint prefix_length = sequence_length - suffix_length;- const uint suffix_position =- is_kv_cache_ring ? uint(ring_params.ring_length) : prefix_length;+ const uint suffix_position = is_kv_cache_ring ? uint(ring_params.ring_length) : prefix_length;- const uint query_position = is_trie ? suffix_position + trie[q_seq_idx].height- : suffix_position + q_seq_idx;+ const uint query_position = is_trie ? suffix_position + trie[q_seq_idx].height : suffix_position + q_seq_idx;- queries += q_offset * HEAD_DIM +- thread_context.simd_lane_id * qk_elements_per_thread;- keys += kv_head_idx * k_head_stride +- thread_context.simdgroup_index * k_seq_stride ++ queries += q_offset * HEAD_DIM + thread_context.simd_lane_id * qk_elements_per_thread;+ keys += kv_head_idx * k_head_stride + thread_context.simdgroup_index * k_seq_stride +thread_context.simd_lane_id * qk_elements_per_thread;- values += kv_head_idx * v_head_stride +- thread_context.simdgroup_index * v_seq_stride ++ values += kv_head_idx * v_head_stride + thread_context.simdgroup_index * v_seq_stride +thread_context.simd_lane_id * value_elements_per_thread;- out += o_offset * value_dim +- thread_context.simdgroup_index * value_elements_per_thread;+ out += o_offset * value_dim + thread_context.simdgroup_index * value_elements_per_thread;// Read the query and 0 the output accumulatorfor (int i = 0; i < qk_elements_per_thread; i++) {@@ -103,8 +96,7 @@ PUBLIC KERNEL(AttentionSinglePass)(}// For each key- for (uint i = thread_context.simdgroup_index; i < sequence_length;- i += SEQUENCE_BLOCK_SIZE) {+ for (uint i = thread_context.simdgroup_index; i < sequence_length; i += SEQUENCE_BLOCK_SIZE) {if (should_use_key(ring_params,trie,@@ -159,20 +151,14 @@ PUBLIC KERNEL(AttentionSinglePass)(max_score = shared_max_scores[thread_context.simd_lane_id];U new_max = simd_max(max_score);U factor = fast::exp(max_score - new_max);- sum_exp_score =- simd_sum(shared_sum_exp_scores[thread_context.simd_lane_id] * factor);+ sum_exp_score = simd_sum(shared_sum_exp_scores[thread_context.simd_lane_id] * factor);// Now we need to aggregate all the outputsfor (int i = 0; i < value_elements_per_thread; i++) {- shared_outputs- [thread_context.simd_lane_id * HEAD_BLOCK_SIZE +- thread_context.simdgroup_index] = o[i];+ shared_outputs[thread_context.simd_lane_id * HEAD_BLOCK_SIZE + thread_context.simdgroup_index] = o[i];threadgroup_barrier(mem_flags::mem_threadgroup);o[i] = simd_sum(- shared_outputs- [thread_context.simdgroup_index * HEAD_BLOCK_SIZE +- thread_context.simd_lane_id] *- factor+ shared_outputs[thread_context.simdgroup_index * HEAD_BLOCK_SIZE + thread_context.simd_lane_id] * factor) /sum_exp_score;threadgroup_barrier(mem_flags::mem_threadgroup);diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_two_pass.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_two_pass.metalindex 5f4b95300..a88f9433e 100644--- a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_two_pass.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_two_pass.metal@@ -56,8 +56,7 @@ PUBLIC KERNEL(AttentionTwoPass1)(typedef float U;// Flat query head mapping — fixed hardware shape, dynamic logical mapping- const uint head_idx = head_group_idx * SIMDGROUPS_PER_THREADGROUP +- thread_context.simdgroup_index;+ const uint head_idx = head_group_idx * SIMDGROUPS_PER_THREADGROUP + thread_context.simdgroup_index;if (head_idx >= num_heads)return;@@ -66,20 +65,15 @@ PUBLIC KERNEL(AttentionTwoPass1)(const uint q_offset = head_idx * suffix_length + q_seq_idx;const uint prefix_length = sequence_length - suffix_length;- const uint suffix_position =- is_kv_cache_ring ? uint(ring_params.ring_length) : prefix_length;- const uint query_position = is_trie ? suffix_position + trie[q_seq_idx].height- : suffix_position + q_seq_idx;+ const uint suffix_position = is_kv_cache_ring ? uint(ring_params.ring_length) : prefix_length;+ const uint query_position = is_trie ? suffix_position + trie[q_seq_idx].height : suffix_position + q_seq_idx;thread U q[elements_per_thread];thread U o[elements_per_thread] = {0};- queries +=- q_offset * HEAD_DIM + thread_context.simd_lane_id * elements_per_thread;- keys += kv_head_idx * k_head_stride + block_idx * k_seq_stride +- thread_context.simd_lane_id * elements_per_thread;- values += kv_head_idx * v_head_stride + block_idx * v_seq_stride +- thread_context.simd_lane_id * elements_per_thread;+ queries += q_offset * HEAD_DIM + thread_context.simd_lane_id * elements_per_thread;+ keys += kv_head_idx * k_head_stride + block_idx * k_seq_stride + thread_context.simd_lane_id * elements_per_thread;+ values += kv_head_idx * v_head_stride + block_idx * v_seq_stride + thread_context.simd_lane_id * elements_per_thread;out += o_offset * TOTAL_BLOCKS_COUNT * HEAD_DIM + block_idx * HEAD_DIM +thread_context.simd_lane_id * elements_per_thread;sums += o_offset * TOTAL_BLOCKS_COUNT + block_idx;@@ -170,16 +164,14 @@ PUBLIC KERNEL(AttentionTwoPass2)(// Find global max across all blocks (each lane handles strided blocks)U max_score = -1e9;- for (uint b = thread_context.simd_lane_id; b < TOTAL_BLOCKS_COUNT;- b += METAL_SIMD_SIZE) {+ for (uint b = thread_context.simd_lane_id; b < TOTAL_BLOCKS_COUNT; b += METAL_SIMD_SIZE) {max_score = max(max_score, maxs[b]);}U new_max = simd_max(max_score);// Find global sum across all blocksU sum_exp = 0;- for (uint b = thread_context.simd_lane_id; b < TOTAL_BLOCKS_COUNT;- b += METAL_SIMD_SIZE) {+ for (uint b = thread_context.simd_lane_id; b < TOTAL_BLOCKS_COUNT; b += METAL_SIMD_SIZE) {sum_exp += sums[b] * fast::exp(maxs[b] - new_max);}U sum_exp_score = simd_sum(sum_exp);@@ -190,14 +182,10 @@ PUBLIC KERNEL(AttentionTwoPass2)(for (uint chunk = 0; chunk < TOTAL_BLOCKS_COUNT; chunk += METAL_SIMD_SIZE) {uint block_idx = chunk + thread_context.simd_lane_id;- U factor = (block_idx < TOTAL_BLOCKS_COUNT)- ? fast::exp(maxs[block_idx] - new_max)- : 0;+ U factor = (block_idx < TOTAL_BLOCKS_COUNT) ? fast::exp(maxs[block_idx] - new_max) : 0;- const device float* block_partials =- partials + o_offset * TOTAL_BLOCKS_COUNT * HEAD_DIM +- block_idx * HEAD_DIM +- thread_context.simdgroup_index * elements_per_thread;+ const device float* block_partials = partials + o_offset * TOTAL_BLOCKS_COUNT * HEAD_DIM + block_idx * HEAD_DIM ++ thread_context.simdgroup_index * elements_per_thread;for (uint i = 0; i < elements_per_thread; i++) {U val = (block_idx < TOTAL_BLOCKS_COUNT) ? block_partials[i] * factor : 0;@@ -207,8 +195,7 @@ PUBLIC KERNEL(AttentionTwoPass2)(// Write the outputif (thread_context.simd_lane_id == 0) {- out += o_offset * HEAD_DIM +- thread_context.simdgroup_index * elements_per_thread;+ out += o_offset * HEAD_DIM + thread_context.simdgroup_index * elements_per_thread;for (uint i = 0; i < elements_per_thread; i++) {out[i] = static_cast<T>(o[i] / sum_exp_score);}diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_update_kv_cache.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_update_kv_cache.metalindex f38009c50..f8dcd5365 100644--- a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_update_kv_cache.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_update_kv_cache.metal@@ -28,22 +28,17 @@ PUBLIC KERNEL(AttentionUpdateKVCache)(// keys_in_place=true: destination shares rotated_keys' group-major layout.// Otherwise, KV cache is token-major:// [max_sequence_length, num_groups, head_dim].- const uint cacheOffset =- keys_in_place- ? ((group_index * max_sequence_length + cacheTokenIndex) * head_dim +- dim_index)- : ((cacheTokenIndex * num_groups + group_index) * head_dim +- dim_index);+ const uint cacheOffset = keys_in_place+ ? ((group_index * max_sequence_length + cacheTokenIndex) * head_dim + dim_index)+ : ((cacheTokenIndex * num_groups + group_index) * head_dim + dim_index);- const uint rotatedKeyOffset =- (group_index * suffix_length + token_index) * head_dim + dim_index;+ const uint rotatedKeyOffset = (group_index * suffix_length + token_index) * head_dim + dim_index;key_cache[cacheOffset] = rotated_keys[rotatedKeyOffset];// qkv layout: [suffix_length, (num_heads + 2*num_groups) * head_dim]// Values start at offset: (num_heads * head_dim) + (num_groups * head_dim)const uint qkvStride = (num_heads + 2 * num_groups) * head_dim;const uint valueBaseOffset = (num_heads + num_groups) * head_dim;- const uint valueOffset = token_index * qkvStride + valueBaseOffset +- group_index * head_dim + dim_index;+ const uint valueOffset = token_index * qkvStride + valueBaseOffset + group_index * head_dim + dim_index;value_cache[cacheOffset] = qkv[valueOffset];}\ No newline at end of filediff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/mask.h b/crates/backend-uzu/src/backends/metal/kernel/attention/mask.hindex e553eabc0..8b6f4cb5e 100644--- a/crates/backend-uzu/src/backends/metal/kernel/attention/mask.h+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/mask.h@@ -35,8 +35,7 @@ static inline bool should_use_key(key_position = suffix_position + key_trie.height;if (is_causal) {- use_key &=- q_seq_idx >= key_trie.trie_start && q_seq_idx <= key_trie.trie_end;+ use_key &= q_seq_idx >= key_trie.trie_start && q_seq_idx <= key_trie.trie_end;}} else {// Non-trie, position equals index in the suffix@@ -51,8 +50,7 @@ static inline bool should_use_key(// We're in the prefixif (is_kv_cache_ring) {// Ring, calculate position in the ring- key_position =- (prefix_length + i - ring_params.ring_offset) % prefix_length;+ key_position = (prefix_length + i - ring_params.ring_offset) % prefix_length;// Ring also skips entries that aren't filled yetuse_key &= key_position < ring_params.ring_length;} else {@@ -63,8 +61,7 @@ static inline bool should_use_key(if (is_sliding_window) {if (is_causal) {- use_key &= key_position <= query_position &&- (query_position - key_position) < sliding_window_size;+ use_key &= key_position <= query_position && (query_position - key_position) < sliding_window_size;} else {if (key_position <= query_position) {use_key &= (query_position - key_position) <= (sliding_window_size / 2);diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/qk_unpack.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/qk_unpack.metalindex 23dab48e5..b2b19ff5e 100644--- a/crates/backend-uzu/src/backends/metal/kernel/attention/qk_unpack.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/qk_unpack.metal@@ -20,14 +20,11 @@ PUBLIC KERNEL(QkUnpack)(const uint total_heads = num_heads + 2 * num_groups;const uint first_head_in_group = group_index * (num_heads / num_groups);- TensorView3D<const T> qkv_tensor_view =- TensorView3D<const T>(qkv).shaped(suffix_length, total_heads, head_dim);+ TensorView3D<const T> qkv_tensor_view = TensorView3D<const T>(qkv).shaped(suffix_length, total_heads, head_dim);TensorView3D<T> unpacked_queries_tensor_view =- TensorView3D<T>(unpacked_queries)- .shaped(num_heads, suffix_length, head_dim);+ TensorView3D<T>(unpacked_queries).shaped(num_heads, suffix_length, head_dim);TensorView3D<T> unpacked_keys_tensor_view =- TensorView3D<T>(unpacked_keys)- .shaped(num_groups, suffix_length, head_dim);+ TensorView3D<T>(unpacked_keys).shaped(num_groups, suffix_length, head_dim);unpacked_queries_tensor_view(head_index, token_index, dimension_index) =qkv_tensor_view(token_index, head_index, dimension_index);diff --git a/crates/backend-uzu/src/backends/metal/kernel/audio/add.metal b/crates/backend-uzu/src/backends/metal/kernel/audio/add.metalindex 545920cd5..a2cb4ffe5 100644--- a/crates/backend-uzu/src/backends/metal/kernel/audio/add.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/audio/add.metal@@ -4,13 +4,7 @@using namespace metal;template <typename T>-void add_tensors(- device const T* a,- device const T* b,- device T* out,- const constant int& n,- const uint tid-) {+void add_tensors(device const T* a, device const T* b, device T* out, const constant int& n, const uint tid) {if ((int)tid >= n) {return;}diff --git a/crates/backend-uzu/src/backends/metal/kernel/audio/causal_conv1d.metal b/crates/backend-uzu/src/backends/metal/kernel/audio/causal_conv1d.metalindex cfcd6e82f..2f29db10e 100644--- a/crates/backend-uzu/src/backends/metal/kernel/audio/causal_conv1d.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/audio/causal_conv1d.metal@@ -74,8 +74,7 @@ void causal_conv1d(acc2 += w * float(input[x_idx + 2]);acc3 += w * float(input[x_idx + 3]);} else {- const uint x_idx0 =- (b * (uint)seq_len + (uint)x_t) * (uint)cin + (uint)ic;+ const uint x_idx0 = (b * (uint)seq_len + (uint)x_t) * (uint)cin + (uint)ic;acc0 += w * float(input[x_idx0]);acc1 += w * float(input[x_idx0 + (uint)cin]);acc2 += w * float(input[x_idx0 + (uint)(2 * cin)]);@@ -84,34 +83,27 @@ void causal_conv1d(continue;}if (valid_count > 0 && x_t >= 0 && x_t < seq_len) {- const uint x_idx =- (input_layout == AUDIO_LAYOUT_NCS)- ? (x_base_ncs + (uint)x_t)- : ((b * (uint)seq_len + (uint)x_t) * (uint)cin + (uint)ic);+ const uint x_idx = (input_layout == AUDIO_LAYOUT_NCS)+ ? (x_base_ncs + (uint)x_t)+ : ((b * (uint)seq_len + (uint)x_t) * (uint)cin + (uint)ic);acc0 += w * float(input[x_idx]);}if (valid_count > 1 && x_t + 1 >= 0 && x_t + 1 < seq_len) {- const uint x_idx =- (input_layout == AUDIO_LAYOUT_NCS)- ? (x_base_ncs + (uint)(x_t + 1))- : ((b * (uint)seq_len + (uint)(x_t + 1)) * (uint)cin +- (uint)ic);+ const uint x_idx = (input_layout == AUDIO_LAYOUT_NCS)+ ? (x_base_ncs + (uint)(x_t + 1))+ : ((b * (uint)seq_len + (uint)(x_t + 1)) * (uint)cin + (uint)ic);acc1 += w * float(input[x_idx]);}if (valid_count > 2 && x_t + 2 >= 0 && x_t + 2 < seq_len) {- const uint x_idx =- (input_layout == AUDIO_LAYOUT_NCS)- ? (x_base_ncs + (uint)(x_t + 2))- : ((b * (uint)seq_len + (uint)(x_t + 2)) * (uint)cin +- (uint)ic);+ const uint x_idx = (input_layout == AUDIO_LAYOUT_NCS)+ ? (x_base_ncs + (uint)(x_t + 2))+ : ((b * (uint)seq_len + (uint)(x_t + 2)) * (uint)cin + (uint)ic);acc2 += w * float(input[x_idx]);}if (valid_count > 3 && x_t + 3 >= 0 && x_t + 3 < seq_len) {- const uint x_idx =- (input_layout == AUDIO_LAYOUT_NCS)- ? (x_base_ncs + (uint)(x_t + 3))- : ((b * (uint)seq_len + (uint)(x_t + 3)) * (uint)cin +- (uint)ic);+ const uint x_idx = (input_layout == AUDIO_LAYOUT_NCS)+ ? (x_base_ncs + (uint)(x_t + 3))+ : ((b * (uint)seq_len + (uint)(x_t + 3)) * (uint)cin + (uint)ic);acc3 += w * float(input[x_idx]);}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/audio/causal_conv1d_grouped.metal b/crates/backend-uzu/src/backends/metal/kernel/audio/causal_conv1d_grouped.metalindex 5e1a5ef4e..35ed3ea05 100644--- a/crates/backend-uzu/src/backends/metal/kernel/audio/causal_conv1d_grouped.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/audio/causal_conv1d_grouped.metal@@ -71,9 +71,7 @@ void causal_conv1d_grouped_ncs(const bool full_tile = valid_count == (int)AUDIO_TIME_TILE;for (int ic_local = 0; ic_local < cin_per_group; ++ic_local) {const uint w_base =- ((group_idx * cout_per_group + oc_in_group) * (uint)cin_per_group +- (uint)ic_local) *- (uint)kernel_size;+ ((group_idx * cout_per_group + oc_in_group) * (uint)cin_per_group + (uint)ic_local) * (uint)kernel_size;const int ic = in_begin + ic_local;const uint x_base_ncs = (b * (uint)cin + (uint)ic) * (uint)seq_len;@@ -216,16 +214,13 @@ void causal_conv1d_grouped_nsc(for (int ic_local = 0; ic_local < cin_per_group; ++ic_local) {const int ic = in_begin + ic_local;const uint w_base =- ((group_idx * cout_per_group + oc_in_group) * (uint)cin_per_group +- (uint)ic_local) *- (uint)kernel_size;+ ((group_idx * cout_per_group + oc_in_group) * (uint)cin_per_group + (uint)ic_local) * (uint)kernel_size;for (int k = 0; k < kernel_size; ++k) {const float w = float(weight[w_base + (uint)k]);const int x_t = (int)t + k * dilation - pad;if (full_tile && x_t >= 0 && x_t + 7 < seq_len) {- const uint x_idx0 =- (b * (uint)seq_len + (uint)x_t) * (uint)cin + (uint)ic;+ const uint x_idx0 = (b * (uint)seq_len + (uint)x_t) * (uint)cin + (uint)ic;acc0 += w * float(input[x_idx0]);acc1 += w * float(input[x_idx0 + (uint)cin]);acc2 += w * float(input[x_idx0 + (uint)(2 * cin)]);@@ -237,43 +232,35 @@ void causal_conv1d_grouped_nsc(continue;}if (valid_count > 0 && x_t >= 0 && x_t < seq_len) {- const uint x_idx =- (b * (uint)seq_len + (uint)x_t) * (uint)cin + (uint)ic;+ const uint x_idx = (b * (uint)seq_len + (uint)x_t) * (uint)cin + (uint)ic;acc0 += w * float(input[x_idx]);}if (valid_count > 1 && x_t + 1 >= 0 && x_t + 1 < seq_len) {- const uint x_idx =- (b * (uint)seq_len + (uint)(x_t + 1)) * (uint)cin + (uint)ic;+ const uint x_idx = (b * (uint)seq_len + (uint)(x_t + 1)) * (uint)cin + (uint)ic;acc1 += w * float(input[x_idx]);}if (valid_count > 2 && x_t + 2 >= 0 && x_t + 2 < seq_len) {- const uint x_idx =- (b * (uint)seq_len + (uint)(x_t + 2)) * (uint)cin + (uint)ic;+ const uint x_idx = (b * (uint)seq_len + (uint)(x_t + 2)) * (uint)cin + (uint)ic;acc2 += w * float(input[x_idx]);}if (valid_count > 3 && x_t + 3 >= 0 && x_t + 3 < seq_len) {- const uint x_idx =- (b * (uint)seq_len + (uint)(x_t + 3)) * (uint)cin + (uint)ic;+ const uint x_idx = (b * (uint)seq_len + (uint)(x_t + 3)) * (uint)cin + (uint)ic;acc3 += w * float(input[x_idx]);}if (valid_count > 4 && x_t + 4 >= 0 && x_t + 4 < seq_len) {- const uint x_idx =- (b * (uint)seq_len + (uint)(x_t + 4)) * (uint)cin + (uint)ic;+ const uint x_idx = (b * (uint)seq_len + (uint)(x_t + 4)) * (uint)cin + (uint)ic;acc4 += w * float(input[x_idx]);}if (valid_count > 5 && x_t + 5 >= 0 && x_t + 5 < seq_len) {- const uint x_idx =- (b * (uint)seq_len + (uint)(x_t + 5)) * (uint)cin + (uint)ic;+ const uint x_idx = (b * (uint)seq_len + (uint)(x_t + 5)) * (uint)cin + (uint)ic;acc5 += w * float(input[x_idx]);}if (valid_count > 6 && x_t + 6 >= 0 && x_t + 6 < seq_len) {- const uint x_idx =- (b * (uint)seq_len + (uint)(x_t + 6)) * (uint)cin + (uint)ic;+ const uint x_idx = (b * (uint)seq_len + (uint)(x_t + 6)) * (uint)cin + (uint)ic;acc6 += w * float(input[x_idx]);}if (valid_count > 7 && x_t + 7 >= 0 && x_t + 7 < seq_len) {- const uint x_idx =- (b * (uint)seq_len + (uint)(x_t + 7)) * (uint)cin + (uint)ic;+ const uint x_idx = (b * (uint)seq_len + (uint)(x_t + 7)) * (uint)cin + (uint)ic;acc7 += w * float(input[x_idx]);}}@@ -327,34 +314,10 @@ PUBLIC KERNEL(AudioCausalConv1dGrouped)(uint b AXIS(batch_size, 1)) {if (input_layout == AUDIO_LAYOUT_NCS) {- causal_conv1d_grouped_ncs<T>(- input,- weight,- bias,- output,- lengths,- cin,- cout,- seq_len,- kernel_size,- dilation,- groups,- uint3(t, oc, b)- );+ causal_conv1d_grouped_ncs<+ T>(input, weight, bias, output, lengths, cin, cout, seq_len, kernel_size, dilation, groups, uint3(t, oc, b));} else {- causal_conv1d_grouped_nsc<T>(- input,- weight,- bias,- output,- lengths,- cin,- cout,- seq_len,- kernel_size,- dilation,- groups,- uint3(t, oc, b)- );+ causal_conv1d_grouped_nsc<+ T>(input, weight, bias, output, lengths, cin, cout, seq_len, kernel_size, dilation, groups, uint3(t, oc, b));}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/audio/causal_conv1d_grouped_residual.metal b/crates/backend-uzu/src/backends/metal/kernel/audio/causal_conv1d_grouped_residual.metalindex bfc694c9f..e472b03ac 100644--- a/crates/backend-uzu/src/backends/metal/kernel/audio/causal_conv1d_grouped_residual.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/audio/causal_conv1d_grouped_residual.metal@@ -80,9 +80,7 @@ void causal_conv1d_grouped_residual(for (int ic_local = 0; ic_local < cin_per_group; ++ic_local) {const int ic = in_begin + ic_local;const uint w_base =- ((group_idx * cout_per_group + oc_in_group) * (uint)cin_per_group +- (uint)ic_local) *- (uint)kernel_size;+ ((group_idx * cout_per_group + oc_in_group) * (uint)cin_per_group + (uint)ic_local) * (uint)kernel_size;const uint x_base = (b * (uint)cin + (uint)ic) * (uint)seq_len;for (int k = 0; k < kernel_size; ++k) {diff --git a/crates/backend-uzu/src/backends/metal/kernel/audio/causal_conv_transpose1d.metal b/crates/backend-uzu/src/backends/metal/kernel/audio/causal_conv_transpose1d.metalindex 016716aaa..6e1ee11f2 100644--- a/crates/backend-uzu/src/backends/metal/kernel/audio/causal_conv_transpose1d.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/audio/causal_conv_transpose1d.metal@@ -54,16 +54,14 @@ void causal_conv_transpose1d(// Weight layout: [Cout, Cin_per_group, K]const uint ic_local = (uint)(ic - ic_begin);- const uint w_base =- ((uint)oc * (uint)cin_per_group + ic_local) * (uint)(2 * stride);+ const uint w_base = ((uint)oc * (uint)cin_per_group + ic_local) * (uint)(2 * stride);// Contribution from input[q] with k=racc += float(weight[w_base + (uint)r]) * float(input[in_base + (uint)q]);// Contribution from input[q-1] with k=stride+rif (q > 0) {- acc += float(weight[w_base + (uint)(stride + r)]) *- float(input[in_base + (uint)(q - 1)]);+ acc += float(weight[w_base + (uint)(stride + r)]) * float(input[in_base + (uint)(q - 1)]);}}@@ -89,18 +87,6 @@ PUBLIC KERNEL(AudioCausalConvTranspose1d)(uint oc AXIS(cout, 1),uint b AXIS(batch_size, 1)) {- causal_conv_transpose1d<T>(- input,- weight,- bias,- output,- lengths,- cin,- cout,- seq_len_in,- seq_len_out,- stride,- groups,- uint3(t_out, oc, b)- );+ causal_conv_transpose1d<+ T>(input, weight, bias, output, lengths, cin, cout, seq_len_in, seq_len_out, stride, groups, uint3(t_out, oc, b));}diff --git a/crates/backend-uzu/src/backends/metal/kernel/audio/causal_conv_transpose1d_causal_pad.metal b/crates/backend-uzu/src/backends/metal/kernel/audio/causal_conv_transpose1d_causal_pad.metalindex 46d3bed81..2481064e6 100644--- a/crates/backend-uzu/src/backends/metal/kernel/audio/causal_conv_transpose1d_causal_pad.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/audio/causal_conv_transpose1d_causal_pad.metal@@ -40,9 +40,8 @@ void causal_conv_transpose1d_causal_pad(if (t_out >= (uint)seq_len_out || oc >= (uint)cout) {return;}- if ((input_layout != AUDIO_LAYOUT_NCS && input_layout != AUDIO_LAYOUT_NSC) ||- groups <= 0 || stride <= 0 || kernel_size <= 0 || (cin % groups) != 0 ||- (cout % groups) != 0) {+ if ((input_layout != AUDIO_LAYOUT_NCS && input_layout != AUDIO_LAYOUT_NSC) || groups <= 0 || stride <= 0 ||+ kernel_size <= 0 || (cin % groups) != 0 || (cout % groups) != 0) {return;}@@ -69,8 +68,7 @@ void causal_conv_transpose1d_causal_pad(const int ic_begin = group_idx * cin_per_group;const int ic_end = ic_begin + cin_per_group;- const int seq_len_expanded =- (seq_len_in > 0) ? (((seq_len_in - 1) * stride) + 1) : 0;+ const int seq_len_expanded = (seq_len_in > 0) ? (((seq_len_in - 1) * stride) + 1) : 0;const int left_pad = kernel_size - 1;const bool fast_two_tap = (kernel_size == (stride * 2));@@ -92,8 +90,7 @@ void causal_conv_transpose1d_causal_pad(for (int ic = ic_begin; ic < ic_end; ++ic) {const uint in_base = (b * (uint)cin + (uint)ic) * (uint)seq_len_in;const uint ic_local = (uint)(ic - ic_begin);- const uint w_base =- ((uint)oc * (uint)cin_per_group + ic_local) * (uint)kernel_size;+ const uint w_base = ((uint)oc * (uint)cin_per_group + ic_local) * (uint)kernel_size;if (fast_two_tap) {// When stride >= AUDIO_TIME_TILE (common for decoder blocks with// strides 4,8), all 4 lanes share the same q value. This means@@ -103,42 +100,30 @@ void causal_conv_transpose1d_causal_pad(if (stride >= (int)AUDIO_TIME_TILE) {// All lanes have same q since t_out+3 < t_out+stride means// (t_out+3)/stride == t_out/stride.- const float in_qm1 = (q0 > 0 && q0 - 1 < seq_len_in)- ? float(input[in_base + (uint)(q0 - 1)])- : 0.0f;- const float in_q0 = (q0 >= 0 && q0 < seq_len_in)- ? float(input[in_base + (uint)q0])- : 0.0f;+ const float in_qm1 = (q0 > 0 && q0 - 1 < seq_len_in) ? float(input[in_base + (uint)(q0 - 1)]) : 0.0f;+ const float in_q0 = (q0 >= 0 && q0 < seq_len_in) ? float(input[in_base + (uint)q0]) : 0.0f;// Lane 0: r = r0if (valid_count > 0) {- acc0 +=- in_qm1 * float(weight[w_base + (uint)((stride - 1) - r0)]);- acc0 += in_q0 *- float(weight[w_base + (uint)((kernel_size - 1) - r0)]);+ acc0 += in_qm1 * float(weight[w_base + (uint)((stride - 1) - r0)]);+ acc0 += in_q0 * float(weight[w_base + (uint)((kernel_size - 1) - r0)]);}// Lane 1: r = r0+1if (valid_count > 1) {const int r1 = r0 + 1;- acc1 +=- in_qm1 * float(weight[w_base + (uint)((stride - 1) - r1)]);- acc1 += in_q0 *- float(weight[w_base + (uint)((kernel_size - 1) - r1)]);+ acc1 += in_qm1 * float(weight[w_base + (uint)((stride - 1) - r1)]);+ acc1 += in_q0 * float(weight[w_base + (uint)((kernel_size - 1) - r1)]);}// Lane 2: r = r0+2if (valid_count > 2) {const int r2 = r0 + 2;- acc2 +=- in_qm1 * float(weight[w_base + (uint)((stride - 1) - r2)]);- acc2 += in_q0 *- float(weight[w_base + (uint)((kernel_size - 1) - r2)]);+ acc2 += in_qm1 * float(weight[w_base + (uint)((stride - 1) - r2)]);+ acc2 += in_q0 * float(weight[w_base + (uint)((kernel_size - 1) - r2)]);}// Lane 3: r = r0+3if (valid_count > 3) {const int r3 = r0 + 3;- acc3 +=- in_qm1 * float(weight[w_base + (uint)((stride - 1) - r3)]);- acc3 += in_q0 *- float(weight[w_base + (uint)((kernel_size - 1) - r3)]);+ acc3 += in_qm1 * float(weight[w_base + (uint)((stride - 1) - r3)]);+ acc3 += in_q0 * float(weight[w_base + (uint)((kernel_size - 1) - r3)]);}} else {// stride < 4 (e.g., stride=2 for upsampler blocks).@@ -147,48 +132,40 @@ void causal_conv_transpose1d_causal_pad(const int k_lo = (stride - 1) - r0;const int k_hi = (kernel_size - 1) - r0;if (q0 > 0 && q0 - 1 < seq_len_in) {- acc0 += float(input[in_base + (uint)(q0 - 1)]) *- float(weight[w_base + (uint)k_lo]);+ acc0 += float(input[in_base + (uint)(q0 - 1)]) * float(weight[w_base + (uint)k_lo]);}if (q0 >= 0 && q0 < seq_len_in) {- acc0 += float(input[in_base + (uint)q0]) *- float(weight[w_base + (uint)k_hi]);+ acc0 += float(input[in_base + (uint)q0]) * float(weight[w_base + (uint)k_hi]);}}if (valid_count > 1) {const int q1 = (t0 + 1) / stride;const int r1 = (t0 + 1) % stride;if (q1 > 0 && q1 - 1 < seq_len_in) {- acc1 += float(input[in_base + (uint)(q1 - 1)]) *- float(weight[w_base + (uint)((stride - 1) - r1)]);+ acc1 += float(input[in_base + (uint)(q1 - 1)]) * float(weight[w_base + (uint)((stride - 1) - r1)]);}if (q1 >= 0 && q1 < seq_len_in) {- acc1 += float(input[in_base + (uint)q1]) *- float(weight[w_base + (uint)((kernel_size - 1) - r1)]);+ acc1 += float(input[in_base + (uint)q1]) * float(weight[w_base + (uint)((kernel_size - 1) - r1)]);}}if (valid_count > 2) {const int q2 = (t0 + 2) / stride;const int r2 = (t0 + 2) % stride;if (q2 > 0 && q2 - 1 < seq_len_in) {- acc2 += float(input[in_base + (uint)(q2 - 1)]) *- float(weight[w_base + (uint)((stride - 1) - r2)]);+ acc2 += float(input[in_base + (uint)(q2 - 1)]) * float(weight[w_base + (uint)((stride - 1) - r2)]);}if (q2 >= 0 && q2 < seq_len_in) {- acc2 += float(input[in_base + (uint)q2]) *- float(weight[w_base + (uint)((kernel_size - 1) - r2)]);+ acc2 += float(input[in_base + (uint)q2]) * float(weight[w_base + (uint)((kernel_size - 1) - r2)]);}}if (valid_count > 3) {const int q3 = (t0 + 3) / stride;const int r3 = (t0 + 3) % stride;if (q3 > 0 && q3 - 1 < seq_len_in) {- acc3 += float(input[in_base + (uint)(q3 - 1)]) *- float(weight[w_base + (uint)((stride - 1) - r3)]);+ acc3 += float(input[in_base + (uint)(q3 - 1)]) * float(weight[w_base + (uint)((stride - 1) - r3)]);}if (q3 >= 0 && q3 < seq_len_in) {- acc3 += float(input[in_base + (uint)q3]) *- float(weight[w_base + (uint)((kernel_size - 1) - r3)]);+ acc3 += float(input[in_base + (uint)q3]) * float(weight[w_base + (uint)((kernel_size - 1) - r3)]);}}}@@ -198,8 +175,7 @@ void causal_conv_transpose1d_causal_pad(// Lane 0if (valid_count > 0) {const int expanded_time0 = (int)t_out + k - left_pad;- if (expanded_time0 >= 0 && expanded_time0 < seq_len_expanded &&- (expanded_time0 % stride) == 0) {+ if (expanded_time0 >= 0 && expanded_time0 < seq_len_expanded && (expanded_time0 % stride) == 0) {const int src_time0 = expanded_time0 / stride;if (src_time0 >= 0 && src_time0 < seq_len_in) {acc0 += float(input[in_base + (uint)src_time0]) * w;@@ -209,8 +185,7 @@ void causal_conv_transpose1d_causal_pad(// Lane 1if (valid_count > 1) {const int expanded_time1 = (int)t_out + 1 + k - left_pad;- if (expanded_time1 >= 0 && expanded_time1 < seq_len_expanded &&- (expanded_time1 % stride) == 0) {+ if (expanded_time1 >= 0 && expanded_time1 < seq_len_expanded && (expanded_time1 % stride) == 0) {const int src_time1 = expanded_time1 / stride;if (src_time1 >= 0 && src_time1 < seq_len_in) {acc1 += float(input[in_base + (uint)src_time1]) * w;@@ -220,8 +195,7 @@ void causal_conv_transpose1d_causal_pad(// Lane 2if (valid_count > 2) {const int expanded_time2 = (int)t_out + 2 + k - left_pad;- if (expanded_time2 >= 0 && expanded_time2 < seq_len_expanded &&- (expanded_time2 % stride) == 0) {+ if (expanded_time2 >= 0 && expanded_time2 < seq_len_expanded && (expanded_time2 % stride) == 0) {const int src_time2 = expanded_time2 / stride;if (src_time2 >= 0 && src_time2 < seq_len_in) {acc2 += float(input[in_base + (uint)src_time2]) * w;@@ -231,8 +205,7 @@ void causal_conv_transpose1d_causal_pad(// Lane 3if (valid_count > 3) {const int expanded_time3 = (int)t_out + 3 + k - left_pad;- if (expanded_time3 >= 0 && expanded_time3 < seq_len_expanded &&- (expanded_time3 % stride) == 0) {+ if (expanded_time3 >= 0 && expanded_time3 < seq_len_expanded && (expanded_time3 % stride) == 0) {const int src_time3 = expanded_time3 / stride;if (src_time3 >= 0 && src_time3 < seq_len_in) {acc3 += float(input[in_base + (uint)src_time3]) * w;@@ -246,117 +219,81 @@ void causal_conv_transpose1d_causal_pad(// NSC layoutfor (int ic = ic_begin; ic < ic_end; ++ic) {const uint ic_local = (uint)(ic - ic_begin);- const uint w_base =- ((uint)oc * (uint)cin_per_group + ic_local) * (uint)kernel_size;+ const uint w_base = ((uint)oc * (uint)cin_per_group + ic_local) * (uint)kernel_size;if (fast_two_tap) {if (stride >= (int)AUDIO_TIME_TILE) {// All lanes share q0; load input once per (q-1, q) pair.- const float in_qm1 =- (q0 > 0 && q0 - 1 < seq_len_in)- ? float(input- [(b * (uint)seq_len_in + (uint)(q0 - 1)) *- (uint)cin +- (uint)ic])- : 0.0f;- const float in_q0 =- (q0 >= 0 && q0 < seq_len_in)- ? float(input- [(b * (uint)seq_len_in + (uint)q0) * (uint)cin +- (uint)ic])- : 0.0f;+ const float in_qm1 = (q0 > 0 && q0 - 1 < seq_len_in)+ ? float(input[(b * (uint)seq_len_in + (uint)(q0 - 1)) * (uint)cin + (uint)ic])+ : 0.0f;+ const float in_q0 = (q0 >= 0 && q0 < seq_len_in)+ ? float(input[(b * (uint)seq_len_in + (uint)q0) * (uint)cin + (uint)ic])+ : 0.0f;if (valid_count > 0) {- acc0 +=- in_qm1 * float(weight[w_base + (uint)((stride - 1) - r0)]);- acc0 += in_q0 *- float(weight[w_base + (uint)((kernel_size - 1) - r0)]);+ acc0 += in_qm1 * float(weight[w_base + (uint)((stride - 1) - r0)]);+ acc0 += in_q0 * float(weight[w_base + (uint)((kernel_size - 1) - r0)]);}if (valid_count > 1) {const int r1 = r0 + 1;- acc1 +=- in_qm1 * float(weight[w_base + (uint)((stride - 1) - r1)]);- acc1 += in_q0 *- float(weight[w_base + (uint)((kernel_size - 1) - r1)]);+ acc1 += in_qm1 * float(weight[w_base + (uint)((stride - 1) - r1)]);+ acc1 += in_q0 * float(weight[w_base + (uint)((kernel_size - 1) - r1)]);}if (valid_count > 2) {const int r2 = r0 + 2;- acc2 +=- in_qm1 * float(weight[w_base + (uint)((stride - 1) - r2)]);- acc2 += in_q0 *- float(weight[w_base + (uint)((kernel_size - 1) - r2)]);+ acc2 += in_qm1 * float(weight[w_base + (uint)((stride - 1) - r2)]);+ acc2 += in_q0 * float(weight[w_base + (uint)((kernel_size - 1) - r2)]);}if (valid_count > 3) {const int r3 = r0 + 3;- acc3 +=- in_qm1 * float(weight[w_base + (uint)((stride - 1) - r3)]);- acc3 += in_q0 *- float(weight[w_base + (uint)((kernel_size - 1) - r3)]);+ acc3 += in_qm1 * float(weight[w_base + (uint)((stride - 1) - r3)]);+ acc3 += in_q0 * float(weight[w_base + (uint)((kernel_size - 1) - r3)]);}} else {// stride < 4: lanes may have different q values.if (valid_count > 0) {if (q0 > 0 && q0 - 1 < seq_len_in) {- const uint in_idx =- (b * (uint)seq_len_in + (uint)(q0 - 1)) * (uint)cin +- (uint)ic;- acc0 += float(input[in_idx]) *- float(weight[w_base + (uint)((stride - 1) - r0)]);+ const uint in_idx = (b * (uint)seq_len_in + (uint)(q0 - 1)) * (uint)cin + (uint)ic;+ acc0 += float(input[in_idx]) * float(weight[w_base + (uint)((stride - 1) - r0)]);}if (q0 >= 0 && q0 < seq_len_in) {- const uint in_idx =- (b * (uint)seq_len_in + (uint)q0) * (uint)cin + (uint)ic;- acc0 += float(input[in_idx]) *- float(weight[w_base + (uint)((kernel_size - 1) - r0)]);+ const uint in_idx = (b * (uint)seq_len_in + (uint)q0) * (uint)cin + (uint)ic;+ acc0 += float(input[in_idx]) * float(weight[w_base + (uint)((kernel_size - 1) - r0)]);}}if (valid_count > 1) {const int q1 = (t0 + 1) / stride;const int r1 = (t0 + 1) % stride;if (q1 > 0 && q1 - 1 < seq_len_in) {- const uint in_idx =- (b * (uint)seq_len_in + (uint)(q1 - 1)) * (uint)cin +- (uint)ic;- acc1 += float(input[in_idx]) *- float(weight[w_base + (uint)((stride - 1) - r1)]);+ const uint in_idx = (b * (uint)seq_len_in + (uint)(q1 - 1)) * (uint)cin + (uint)ic;+ acc1 += float(input[in_idx]) * float(weight[w_base + (uint)((stride - 1) - r1)]);}if (q1 >= 0 && q1 < seq_len_in) {- const uint in_idx =- (b * (uint)seq_len_in + (uint)q1) * (uint)cin + (uint)ic;- acc1 += float(input[in_idx]) *- float(weight[w_base + (uint)((kernel_size - 1) - r1)]);+ const uint in_idx = (b * (uint)seq_len_in + (uint)q1) * (uint)cin + (uint)ic;+ acc1 += float(input[in_idx]) * float(weight[w_base + (uint)((kernel_size - 1) - r1)]);}}if (valid_count > 2) {const int q2 = (t0 + 2) / stride;const int r2 = (t0 + 2) % stride;if (q2 > 0 && q2 - 1 < seq_len_in) {- const uint in_idx =- (b * (uint)seq_len_in + (uint)(q2 - 1)) * (uint)cin +- (uint)ic;- acc2 += float(input[in_idx]) *- float(weight[w_base + (uint)((stride - 1) - r2)]);+ const uint in_idx = (b * (uint)seq_len_in + (uint)(q2 - 1)) * (uint)cin + (uint)ic;+ acc2 += float(input[in_idx]) * float(weight[w_base + (uint)((stride - 1) - r2)]);}if (q2 >= 0 && q2 < seq_len_in) {- const uint in_idx =- (b * (uint)seq_len_in + (uint)q2) * (uint)cin + (uint)ic;- acc2 += float(input[in_idx]) *- float(weight[w_base + (uint)((kernel_size - 1) - r2)]);+ const uint in_idx = (b * (uint)seq_len_in + (uint)q2) * (uint)cin + (uint)ic;+ acc2 += float(input[in_idx]) * float(weight[w_base + (uint)((kernel_size - 1) - r2)]);}}if (valid_count > 3) {const int q3 = (t0 + 3) / stride;const int r3 = (t0 + 3) % stride;if (q3 > 0 && q3 - 1 < seq_len_in) {- const uint in_idx =- (b * (uint)seq_len_in + (uint)(q3 - 1)) * (uint)cin +- (uint)ic;- acc3 += float(input[in_idx]) *- float(weight[w_base + (uint)((stride - 1) - r3)]);+ const uint in_idx = (b * (uint)seq_len_in + (uint)(q3 - 1)) * (uint)cin + (uint)ic;+ acc3 += float(input[in_idx]) * float(weight[w_base + (uint)((stride - 1) - r3)]);}if (q3 >= 0 && q3 < seq_len_in) {- const uint in_idx =- (b * (uint)seq_len_in + (uint)q3) * (uint)cin + (uint)ic;- acc3 += float(input[in_idx]) *- float(weight[w_base + (uint)((kernel_size - 1) - r3)]);+ const uint in_idx = (b * (uint)seq_len_in + (uint)q3) * (uint)cin + (uint)ic;+ acc3 += float(input[in_idx]) * float(weight[w_base + (uint)((kernel_size - 1) - r3)]);}}}@@ -366,13 +303,10 @@ void causal_conv_transpose1d_causal_pad(// Lane 0if (valid_count > 0) {const int expanded_time0 = (int)t_out + k - left_pad;- if (expanded_time0 >= 0 && expanded_time0 < seq_len_expanded &&- (expanded_time0 % stride) == 0) {+ if (expanded_time0 >= 0 && expanded_time0 < seq_len_expanded && (expanded_time0 % stride) == 0) {const int src_time0 = expanded_time0 / stride;if (src_time0 >= 0 && src_time0 < seq_len_in) {- const uint in_idx =- (b * (uint)seq_len_in + (uint)src_time0) * (uint)cin +- (uint)ic;+ const uint in_idx = (b * (uint)seq_len_in + (uint)src_time0) * (uint)cin + (uint)ic;acc0 += float(input[in_idx]) * w;}}@@ -380,13 +314,10 @@ void causal_conv_transpose1d_causal_pad(// Lane 1if (valid_count > 1) {const int expanded_time1 = (int)t_out + 1 + k - left_pad;- if (expanded_time1 >= 0 && expanded_time1 < seq_len_expanded &&- (expanded_time1 % stride) == 0) {+ if (expanded_time1 >= 0 && expanded_time1 < seq_len_expanded && (expanded_time1 % stride) == 0) {const int src_time1 = expanded_time1 / stride;if (src_time1 >= 0 && src_time1 < seq_len_in) {- const uint in_idx =- (b * (uint)seq_len_in + (uint)src_time1) * (uint)cin +- (uint)ic;+ const uint in_idx = (b * (uint)seq_len_in + (uint)src_time1) * (uint)cin + (uint)ic;acc1 += float(input[in_idx]) * w;}}@@ -394,13 +325,10 @@ void causal_conv_transpose1d_causal_pad(// Lane 2if (valid_count > 2) {const int expanded_time2 = (int)t_out + 2 + k - left_pad;- if (expanded_time2 >= 0 && expanded_time2 < seq_len_expanded &&- (expanded_time2 % stride) == 0) {+ if (expanded_time2 >= 0 && expanded_time2 < seq_len_expanded && (expanded_time2 % stride) == 0) {const int src_time2 = expanded_time2 / stride;if (src_time2 >= 0 && src_time2 < seq_len_in) {- const uint in_idx =- (b * (uint)seq_len_in + (uint)src_time2) * (uint)cin +- (uint)ic;+ const uint in_idx = (b * (uint)seq_len_in + (uint)src_time2) * (uint)cin + (uint)ic;acc2 += float(input[in_idx]) * w;}}@@ -408,13 +336,10 @@ void causal_conv_transpose1d_causal_pad(// Lane 3if (valid_count > 3) {const int expanded_time3 = (int)t_out + 3 + k - left_pad;- if (expanded_time3 >= 0 && expanded_time3 < seq_len_expanded &&- (expanded_time3 % stride) == 0) {+ if (expanded_time3 >= 0 && expanded_time3 < seq_len_expanded && (expanded_time3 % stride) == 0) {const int src_time3 = expanded_time3 / stride;if (src_time3 >= 0 && src_time3 < seq_len_in) {- const uint in_idx =- (b * (uint)seq_len_in + (uint)src_time3) * (uint)cin +- (uint)ic;+ const uint in_idx = (b * (uint)seq_len_in + (uint)src_time3) * (uint)cin + (uint)ic;acc3 += float(input[in_idx]) * w;}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/audio/fsq_decode.metal b/crates/backend-uzu/src/backends/metal/kernel/audio/fsq_decode.metalindex 95e3fb2db..486c4f9ea 100644--- a/crates/backend-uzu/src/backends/metal/kernel/audio/fsq_decode.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/audio/fsq_decode.metal@@ -69,15 +69,5 @@ PUBLIC KERNEL(AudioFsqDecode)(uint gd AXIS(num_groups * codebook_dim, 1),uint b AXIS(batch_size, 1)) {- fsq_decode<T>(- tokens,- out,- lengths,- num_groups,- seq_len,- codebook_dim,- num_levels,- dim_base_index,- uint3(t, gd, b)- );+ fsq_decode<T>(tokens, out, lengths, num_groups, seq_len, codebook_dim, num_levels, dim_base_index, uint3(t, gd, b));}diff --git a/crates/backend-uzu/src/backends/metal/kernel/audio/fsq_encode.metal b/crates/backend-uzu/src/backends/metal/kernel/audio/fsq_encode.metalindex 34320fcae..789bddeee 100644--- a/crates/backend-uzu/src/backends/metal/kernel/audio/fsq_encode.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/audio/fsq_encode.metal@@ -44,10 +44,7 @@ void fsq_encode(return;}- const uint in_base =- (b * (uint)(num_groups * codebook_dim) + g * (uint)codebook_dim) *- (uint)seq_len +- t;+ const uint in_base = (b * (uint)(num_groups * codebook_dim) + g * (uint)codebook_dim) * (uint)seq_len + t;int token = 0;for (int d = 0; d < codebook_dim; ++d) {@@ -58,8 +55,7 @@ void fsq_encode(const float input_shift = tan(output_offset / output_scale);const float x = float(input[in_base + (uint)d * (uint)seq_len]);- const float compressed =- output_scale * tanh(x + input_shift) - output_offset;+ const float compressed = output_scale * tanh(x + input_shift) - output_offset;const float rounded = round_ties_to_even(compressed);int code_nonneg = (int)rounded + scale_i;@@ -87,16 +83,6 @@ PUBLIC KERNEL(AudioFsqEncode)(uint g AXIS(num_groups, 1),uint b AXIS(batch_size, 1)) {- fsq_encode<T>(- input,- tokens,- lengths,- num_groups,- seq_len,- codebook_dim,- num_levels,- dim_base_index,- eps,- uint3(t, g, b)- );+ fsq_encode<+ T>(input, tokens, lengths, num_groups, seq_len, codebook_dim, num_levels, dim_base_index, eps, uint3(t, g, b));}diff --git a/crates/backend-uzu/src/backends/metal/kernel/audio/half_snake.metal b/crates/backend-uzu/src/backends/metal/kernel/audio/half_snake.metalindex 84e866842..5ac6454e3 100644--- a/crates/backend-uzu/src/backends/metal/kernel/audio/half_snake.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/audio/half_snake.metal@@ -54,15 +54,5 @@ PUBLIC KERNEL(AudioHalfSnake)(uint c AXIS(channels, 1),uint b AXIS(batch_size, 1)) {- half_snake<T>(- input,- alpha,- output,- channels,- seq_len,- snake_channels,- negative_slope,- eps,- uint3(t, c, b)- );+ half_snake<T>(input, alpha, output, channels, seq_len, snake_channels, negative_slope, eps, uint3(t, c, b));}diff --git a/crates/backend-uzu/src/backends/metal/kernel/audio/norm_ncs.metal b/crates/backend-uzu/src/backends/metal/kernel/audio/norm_ncs.metalindex 7650656d7..3baa8e414 100644--- a/crates/backend-uzu/src/backends/metal/kernel/audio/norm_ncs.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/audio/norm_ncs.metal@@ -48,9 +48,11 @@ void norm_ncs(}}- const float sum = threadgroup_cooperative_reduce<- SimdReduceSum<float>,- AUDIO_NORM_NCS_BLOCK_SIZE>(partial_sum, shared_mean, thread_context);+ const float sum = threadgroup_cooperative_reduce<SimdReduceSum<float>, AUDIO_NORM_NCS_BLOCK_SIZE>(+ partial_sum,+ shared_mean,+ thread_context+ );const float mean = (subtract_mean != 0) ? (sum / (float)channels) : 0.0f;float partial_variance = 0.0f;@@ -61,9 +63,7 @@ void norm_ncs(partial_variance += centered * centered;}- const float variance_sum = threadgroup_cooperative_reduce<- SimdReduceSum<float>,- AUDIO_NORM_NCS_BLOCK_SIZE>(+ const float variance_sum = threadgroup_cooperative_reduce<SimdReduceSum<float>, AUDIO_NORM_NCS_BLOCK_SIZE>(partial_variance,shared_variance,thread_contextdiff --git a/crates/backend-uzu/src/backends/metal/kernel/audio/quantizer_decode.metal b/crates/backend-uzu/src/backends/metal/kernel/audio/quantizer_decode.metalindex ec147ca1d..78b0ebc11 100644--- a/crates/backend-uzu/src/backends/metal/kernel/audio/quantizer_decode.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/audio/quantizer_decode.metal@@ -5,16 +5,15 @@ using namespace metal;template <typename T>void quantizer_decode(- device const uint* tokens, // [B, K, T]- device const int* lengths, // [B]- device const T* semantic_codebook, // [semantic_cardinality, codebook_dim]- device const T* semantic_out_proj, // [input_dim, codebook_dim]- device const T* semantic_out_bias, // [input_dim]- device const T*- residual_codebooks, // [R, residual_cardinality, codebook_dim]- device const T* residual_out_proj, // [R, input_dim, codebook_dim]- device const T* residual_out_bias, // [R, input_dim]- device T* output, // [B, T, input_dim]+ device const uint* tokens, // [B, K, T]+ device const int* lengths, // [B]+ device const T* semantic_codebook, // [semantic_cardinality, codebook_dim]+ device const T* semantic_out_proj, // [input_dim, codebook_dim]+ device const T* semantic_out_bias, // [input_dim]+ device const T* residual_codebooks, // [R, residual_cardinality, codebook_dim]+ device const T* residual_out_proj, // [R, input_dim, codebook_dim]+ device const T* residual_out_bias, // [R, input_dim]+ device T* output, // [B, T, input_dim]const constant int& batch_size,const constant int& total_codebooks,const constant int& seq_len,@@ -48,8 +47,7 @@ void quantizer_decode(const uint semantic_code_base = (uint)semantic_token * (uint)codebook_dim;const uint semantic_proj_base = d * (uint)codebook_dim;for (uint k = 0; k < (uint)codebook_dim; ++k) {- acc += float(semantic_out_proj[semantic_proj_base + k]) *- float(semantic_codebook[semantic_code_base + k]);+ acc += float(semantic_out_proj[semantic_proj_base + k]) * float(semantic_codebook[semantic_code_base + k]);}for (int r = 0; r < residual_quantizers; ++r) {@@ -60,13 +58,10 @@ void quantizer_decode(const uint bias_idx = (uint)r * (uint)input_dim + d;acc += float(residual_out_bias[bias_idx]);- const uint code_base =- ((uint)r * (uint)residual_cardinality + (uint)residual_token) *- (uint)codebook_dim;+ const uint code_base = ((uint)r * (uint)residual_cardinality + (uint)residual_token) * (uint)codebook_dim;const uint proj_base = ((uint)r * (uint)input_dim + d) * (uint)codebook_dim;for (uint k = 0; k < (uint)codebook_dim; ++k) {- acc += float(residual_out_proj[proj_base + k]) *- float(residual_codebooks[code_base + k]);+ acc += float(residual_out_proj[proj_base + k]) * float(residual_codebooks[code_base + k]);}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/common/dsl.h b/crates/backend-uzu/src/backends/metal/kernel/common/dsl.hindex 11922d052..d61fe789d 100644--- a/crates/backend-uzu/src/backends/metal/kernel/common/dsl.h+++ b/crates/backend-uzu/src/backends/metal/kernel/common/dsl.h@@ -13,8 +13,7 @@ using namespace metal;#define DSL_STR(X) #X#define DSL_XSTR(X) DSL_STR(X)-#define VARIANTS(TYPENAME, ...) \- DSL_META("dsl.variants", #TYPENAME, #__VA_ARGS__)+#define VARIANTS(TYPENAME, ...) DSL_META("dsl.variants", #TYPENAME, #__VA_ARGS__)#define CONSTRAINT(C) DSL_META("dsl.constraint", #C)#define PUBLIC DSL_META("dsl.public")#define KERNEL(NAME) DSL_META("dsl.kernel") void NAMEdiff --git a/crates/backend-uzu/src/backends/metal/kernel/common/integral_constant.h b/crates/backend-uzu/src/backends/metal/kernel/common/integral_constant.hindex 4263ddeba..fa36461fc 100644--- a/crates/backend-uzu/src/backends/metal/kernel/common/integral_constant.h+++ b/crates/backend-uzu/src/backends/metal/kernel/common/integral_constant.h@@ -23,14 +23,11 @@ using false_type = bool_constant<false>;template <int val>using Int = integral_constant<int, val>;-#define METAL_INTEGRAL_CONST_BINOP(op, fn) \- template <typename T, T tv, typename U, U uv> \- METAL_FUNC constexpr auto fn( \- integral_constant<T, tv>, \- integral_constant<U, uv> \- ) { \- constexpr auto res = tv op uv; \- return integral_constant<decltype(res), res>{}; \+#define METAL_INTEGRAL_CONST_BINOP(op, fn) \+ template <typename T, T tv, typename U, U uv> \+ METAL_FUNC constexpr auto fn(integral_constant<T, tv>, integral_constant<U, uv>) { \+ constexpr auto res = tv op uv; \+ return integral_constant<decltype(res), res>{}; \}METAL_INTEGRAL_CONST_BINOP(+, operator+)@@ -41,10 +38,7 @@ METAL_INTEGRAL_CONST_BINOP(/, operator/)#undef METAL_INTEGRAL_CONST_BINOPtemplate <int start, int step, typename F, int... Is>-METAL_FUNC constexpr void const_for_loop_impl(- F f,- metal::integer_sequence<int, Is...>-) {+METAL_FUNC constexpr void const_for_loop_impl(F f, metal::integer_sequence<int, Is...>) {(f(Int<start + Is * step>{}), ...);}@@ -52,10 +46,7 @@ template <int start, int stop, int step, typename F>METAL_FUNC constexpr void const_for_loop(F f) {static_assert(step > 0 && start <= stop);constexpr int count = (stop - start + step - 1) / step;- const_for_loop_impl<start, step>(- f,- metal::make_integer_sequence<int, count>{}- );+ const_for_loop_impl<start, step>(f, metal::make_integer_sequence<int, count>{});}template <typename F>diff --git a/crates/backend-uzu/src/backends/metal/kernel/common/tensor_view.h b/crates/backend-uzu/src/backends/metal/kernel/common/tensor_view.hindex 96030c488..76e67c935 100644--- a/crates/backend-uzu/src/backends/metal/kernel/common/tensor_view.h+++ b/crates/backend-uzu/src/backends/metal/kernel/common/tensor_view.h@@ -40,16 +40,14 @@ struct TensorView3D {TensorView3D(device T* ptr) : data(ptr) {}- TensorView3D(device T* ptr, int d0, int d1, int d2)- : data(ptr), dim0(d0), dim1(d1), dim2(d2) {+ TensorView3D(device T* ptr, int d0, int d1, int d2) : data(ptr), dim0(d0), dim1(d1), dim2(d2) {stride0 = d1 * d2;stride1 = d2;stride2 = 1;}TensorView3D(device T* ptr, int d0, int d1, int d2, int s0, int s1, int s2)- : data(ptr), dim0(d0), dim1(d1), dim2(d2), stride0(s0), stride1(s1),- stride2(s2) {}+ : data(ptr), dim0(d0), dim1(d1), dim2(d2), stride0(s0), stride1(s1), stride2(s2) {}thread TensorView3D& shaped(int d0, int d1, int d2) {dim0 = d0;@@ -61,9 +59,7 @@ struct TensorView3D {return *this;}- device T& at(uint i, uint j, uint k) const {- return data[i * stride0 + j * stride1 + k * stride2];- }+ device T& at(uint i, uint j, uint k) const { return data[i * stride0 + j * stride1 + k * stride2]; }device T& operator()(uint i, uint j, uint k) const { return at(i, j, k); }};diff --git a/crates/backend-uzu/src/backends/metal/kernel/common/thread_context.h b/crates/backend-uzu/src/backends/metal/kernel/common/thread_context.hindex 4c0691786..b73d4048d 100644--- a/crates/backend-uzu/src/backends/metal/kernel/common/thread_context.h+++ b/crates/backend-uzu/src/backends/metal/kernel/common/thread_context.h@@ -4,10 +4,10 @@using namespace metal;struct ThreadContext {- uint simd_lane_id; // [[thread_index_in_simdgroup]] — 0..simdgroup_size- uint simdgroup_index; // [[simdgroup_index_in_threadgroup]] —- // 0..simdgroups_per_threadgroup- uint simdgroup_size; // [[threads_per_simdgroup]] — typically 32+ uint simd_lane_id; // [[thread_index_in_simdgroup]] — 0..simdgroup_size+ uint simdgroup_index; // [[simdgroup_index_in_threadgroup]] —+ // 0..simdgroups_per_threadgroup+ uint simdgroup_size; // [[threads_per_simdgroup]] — typically 32uint simdgroups_per_threadgroup; // [[simdgroups_per_threadgroup]]uint3 threadgroup_position; // [[threadgroup_position_in_grid]]uint3 threadgroup_size;diff --git a/crates/backend-uzu/src/backends/metal/kernel/common/threadgroup_reduce.h b/crates/backend-uzu/src/backends/metal/kernel/common/threadgroup_reduce.hindex df37462bc..85756d118 100644--- a/crates/backend-uzu/src/backends/metal/kernel/common/threadgroup_reduce.h+++ b/crates/backend-uzu/src/backends/metal/kernel/common/threadgroup_reduce.h@@ -15,18 +15,16 @@ struct SimdReduceSum {template <typename T>struct SimdReduceMax {using value_type = T;- static constant constexpr T identity = numeric_limits<T>::has_infinity- ? -numeric_limits<T>::infinity()- : numeric_limits<T>::lowest();+ static constant constexpr T identity =+ numeric_limits<T>::has_infinity ? -numeric_limits<T>::infinity() : numeric_limits<T>::lowest();static T simd_reduce(T x) { return simd_max(x); }};template <typename T>struct SimdReduceMin {using value_type = T;- static constant constexpr T identity = numeric_limits<T>::has_infinity- ? numeric_limits<T>::infinity()- : numeric_limits<T>::max();+ static constant constexpr T identity =+ numeric_limits<T>::has_infinity ? numeric_limits<T>::infinity() : numeric_limits<T>::max();static T simd_reduce(T x) { return simd_min(x); }};@@ -47,9 +45,7 @@ static typename Op::value_type threadgroup_cooperative_reduce(// Phase 2: first simdgroup reduces across simdgropusif (thread_context.simdgroup_index == 0) {typename Op::value_type total =- thread_context.simd_lane_id <- (BLOCK_SIZE + thread_context.simdgroup_size - 1) /- thread_context.simdgroup_size+ thread_context.simd_lane_id < (BLOCK_SIZE + thread_context.simdgroup_size - 1) / thread_context.simdgroup_size? shared[thread_context.simd_lane_id]: Op::identity;total = Op::simd_reduce(total);diff --git a/crates/backend-uzu/src/backends/metal/kernel/delta_net/norm_gate.metal b/crates/backend-uzu/src/backends/metal/kernel/delta_net/norm_gate.metalindex f8811bd0b..7933c475f 100644--- a/crates/backend-uzu/src/backends/metal/kernel/delta_net/norm_gate.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/delta_net/norm_gate.metal@@ -42,18 +42,13 @@ PUBLIC KERNEL(DeltaNetNormGate)(// RMSNorm: cooperative sum of squaresfloat o_sq = active ? o_i * o_i : 0.0f;float o_sumsq =- threadgroup_cooperative_reduce<SimdReduceSum<float>, HEAD_V_DIM>(- o_sq,- shared_scratch,- thread_context- );+ threadgroup_cooperative_reduce<SimdReduceSum<float>, HEAD_V_DIM>(o_sq, shared_scratch, thread_context);float inv_rms = rsqrt(o_sumsq / float(head_v_dim) + norm_epsilon);// Apply norm + SiLU gate (in-place)if (active) {float nw = float(norm_weight[tid]);- uint z_idx =- token_idx * total_proj_dim + conv_dim + hv_idx * head_v_dim + tid;+ uint z_idx = token_idx * total_proj_dim + conv_dim + hv_idx * head_v_dim + tid;float z_silu = activate_silu(float(in_proj[z_idx]));in_out[base + tid] = static_cast<T>(o_i * inv_rms * nw * z_silu);}diff --git a/crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill.metal b/crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill.metalindex aebfcdfe6..822cf8918 100644--- a/crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill.metal@@ -10,10 +10,7 @@ using namespace metal;#define PREFILL_THREADS 128#define DV_PER_SIMDGROUP 4-static_assert(- PREFILL_THREADS % METAL_SIMD_SIZE == 0,- "PREFILL_THREADS must be a multiple of METAL_SIMD_SIZE"-);+static_assert(PREFILL_THREADS % METAL_SIMD_SIZE == 0, "PREFILL_THREADS must be a multiple of METAL_SIMD_SIZE");template <typename T, uint HEAD_K_DIM>VARIANTS(T, float, half, bfloat)@@ -37,10 +34,7 @@ PUBLIC KERNEL(DeltaNetPrefill)(const uint dv_group GROUPS(num_dv_groups),const uint tid THREADS(PREFILL_THREADS)) {- static_assert(- HEAD_K_DIM % METAL_SIMD_SIZE == 0,- "HEAD_K_DIM must be a multiple of METAL_SIMD_SIZE"- );+ static_assert(HEAD_K_DIM % METAL_SIMD_SIZE == 0, "HEAD_K_DIM must be a multiple of METAL_SIMD_SIZE");constexpr uint ELEMS = HEAD_K_DIM / METAL_SIMD_SIZE;constexpr uint NUM_SG = PREFILL_THREADS / METAL_SIMD_SIZE;static_assert(ELEMS == 4, "float4 prefill requires ELEMS == 4");@@ -66,8 +60,7 @@ PUBLIC KERNEL(DeltaNetPrefill)(// State layout: [Hv, Dv, Dk], contiguous along Dk (HEAD_K_DIM floats/row).// Unrolled by hand: METAL_PRAGMA_UNROLL is ignored under metal4.0 -O2, so a// loop over s[i] keeps the array in an alloca and reloads it every token.- device const float* s_in =- state + (hv_idx * head_v_dim + dv_idx) * HEAD_K_DIM + dk_base;+ device const float* s_in = state + (hv_idx * head_v_dim + dv_idx) * HEAD_K_DIM + dk_base;float4 s[DV_PER_SIMDGROUP];s[0] = *reinterpret_cast<device const float4*>(s_in + 0 * HEAD_K_DIM);s[1] = *reinterpret_cast<device const float4*>(s_in + 1 * HEAD_K_DIM);@@ -84,16 +77,10 @@ PUBLIC KERNEL(DeltaNetPrefill)(s[1] *= decay;s[2] *= decay;s[3] *= decay;- float4 kv_mem =- float4(dot(s[0], k), dot(s[1], k), dot(s[2], k), dot(s[3], k));+ float4 kv_mem = float4(dot(s[0], k), dot(s[1], k), dot(s[2], k), dot(s[3], k));kv_mem = simd_sum(kv_mem);- float4 v_val = float4(- float(v_row[0]),- float(v_row[1]),- float(v_row[2]),- float(v_row[3])- );+ float4 v_val = float4(float(v_row[0]), float(v_row[1]), float(v_row[2]), float(v_row[3]));float4 delta = beta * (v_val - kv_mem);s[0] += k * delta[0];s[1] += k * delta[1];@@ -117,8 +104,7 @@ PUBLIC KERNEL(DeltaNetPrefill)(out_row += value_dim;}- device float* s_out =- state + (hv_idx * head_v_dim + dv_idx) * HEAD_K_DIM + dk_base;+ device float* s_out = state + (hv_idx * head_v_dim + dv_idx) * HEAD_K_DIM + dk_base;*reinterpret_cast<device float4*>(s_out + 0 * HEAD_K_DIM) = s[0];*reinterpret_cast<device float4*>(s_out + 1 * HEAD_K_DIM) = s[1];*reinterpret_cast<device float4*>(s_out + 2 * HEAD_K_DIM) = s[2];diff --git a/crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill_prep.metal b/crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill_prep.metalindex 937d73719..f78cf62e8 100644--- a/crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill_prep.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/delta_net/prefill_prep.metal@@ -29,10 +29,7 @@ PUBLIC KERNEL(DeltaNetPrefillPrep)(const uint hk_idx GROUPS(num_k_heads),const uint lane THREADS(METAL_SIMD_SIZE)) {- static_assert(- HEAD_K_DIM % METAL_SIMD_SIZE == 0,- "HEAD_K_DIM must be a multiple of METAL_SIMD_SIZE"- );+ static_assert(HEAD_K_DIM % METAL_SIMD_SIZE == 0, "HEAD_K_DIM must be a multiple of METAL_SIMD_SIZE");constexpr uint elems_per_thread = HEAD_K_DIM / METAL_SIMD_SIZE;const uint conv_dim = 2 * key_dim + value_dim;@@ -86,8 +83,7 @@ PUBLIC KERNEL(DeltaNetPrefillPrep)(float beta_raw = float(in_proj[tok_offset + conv_dim + value_dim + hv]);float beta = 1.0f / (1.0f + fast::exp(-beta_raw));- float a_raw =- float(in_proj[tok_offset + conv_dim + value_dim + num_v_heads + hv]);+ float a_raw = float(in_proj[tok_offset + conv_dim + value_dim + num_v_heads + hv]);float sp = activate_softplus(a_raw + float(dt_bias[hv]));float decay = fast::exp(-fast::exp(float(a_log[hv])) * sp);diff --git a/crates/backend-uzu/src/backends/metal/kernel/delta_net/update.metal b/crates/backend-uzu/src/backends/metal/kernel/delta_net/update.metalindex 9a6dc76bf..9790551f8 100644--- a/crates/backend-uzu/src/backends/metal/kernel/delta_net/update.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/delta_net/update.metal@@ -35,14 +35,9 @@ PUBLIC KERNEL(DeltaNetUpdate)(const uint hv_idx GROUPS(num_v_heads),const uint tid THREADS(UPDATE_THREADS)) {- static_assert(- HEAD_K_DIM % METAL_SIMD_SIZE == 0,- "HEAD_K_DIM must be a multiple of METAL_SIMD_SIZE"- );- constexpr uint ELEMS =- HEAD_K_DIM / METAL_SIMD_SIZE; // Dk per lane (128/32 = 4)- constexpr uint NUM_SG =- UPDATE_THREADS / METAL_SIMD_SIZE; // simd groups / tg (16)+ static_assert(HEAD_K_DIM % METAL_SIMD_SIZE == 0, "HEAD_K_DIM must be a multiple of METAL_SIMD_SIZE");+ constexpr uint ELEMS = HEAD_K_DIM / METAL_SIMD_SIZE; // Dk per lane (128/32 = 4)+ constexpr uint NUM_SG = UPDATE_THREADS / METAL_SIMD_SIZE; // simd groups / tg (16)const uint lane = tid % METAL_SIMD_SIZE;const uint sg = tid / METAL_SIMD_SIZE;@@ -78,8 +73,7 @@ PUBLIC KERNEL(DeltaNetUpdate)(// beta / decay (scalar per head)const float beta_raw = float(in_proj[conv_dim + value_dim + hv_idx]);const float beta = 1.0f / (1.0f + fast::exp(-beta_raw));- const float a_raw =- float(in_proj[conv_dim + value_dim + num_v_heads + hv_idx]);+ const float a_raw = float(in_proj[conv_dim + value_dim + num_v_heads + hv_idx]);const float sp = activate_softplus(a_raw + float(dt_bias[hv_idx]));const float decay = fast::exp(-fast::exp(float(a_log[hv_idx])) * sp);@@ -122,11 +116,7 @@ PUBLIC KERNEL(DeltaNetUpdate)(o_sq += o * o;}const float o_sumsq =- threadgroup_cooperative_reduce<SimdReduceSum<float>, UPDATE_THREADS>(- o_sq,- shared_scratch,- thread_context- );+ threadgroup_cooperative_reduce<SimdReduceSum<float>, UPDATE_THREADS>(o_sq, shared_scratch, thread_context);const float inv_rms = rsqrt(o_sumsq / float(head_v_dim) + norm_epsilon);for (uint dv = sg; dv < head_v_dim; dv += NUM_SG) {@@ -134,8 +124,7 @@ PUBLIC KERNEL(DeltaNetUpdate)(const float nw = float(norm_weight[dv]);const float z_i = float(in_proj[conv_dim + hv_idx * head_v_dim + dv]);const float z_silu = activate_silu(z_i);- out[hv_idx * head_v_dim + dv] =- static_cast<T>(shared_o[dv] * inv_rms * nw * z_silu);+ out[hv_idx * head_v_dim + dv] = static_cast<T>(shared_o[dv] * inv_rms * nw * z_silu);}}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/embedding/quant_embedding.metal b/crates/backend-uzu/src/backends/metal/kernel/embedding/quant_embedding.metalindex b7521d1a2..5a885f0ce 100644--- a/crates/backend-uzu/src/backends/metal/kernel/embedding/quant_embedding.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/embedding/quant_embedding.metal@@ -37,14 +37,11 @@ PUBLIC KERNEL(QuantizedEmbeddingLookup) (const uint group_idx = dim_idx / group_size;const uint num_groups = (model_dim + group_size - 1) / group_size;- const uint zp_stride = quantization_mode == QuantizationMode::U4- ? (num_groups + 1) / 2- : num_groups;+ const uint zp_stride = quantization_mode == QuantizationMode::U4 ? (num_groups + 1) / 2 : num_groups;const T scale = scales[token_id * num_groups + group_idx];- const uint packing_divisor =- quantization_mode == QuantizationMode::U4 ? 2 : 1;+ const uint packing_divisor = quantization_mode == QuantizationMode::U4 ? 2 : 1;const uint weights_stride = model_dim / packing_divisor;int quantized_value = 0;@@ -61,8 +58,7 @@ PUBLIC KERNEL(QuantizedEmbeddingLookup) (};case QuantizationMode::I8: {const uint elem_idx = token_id * weights_stride + dim_idx;- const device int8_t* weights_i8 =- reinterpret_cast<const device int8_t*>(weights);+ const device int8_t* weights_i8 = reinterpret_cast<const device int8_t*>(weights);quantized_value = int(weights_i8[elem_idx]);break;};@@ -80,8 +76,7 @@ PUBLIC KERNEL(QuantizedEmbeddingLookup) (uint zero_point = 0;if (quantization_mode == QuantizationMode::U4) {const uint8_t packed = zero_points[token_id * zp_stride + group_idx / 2];- zero_point =- (group_idx & 1) == 0 ? (packed & 0x0F) : ((packed >> 4) & 0x0F);+ zero_point = (group_idx & 1) == 0 ? (packed & 0x0F) : ((packed >> 4) & 0x0F);} else {zero_point = zero_points[token_id * zp_stride + group_idx];}diff --git a/crates/backend-uzu/src/backends/metal/kernel/gated_act_mul/gated_act_mul.metal b/crates/backend-uzu/src/backends/metal/kernel/gated_act_mul/gated_act_mul.metalindex 8ba468d2d..ef6ca3413 100644--- a/crates/backend-uzu/src/backends/metal/kernel/gated_act_mul/gated_act_mul.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/gated_act_mul/gated_act_mul.metal@@ -29,10 +29,8 @@ PUBLIC KERNEL(GatedActMul) (value = act_operand[base + gated_idx];activated = activate(act_operand[base + gated_dim + gated_idx], act_type);} else {- activated =- activate(act_operand[batch_idx * gated_dim + gated_idx], act_type);- value =- value_operand[batch_idx * value_row_stride + value_offset + gated_idx];+ activated = activate(act_operand[batch_idx * gated_dim + gated_idx], act_type);+ value = value_operand[batch_idx * value_row_stride + value_offset + gated_idx];}T result = value * activated;diff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/gemm.h b/crates/backend-uzu/src/backends/metal/kernel/generated/gemm.hindex a2e90ab48..05d191ae7 100644--- a/crates/backend-uzu/src/backends/metal/kernel/generated/gemm.h+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/gemm.h@@ -20,12 +20,8 @@ struct GemmDTransform {static constant constexpr uint ACCUMULATE = 1 << 1;static constant constexpr uint BIAS = 1 << 2;static constant constexpr uint RHT = 1 << 3;- constexpr bool contains(uint flag) const thread {- return (raw_value & flag) != 0;- }- constexpr bool contains(uint flag) const constant {- return (raw_value & flag) != 0;- }+ constexpr bool contains(uint flag) const thread { return (raw_value & flag) != 0; }+ constexpr bool contains(uint flag) const constant { return (raw_value & flag) != 0; }constexpr uint bits() const thread { return raw_value; }constexpr uint bits() const constant { return raw_value; }};@@ -49,12 +45,8 @@ struct GemmAlignment {static constant constexpr uint M = 1 << 0;static constant constexpr uint N = 1 << 1;static constant constexpr uint K = 1 << 2;- constexpr bool contains(uint flag) const thread {- return (raw_value & flag) != 0;- }- constexpr bool contains(uint flag) const constant {- return (raw_value & flag) != 0;- }+ constexpr bool contains(uint flag) const thread { return (raw_value & flag) != 0; }+ constexpr bool contains(uint flag) const constant { return (raw_value & flag) != 0; }constexpr uint bits() const thread { return raw_value; }constexpr uint bits() const constant { return raw_value; }};diff --git a/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.h b/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.hindex 10bc5ce2f..810c91953 100644--- a/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.h+++ b/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.h@@ -6,39 +6,21 @@using namespace metal;-static METAL_FUNC float simdgroup_hadamard_transform(- ushort lane_index,- float lane_value-) {+static METAL_FUNC float simdgroup_hadamard_transform(ushort lane_index, float lane_value) {for (ushort stride = 1; stride < METAL_SIMD_SIZE; stride <<= 1) {float partner_lane_value = simd_shuffle_xor(lane_value, stride);- lane_value = (lane_index & stride) ? (partner_lane_value - lane_value)- : (partner_lane_value + lane_value);+ lane_value = (lane_index & stride) ? (partner_lane_value - lane_value) : (partner_lane_value + lane_value);}return lane_value / sqrt((float)METAL_SIMD_SIZE);}template <typename T>-static METAL_FUNC T simdgroup_input_random_hadamard_transform(- ushort lane_index,- T lane_value,- int32_t lane_factor-) {- return T(simdgroup_hadamard_transform(- lane_index,- float(lane_value) * float(lane_factor)- ));+static METAL_FUNC T simdgroup_input_random_hadamard_transform(ushort lane_index, T lane_value, int32_t lane_factor) {+ return T(simdgroup_hadamard_transform(lane_index, float(lane_value) * float(lane_factor)));}template <typename T>-static METAL_FUNC T simdgroup_output_random_hadamard_transform(- ushort lane_index,- T lane_value,- int32_t lane_factor-) {- return T(- simdgroup_hadamard_transform(lane_index, float(lane_value)) *- float(lane_factor)- );+static METAL_FUNC T simdgroup_output_random_hadamard_transform(ushort lane_index, T lane_value, int32_t lane_factor) {+ return T(simdgroup_hadamard_transform(lane_index, float(lane_value)) * float(lane_factor));}diff --git a/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.metal b/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.metalindex 6678f8060..f09eb4b60 100644--- a/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.metal@@ -23,16 +23,10 @@ PUBLIC KERNEL(HadamardTransform)(uint element_index = batch_index * hidden_dim + factor_index;if (transform_order == HadamardTransformOrder::Input) {- data[element_index] = simdgroup_input_random_hadamard_transform(- lane_index,- data[element_index],- factors[factor_index]- );+ data[element_index] =+ simdgroup_input_random_hadamard_transform(lane_index, data[element_index], factors[factor_index]);} else {- data[element_index] = simdgroup_output_random_hadamard_transform(- lane_index,- data[element_index],- factors[factor_index]- );+ data[element_index] =+ simdgroup_output_random_hadamard_transform(lane_index, data[element_index], factors[factor_index]);}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal b/crates/backend-uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metalindex 9293e927a..76582bca9 100644--- a/crates/backend-uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal@@ -27,10 +27,8 @@ PUBLIC KERNEL(KVCacheUpdate) (// Offset = token_idx * num_heads * head_dim +// head_idx * head_dim + channel_idxconst uint head_offset = head_idx * head_dim;- const uint sourceIdx =- swaps[i].source * num_heads * head_dim + head_offset + channel_idx;- const uint destIdx =- swaps[i].destination * num_heads * head_dim + head_offset + channel_idx;+ const uint sourceIdx = swaps[i].source * num_heads * head_dim + head_offset + channel_idx;+ const uint destIdx = swaps[i].destination * num_heads * head_dim + head_offset + channel_idx;swap(in_place_keys, sourceIdx, destIdx);swap(in_place_values, sourceIdx, destIdx);diff --git a/crates/backend-uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal b/crates/backend-uzu/src/backends/metal/kernel/layer_norm/layer_norm.metalindex 92c946f22..902c624c2 100644--- a/crates/backend-uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal@@ -25,8 +25,7 @@ void layer_norm_core(AccumT partial_sum = static_cast<AccumT>(0.0f);// Compute mean: sum all elements- for (uint base_i = thread_in_row * GRAIN_SIZE; base_i < element_count;- base_i += BLOCK_SIZE * GRAIN_SIZE) {+ for (uint base_i = thread_in_row * GRAIN_SIZE; base_i < element_count; base_i += BLOCK_SIZE * GRAIN_SIZE) {for (uint j = 0; j < GRAIN_SIZE; ++j) {uint i = base_i + j;if (i < element_count) {@@ -37,17 +36,12 @@ void layer_norm_core(// Reduce to get total meanAccumT total_sum =- threadgroup_cooperative_reduce<SimdReduceSum<AccumT>, BLOCK_SIZE>(- partial_sum,- shared_mean,- thread_context- );+ threadgroup_cooperative_reduce<SimdReduceSum<AccumT>, BLOCK_SIZE>(partial_sum, shared_mean, thread_context);AccumT mean = total_sum / static_cast<AccumT>(element_count);// Compute variance: sum of squared deviationsAccumT partial_var_sum = static_cast<AccumT>(0.0f);- for (uint base_i = thread_in_row * GRAIN_SIZE; base_i < element_count;- base_i += BLOCK_SIZE * GRAIN_SIZE) {+ for (uint base_i = thread_in_row * GRAIN_SIZE; base_i < element_count; base_i += BLOCK_SIZE * GRAIN_SIZE) {for (uint j = 0; j < GRAIN_SIZE; ++j) {uint i = base_i + j;if (i < element_count) {@@ -58,27 +52,23 @@ void layer_norm_core(}// Reduce to get total variance- AccumT total_var =- threadgroup_cooperative_reduce<SimdReduceSum<AccumT>, BLOCK_SIZE>(- partial_var_sum,- shared_variance,- thread_context- );+ AccumT total_var = threadgroup_cooperative_reduce<SimdReduceSum<AccumT>, BLOCK_SIZE>(+ partial_var_sum,+ shared_variance,+ thread_context+ );AccumT variance = total_var / static_cast<AccumT>(element_count);AccumT inv_std = rsqrt(variance + static_cast<AccumT>(epsilon));// Apply normalization and scaling- for (uint base_i = thread_in_row * GRAIN_SIZE; base_i < element_count;- base_i += BLOCK_SIZE * GRAIN_SIZE) {+ for (uint base_i = thread_in_row * GRAIN_SIZE; base_i < element_count; base_i += BLOCK_SIZE * GRAIN_SIZE) {AccumT vals[GRAIN_SIZE];AccumT scaled_vals[GRAIN_SIZE];// Load and centerfor (uint j = 0; j < GRAIN_SIZE; ++j) {uint i = base_i + j;- vals[j] = (i < element_count)- ? (static_cast<AccumT>(input_data[i]) - mean)- : 0.0f;+ vals[j] = (i < element_count) ? (static_cast<AccumT>(input_data[i]) - mean) : 0.0f;}// Normalize and scale@@ -91,15 +81,12 @@ void layer_norm_core(AccumT normalized_high = vals[j] * inv_std;if (full_layer) {- AccumT scale_value_high = static_cast<AccumT>(scales_data[i]) +- static_cast<AccumT>(scale_offset);+ AccumT scale_value_high = static_cast<AccumT>(scales_data[i]) + static_cast<AccumT>(scale_offset);scaled_vals[j] = normalized_high * scale_value_high;} else {OutputT normalized_low = static_cast<OutputT>(normalized_high);- OutputT scale_value_low = static_cast<OutputT>(- static_cast<AccumT>(scales_data[i]) +- static_cast<AccumT>(scale_offset)- );+ OutputT scale_value_low =+ static_cast<OutputT>(static_cast<AccumT>(scales_data[i]) + static_cast<AccumT>(scale_offset));OutputT product_low = normalized_low * scale_value_low;scaled_vals[j] = static_cast<AccumT>(product_low);}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/defines.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/defines.hindex 5781cdf8c..93691392c 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/defines.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/defines.h@@ -35,12 +35,8 @@ template <typename OutputType, typename InputType>struct TransformNone {TransformNone(float = 1.0f, float = 0.0f) {}- static METAL_FUNC OutputType apply(InputType x) {- return static_cast<OutputType>(x);- }- METAL_FUNC OutputType apply(InputType x, OutputType) const {- return static_cast<OutputType>(x);- }+ static METAL_FUNC OutputType apply(InputType x) { return static_cast<OutputType>(x); }+ METAL_FUNC OutputType apply(InputType x, OutputType) const { return static_cast<OutputType>(x); }};template <typename OutputType, typename InputType>@@ -48,18 +44,14 @@ struct TransformScaleAccumulate {const float alpha;const float beta;- TransformScaleAccumulate(float alpha, float beta)- : alpha(alpha), beta(beta) {}+ TransformScaleAccumulate(float alpha, float beta) : alpha(alpha), beta(beta) {}- static METAL_FUNC OutputType apply(InputType x) {- return static_cast<OutputType>(x);- }+ static METAL_FUNC OutputType apply(InputType x) { return static_cast<OutputType>(x); }METAL_FUNC OutputType apply(InputType x, OutputType c) const {return static_cast<OutputType>(x * static_cast<InputType>(alpha) +- (beta != 0.0f ? (static_cast<OutputType>(beta) * c)- : static_cast<OutputType>(0.0))+ (beta != 0.0f ? (static_cast<OutputType>(beta) * c) : static_cast<OutputType>(0.0)));}};diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.hindex 3f2973d06..6c3f08682 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h@@ -28,8 +28,7 @@ struct Fragment {"Ops must expose positive FRAGMENT_ROWS/FRAGMENT_COLS");static_assert(- Ops::ELEMENTS_PER_THREAD ==- (Ops::FRAGMENT_ROWS * Ops::FRAGMENT_COLS) / METAL_SIMD_SIZE,+ Ops::ELEMENTS_PER_THREAD == (Ops::FRAGMENT_ROWS * Ops::FRAGMENT_COLS) / METAL_SIMD_SIZE,"Ops::ELEMENTS_PER_THREAD must equal ""(FRAGMENT_ROWS * FRAGMENT_COLS) / METAL_SIMD_SIZE");@@ -51,26 +50,20 @@ struct Fragment {ThreadVectorType fragment_data[NUM_FRAGS];ThreadContext thread_context;- METAL_FUNC Fragment(const thread ThreadContext& thread_context) thread- : thread_context(thread_context) {}+ METAL_FUNC Fragment(const thread ThreadContext& thread_context) thread : thread_context(thread_context) {}// Lane origin (row, col) within a single FRAGMENT_ROWS x FRAGMENT_COLS// sub-tile. The mapping depends only on Ops::THREAD_ELEMENT_COLS, so it is// identical for every Fragment sharing that Ops type.- METAL_FUNC static constexpr short2 get_position(- const thread ThreadContext& thread_context- ) {+ METAL_FUNC static constexpr short2 get_position(const thread ThreadContext& thread_context) {const ushort simd_lane_id = ushort(thread_context.simd_lane_id);const short quad = simd_lane_id / 4;const short row = (quad & 4) + (simd_lane_id / 2) % 4;- const short col =- ((quad & 2) + simd_lane_id % 2) * Ops::THREAD_ELEMENT_COLS;+ const short col = ((quad & 2) + simd_lane_id % 2) * Ops::THREAD_ELEMENT_COLS;return short2{col, row};}- METAL_FUNC short2 get_position() const thread {- return get_position(thread_context);- }+ METAL_FUNC short2 get_position() const thread { return get_position(thread_context); }METAL_FUNC constexpr void clear() {METAL_PRAGMA_UNROLL@@ -79,10 +72,7 @@ struct Fragment {}}- METAL_FUNC constexpr thread ThreadVectorType& fragment_at(- const ushort row_index,- const ushort col_index- ) {+ METAL_FUNC constexpr thread ThreadVectorType& fragment_at(const ushort row_index, const ushort col_index) {return fragment_data[row_index * TILE_COLS + col_index];}@@ -99,9 +89,7 @@ struct Fragment {}}- METAL_FUNC thread ElementType* elements() {- return reinterpret_cast<thread ElementType*>(fragment_data);- }+ METAL_FUNC thread ElementType* elements() { return reinterpret_cast<thread ElementType*>(fragment_data); }// Unsafe load: copy a (TILE_ROWS * FRAGMENT_ROWS) x (TILE_COLS *// FRAGMENT_COLS) block from device memory into fragment registers. ColStride@@ -121,22 +109,11 @@ struct Fragment {TileRowStride tile_row_stride = {},TileColStride tile_col_stride = {}) thread {- transfer<LOAD, UNSAFE>(- source,- row_stride,- col_stride,- Int<0>{},- Int<0>{},- tile_row_stride,- tile_col_stride- );+ transfer<LOAD, UNSAFE>(source, row_stride, col_stride, Int<0>{}, Int<0>{}, tile_row_stride, tile_col_stride);}// Safe load: col_stride is implicitly 1; out-of-bounds elements become T(0).- template <- class Ptr,- class TileRowStride = Int<1>,- class TileColStride = Int<1>>+ template <class Ptr, class TileRowStride = Int<1>, class TileColStride = Int<1>>METAL_FUNC void load_safe(Ptr source,const int leading_dimension,@@ -169,22 +146,11 @@ struct Fragment {TileRowStride tile_row_stride = {},TileColStride tile_col_stride = {}) thread {- transfer<STORE, UNSAFE>(- destination,- row_stride,- col_stride,- Int<0>{},- Int<0>{},- tile_row_stride,- tile_col_stride- );+ transfer<STORE, UNSAFE>(destination, row_stride, col_stride, Int<0>{}, Int<0>{}, tile_row_stride, tile_col_stride);}// Safe store: col_stride is implicitly 1; out-of-bounds elements are skipped.- template <- class Ptr,- class TileRowStride = Int<1>,- class TileColStride = Int<1>>+ template <class Ptr, class TileRowStride = Int<1>, class TileColStride = Int<1>>METAL_FUNC void store_safe(Ptr destination,const int leading_dimension,@@ -212,9 +178,7 @@ private:template <class Fn>METAL_FUNC void for_each_fragment(Fn fn) thread {const_for_loop<0, TILE_ROWS, 1>([&](auto row_index) {- const_for_loop<0, TILE_COLS, 1>([&](auto col_index) {- fn(row_index, col_index);- });+ const_for_loop<0, TILE_COLS, 1>([&](auto col_index) { fn(row_index, col_index); });});}@@ -266,16 +230,12 @@ private:for (ushort j = 0; j < Ops::THREAD_ELEMENT_COLS; j++) {const ushort element_index = i * Ops::THREAD_ELEMENT_COLS + j;const auto col = col_base + j;- const auto offset = col_stride_is_one- ? (row * row_stride + col)- : (row * row_stride + col * col_stride);+ const auto offset = col_stride_is_one ? (row * row_stride + col) : (row * row_stride + col * col_stride);if constexpr (IS_LOAD) {if constexpr (IS_SAFE) {- const bool in_bounds =- (row < local_row_limit) && (col < local_col_limit);- frag[element_index] =- in_bounds ? static_cast<T>(ptr[offset]) : T(0);+ const bool in_bounds = (row < local_row_limit) && (col < local_col_limit);+ frag[element_index] = in_bounds ? static_cast<T>(ptr[offset]) : T(0);} else {frag[element_index] = static_cast<T>(ptr[offset]);}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/loader.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/loader.hindex a1a2f1a9b..abf7c3c6a 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/loader.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/loader.h@@ -20,13 +20,11 @@ template <ushort REDUCTION_DIMENSION,ushort THREADGROUP_SIZE,ushort ALIGNMENT = 1,- ushort READS_PER_THREAD =- (THREADGROUP_TILE_COLS * THREADGROUP_TILE_ROWS) / (THREADGROUP_SIZE),+ ushort READS_PER_THREAD = (THREADGROUP_TILE_COLS * THREADGROUP_TILE_ROWS) / (THREADGROUP_SIZE),ushort THREAD_COLS = THREADGROUP_TILE_COLS / READS_PER_THREAD,ushort THREAD_ROWS = THREADGROUP_SIZE / THREAD_COLS>struct ThreadgroupLoader {- METAL_CONST ushort ROW_ITERATIONS =- (THREADGROUP_TILE_ROWS + THREAD_ROWS - 1) / THREAD_ROWS;+ METAL_CONST ushort ROW_ITERATIONS = (THREADGROUP_TILE_ROWS + THREAD_ROWS - 1) / THREAD_ROWS;const int source_leading_dimension;const int tile_stride;@@ -49,23 +47,11 @@ struct ThreadgroupLoader {const thread ThreadContext& thread_context): source_leading_dimension(source_leading_dim),- tile_stride(- REDUCTION_DIMENSION ? THREADGROUP_TILE_COLS- : THREADGROUP_TILE_ROWS * source_leading_dim- ),- thread_index(- thread_context.simdgroup_index * METAL_SIMD_SIZE +- thread_context.simd_lane_id- ),- tile_row_index(thread_index / THREAD_COLS),- tile_col_index(READS_PER_THREAD * (thread_index % THREAD_COLS)),- destination(- destination_ptr + tile_row_index * DESTINATION_LEADING_DIMENSION +- tile_col_index- ),- source(- source_ptr + tile_row_index * source_leading_dim + tile_col_index- ) {}+ tile_stride(REDUCTION_DIMENSION ? THREADGROUP_TILE_COLS : THREADGROUP_TILE_ROWS * source_leading_dim),+ thread_index(thread_context.simdgroup_index * METAL_SIMD_SIZE + thread_context.simd_lane_id),+ tile_row_index(thread_index / THREAD_COLS), tile_col_index(READS_PER_THREAD * (thread_index % THREAD_COLS)),+ destination(destination_ptr + tile_row_index * DESTINATION_LEADING_DIMENSION + tile_col_index),+ source(source_ptr + tile_row_index * source_leading_dim + tile_col_index) {}template <typename UnaryOp>METAL_FUNC void apply_inplace_op(thread const UnaryOp& operation) const {@@ -82,18 +68,13 @@ struct ThreadgroupLoader {METAL_FUNC void load_unsafe() const {METAL_PRAGMA_UNROLLfor (ushort i = 0; i < THREADGROUP_TILE_ROWS; i += THREAD_ROWS) {- *reinterpret_cast<threadgroup ReadVector*>(- &destination[i * DESTINATION_LEADING_DIMENSION]- ) =- *reinterpret_cast<const device ReadVector*>(- &source[i * source_leading_dimension]- );+ *reinterpret_cast<threadgroup ReadVector*>(&destination[i * DESTINATION_LEADING_DIMENSION]) =+ *reinterpret_cast<const device ReadVector*>(&source[i * source_leading_dimension]);}}METAL_FUNC void load_safe(short2 source_tile_dimensions) const {- source_tile_dimensions =- source_tile_dimensions - short2(tile_col_index, tile_row_index);+ source_tile_dimensions = source_tile_dimensions - short2(tile_col_index, tile_row_index);if (source_tile_dimensions.x <= 0 || source_tile_dimensions.y <= 0) {METAL_PRAGMA_UNROLL@@ -113,14 +94,12 @@ struct ThreadgroupLoader {for (ushort i = 0; i < THREADGROUP_TILE_ROWS; i += THREAD_ROWS) {METAL_PRAGMA_UNROLLfor (ushort j = 0; j < READS_PER_THREAD; j++) {- valid_mask[j] =- (i < source_tile_dimensions.y) && (j < source_tile_dimensions.x);+ valid_mask[j] = (i < source_tile_dimensions.y) && (j < source_tile_dimensions.x);}METAL_PRAGMA_UNROLLfor (ushort j = 0; j < READS_PER_THREAD; j++) {- loaded_values[j] =- source[(valid_mask[j] ? i * source_leading_dimension + j : 0)];+ loaded_values[j] = source[(valid_mask[j] ? i * source_leading_dimension + j : 0)];}METAL_PRAGMA_UNROLLdiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment_ops.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment_ops.hindex 9b31e884a..cce0638d8 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment_ops.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment_ops.h@@ -21,8 +21,7 @@ struct MxuFragmentOps {METAL_CONST ushort FRAGMENT_ROWS = 16;METAL_CONST ushort FRAGMENT_COLS = 16;- METAL_CONST ushort ELEMENTS_PER_THREAD =- (FRAGMENT_ROWS * FRAGMENT_COLS) / METAL_SIMD_SIZE;+ METAL_CONST ushort ELEMENTS_PER_THREAD = (FRAGMENT_ROWS * FRAGMENT_COLS) / METAL_SIMD_SIZE;METAL_CONST ushort THREAD_ELEMENT_ROWS = 2;METAL_CONST ushort THREAD_ELEMENT_COLS = 4;@@ -41,13 +40,7 @@ struct MxuFragmentOps {// supplies a `marshal_inputs` callable that copies its fragments into the// cooperative left/right tensors; the descriptor and output marshalling are// shared between all call shapes.- template <- typename CType,- typename AType,- typename BType,- bool transpose_a,- bool transpose_b,- typename MarshalInputs>+ template <typename CType, typename AType, typename BType, bool transpose_a, bool transpose_b, typename MarshalInputs>METAL_FUNC static void mma_impl(thread ThreadVector<CType>& output_0,thread ThreadVector<CType>& output_1,@@ -65,17 +58,12 @@ struct MxuFragmentOps {mpp::tensor_ops::matmul2d<descriptor, metal::execution_simdgroup> matmul_op;- auto cooperative_left =- matmul_op- .template get_left_input_cooperative_tensor<AType, BType, CType>();- auto cooperative_right =- matmul_op- .template get_right_input_cooperative_tensor<AType, BType, CType>();- auto cooperative_output =- matmul_op.template get_destination_cooperative_tensor<- decltype(cooperative_left),- decltype(cooperative_right),- CType>();+ auto cooperative_left = matmul_op.template get_left_input_cooperative_tensor<AType, BType, CType>();+ auto cooperative_right = matmul_op.template get_right_input_cooperative_tensor<AType, BType, CType>();+ auto cooperative_output = matmul_op.template get_destination_cooperative_tensor<+ decltype(cooperative_left),+ decltype(cooperative_right),+ CType>();marshal_inputs(cooperative_left, cooperative_right);@@ -95,12 +83,7 @@ struct MxuFragmentOps {}// N-paired: output is two columns, left is one fragment, right is two.- template <- typename CType,- typename AType,- typename BType,- bool transpose_a = false,- bool transpose_b = false>+ template <typename CType, typename AType, typename BType, bool transpose_a = false, bool transpose_b = false>METAL_FUNC static void mma(thread ThreadVector<CType>& output_col_0,thread ThreadVector<CType>& output_col_1,@@ -125,12 +108,7 @@ struct MxuFragmentOps {}// M-paired: output is two rows, left is two fragments, right is one.- template <- typename CType,- typename AType,- typename BType,- bool transpose_a = false,- bool transpose_b = false>+ template <typename CType, typename AType, typename BType, bool transpose_a = false, bool transpose_b = false>METAL_FUNC static void mma(thread ThreadVector<CType>& output_row_0,thread ThreadVector<CType>& output_row_1,@@ -154,41 +132,19 @@ struct MxuFragmentOps {);}- template <- bool transpose_a,- bool transpose_b,- class OutputTile,- class LeftTile,- class RightTile>- METAL_FUNC static void tile_matmul(- thread OutputTile& output,- thread LeftTile& left,- thread RightTile& right- ) {- constexpr ushort left_tile_m =- transpose_a ? LeftTile::TILE_COLS : LeftTile::TILE_ROWS;+ template <bool transpose_a, bool transpose_b, class OutputTile, class LeftTile, class RightTile>+ METAL_FUNC static void tile_matmul(thread OutputTile& output, thread LeftTile& left, thread RightTile& right) {+ constexpr ushort left_tile_m = transpose_a ? LeftTile::TILE_COLS : LeftTile::TILE_ROWS;constexpr ushort tile_m = OutputTile::TILE_ROWS;- static_assert(- left_tile_m == tile_m,- "tile matmul: M dimensions do not match"- );+ static_assert(left_tile_m == tile_m, "tile matmul: M dimensions do not match");- constexpr ushort right_tile_n =- transpose_b ? RightTile::TILE_ROWS : RightTile::TILE_COLS;+ constexpr ushort right_tile_n = transpose_b ? RightTile::TILE_ROWS : RightTile::TILE_COLS;constexpr ushort tile_n = OutputTile::TILE_COLS;- static_assert(- right_tile_n == tile_n,- "tile matmul: N dimensions do not match"- );+ static_assert(right_tile_n == tile_n, "tile matmul: N dimensions do not match");- constexpr ushort left_tile_k =- transpose_a ? LeftTile::TILE_ROWS : LeftTile::TILE_COLS;- constexpr ushort tile_k =- transpose_b ? RightTile::TILE_COLS : RightTile::TILE_ROWS;- static_assert(- left_tile_k == tile_k,- "tile matmul: K dimensions do not match"- );+ constexpr ushort left_tile_k = transpose_a ? LeftTile::TILE_ROWS : LeftTile::TILE_COLS;+ constexpr ushort tile_k = transpose_b ? RightTile::TILE_COLS : RightTile::TILE_ROWS;+ static_assert(left_tile_k == tile_k, "tile matmul: K dimensions do not match");constexpr auto transpose_left = metal::bool_constant<transpose_a>{};constexpr auto transpose_right = metal::bool_constant<transpose_b>{};diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.hindex a62acb567..4fd31c489 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.h@@ -43,9 +43,7 @@ METAL_FUNC auto gemm_loop(constexpr ushort RIGHT_TILE_ROWS = transpose_b ? TILES_N : TILES_K;constexpr ushort RIGHT_TILE_COLS = transpose_b ? TILES_K : TILES_N;- Fragment<AccumulatorType, TILES_M, TILES_N, MxuFragmentOps> accumulator(- thread_context- );+ Fragment<AccumulatorType, TILES_M, TILES_N, MxuFragmentOps> accumulator(thread_context);accumulator.clear();METAL_PRAGMA_NO_UNROLL@@ -54,53 +52,31 @@ METAL_FUNC auto gemm_loop(METAL_PRAGMA_NO_UNROLLfor (int inner_k = 0; inner_k < BLOCK_K; inner_k += SIMDGROUP_BLOCK_K) {- Fragment<AT, LEFT_TILE_ROWS, LEFT_TILE_COLS, MxuFragmentOps> left_tile(- thread_context- );- Fragment<BT, RIGHT_TILE_ROWS, RIGHT_TILE_COLS, MxuFragmentOps> right_tile(- thread_context- );+ Fragment<AT, LEFT_TILE_ROWS, LEFT_TILE_COLS, MxuFragmentOps> left_tile(thread_context);+ Fragment<BT, RIGHT_TILE_ROWS, RIGHT_TILE_COLS, MxuFragmentOps> right_tile(thread_context);volatile int mxu_iteration_fence;- const int left_offset =- transpose_a ? inner_k * leading_dimension_a : inner_k;- const int right_offset =- transpose_b ? inner_k : inner_k * leading_dimension_b;+ const int left_offset = transpose_a ? inner_k * leading_dimension_a : inner_k;+ const int right_offset = transpose_b ? inner_k : inner_k * leading_dimension_b;if constexpr (aligned_m) {left_tile.load(left_ptr + left_offset, leading_dimension_a);} else {- const short row_limit =- transpose_a ? SIMDGROUP_BLOCK_K : simdgroup_limit_m;- const short col_limit =- transpose_a ? simdgroup_limit_m : SIMDGROUP_BLOCK_K;- left_tile.load_safe(- left_ptr + left_offset,- leading_dimension_a,- short2(col_limit, row_limit)- );+ const short row_limit = transpose_a ? SIMDGROUP_BLOCK_K : simdgroup_limit_m;+ const short col_limit = transpose_a ? simdgroup_limit_m : SIMDGROUP_BLOCK_K;+ left_tile.load_safe(left_ptr + left_offset, leading_dimension_a, short2(col_limit, row_limit));}if constexpr (aligned_n) {right_tile.load(right_ptr + right_offset, leading_dimension_b);} else {- const short row_limit =- transpose_b ? simdgroup_limit_n : SIMDGROUP_BLOCK_K;- const short col_limit =- transpose_b ? SIMDGROUP_BLOCK_K : simdgroup_limit_n;- right_tile.load_safe(- right_ptr + right_offset,- leading_dimension_b,- short2(col_limit, row_limit)- );+ const short row_limit = transpose_b ? simdgroup_limit_n : SIMDGROUP_BLOCK_K;+ const short col_limit = transpose_b ? SIMDGROUP_BLOCK_K : simdgroup_limit_n;+ right_tile.load_safe(right_ptr + right_offset, leading_dimension_b, short2(col_limit, row_limit));}- MxuFragmentOps::template tile_matmul<transpose_a, transpose_b>(- accumulator,- left_tile,- right_tile- );+ MxuFragmentOps::template tile_matmul<transpose_a, transpose_b>(accumulator, left_tile, right_tile);(void)mxu_iteration_fence;}@@ -116,40 +92,21 @@ METAL_FUNC auto gemm_loop(METAL_PRAGMA_NO_UNROLLfor (int inner_k = 0; inner_k < remaining_k; inner_k += SIMDGROUP_BLOCK_K) {- Fragment<AT, LEFT_TILE_ROWS, LEFT_TILE_COLS, MxuFragmentOps> left_tile(- thread_context- );- Fragment<BT, RIGHT_TILE_ROWS, RIGHT_TILE_COLS, MxuFragmentOps> right_tile(- thread_context- );+ Fragment<AT, LEFT_TILE_ROWS, LEFT_TILE_COLS, MxuFragmentOps> left_tile(thread_context);+ Fragment<BT, RIGHT_TILE_ROWS, RIGHT_TILE_COLS, MxuFragmentOps> right_tile(thread_context);const short safe_k = max(short(0), short(remaining_k - inner_k));- const short2 left_limits = transpose_a- ? short2(simdgroup_limit_m, safe_k)- : short2(safe_k, simdgroup_limit_m);- const short2 right_limits = transpose_b- ? short2(safe_k, simdgroup_limit_n)- : short2(simdgroup_limit_n, safe_k);-- const int left_offset =- transpose_a ? inner_k * leading_dimension_a : inner_k;- const int right_offset =- transpose_b ? inner_k : inner_k * leading_dimension_b;-- left_tile- .load_safe(left_ptr + left_offset, leading_dimension_a, left_limits);- right_tile.load_safe(- right_ptr + right_offset,- leading_dimension_b,- right_limits- );-- MxuFragmentOps::template tile_matmul<transpose_a, transpose_b>(- accumulator,- left_tile,- right_tile- );+ const short2 left_limits = transpose_a ? short2(simdgroup_limit_m, safe_k) : short2(safe_k, simdgroup_limit_m);+ const short2 right_limits = transpose_b ? short2(safe_k, simdgroup_limit_n) : short2(simdgroup_limit_n, safe_k);++ const int left_offset = transpose_a ? inner_k * leading_dimension_a : inner_k;+ const int right_offset = transpose_b ? inner_k : inner_k * leading_dimension_b;++ left_tile.load_safe(left_ptr + left_offset, leading_dimension_a, left_limits);+ right_tile.load_safe(right_ptr + right_offset, leading_dimension_b, right_limits);++ MxuFragmentOps::template tile_matmul<transpose_a, transpose_b>(accumulator, left_tile, right_tile);}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/qdot.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/qdot.hindex 7f1b2f305..f98167440 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/qdot.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/qdot.h@@ -18,11 +18,7 @@ namespace gemm {template <typename T, typename U, int VALUES_PER_THREAD, int BITS>METAL_FUNC U load_vector(const device T* x, thread U* x_thread) {using U4 = vec<U, 4>;- const U4 inv =- U4(U(1),- U(1) / U(1u << BITS),- U(1) / U(1u << (2u * BITS)),- U(1) / U(1u << (3u * BITS)));+ const U4 inv = U4(U(1), U(1) / U(1u << BITS), U(1) / U(1u << (2u * BITS)), U(1) / U(1u << (3u * BITS)));U sum = 0;thread U4* x_vec4 = reinterpret_cast<thread U4*>(x_thread);METAL_PRAGMA_UNROLL@@ -50,30 +46,21 @@ METAL_FUNC U load_vector_safe(const device T* x, thread U* x_thread, int N) {}template <typename U, int VALUES_PER_THREAD, int BITS>-METAL_FUNC U qdot(- const device uint8_t* w,- const thread U* x_thread,- U scale,- U bias,- U sum-) {+METAL_FUNC U qdot(const device uint8_t* w, const thread U* x_thread, U scale, U bias, U sum) {static_assert(BITS == 4 || BITS == 8, "Only int4 and int8 supported");U accumulator = 0;if constexpr (BITS == 4) {using U4 = vec<U, 4>;- const device ushort* weight_words =- reinterpret_cast<const device ushort*>(w);+ const device ushort* weight_words = reinterpret_cast<const device ushort*>(w);const thread U4* x_vec4 = reinterpret_cast<const thread U4*>(x_thread);METAL_PRAGMA_UNROLLfor (int i = 0; i < (VALUES_PER_THREAD / 4); i++) {// Mask each nibble in place (no shifts); value of lane k is n_k << (4*k),// i.e. n_k * 16^k, which is < 2^23 so the magic-number convert is valid.// The matching x lane was pre-divided by 16^k in load_vector.- const uint4 lanes =- uint4(weight_words[i]) & uint4(0x000fu, 0x00f0u, 0x0f00u, 0xf000u);- const U4 weight_vec4 =- U4(as_type<float4>(lanes | uint4(0x4b000000u)) - float4(8388608.0f));+ const uint4 lanes = uint4(weight_words[i]) & uint4(0x000fu, 0x00f0u, 0x0f00u, 0xf000u);+ const U4 weight_vec4 = U4(as_type<float4>(lanes | uint4(0x4b000000u)) - float4(8388608.0f));accumulator += dot(x_vec4[i], weight_vec4);}} else if constexpr (BITS == 8) {@@ -85,9 +72,7 @@ METAL_FUNC U qdot(// Mask each byte in place (no shifts); lane k value is b_k * 256^k. This// exceeds the magic-number range for k=3, so use the hardware convert// (exact for b_k * 256^k). x lane k was pre-divided by 256^k.- const uint4 lanes =- uint4(weight_words[i]) &- uint4(0x000000ffu, 0x0000ff00u, 0x00ff0000u, 0xff000000u);+ const uint4 lanes = uint4(weight_words[i]) & uint4(0x000000ffu, 0x0000ff00u, 0x00ff0000u, 0xff000000u);const U4 weight_vec4 = U4(float4(lanes));accumulator += dot(x_vec4[i], weight_vec4);}@@ -96,32 +81,19 @@ METAL_FUNC U qdot(}template <typename U, int VALUES_PER_THREAD, int BITS>-METAL_FUNC U qdot_safe(- const device uint8_t* w,- const thread U* x_thread,- U scale,- U bias,- U sum,- int N-) {+METAL_FUNC U qdot_safe(const device uint8_t* w, const thread U* x_thread, U scale, U bias, U sum, int N) {static_assert(BITS == 4 || BITS == 8, "Only int4 and int8 supported");U accumulator = 0;if constexpr (BITS == 4) {using U4 = vec<U, 4>;- const device uint16_t* weight_words =- reinterpret_cast<const device uint16_t*>(w);+ const device uint16_t* weight_words = reinterpret_cast<const device uint16_t*>(w);const thread U4* x_vec4 = reinterpret_cast<const thread U4*>(x_thread);int full_chunks = N / 4;for (int i = 0; i < full_chunks; i++) {uint16_t weight_word = weight_words[i];- U4 weight_vec4 = uint4_to_fp4<U, 4>(uint4(- weight_word,- weight_word >> 4,- weight_word >> 8,- weight_word >> 12- ));+ U4 weight_vec4 = uint4_to_fp4<U, 4>(uint4(weight_word, weight_word >> 4, weight_word >> 8, weight_word >> 12));accumulator += dot(x_vec4[i], weight_vec4);}@@ -131,11 +103,9 @@ METAL_FUNC U qdot_safe(int base_index = 4 * full_chunks;accumulator += x_thread[base_index] * uint_to_fp<U>(weight_word & 0xf);if (remainder > 1)- accumulator +=- x_thread[base_index + 1] * uint_to_fp<U>((weight_word >> 4) & 0xf);+ accumulator += x_thread[base_index + 1] * uint_to_fp<U>((weight_word >> 4) & 0xf);if (remainder > 2)- accumulator +=- x_thread[base_index + 2] * uint_to_fp<U>((weight_word >> 8) & 0xf);+ accumulator += x_thread[base_index + 2] * uint_to_fp<U>((weight_word >> 8) & 0xf);}} else if constexpr (BITS == 8) {for (int i = 0; i < N; i++) {diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/quant_unpack.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/quant_unpack.hindex 4735576f3..137cdb4fd 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/quant_unpack.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/quant_unpack.h@@ -51,12 +51,7 @@ METAL_FUNC bfloat4 uint4_to_fp4<bfloat, 8>(uint4 n) {}template <typename U, int N, int bits>-METAL_FUNC void dequantize(- const device uint8_t* w,- U scale,- U bias,- threadgroup U* w_local-) {+METAL_FUNC void dequantize(const device uint8_t* w, U scale, U bias, threadgroup U* w_local) {static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");if constexpr (bits == 4) {@@ -100,8 +95,7 @@ METAL_FUNC void dequantize<bfloat, 8, 4>(v0 = v0 * scale + bias;v1 = v1 * scale + bias;- threadgroup bfloat4* out_ptr =- reinterpret_cast<threadgroup bfloat4*>(w_local);+ threadgroup bfloat4* out_ptr = reinterpret_cast<threadgroup bfloat4*>(w_local);out_ptr[0] = v0;out_ptr[1] = v1;}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment.hindex 2db44e7a1..cae407263 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment.h@@ -15,17 +15,13 @@ template <typename T,int GRID_ROWS_,int GRID_COLS_,- class SimdgroupMultiplyAccumulateOps_ =- SimdgroupMultiplyAccumulate<T, 8, 8>>+ class SimdgroupMultiplyAccumulateOps_ = SimdgroupMultiplyAccumulate<T, 8, 8>>struct SimdgroupFragment {using SimdgroupMultiplyAccumulateOpsType = SimdgroupMultiplyAccumulateOps_;using ElementType = T;- METAL_CONST int MULTIPLY_ACCUMULATE_ROWS =- SimdgroupMultiplyAccumulateOpsType::ROWS;- METAL_CONST int MULTIPLY_ACCUMULATE_COLS =- SimdgroupMultiplyAccumulateOpsType::COLS;- METAL_CONST int ELEMENTS_PER_MULTIPLY_ACCUMULATE =- SimdgroupMultiplyAccumulateOpsType::THREAD_ELEMENT_COUNT;+ METAL_CONST int MULTIPLY_ACCUMULATE_ROWS = SimdgroupMultiplyAccumulateOpsType::ROWS;+ METAL_CONST int MULTIPLY_ACCUMULATE_COLS = SimdgroupMultiplyAccumulateOpsType::COLS;+ METAL_CONST int ELEMENTS_PER_MULTIPLY_ACCUMULATE = SimdgroupMultiplyAccumulateOpsType::THREAD_ELEMENT_COUNT;METAL_CONST int GRID_ROWS = GRID_ROWS_;METAL_CONST int GRID_COLS = GRID_COLS_;@@ -34,17 +30,12 @@ struct SimdgroupFragment {METAL_CONST int TOTAL_COLS = GRID_COLS * MULTIPLY_ACCUMULATE_COLS;METAL_CONST int MULTIPLY_ACCUMULATE_COUNT = GRID_ROWS * GRID_COLS;- METAL_CONST int ELEMENTS_PER_FRAGMENT =- MULTIPLY_ACCUMULATE_COUNT * ELEMENTS_PER_MULTIPLY_ACCUMULATE;+ METAL_CONST int ELEMENTS_PER_FRAGMENT = MULTIPLY_ACCUMULATE_COUNT * ELEMENTS_PER_MULTIPLY_ACCUMULATE;- typedef typename SimdgroupMultiplyAccumulateOpsType::SimdgroupMatrixType- SimdgroupMatrixType;- typedef typename SimdgroupMultiplyAccumulateOpsType::ThreadDataType- ThreadDataType;+ typedef typename SimdgroupMultiplyAccumulateOpsType::SimdgroupMatrixType SimdgroupMatrixType;+ typedef typename SimdgroupMultiplyAccumulateOpsType::ThreadDataType ThreadDataType;- ThreadDataType multiply_accumulate_data[MULTIPLY_ACCUMULATE_COUNT] = {- ThreadDataType(0)- };+ ThreadDataType multiply_accumulate_data[MULTIPLY_ACCUMULATE_COUNT] = {ThreadDataType(0)};METAL_FUNC SimdgroupFragment() thread {}@@ -55,36 +46,21 @@ struct SimdgroupFragment {}}- METAL_FUNC constexpr thread ThreadDataType& multiply_accumulate_at(- const ushort i,- const ushort j- ) {+ METAL_FUNC constexpr thread ThreadDataType& multiply_accumulate_at(const ushort i, const ushort j) {return multiply_accumulate_data[i * GRID_COLS + j];}- METAL_FUNC constexpr const thread ThreadDataType& multiply_accumulate_at(- const ushort i,- const ushort j- ) const {+ METAL_FUNC constexpr const thread ThreadDataType& multiply_accumulate_at(const ushort i, const ushort j) const {return multiply_accumulate_data[i * GRID_COLS + j];}- METAL_FUNC thread ElementType* elements() {- return reinterpret_cast<thread ElementType*>(multiply_accumulate_data);- }+ METAL_FUNC thread ElementType* elements() { return reinterpret_cast<thread ElementType*>(multiply_accumulate_data); }METAL_FUNC const thread ElementType* elements() const {- return reinterpret_cast<const thread ElementType*>(- multiply_accumulate_data- );+ return reinterpret_cast<const thread ElementType*>(multiply_accumulate_data);}- template <- typename U,- int SIMDGROUP_STRIDE_X,- int SIMDGROUP_STRIDE_Y,- int STRIDE_X,- int STRIDE_Y>+ template <typename U, int SIMDGROUP_STRIDE_X, int SIMDGROUP_STRIDE_Y, int STRIDE_X, int STRIDE_Y>METAL_FUNC void load(const threadgroup U* source) {METAL_PRAGMA_UNROLLfor (ushort i = 0; i < GRID_ROWS; ++i) {@@ -93,10 +69,8 @@ struct SimdgroupFragment {SimdgroupMultiplyAccumulateOpsType::load(multiply_accumulate_at(i, j),&(source- [(i * MULTIPLY_ACCUMULATE_ROWS) * SIMDGROUP_STRIDE_X *- STRIDE_X +- (j * MULTIPLY_ACCUMULATE_COLS) * SIMDGROUP_STRIDE_Y *- STRIDE_Y]),+ [(i * MULTIPLY_ACCUMULATE_ROWS) * SIMDGROUP_STRIDE_X * STRIDE_X ++ (j * MULTIPLY_ACCUMULATE_COLS) * SIMDGROUP_STRIDE_Y * STRIDE_Y]),STRIDE_X,STRIDE_Y);@@ -105,10 +79,7 @@ struct SimdgroupFragment {}template <typename U, int SIMDGROUP_STRIDE_X, int SIMDGROUP_STRIDE_Y>- METAL_FUNC void store(- device U* destination,- const int leading_dimension- ) const {+ METAL_FUNC void store(device U* destination, const int leading_dimension) const {METAL_PRAGMA_UNROLLfor (ushort i = 0; i < GRID_ROWS; ++i) {METAL_PRAGMA_UNROLL@@ -116,8 +87,7 @@ struct SimdgroupFragment {SimdgroupMultiplyAccumulateOpsType::store(multiply_accumulate_at(i, j),&(destination- [(i * MULTIPLY_ACCUMULATE_ROWS) * SIMDGROUP_STRIDE_X *- leading_dimension ++ [(i * MULTIPLY_ACCUMULATE_ROWS) * SIMDGROUP_STRIDE_X * leading_dimension +(j * MULTIPLY_ACCUMULATE_COLS) * SIMDGROUP_STRIDE_Y]),leading_dimension,1@@ -169,13 +139,12 @@ METAL_FUNC void tile_multiply_accumulate(const ushort column_serpentine = (m % 2) ? (N - 1 - n) : n;METAL_PRAGMA_UNROLLfor (ushort k = 0; k < K; ++k) {- SimdgroupFragment<T, M, N>::SimdgroupMultiplyAccumulateOpsType::- multiply_accumulate(- D.multiply_accumulate_at(m, column_serpentine),- A.multiply_accumulate_at(m, k),- B.multiply_accumulate_at(k, column_serpentine),- C.multiply_accumulate_at(m, column_serpentine)- );+ SimdgroupFragment<T, M, N>::SimdgroupMultiplyAccumulateOpsType::multiply_accumulate(+ D.multiply_accumulate_at(m, column_serpentine),+ A.multiply_accumulate_at(m, k),+ B.multiply_accumulate_at(k, column_serpentine),+ C.multiply_accumulate_at(m, column_serpentine)+ );}}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_multiply_accumulate.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_multiply_accumulate.hindex 27860a3ac..939635b80 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_multiply_accumulate.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_multiply_accumulate.h@@ -13,14 +13,8 @@ namespace matmul {template <typename T, int ROWS_, int COLS_>struct SimdgroupMultiplyAccumulate {- static_assert(- ROWS_ == 8,- "Only 8 x 8 fragment matrices are currently supported"- );- static_assert(- COLS_ == 8,- "Only 8 x 8 fragment matrices are currently supported"- );+ static_assert(ROWS_ == 8, "Only 8 x 8 fragment matrices are currently supported");+ static_assert(COLS_ == 8, "Only 8 x 8 fragment matrices are currently supported");};template <typename T>@@ -39,9 +33,7 @@ struct SimdgroupMultiplyAccumulate<T, 8, 8> {typedef metal::simdgroup_matrix<T, ROWS, COLS> SimdgroupMatrixType;typedef metal::vec<T, THREAD_ELEMENT_COUNT> ThreadDataType;- METAL_FUNC static constexpr short2 get_lane_coordinates(- ushort simd_lane_id [[thread_index_in_simdgroup]]- ) {+ METAL_FUNC static constexpr short2 get_lane_coordinates(ushort simd_lane_id [[thread_index_in_simdgroup]]) {const short quad_index = simd_lane_id / 4;const short lane_row = (quad_index & 4) + ((simd_lane_id / 2) % 4);const short lane_col = (quad_index & 2) * 2 + (simd_lane_id % 2) * 2;@@ -59,8 +51,7 @@ struct SimdgroupMultiplyAccumulate<T, 8, 8> {for (ushort i = 0; i < THREAD_ELEMENT_ROWS; i++) {METAL_PRAGMA_UNROLLfor (ushort j = 0; j < THREAD_ELEMENT_COLS; j++) {- destination[i * THREAD_ELEMENT_COLS + j] =- static_cast<T>(source[i * row_stride + j * col_stride]);+ destination[i * THREAD_ELEMENT_COLS + j] = static_cast<T>(source[i * row_stride + j * col_stride]);}}}@@ -89,9 +80,7 @@ struct SimdgroupMultiplyAccumulate<T, 8, 8> {for (ushort j = 0; j < THREAD_ELEMENT_COLS; j++) {if ((row_offset + i) < row_limit && (col_offset + j) < col_limit) {destination[i * THREAD_ELEMENT_COLS + j] =- static_cast<T>(source- [(row_offset + i) * row_stride +- (col_offset + j) * col_stride]);+ static_cast<T>(source[(row_offset + i) * row_stride + (col_offset + j) * col_stride]);} else {destination[i * THREAD_ELEMENT_COLS + j] = T(0);}@@ -99,10 +88,7 @@ struct SimdgroupMultiplyAccumulate<T, 8, 8> {}}- template <- typename DestinationPointerType,- typename RowStride,- typename ColStride>+ template <typename DestinationPointerType, typename RowStride, typename ColStride>METAL_FUNC static constexpr void store(const thread ThreadDataType& source,DestinationPointerType destination,@@ -115,8 +101,7 @@ struct SimdgroupMultiplyAccumulate<T, 8, 8> {for (ushort i = 0; i < THREAD_ELEMENT_ROWS; i++) {METAL_PRAGMA_UNROLLfor (ushort j = 0; j < THREAD_ELEMENT_COLS; j++) {- destination[i * row_stride + j * col_stride] =- static_cast<U>(source[i * THREAD_ELEMENT_COLS + j]);+ destination[i * row_stride + j * col_stride] = static_cast<U>(source[i * THREAD_ELEMENT_COLS + j]);}}}@@ -146,9 +131,8 @@ struct SimdgroupMultiplyAccumulate<T, 8, 8> {METAL_PRAGMA_UNROLLfor (ushort j = 0; j < THREAD_ELEMENT_COLS; j++) {if ((row_offset + i) < row_limit && (col_offset + j) < col_limit) {- destination- [(row_offset + i) * row_stride + (col_offset + j) * col_stride] =- static_cast<U>(source[i * THREAD_ELEMENT_COLS + j]);+ destination[(row_offset + i) * row_stride + (col_offset + j) * col_stride] =+ static_cast<U>(source[i * THREAD_ELEMENT_COLS + j]);}}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.hindex 2552c3a0a..451bc066b 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h@@ -29,52 +29,27 @@ template <typename Epilogue = TransformNone<DT, AccumulatorType>>struct ThreadgroupTile {METAL_CONST ushort SIMDGROUP_BLOCK_SIZE = 8;- using SimdgroupMultiplyAccumulateType = SimdgroupMultiplyAccumulate<- AccumulatorType,- SIMDGROUP_BLOCK_SIZE,- SIMDGROUP_BLOCK_SIZE>;-- METAL_CONST ushort TILE_ROW_STRIDE =- SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW;- METAL_CONST ushort TILE_COL_STRIDE =- SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN;-- METAL_CONST ushort TILE_ROWS =- BLOCK_ROWS / (SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW);- METAL_CONST ushort TILE_COLS =- BLOCK_COLS / (SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN);-- METAL_CONST ushort A_STRIDE_ROW =- transpose_a ? 1 : THREADGROUP_LEADING_DIMENSION_A;- METAL_CONST ushort A_STRIDE_INNER =- transpose_a ? THREADGROUP_LEADING_DIMENSION_A : 1;-- METAL_CONST ushort B_STRIDE_INNER =- transpose_b ? 1 : THREADGROUP_LEADING_DIMENSION_B;- METAL_CONST ushort B_STRIDE_COL =- transpose_b ? THREADGROUP_LEADING_DIMENSION_B : 1;+ using SimdgroupMultiplyAccumulateType =+ SimdgroupMultiplyAccumulate<AccumulatorType, SIMDGROUP_BLOCK_SIZE, SIMDGROUP_BLOCK_SIZE>;++ METAL_CONST ushort TILE_ROW_STRIDE = SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW;+ METAL_CONST ushort TILE_COL_STRIDE = SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN;++ METAL_CONST ushort TILE_ROWS = BLOCK_ROWS / (SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW);+ METAL_CONST ushort TILE_COLS = BLOCK_COLS / (SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN);++ METAL_CONST ushort A_STRIDE_ROW = transpose_a ? 1 : THREADGROUP_LEADING_DIMENSION_A;+ METAL_CONST ushort A_STRIDE_INNER = transpose_a ? THREADGROUP_LEADING_DIMENSION_A : 1;++ METAL_CONST ushort B_STRIDE_INNER = transpose_b ? 1 : THREADGROUP_LEADING_DIMENSION_B;+ METAL_CONST ushort B_STRIDE_COL = transpose_b ? THREADGROUP_LEADING_DIMENSION_B : 1;METAL_CONST ushort TILE_STRIDE_A = SIMDGROUP_BLOCK_SIZE * A_STRIDE_INNER;METAL_CONST ushort TILE_STRIDE_B = SIMDGROUP_BLOCK_SIZE * B_STRIDE_INNER;- SimdgroupFragment<- AccumulatorType,- TILE_ROWS,- 1,- SimdgroupMultiplyAccumulateType>- a_fragment;- SimdgroupFragment<- AccumulatorType,- 1,- TILE_COLS,- SimdgroupMultiplyAccumulateType>- b_fragment;- SimdgroupFragment<- AccumulatorType,- TILE_ROWS,- TILE_COLS,- SimdgroupMultiplyAccumulateType>- c_fragment;+ SimdgroupFragment<AccumulatorType, TILE_ROWS, 1, SimdgroupMultiplyAccumulateType> a_fragment;+ SimdgroupFragment<AccumulatorType, 1, TILE_COLS, SimdgroupMultiplyAccumulateType> b_fragment;+ SimdgroupFragment<AccumulatorType, TILE_ROWS, TILE_COLS, SimdgroupMultiplyAccumulateType> c_fragment;ushort simdgroup_row_offset;ushort simdgroup_col_offset;@@ -83,53 +58,30 @@ struct ThreadgroupTile {ushort b_shared_offset;METAL_FUNC ThreadgroupTile(const thread ThreadContext& thread_context) {- ushort tile_row_base =- SIMDGROUP_BLOCK_SIZE *- (thread_context.simdgroup_index / SIMDGROUPS_PER_COLUMN);- ushort tile_col_base =- SIMDGROUP_BLOCK_SIZE *- (thread_context.simdgroup_index % SIMDGROUPS_PER_COLUMN);-- const ushort2 simdgroup_coordinates = ushort2(- SimdgroupMultiplyAccumulateType::get_lane_coordinates(- thread_context.simd_lane_id- )- );+ ushort tile_row_base = SIMDGROUP_BLOCK_SIZE * (thread_context.simdgroup_index / SIMDGROUPS_PER_COLUMN);+ ushort tile_col_base = SIMDGROUP_BLOCK_SIZE * (thread_context.simdgroup_index % SIMDGROUPS_PER_COLUMN);++ const ushort2 simdgroup_coordinates =+ ushort2(SimdgroupMultiplyAccumulateType::get_lane_coordinates(thread_context.simd_lane_id));simdgroup_row_offset = simdgroup_coordinates.y;simdgroup_col_offset = simdgroup_coordinates.x;- a_shared_offset = (tile_row_base + simdgroup_row_offset) * A_STRIDE_ROW +- (simdgroup_col_offset)*A_STRIDE_INNER;- b_shared_offset = (simdgroup_row_offset)*B_STRIDE_INNER +- (tile_col_base + simdgroup_col_offset) * B_STRIDE_COL;+ a_shared_offset = (tile_row_base + simdgroup_row_offset) * A_STRIDE_ROW + (simdgroup_col_offset)*A_STRIDE_INNER;+ b_shared_offset = (simdgroup_row_offset)*B_STRIDE_INNER + (tile_col_base + simdgroup_col_offset) * B_STRIDE_COL;simdgroup_row_offset += tile_row_base;simdgroup_col_offset += tile_col_base;}- METAL_FUNC void multiply_accumulate(- const threadgroup AT* a_shared,- const threadgroup BT* b_shared- ) {+ METAL_FUNC void multiply_accumulate(const threadgroup AT* a_shared, const threadgroup BT* b_shared) {a_shared += a_shared_offset;b_shared += b_shared_offset;METAL_PRAGMA_UNROLL- for (ushort k_block_index = 0; k_block_index < BLOCK_DEPTH;- k_block_index += SIMDGROUP_BLOCK_SIZE) {- a_fragment.template load<- AT,- SIMDGROUPS_PER_ROW,- 1,- A_STRIDE_ROW,- A_STRIDE_INNER>(a_shared);-- b_fragment.template load<- BT,- 1,- SIMDGROUPS_PER_COLUMN,- B_STRIDE_INNER,- B_STRIDE_COL>(b_shared);+ for (ushort k_block_index = 0; k_block_index < BLOCK_DEPTH; k_block_index += SIMDGROUP_BLOCK_SIZE) {+ a_fragment.template load<AT, SIMDGROUPS_PER_ROW, 1, A_STRIDE_ROW, A_STRIDE_INNER>(a_shared);++ b_fragment.template load<BT, 1, SIMDGROUPS_PER_COLUMN, B_STRIDE_INNER, B_STRIDE_COL>(b_shared);tile_multiply_accumulate(c_fragment, a_fragment, b_fragment, c_fragment);@@ -146,36 +98,26 @@ struct ThreadgroupTile {D += simdgroup_row_offset * leading_dimension_d + simdgroup_col_offset;- c_fragment.template store<DT, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN>(- D,- leading_dimension_d- );+ c_fragment.template store<DT, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN>(D, leading_dimension_d);}- METAL_FUNC void store_result_safe(- device DT* D,- const int leading_dimension_d,- short2 destination_tile_dimensions- ) {+ METAL_FUNC void store_result_safe(device DT* D, const int leading_dimension_d, short2 destination_tile_dimensions) {METAL_PRAGMA_UNROLLfor (ushort i = 0; i < decltype(c_fragment)::ELEMENTS_PER_FRAGMENT; i++) {c_fragment.elements()[i] = Epilogue::apply(c_fragment.elements()[i]);}D += simdgroup_row_offset * leading_dimension_d + simdgroup_col_offset;- destination_tile_dimensions -=- short2(simdgroup_col_offset, simdgroup_row_offset);+ destination_tile_dimensions -= short2(simdgroup_col_offset, simdgroup_row_offset);- if (destination_tile_dimensions.x <= 0 ||- destination_tile_dimensions.y <= 0)+ if (destination_tile_dimensions.x <= 0 || destination_tile_dimensions.y <= 0)return;- c_fragment- .template store_safe<DT, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN>(- D,- leading_dimension_d,- destination_tile_dimensions- );+ c_fragment.template store_safe<DT, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN>(+ D,+ leading_dimension_d,+ destination_tile_dimensions+ );}template <typename EpilogueOp>@@ -185,9 +127,8 @@ struct ThreadgroupTile {const int column_stride_c,thread const EpilogueOp& epilogue_operation) {- const device DT* c_pointer = C +- simdgroup_row_offset * leading_dimension_c +- simdgroup_col_offset * column_stride_c;+ const device DT* c_pointer =+ C + simdgroup_row_offset * leading_dimension_c + simdgroup_col_offset * column_stride_c;METAL_PRAGMA_UNROLLfor (ushort i = 0; i < TILE_ROWS; i++) {@@ -195,20 +136,11 @@ struct ThreadgroupTile {for (ushort j = 0; j < TILE_COLS; j++) {thread auto& block_data = c_fragment.multiply_accumulate_at(i, j);METAL_PRAGMA_UNROLL- for (ushort k = 0;- k < decltype(c_fragment)::ELEMENTS_PER_MULTIPLY_ACCUMULATE;- k++) {- const ushort row_offset =- i * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW;- const ushort col_offset =- j * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN;- DT c_value = c_pointer- [row_offset * leading_dimension_c + col_offset * column_stride_c +- k];- block_data[k] = epilogue_operation.apply(- block_data[k],- static_cast<AccumulatorType>(c_value)- );+ for (ushort k = 0; k < decltype(c_fragment)::ELEMENTS_PER_MULTIPLY_ACCUMULATE; k++) {+ const ushort row_offset = i * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW;+ const ushort col_offset = j * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN;+ DT c_value = c_pointer[row_offset * leading_dimension_c + col_offset * column_stride_c + k];+ block_data[k] = epilogue_operation.apply(block_data[k], static_cast<AccumulatorType>(c_value));}}}@@ -222,9 +154,8 @@ struct ThreadgroupTile {short2 tile_dimensions,thread const EpilogueOp& epilogue_operation) {- const device DT* c_pointer = C +- simdgroup_row_offset * leading_dimension_c +- simdgroup_col_offset * column_stride_c;+ const device DT* c_pointer =+ C + simdgroup_row_offset * leading_dimension_c + simdgroup_col_offset * column_stride_c;tile_dimensions -= short2(simdgroup_col_offset, simdgroup_row_offset);METAL_PRAGMA_UNROLL@@ -233,21 +164,12 @@ struct ThreadgroupTile {for (ushort j = 0; j < TILE_COLS; j++) {thread auto& block_data = c_fragment.multiply_accumulate_at(i, j);const ushort row_offset = i * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW;- const ushort col_offset =- j * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN;+ const ushort col_offset = j * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN;METAL_PRAGMA_UNROLL- for (ushort k = 0;- k < decltype(c_fragment)::ELEMENTS_PER_MULTIPLY_ACCUMULATE;- k++) {- if (row_offset < tile_dimensions.y &&- col_offset + k < tile_dimensions.x) {- DT c_value = c_pointer- [row_offset * leading_dimension_c +- col_offset * column_stride_c + k];- block_data[k] = epilogue_operation.apply(- block_data[k],- static_cast<AccumulatorType>(c_value)- );+ for (ushort k = 0; k < decltype(c_fragment)::ELEMENTS_PER_MULTIPLY_ACCUMULATE; k++) {+ if (row_offset < tile_dimensions.y && col_offset + k < tile_dimensions.x) {+ DT c_value = c_pointer[row_offset * leading_dimension_c + col_offset * column_stride_c + k];+ block_data[k] = epilogue_operation.apply(block_data[k], static_cast<AccumulatorType>(c_value));}}}@@ -262,23 +184,16 @@ struct ThreadgroupTile {METAL_PRAGMA_UNROLLfor (ushort j = 0; j < TILE_COLS; j++) {thread auto& block_data = c_fragment.multiply_accumulate_at(i, j);- const ushort col_offset =- j * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN;+ const ushort col_offset = j * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN;METAL_PRAGMA_UNROLL- for (ushort k = 0;- k < decltype(c_fragment)::ELEMENTS_PER_MULTIPLY_ACCUMULATE;- k++) {- block_data[k] +=- static_cast<AccumulatorType>(bias_pointer[col_offset + k]);+ for (ushort k = 0; k < decltype(c_fragment)::ELEMENTS_PER_MULTIPLY_ACCUMULATE; k++) {+ block_data[k] += static_cast<AccumulatorType>(bias_pointer[col_offset + k]);}}}}- METAL_FUNC void apply_bias_safe(- const device BT* bias,- short2 tile_dimensions- ) {+ METAL_FUNC void apply_bias_safe(const device BT* bias, short2 tile_dimensions) {const device BT* bias_pointer = bias + simdgroup_col_offset;tile_dimensions -= short2(simdgroup_col_offset, simdgroup_row_offset);@@ -288,16 +203,11 @@ struct ThreadgroupTile {for (ushort j = 0; j < TILE_COLS; j++) {thread auto& block_data = c_fragment.multiply_accumulate_at(i, j);const ushort row_offset = i * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW;- const ushort col_offset =- j * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN;+ const ushort col_offset = j * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN;METAL_PRAGMA_UNROLL- for (ushort k = 0;- k < decltype(c_fragment)::ELEMENTS_PER_MULTIPLY_ACCUMULATE;- k++) {- if (row_offset < tile_dimensions.y &&- col_offset + k < tile_dimensions.x) {- block_data[k] +=- static_cast<AccumulatorType>(bias_pointer[col_offset + k]);+ for (ushort k = 0; k < decltype(c_fragment)::ELEMENTS_PER_MULTIPLY_ACCUMULATE; k++) {+ if (row_offset < tile_dimensions.y && col_offset + k < tile_dimensions.x) {+ block_data[k] += static_cast<AccumulatorType>(bias_pointer[col_offset + k]);}}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/block_geometry.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/block_geometry.hindex 84b37753a..19ba79a27 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/block_geometry.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/block_geometry.h@@ -20,16 +20,10 @@ static METAL_FUNC uint morton_expand_bits(uint x) {// Morton-unpacks a 1D dispatch index (`fp.rs` sets `group_count_y = 1`) into// 2D.static METAL_FUNC uint2 morton_tile_id(uint2 threadgroup_position) {- return uint2(- morton_expand_bits(threadgroup_position.x),- morton_expand_bits(threadgroup_position.x >> 1)- );+ return uint2(morton_expand_bits(threadgroup_position.x), morton_expand_bits(threadgroup_position.x >> 1));}-static METAL_FUNC uint2 tile_id(- uint2 threadgroup_position,- const constant uzu::matmul::GemmParams* params-) {+static METAL_FUNC uint2 tile_id(uint2 threadgroup_position, const constant uzu::matmul::GemmParams* params) {if (params->use_morton) {return morton_tile_id(threadgroup_position);}@@ -44,12 +38,11 @@ struct ThreadgroupTileGeometry {uint block_col_start;bool out_of_bounds;- static METAL_FUNC ThreadgroupTileGeometry- compute(uint2 tile_id, const constant uzu::matmul::GemmParams* params) {+ static METAL_FUNC ThreadgroupTileGeometry compute(uint2 tile_id, const constant uzu::matmul::GemmParams* params) {ThreadgroupTileGeometry geometry;geometry.tile_id = tile_id;- geometry.out_of_bounds = (tile_id.x >= params->threadgroups_per_row) ||- (tile_id.y >= params->threadgroups_per_column);+ geometry.out_of_bounds =+ (tile_id.x >= params->threadgroups_per_row) || (tile_id.y >= params->threadgroups_per_column);geometry.block_row_start = tile_id.y * THREADGROUP_BLOCK_M;geometry.block_col_start = tile_id.x * THREADGROUP_BLOCK_N;return geometry;diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/gemm_rht.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/gemm_rht.hindex 864a0af1c..35ba6d7c7 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/gemm_rht.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/gemm_rht.h@@ -18,17 +18,12 @@ static METAL_FUNC void apply_output_random_hadamard_transform(const ushort stripes_per_row = tile_block_cols / METAL_SIMD_SIZE;const ushort simd_lane = thread_context.simd_lane_id;const uint total_work = uint(tile_block_rows) * uint(stripes_per_row);- for (uint cell = thread_context.simdgroup_index; cell < total_work;- cell += simdgroup_count) {+ for (uint cell = thread_context.simdgroup_index; cell < total_work; cell += simdgroup_count) {const ushort row_local = ushort(cell / stripes_per_row);const ushort stripe = ushort(cell % stripes_per_row);const ushort col_local = stripe * ushort(METAL_SIMD_SIZE) + simd_lane;- const size_t output_index =- size_t(row_local) * size_t(leading_dimension_d) + size_t(col_local);- output_block[output_index] = simdgroup_output_random_hadamard_transform(- simd_lane,- output_block[output_index],- rht_factors_block[col_local]- );+ const size_t output_index = size_t(row_local) * size_t(leading_dimension_d) + size_t(col_local);+ output_block[output_index] =+ simdgroup_output_random_hadamard_transform(simd_lane, output_block[output_index], rht_factors_block[col_local]);}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.hindex 82202985f..17112a7fb 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h@@ -32,42 +32,30 @@ template <struct MxuMmaCore {METAL_CONST ushort THREADGROUP_BLOCK_M = gemm_tiling_block_m(GEMM_TILING);METAL_CONST ushort THREADGROUP_BLOCK_N = gemm_tiling_block_n(GEMM_TILING);- METAL_CONST ushort SIMDGROUPS_PER_ROW =- gemm_tiling_simdgroups_per_row(GEMM_TILING);- METAL_CONST ushort SIMDGROUPS_PER_COLUMN =- gemm_tiling_simdgroups_per_column(GEMM_TILING);- METAL_CONST ushort SIMDGROUP_BLOCK_M =- THREADGROUP_BLOCK_M / SIMDGROUPS_PER_ROW;- METAL_CONST ushort SIMDGROUP_BLOCK_N =- THREADGROUP_BLOCK_N / SIMDGROUPS_PER_COLUMN;+ METAL_CONST ushort SIMDGROUPS_PER_ROW = gemm_tiling_simdgroups_per_row(GEMM_TILING);+ METAL_CONST ushort SIMDGROUPS_PER_COLUMN = gemm_tiling_simdgroups_per_column(GEMM_TILING);+ METAL_CONST ushort SIMDGROUP_BLOCK_M = THREADGROUP_BLOCK_M / SIMDGROUPS_PER_ROW;+ METAL_CONST ushort SIMDGROUP_BLOCK_N = THREADGROUP_BLOCK_N / SIMDGROUPS_PER_COLUMN;METAL_CONST ushort SIMDGROUP_BLOCK_K = 32;METAL_CONST ushort THREADGROUP_BLOCK_K_FP = gemm_tiling_block_k(GEMM_TILING);static_assert(THREADGROUP_BLOCK_K_FP % SIMDGROUP_BLOCK_K == 0,"FP THREADGROUP_BLOCK_K must be a multiple of SIMDGROUP_BLOCK_K");- METAL_CONST ushort TILES_M =- SIMDGROUP_BLOCK_M / uzu::matmul::MxuFragmentOps::FRAGMENT_ROWS;- METAL_CONST ushort TILES_N =- SIMDGROUP_BLOCK_N / uzu::matmul::MxuFragmentOps::FRAGMENT_COLS;- METAL_CONST ushort TILES_K =- SIMDGROUP_BLOCK_K / uzu::matmul::MxuFragmentOps::FRAGMENT_ROWS;-- METAL_CONST ushort QUANT_BK =- (B_PROLOGUE == GemmBPrologueKind::FullPrecision) ? 0 : GROUP_SIZE;+ METAL_CONST ushort TILES_M = SIMDGROUP_BLOCK_M / uzu::matmul::MxuFragmentOps::FRAGMENT_ROWS;+ METAL_CONST ushort TILES_N = SIMDGROUP_BLOCK_N / uzu::matmul::MxuFragmentOps::FRAGMENT_COLS;+ METAL_CONST ushort TILES_K = SIMDGROUP_BLOCK_K / uzu::matmul::MxuFragmentOps::FRAGMENT_ROWS;++ METAL_CONST ushort QUANT_BK = (B_PROLOGUE == GemmBPrologueKind::FullPrecision) ? 0 : GROUP_SIZE;METAL_CONST ushort PADDING_B = 16 / sizeof(BT);- METAL_CONST ushort SHARED_STRIDE_B =- (QUANT_BK > 0) ? (QUANT_BK + PADDING_B) : 1;- METAL_CONST ushort THREADGROUP_THREADS =- SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * METAL_SIMD_SIZE;+ METAL_CONST ushort SHARED_STRIDE_B = (QUANT_BK > 0) ? (QUANT_BK + PADDING_B) : 1;+ METAL_CONST ushort THREADGROUP_THREADS = SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * METAL_SIMD_SIZE;static_assert(- B_PROLOGUE == GemmBPrologueKind::FullPrecision ||- QUANT_BK % SIMDGROUP_BLOCK_K == 0,+ B_PROLOGUE == GemmBPrologueKind::FullPrecision || QUANT_BK % SIMDGROUP_BLOCK_K == 0,"QUANT_BK must be a multiple of SIMDGROUP_BLOCK_K");static_assert(- B_PROLOGUE == GemmBPrologueKind::FullPrecision ||- THREADGROUP_BLOCK_K_FP % QUANT_BK == 0,+ B_PROLOGUE == GemmBPrologueKind::FullPrecision || THREADGROUP_BLOCK_K_FP % QUANT_BK == 0,"Tile block_k must be a multiple of QUANT_BK");@@ -102,8 +90,7 @@ struct MxuMmaCore {(BITS > 0) ? BITS : 4,true>;- using AccumFragment = uzu::matmul::- Fragment<AccumulatorType, TILES_M, TILES_N, uzu::matmul::MxuFragmentOps>;+ using AccumFragment = uzu::matmul::Fragment<AccumulatorType, TILES_M, TILES_N, uzu::matmul::MxuFragmentOps>;template <bool ALIGNED_M, bool ALIGNED_N, typename Loader>static METAL_FUNC AccumFragment quant_k_loop(@@ -121,8 +108,7 @@ struct MxuMmaCore {AccumFragment accumulator(thread_context);accumulator.clear();- threadgroup BT* b_shared_simdgroup =- b_shared + tile_col_offset * SHARED_STRIDE_B;+ threadgroup BT* b_shared_simdgroup = b_shared + tile_col_offset * SHARED_STRIDE_B;const short2 tile_dimensions_b = short2(QUANT_BK, tile_block_cols);METAL_PRAGMA_NO_UNROLL@@ -137,29 +123,20 @@ struct MxuMmaCore {METAL_PRAGMA_NO_UNROLLfor (int inner_k = 0; inner_k < QUANT_BK; inner_k += SIMDGROUP_BLOCK_K) {- uzu::matmul::Fragment<AT, TILES_M, TILES_K, uzu::matmul::MxuFragmentOps>- left_tile(thread_context);- uzu::matmul::Fragment<BT, TILES_N, TILES_K, uzu::matmul::MxuFragmentOps>- right_tile(thread_context);+ uzu::matmul::Fragment<AT, TILES_M, TILES_K, uzu::matmul::MxuFragmentOps> left_tile(thread_context);+ uzu::matmul::Fragment<BT, TILES_N, TILES_K, uzu::matmul::MxuFragmentOps> right_tile(thread_context);const int left_offset = inner_k;if constexpr (ALIGNED_M) {left_tile.load(a_simdgroup + left_offset, leading_dimension_a);} else {- left_tile.load_safe(- a_simdgroup + left_offset,- leading_dimension_a,- short2(SIMDGROUP_BLOCK_K, simdgroup_limit_m)- );+ left_tile+ .load_safe(a_simdgroup + left_offset, leading_dimension_a, short2(SIMDGROUP_BLOCK_K, simdgroup_limit_m));}right_tile.load(b_shared_simdgroup + inner_k, int(SHARED_STRIDE_B));- uzu::matmul::MxuFragmentOps::template tile_matmul<false, true>(- accumulator,- left_tile,- right_tile- );+ uzu::matmul::MxuFragmentOps::template tile_matmul<false, true>(accumulator, left_tile, right_tile);}a_simdgroup += QUANT_BK;@@ -187,11 +164,8 @@ struct MxuMmaCore {const uint partition = thread_context.threadgroup_position.z;const uint tile_y = thread_context.threadgroup_position.y;- const uint2 tile =- tile_id(uint2(thread_context.threadgroup_position.x, tile_y), params);- const auto geometry =- ThreadgroupTileGeometry<THREADGROUP_BLOCK_M, THREADGROUP_BLOCK_N>::- compute(tile, params);+ const uint2 tile = tile_id(uint2(thread_context.threadgroup_position.x, tile_y), params);+ const auto geometry = ThreadgroupTileGeometry<THREADGROUP_BLOCK_M, THREADGROUP_BLOCK_N>::compute(tile, params);if (geometry.out_of_bounds) {return;}@@ -200,69 +174,43 @@ struct MxuMmaCore {const size_t block_col = size_t(geometry.block_col_start);const uint k_offset_per_block =- (B_PROLOGUE == GemmBPrologueKind::FullPrecision)- ? uint(THREADGROUP_BLOCK_K_FP)- : uint(QUANT_BK);- const uint k_offset =- partition * params->aligned_inner_iterations * k_offset_per_block;-- const device AT* a_block =- a + block_row * params->leading_dimension_a + k_offset;- const device BT* b_block_fp =- b +- (TRANSPOSE_B ? block_col * params->leading_dimension_b : block_col) +- (TRANSPOSE_B ? k_offset : k_offset * uint(params->leading_dimension_b));-- const ushort tile_row_offset =- SIMDGROUP_BLOCK_M *- (thread_context.simdgroup_index / SIMDGROUPS_PER_COLUMN);- const ushort tile_col_offset =- SIMDGROUP_BLOCK_N *- (thread_context.simdgroup_index % SIMDGROUPS_PER_COLUMN);-- device DT* d_simdgroup =- d + size_t(partition) * size_t(params->M) * size_t(params->N) +- block_row * params->leading_dimension_d + block_col +- tile_row_offset * params->leading_dimension_d + tile_col_offset;+ (B_PROLOGUE == GemmBPrologueKind::FullPrecision) ? uint(THREADGROUP_BLOCK_K_FP) : uint(QUANT_BK);+ const uint k_offset = partition * params->aligned_inner_iterations * k_offset_per_block;++ const device AT* a_block = a + block_row * params->leading_dimension_a + k_offset;+ const device BT* b_block_fp = b + (TRANSPOSE_B ? block_col * params->leading_dimension_b : block_col) ++ (TRANSPOSE_B ? k_offset : k_offset * uint(params->leading_dimension_b));++ const ushort tile_row_offset = SIMDGROUP_BLOCK_M * (thread_context.simdgroup_index / SIMDGROUPS_PER_COLUMN);+ const ushort tile_col_offset = SIMDGROUP_BLOCK_N * (thread_context.simdgroup_index % SIMDGROUPS_PER_COLUMN);++ device DT* d_simdgroup = d + size_t(partition) * size_t(params->M) * size_t(params->N) ++ block_row * params->leading_dimension_d + block_col ++ tile_row_offset * params->leading_dimension_d + tile_col_offset;const short simdgroup_limit_m =alignment.contains(GemmAlignment::M)? SIMDGROUP_BLOCK_M- : short(- min(int(SIMDGROUP_BLOCK_M),- int(params->M) -- int(geometry.block_row_start + tile_row_offset))- );+ : short(min(int(SIMDGROUP_BLOCK_M), int(params->M) - int(geometry.block_row_start + tile_row_offset)));const short simdgroup_limit_n =alignment.contains(GemmAlignment::N)? SIMDGROUP_BLOCK_N- : short(- min(int(SIMDGROUP_BLOCK_N),- int(params->N) -- int(geometry.block_col_start + tile_col_offset))- );-- const device AT* a_simdgroup =- a_block + size_t(tile_row_offset) * params->leading_dimension_a;+ : short(min(int(SIMDGROUP_BLOCK_N), int(params->N) - int(geometry.block_col_start + tile_col_offset)));++ const device AT* a_simdgroup = a_block + size_t(tile_row_offset) * params->leading_dimension_a;const device BT* b_simdgroup_fp =- b_block_fp + (TRANSPOSE_B ? size_t(tile_col_offset) *- int(params->leading_dimension_b)- : size_t(tile_col_offset));+ b_block_fp ++ (TRANSPOSE_B ? size_t(tile_col_offset) * int(params->leading_dimension_b) : size_t(tile_col_offset));- const ushort tile_block_cols = ushort(- min(int(THREADGROUP_BLOCK_N),- int(params->N) - int(geometry.block_col_start))- );+ const ushort tile_block_cols =+ ushort(min(int(THREADGROUP_BLOCK_N), int(params->N) - int(geometry.block_col_start)));const bool apply_scale = output_transform.contains(GemmDTransform::SCALE);- const bool apply_accumulate =- output_transform.contains(GemmDTransform::ACCUMULATE);+ const bool apply_accumulate = output_transform.contains(GemmDTransform::ACCUMULATE);const bool apply_bias = output_transform.contains(GemmDTransform::BIAS);- const device BT* bias_simdgroup =- output_bias + size_t(block_col) + size_t(tile_col_offset);- using FragType = uzu::matmul::- Fragment<DT, TILES_M, TILES_N, uzu::matmul::MxuFragmentOps>;+ const device BT* bias_simdgroup = output_bias + size_t(block_col) + size_t(tile_col_offset);+ using FragType = uzu::matmul::Fragment<DT, TILES_M, TILES_N, uzu::matmul::MxuFragmentOps>;const short2 thread_position = FragType::get_position(thread_context);auto dispatch_aligned_k = [&](auto body) {@@ -274,19 +222,14 @@ struct MxuMmaCore {};dispatch_aligned_k([&](auto aligned_k) {dispatch_bool(- alignment.contains(GemmAlignment::M) ||- (simdgroup_limit_m == SIMDGROUP_BLOCK_M),+ alignment.contains(GemmAlignment::M) || (simdgroup_limit_m == SIMDGROUP_BLOCK_M),[&](auto aligned_m) {dispatch_bool(- alignment.contains(GemmAlignment::N) ||- (simdgroup_limit_n == SIMDGROUP_BLOCK_N),+ alignment.contains(GemmAlignment::N) || (simdgroup_limit_n == SIMDGROUP_BLOCK_N),[&](auto aligned_n) {auto accumulator_tile = [&]() {- if constexpr (- B_PROLOGUE == GemmBPrologueKind::FullPrecision- ) {- const int aligned_k_iterations_fp =- int(params->aligned_inner_iterations);+ if constexpr (B_PROLOGUE == GemmBPrologueKind::FullPrecision) {+ const int aligned_k_iterations_fp = int(params->aligned_inner_iterations);return uzu::matmul::gemm_loop<AT,BT,@@ -311,30 +254,21 @@ struct MxuMmaCore {thread_context);} else {- const int aligned_k_iterations_q =- int(params->aligned_inner_iterations);+ const int aligned_k_iterations_q = int(params->aligned_inner_iterations);constexpr int pack_factor = get_pack_factor<BITS, 8>();constexpr int bytes_per_pack = get_bytes_per_pack<BITS>();const int k_elements = int(params->K);- const int weights_row_stride_bytes =- k_elements * bytes_per_pack / pack_factor;- const int groups_per_row =- (k_elements + GROUP_SIZE - 1) / GROUP_SIZE;+ const int weights_row_stride_bytes = k_elements * bytes_per_pack / pack_factor;+ const int groups_per_row = (k_elements + GROUP_SIZE - 1) / GROUP_SIZE;const int k_offset_groups = int(k_offset) / GROUP_SIZE;- const device uint8_t* weights_block =- reinterpret_cast<const device uint8_t*>(b) +- block_col * weights_row_stride_bytes +- int(k_offset) * bytes_per_pack / pack_factor;- const device BT* scales_offset =- scales + block_col * groups_per_row + k_offset_groups;+ const device uint8_t* weights_block = reinterpret_cast<const device uint8_t*>(b) ++ block_col * weights_row_stride_bytes ++ int(k_offset) * bytes_per_pack / pack_factor;+ const device BT* scales_offset = scales + block_col * groups_per_row + k_offset_groups;auto loader_b = [&]() {- if constexpr (- B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant- ) {- const device BT* biases_offset =- biases + block_col * groups_per_row +- k_offset_groups;+ if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant) {+ const device BT* biases_offset = biases + block_col * groups_per_row + k_offset_groups;return BLoaderScaleBias(weights_block,scales_offset,@@ -344,18 +278,12 @@ struct MxuMmaCore {thread_context.simdgroup_index,thread_context.simd_lane_id);- } else if constexpr (- B_PROLOGUE ==- GemmBPrologueKind::ScaleZeroPointDequant- ) {+ } else if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant) {const int zero_point_stride_per_row =- (BITS == 4) ? ((groups_per_row + 1) / 2)- : groups_per_row;+ (BITS == 4) ? ((groups_per_row + 1) / 2) : groups_per_row;const device uint8_t* zero_points_row_start =- zero_points +- block_col * zero_point_stride_per_row +- ((BITS == 4) ? (k_offset_groups / 2)- : k_offset_groups);+ zero_points + block_col * zero_point_stride_per_row ++ ((BITS == 4) ? (k_offset_groups / 2) : k_offset_groups);return BLoaderScaleZeroPoint(weights_block,scales_offset,@@ -396,27 +324,19 @@ struct MxuMmaCore {}();if (apply_scale) {- const AccumulatorType scale =- AccumulatorType(params->ab_scale);+ const AccumulatorType scale = AccumulatorType(params->ab_scale);METAL_PRAGMA_UNROLL- for (ushort i = 0; i < accumulator_tile.ELEMENTS_PER_TILE;- i++) {+ for (ushort i = 0; i < accumulator_tile.ELEMENTS_PER_TILE; i++) {accumulator_tile.elements()[i] *= scale;}}if (apply_accumulate) {- uzu::matmul::Fragment<- DT,- TILES_M,- TILES_N,- uzu::matmul::MxuFragmentOps>- existing_output(thread_context);+ uzu::matmul::Fragment<DT, TILES_M, TILES_N, uzu::matmul::MxuFragmentOps> existing_output(+ thread_context+ );if constexpr (aligned_m.value && aligned_n.value) {- existing_output.load(- d_simdgroup,- int(params->leading_dimension_d)- );+ existing_output.load(d_simdgroup, int(params->leading_dimension_d));} else {existing_output.load_safe(d_simdgroup,@@ -425,10 +345,8 @@ struct MxuMmaCore {);}METAL_PRAGMA_UNROLL- for (ushort i = 0; i < accumulator_tile.ELEMENTS_PER_TILE;- i++) {- accumulator_tile.elements()[i] +=- AccumulatorType(existing_output.elements()[i]);+ for (ushort i = 0; i < accumulator_tile.ELEMENTS_PER_TILE; i++) {+ accumulator_tile.elements()[i] += AccumulatorType(existing_output.elements()[i]);}}@@ -436,37 +354,21 @@ struct MxuMmaCore {METAL_PRAGMA_UNROLLfor (ushort tile_row = 0; tile_row < TILES_M; ++tile_row) {METAL_PRAGMA_UNROLL- for (ushort tile_col = 0; tile_col < TILES_N;- ++tile_col) {- thread auto& frag =- accumulator_tile.fragment_at(tile_row, tile_col);- const short col_base = short(- tile_col *- uzu::matmul::MxuFragmentOps::FRAGMENT_COLS +- thread_position.x- );+ for (ushort tile_col = 0; tile_col < TILES_N; ++tile_col) {+ thread auto& frag = accumulator_tile.fragment_at(tile_row, tile_col);+ const short col_base =+ short(tile_col * uzu::matmul::MxuFragmentOps::FRAGMENT_COLS + thread_position.x);METAL_PRAGMA_UNROLL- for (ushort i = 0;- i <- uzu::matmul::MxuFragmentOps::THREAD_ELEMENT_ROWS;- ++i) {+ for (ushort i = 0; i < uzu::matmul::MxuFragmentOps::THREAD_ELEMENT_ROWS; ++i) {METAL_PRAGMA_UNROLL- for (ushort j = 0;- j <- uzu::matmul::MxuFragmentOps::THREAD_ELEMENT_COLS;- ++j) {- const ushort element_index =- i * uzu::matmul::MxuFragmentOps::- THREAD_ELEMENT_COLS +- j;+ for (ushort j = 0; j < uzu::matmul::MxuFragmentOps::THREAD_ELEMENT_COLS; ++j) {+ const ushort element_index = i * uzu::matmul::MxuFragmentOps::THREAD_ELEMENT_COLS + j;const short col_local = short(col_base + j);if constexpr (aligned_n.value) {- frag[element_index] +=- AccumulatorType(bias_simdgroup[col_local]);+ frag[element_index] += AccumulatorType(bias_simdgroup[col_local]);} else {if (col_local < simdgroup_limit_n) {- frag[element_index] +=- AccumulatorType(bias_simdgroup[col_local]);+ frag[element_index] += AccumulatorType(bias_simdgroup[col_local]);}}}@@ -476,10 +378,7 @@ struct MxuMmaCore {}if constexpr (aligned_m.value && aligned_n.value) {- accumulator_tile.store(- d_simdgroup,- int(params->leading_dimension_d)- );+ accumulator_tile.store(d_simdgroup, int(params->leading_dimension_d));} else {accumulator_tile.store_safe(d_simdgroup,@@ -495,14 +394,9 @@ struct MxuMmaCore {if (output_transform.contains(GemmDTransform::RHT)) {threadgroup_barrier(mem_flags::mem_device);- device DT* d_block =- d + block_row * params->leading_dimension_d + block_col;- const ushort tile_block_rows = ushort(- min(int(THREADGROUP_BLOCK_M), int(params->M) - int(block_row))- );- const ushort tile_block_cols = ushort(- min(int(THREADGROUP_BLOCK_N), int(params->N) - int(block_col))- );+ device DT* d_block = d + block_row * params->leading_dimension_d + block_col;+ const ushort tile_block_rows = ushort(min(int(THREADGROUP_BLOCK_M), int(params->M) - int(block_row)));+ const ushort tile_block_cols = ushort(min(int(THREADGROUP_BLOCK_N), int(params->N) - int(block_col)));apply_output_random_hadamard_transform(d_block,rht_factors + block_col,diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.hindex 6e34d9caf..bcdcc86a8 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_bias.h@@ -22,28 +22,18 @@ template <short GROUP_SIZE,short BITS>struct QuantizedBlockLoaderScaleBias {- static_assert(- THREADGROUP_TILE_COLS <= GROUP_SIZE,- "Group size should be larger than columns"- );- static_assert(- GROUP_SIZE % THREADGROUP_TILE_COLS == 0,- "Group size should be divisible by columns"- );+ static_assert(THREADGROUP_TILE_COLS <= GROUP_SIZE, "Group size should be larger than columns");+ static_assert(GROUP_SIZE % THREADGROUP_TILE_COLS == 0, "Group size should be divisible by columns");static_assert(BITS == 4 || BITS == 8, "Only int4 and int8 supported");METAL_CONST short pack_factor = get_pack_factor<BITS, 8>();METAL_CONST short bytes_per_pack = get_bytes_per_pack<BITS>();- METAL_CONST short THREADGROUP_TILE_COLS_PACKED =- THREADGROUP_TILE_COLS / pack_factor;- METAL_CONST short READS_PER_THREAD =- (THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS < THREADGROUP_SIZE)- ? 1- : (THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS) /- THREADGROUP_SIZE;+ METAL_CONST short THREADGROUP_TILE_COLS_PACKED = THREADGROUP_TILE_COLS / pack_factor;+ METAL_CONST short READS_PER_THREAD = (THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS < THREADGROUP_SIZE)+ ? 1+ : (THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS) / THREADGROUP_SIZE;METAL_CONST short GROUP_STEPS_PER_BLOCK = GROUP_SIZE / THREADGROUP_TILE_COLS;- METAL_CONST bool TILE_HAS_IDLE_THREADS =- THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS < THREADGROUP_SIZE;+ METAL_CONST bool TILE_HAS_IDLE_THREADS = THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS < THREADGROUP_SIZE;const int src_leading_dim;const int tile_stride;@@ -71,23 +61,14 @@ struct QuantizedBlockLoaderScaleBias {: src_leading_dim(src_leading_dim_),tile_stride(REDUCTION_DIMENSION ? THREADGROUP_TILE_COLS_PACKED * bytes_per_pack- : THREADGROUP_TILE_ROWS * src_leading_dim_ *- bytes_per_pack / pack_factor+ : THREADGROUP_TILE_ROWS * src_leading_dim_ * bytes_per_pack / pack_factor),- group_step_counter(0),- group_stride(THREADGROUP_TILE_ROWS * src_leading_dim_ / GROUP_SIZE),+ group_step_counter(0), group_stride(THREADGROUP_TILE_ROWS * src_leading_dim_ / GROUP_SIZE),thread_index(simd_group_id * 32 + simd_lane_id),- tile_row_index(- READS_PER_THREAD * thread_index / THREADGROUP_TILE_COLS_PACKED- ),- tile_col_index(- (READS_PER_THREAD * thread_index) % THREADGROUP_TILE_COLS_PACKED- ),- dst(dst_ + tile_row_index * DESTINATION_LEADING_DIMENSION +- tile_col_index * pack_factor),- src(src_ +- tile_row_index * src_leading_dim_ * bytes_per_pack / pack_factor +- tile_col_index * bytes_per_pack),+ tile_row_index(READS_PER_THREAD * thread_index / THREADGROUP_TILE_COLS_PACKED),+ tile_col_index((READS_PER_THREAD * thread_index) % THREADGROUP_TILE_COLS_PACKED),+ dst(dst_ + tile_row_index * DESTINATION_LEADING_DIMENSION + tile_col_index * pack_factor),+ src(src_ + tile_row_index * src_leading_dim_ * bytes_per_pack / pack_factor + tile_col_index * bytes_per_pack),scales(scales_ + tile_row_index * src_leading_dim_ / GROUP_SIZE),biases(biases_ + tile_row_index * src_leading_dim_ / GROUP_SIZE) {}@@ -101,12 +82,7 @@ struct QuantizedBlockLoaderScaleBias {T scale = *scales;T bias = *biases;for (int i = 0; i < READS_PER_THREAD; i++) {- dequantize<T, pack_factor, BITS>(- src + i * bytes_per_pack,- scale,- bias,- dst + i * pack_factor- );+ dequantize<T, pack_factor, BITS>(src + i * bytes_per_pack, scale, bias, dst + i * pack_factor);}}@@ -136,12 +112,7 @@ struct QuantizedBlockLoaderScaleBias {T scale = *scales;T bias = *biases;for (int i = 0; i < READS_PER_THREAD; i++) {- dequantize<T, pack_factor, BITS>(- src + i * bytes_per_pack,- scale,- bias,- dst + i * pack_factor- );+ dequantize<T, pack_factor, BITS>(src + i * bytes_per_pack, scale, bias, dst + i * pack_factor);}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.hindex 5f21844d6..8a36d2549 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h@@ -23,28 +23,18 @@ template <short BITS,bool SCALE_SYMMETRIC = false>struct QuantizedBlockLoaderScaleZeroPoint {- static_assert(- THREADGROUP_TILE_COLS <= GROUP_SIZE,- "Group size should be larger than columns"- );- static_assert(- GROUP_SIZE % THREADGROUP_TILE_COLS == 0,- "Group size should be divisible by columns"- );+ static_assert(THREADGROUP_TILE_COLS <= GROUP_SIZE, "Group size should be larger than columns");+ static_assert(GROUP_SIZE % THREADGROUP_TILE_COLS == 0, "Group size should be divisible by columns");static_assert(BITS == 4 || BITS == 8, "Only int4 and int8 supported");METAL_CONST short pack_factor = get_pack_factor<BITS, 8>();METAL_CONST short bytes_per_pack = get_bytes_per_pack<BITS>();- METAL_CONST short THREADGROUP_TILE_COLS_PACKED =- THREADGROUP_TILE_COLS / pack_factor;- METAL_CONST short READS_PER_THREAD =- (THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS < THREADGROUP_SIZE)- ? 1- : (THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS) /- THREADGROUP_SIZE;+ METAL_CONST short THREADGROUP_TILE_COLS_PACKED = THREADGROUP_TILE_COLS / pack_factor;+ METAL_CONST short READS_PER_THREAD = (THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS < THREADGROUP_SIZE)+ ? 1+ : (THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS) / THREADGROUP_SIZE;METAL_CONST short GROUP_STEPS_PER_BLOCK = GROUP_SIZE / THREADGROUP_TILE_COLS;- METAL_CONST bool TILE_HAS_IDLE_THREADS =- THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS < THREADGROUP_SIZE;+ METAL_CONST bool TILE_HAS_IDLE_THREADS = THREADGROUP_TILE_COLS_PACKED * THREADGROUP_TILE_ROWS < THREADGROUP_SIZE;const int src_leading_dim;const int groups_per_row;@@ -76,48 +66,25 @@ struct QuantizedBlockLoaderScaleZeroPoint {: src_leading_dim(src_leading_dim_), groups_per_row(groups_per_row_),tile_stride(REDUCTION_DIMENSION ? THREADGROUP_TILE_COLS_PACKED * bytes_per_pack- : THREADGROUP_TILE_ROWS * src_leading_dim_ *- bytes_per_pack / pack_factor+ : THREADGROUP_TILE_ROWS * src_leading_dim_ * bytes_per_pack / pack_factor),- group_step_counter(0), k_base(0),- group_stride(THREADGROUP_TILE_ROWS * groups_per_row_),+ group_step_counter(0), k_base(0), group_stride(THREADGROUP_TILE_ROWS * groups_per_row_),thread_index(simd_group_id * 32 + simd_lane_id),- tile_row_index(- READS_PER_THREAD * thread_index / THREADGROUP_TILE_COLS_PACKED- ),- tile_col_index(- (READS_PER_THREAD * thread_index) % THREADGROUP_TILE_COLS_PACKED- ),- dst(dst_ + tile_row_index * DESTINATION_LEADING_DIMENSION +- tile_col_index * pack_factor),- src(src_ +- tile_row_index * src_leading_dim_ * bytes_per_pack / pack_factor +- tile_col_index * bytes_per_pack),- scales(- REDUCTION_DIMENSION == 1- ? (scales_ + tile_row_index * groups_per_row_)- : scales_- ),- scales_row_start(- REDUCTION_DIMENSION == 1- ? (scales_ + tile_row_index * groups_per_row_)- : scales_- ),+ tile_row_index(READS_PER_THREAD * thread_index / THREADGROUP_TILE_COLS_PACKED),+ tile_col_index((READS_PER_THREAD * thread_index) % THREADGROUP_TILE_COLS_PACKED),+ dst(dst_ + tile_row_index * DESTINATION_LEADING_DIMENSION + tile_col_index * pack_factor),+ src(src_ + tile_row_index * src_leading_dim_ * bytes_per_pack / pack_factor + tile_col_index * bytes_per_pack),+ scales(REDUCTION_DIMENSION == 1 ? (scales_ + tile_row_index * groups_per_row_) : scales_),+ scales_row_start(REDUCTION_DIMENSION == 1 ? (scales_ + tile_row_index * groups_per_row_) : scales_),zero_points_row_start(- SCALE_SYMMETRIC- ? nullptr- : (REDUCTION_DIMENSION == 1- ? (zero_points_row_start_ +- tile_row_index * (BITS == 4- ? ((groups_per_row_ + 1) / 2)- : groups_per_row_))- : zero_points_row_start_)+ SCALE_SYMMETRIC ? nullptr+ : (REDUCTION_DIMENSION == 1+ ? (zero_points_row_start_ ++ tile_row_index * (BITS == 4 ? ((groups_per_row_ + 1) / 2) : groups_per_row_))+ : zero_points_row_start_)) {}- inline void current_scale_bias(- thread T& out_scale,- thread T& out_bias- ) const {+ inline void current_scale_bias(thread T& out_scale, thread T& out_bias) const {uint zero_point_value;T scale_value;int group_index;@@ -131,11 +98,9 @@ struct QuantizedBlockLoaderScaleZeroPoint {if constexpr (SCALE_SYMMETRIC) {zero_point_value = 1u << (BITS - 1);} else if constexpr (BITS == 4) {- const device uint8_t* zero_point_ptr =- zero_points_row_start + (group_index >> 1);+ const device uint8_t* zero_point_ptr = zero_points_row_start + (group_index >> 1);uint8_t zero_point_byte = *zero_point_ptr;- zero_point_value =- (uint(zero_point_byte) >> (uint(group_index & 1) * 4u)) & 0x0Fu;+ zero_point_value = (uint(zero_point_byte) >> (uint(group_index & 1) * 4u)) & 0x0Fu;} else {zero_point_value = zero_points_row_start[group_index];}@@ -154,12 +119,7 @@ struct QuantizedBlockLoaderScaleZeroPoint {T bias;current_scale_bias(scale, bias);for (int i = 0; i < READS_PER_THREAD; i++) {- dequantize<T, pack_factor, BITS>(- src + i * bytes_per_pack,- scale,- bias,- dst + i * pack_factor- );+ dequantize<T, pack_factor, BITS>(src + i * bytes_per_pack, scale, bias, dst + i * pack_factor);}}@@ -187,12 +147,7 @@ struct QuantizedBlockLoaderScaleZeroPoint {for (int i = 0; i < READS_PER_THREAD; i++) {int pack_index = tile_col_index + i;if (pack_index < valid_packs) {- dequantize<T, pack_factor, BITS>(- src + i * bytes_per_pack,- scale,- bias,- dst + i * pack_factor- );+ dequantize<T, pack_factor, BITS>(src + i * bytes_per_pack, scale, bias, dst + i * pack_factor);if (pack_index == valid_packs - 1) {int remaining = valid_cols - pack_index * pack_factor;@@ -221,12 +176,7 @@ struct QuantizedBlockLoaderScaleZeroPoint {T bias;current_scale_bias(scale, bias);for (int i = 0; i < READS_PER_THREAD; i++) {- dequantize<T, pack_factor, BITS>(- src + i * bytes_per_pack,- scale,- bias,- dst + i * pack_factor- );+ dequantize<T, pack_factor, BITS>(src + i * bytes_per_pack, scale, bias, dst + i * pack_factor);}}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_unpack.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_unpack.hindex e892d0a2f..654c72165 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_unpack.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_unpack.h@@ -51,12 +51,7 @@ METAL_FUNC bfloat4 uint4_to_fp4<bfloat, 8>(uint4 n) {}template <typename U, int N, int bits>-inline void dequantize(- const device uint8_t* w,- U scale,- U bias,- threadgroup U* w_local-) {+inline void dequantize(const device uint8_t* w, U scale, U bias, threadgroup U* w_local) {static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");if (bits == 4) {@@ -74,18 +69,10 @@ inline void dequantize(}template <>-inline void dequantize<bfloat, 8, 4>(- const device uint8_t* w,- bfloat scale,- bfloat bias,- threadgroup bfloat* w_local-) {+inline void dequantize<bfloat, 8, 4>(const device uint8_t* w, bfloat scale, bfloat bias, threadgroup bfloat* w_local) {const uint32_t packed = *reinterpret_cast<const device uint32_t*>(w);- const bfloat4 lo =- bfloat4(as_type<uchar4>(packed & 0x0f0f0f0fu)) * scale + bias;- const bfloat4 hi = bfloat4(as_type<uchar4>(packed & 0xf0f0f0f0u)) *- (scale * bfloat(0.0625f)) +- bias;+ const bfloat4 lo = bfloat4(as_type<uchar4>(packed & 0x0f0f0f0fu)) * scale + bias;+ const bfloat4 hi = bfloat4(as_type<uchar4>(packed & 0xf0f0f0f0u)) * (scale * bfloat(0.0625f)) + bias;w_local[0] = lo.x;w_local[1] = hi.x;diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.hindex 1ce69ed3b..30be122da 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h@@ -33,25 +33,16 @@ struct SimdgroupMmaCore {METAL_CONST int THREADGROUP_BLOCK_M = gemm_tiling_block_m(GEMM_TILING);METAL_CONST int THREADGROUP_BLOCK_N = gemm_tiling_block_n(GEMM_TILING);METAL_CONST int THREADGROUP_BLOCK_K = gemm_tiling_block_k(GEMM_TILING);- METAL_CONST int SIMDGROUPS_PER_ROW =- gemm_tiling_simdgroups_per_row(GEMM_TILING);- METAL_CONST int SIMDGROUPS_PER_COLUMN =- gemm_tiling_simdgroups_per_column(GEMM_TILING);+ METAL_CONST int SIMDGROUPS_PER_ROW = gemm_tiling_simdgroups_per_row(GEMM_TILING);+ METAL_CONST int SIMDGROUPS_PER_COLUMN = gemm_tiling_simdgroups_per_column(GEMM_TILING);METAL_CONST ushort PADDING_A = 16 / sizeof(AT);METAL_CONST ushort PADDING_B = 16 / sizeof(BT);METAL_CONST ushort SHARED_STRIDE_A = THREADGROUP_BLOCK_K + PADDING_A;- METAL_CONST ushort SHARED_STRIDE_B =- (TRANSPOSE_B ? THREADGROUP_BLOCK_K : THREADGROUP_BLOCK_N) + PADDING_B;- METAL_CONST ushort THREADGROUP_THREADS =- SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * METAL_SIMD_SIZE;+ METAL_CONST ushort SHARED_STRIDE_B = (TRANSPOSE_B ? THREADGROUP_BLOCK_K : THREADGROUP_BLOCK_N) + PADDING_B;+ METAL_CONST ushort THREADGROUP_THREADS = SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * METAL_SIMD_SIZE;- using ALoader = uzu::matmul::ThreadgroupLoader<- AT,- THREADGROUP_BLOCK_M,- THREADGROUP_BLOCK_K,- SHARED_STRIDE_A,- true,- THREADGROUP_THREADS>;+ using ALoader = uzu::matmul::+ ThreadgroupLoader<AT, THREADGROUP_BLOCK_M, THREADGROUP_BLOCK_K, SHARED_STRIDE_A, true, THREADGROUP_THREADS>;using BLoaderFp = uzu::matmul::ThreadgroupLoader<BT,TRANSPOSE_B ? THREADGROUP_BLOCK_N : THREADGROUP_BLOCK_K,@@ -118,8 +109,7 @@ struct SimdgroupMmaCore {constexpr GemmAlignment gemm_alignment{GEMM_ALIGNMENT_RAW};short2 tile_dimensions_a = short2(THREADGROUP_BLOCK_K, tile_block_rows);short2 tile_dimensions_b =- TRANSPOSE_B ? short2(THREADGROUP_BLOCK_K, tile_block_cols)- : short2(tile_block_cols, THREADGROUP_BLOCK_K);+ TRANSPOSE_B ? short2(THREADGROUP_BLOCK_K, tile_block_cols) : short2(tile_block_cols, THREADGROUP_BLOCK_K);for (int k = 0; k < aligned_k_iterations; k++) {threadgroup_barrier(mem_flags::mem_threadgroup);@@ -144,11 +134,9 @@ struct SimdgroupMmaCore {if constexpr (!gemm_alignment.contains(GemmAlignment::K)) {threadgroup_barrier(mem_flags::mem_threadgroup);- short2 last_tile_dimensions_a =- short2(leftover_block_depth, tile_block_rows);+ short2 last_tile_dimensions_a = short2(leftover_block_depth, tile_block_rows);short2 last_tile_dimensions_b =- TRANSPOSE_B ? short2(leftover_block_depth, tile_block_cols)- : short2(tile_block_cols, leftover_block_depth);+ TRANSPOSE_B ? short2(leftover_block_depth, tile_block_cols) : short2(tile_block_cols, leftover_block_depth);loader_a.load_safe(last_tile_dimensions_a);loader_b.load_safe(last_tile_dimensions_b);@@ -166,8 +154,7 @@ struct SimdgroupMmaCore {const thread ushort& tile_block_rows,const thread ushort& tile_block_cols,const bool needs_epilogue,- const thread uzu::matmul::TransformScaleAccumulate<float, float>&- epilogue,+ const thread uzu::matmul::TransformScaleAccumulate<float, float>& epilogue,const device BT* bias_block,const bool needs_bias,const device int32_t* rht_factors_block,@@ -175,10 +162,7 @@ struct SimdgroupMmaCore {const thread ThreadContext& thread_context) {constexpr GemmAlignment gemm_alignment{GEMM_ALIGNMENT_RAW};- if constexpr (- gemm_alignment.contains(GemmAlignment::M) &&- gemm_alignment.contains(GemmAlignment::N)- ) {+ if constexpr (gemm_alignment.contains(GemmAlignment::M) && gemm_alignment.contains(GemmAlignment::N)) {if (needs_epilogue) {accumulator.apply_epilogue(d, params->leading_dimension_d, 1, epilogue);}@@ -188,25 +172,13 @@ struct SimdgroupMmaCore {accumulator.store_result(d, params->leading_dimension_d);} else {if (needs_epilogue) {- accumulator.apply_epilogue_safe(- d,- params->leading_dimension_d,- 1,- short2(tile_block_cols, tile_block_rows),- epilogue- );+ accumulator+ .apply_epilogue_safe(d, params->leading_dimension_d, 1, short2(tile_block_cols, tile_block_rows), epilogue);}if (needs_bias) {- accumulator.apply_bias_safe(- bias_block,- short2(tile_block_cols, tile_block_rows)- );+ accumulator.apply_bias_safe(bias_block, short2(tile_block_cols, tile_block_rows));}- accumulator.store_result_safe(- d,- params->leading_dimension_d,- short2(tile_block_cols, tile_block_rows)- );+ accumulator.store_result_safe(d, params->leading_dimension_d, short2(tile_block_cols, tile_block_rows));}if (needs_rht) {@@ -241,17 +213,13 @@ struct SimdgroupMmaCore {) {const uint partition = thread_context.threadgroup_position.z;const uint tile_row = thread_context.threadgroup_position.y;- const uint2 tile =- tile_id(uint2(thread_context.threadgroup_position.x, tile_row), params);- const auto geometry =- ThreadgroupTileGeometry<THREADGROUP_BLOCK_M, THREADGROUP_BLOCK_N>::- compute(tile, params);+ const uint2 tile = tile_id(uint2(thread_context.threadgroup_position.x, tile_row), params);+ const auto geometry = ThreadgroupTileGeometry<THREADGROUP_BLOCK_M, THREADGROUP_BLOCK_N>::compute(tile, params);if (geometry.out_of_bounds) {return;}- const uint k_offset =- partition * params->aligned_inner_iterations * THREADGROUP_BLOCK_K;+ const uint k_offset = partition * params->aligned_inner_iterations * THREADGROUP_BLOCK_K;threadgroup_barrier(mem_flags::mem_none);@@ -259,27 +227,20 @@ struct SimdgroupMmaCore {const size_t block_col = size_t(geometry.block_col_start);a += block_row * params->leading_dimension_a + k_offset;- d += size_t(partition) * size_t(params->M) * size_t(params->N) +- block_row * params->leading_dimension_d + block_col;+ d +=+ size_t(partition) * size_t(params->M) * size_t(params->N) + block_row * params->leading_dimension_d + block_col;- thread ALoader- loader_a(a, params->leading_dimension_a, a_shared, thread_context);+ thread ALoader loader_a(a, params->leading_dimension_a, a_shared, thread_context);thread TileAccumulator accumulator(thread_context);const ushort tile_block_rows =- min(THREADGROUP_BLOCK_M,- static_cast<int>(params->M) -- static_cast<int>(geometry.block_row_start));+ min(THREADGROUP_BLOCK_M, static_cast<int>(params->M) - static_cast<int>(geometry.block_row_start));const ushort tile_block_cols =- min(THREADGROUP_BLOCK_N,- static_cast<int>(params->N) -- static_cast<int>(geometry.block_col_start));- const ushort leftover_block_depth =- params->K - params->aligned_inner_iterations * THREADGROUP_BLOCK_K;+ min(THREADGROUP_BLOCK_N, static_cast<int>(params->N) - static_cast<int>(geometry.block_col_start));+ const ushort leftover_block_depth = params->K - params->aligned_inner_iterations * THREADGROUP_BLOCK_K;const bool needs_scale = output_transform.contains(GemmDTransform::SCALE);- const bool needs_accumulate =- output_transform.contains(GemmDTransform::ACCUMULATE);+ const bool needs_accumulate = output_transform.contains(GemmDTransform::ACCUMULATE);const bool needs_bias = output_transform.contains(GemmDTransform::BIAS);const bool needs_rht = output_transform.contains(GemmDTransform::RHT);const bool needs_epilogue = needs_scale || needs_accumulate;@@ -291,35 +252,23 @@ struct SimdgroupMmaCore {auto loader_b = [&]() {if constexpr (B_PROLOGUE == GemmBPrologueKind::FullPrecision) {- const device BT* b_block_fp =- b +- (TRANSPOSE_B ? block_col * params->leading_dimension_b- : block_col) +- (TRANSPOSE_B ? k_offset : k_offset * params->leading_dimension_b);- return BLoaderFp(- b_block_fp,- params->leading_dimension_b,- b_shared,- thread_context- );+ const device BT* b_block_fp = b + (TRANSPOSE_B ? block_col * params->leading_dimension_b : block_col) ++ (TRANSPOSE_B ? k_offset : k_offset * params->leading_dimension_b);+ return BLoaderFp(b_block_fp, params->leading_dimension_b, b_shared, thread_context);} else {constexpr int pack_factor = get_pack_factor<BITS, 8>();constexpr int bytes_per_pack = get_bytes_per_pack<BITS>();const int k_elements = int(params->K);- const int weights_row_stride_bytes =- k_elements * bytes_per_pack / pack_factor;+ const int weights_row_stride_bytes = k_elements * bytes_per_pack / pack_factor;const int groups_per_row = (k_elements + GROUP_SIZE - 1) / GROUP_SIZE;const int k_offset_groups = int(k_offset) / GROUP_SIZE;- const device uint8_t* weights_block =- reinterpret_cast<const device uint8_t*>(b) +- block_col * weights_row_stride_bytes +- int(k_offset) * bytes_per_pack / pack_factor;- const device BT* scales_offset =- scales + block_col * groups_per_row + k_offset_groups;+ const device uint8_t* weights_block = reinterpret_cast<const device uint8_t*>(b) ++ block_col * weights_row_stride_bytes ++ int(k_offset) * bytes_per_pack / pack_factor;+ const device BT* scales_offset = scales + block_col * groups_per_row + k_offset_groups;if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant) {- const device BT* biases_offset =- biases + block_col * groups_per_row + k_offset_groups;+ const device BT* biases_offset = biases + block_col * groups_per_row + k_offset_groups;return BLoaderScaleBias(weights_block,scales_offset,@@ -329,14 +278,10 @@ struct SimdgroupMmaCore {thread_context.simdgroup_index,thread_context.simd_lane_id);- } else if constexpr (- B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant- ) {- const int zero_point_stride_per_row =- (BITS == 4) ? ((groups_per_row + 1) / 2) : groups_per_row;- const device uint8_t* zero_points_row_start =- zero_points + block_col * zero_point_stride_per_row +- ((BITS == 4) ? (k_offset_groups / 2) : k_offset_groups);+ } else if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant) {+ const int zero_point_stride_per_row = (BITS == 4) ? ((groups_per_row + 1) / 2) : groups_per_row;+ const device uint8_t* zero_points_row_start = zero_points + block_col * zero_point_stride_per_row ++ ((BITS == 4) ? (k_offset_groups / 2) : k_offset_groups);return BLoaderScaleZeroPoint(weights_block,scales_offset,@@ -362,22 +307,14 @@ struct SimdgroupMmaCore {}}();- const bool all_aligned = ((alignment.contains(GemmAlignment::M)) ||- (tile_block_rows == THREADGROUP_BLOCK_M)) &&- ((alignment.contains(GemmAlignment::N)) ||- (tile_block_cols == THREADGROUP_BLOCK_N)) &&+ const bool all_aligned = ((alignment.contains(GemmAlignment::M)) || (tile_block_rows == THREADGROUP_BLOCK_M)) &&+ ((alignment.contains(GemmAlignment::N)) || (tile_block_cols == THREADGROUP_BLOCK_N)) &&alignment.contains(GemmAlignment::K);- constexpr uint MASK_ALL = static_cast<uint>(GemmAlignment::M) |- static_cast<uint>(GemmAlignment::N) |- static_cast<uint>(GemmAlignment::K);+ constexpr uint MASK_ALL =+ static_cast<uint>(GemmAlignment::M) | static_cast<uint>(GemmAlignment::N) | static_cast<uint>(GemmAlignment::K);const uint dynamic_alignment_mask =- alignment.raw_value |- ((tile_block_rows == THREADGROUP_BLOCK_M)- ? static_cast<uint>(GemmAlignment::M)- : 0u) |- ((tile_block_cols == THREADGROUP_BLOCK_N)- ? static_cast<uint>(GemmAlignment::N)- : 0u);+ alignment.raw_value | ((tile_block_rows == THREADGROUP_BLOCK_M) ? static_cast<uint>(GemmAlignment::M) : 0u) |+ ((tile_block_cols == THREADGROUP_BLOCK_N) ? static_cast<uint>(GemmAlignment::N) : 0u);auto kernel_invoke = [&](auto gemm_alignment_mask) {constexpr uint gemm_alignment = gemm_alignment_mask.value;diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metalindex 62046f8b8..66fe494e6 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal@@ -10,18 +10,12 @@using namespace metal;using namespace uzu::gemm;-#define GEMM_MXU_QUANT \- (USE_MXU && B_PROLOGUE != GemmBPrologueKind::FullPrecision)-#define GEMM_TGA_ELEMENTS \- ((USE_MXU) ? 1 \- : (gemm_tiling_block_m(GEMM_TILING) * \- (gemm_tiling_block_k(GEMM_TILING) + 16 / int(sizeof(AT)))))-#define GEMM_TGB_ELEMENTS \- ((USE_MXU) ? (GEMM_MXU_QUANT ? (gemm_tiling_block_n(GEMM_TILING) * \- (int(GROUP_SIZE) + 16 / int(sizeof(BT)))) \- : 1) \- : (gemm_tiling_block_n(GEMM_TILING) * \- (gemm_tiling_block_k(GEMM_TILING) + 16 / int(sizeof(BT)))))+#define GEMM_MXU_QUANT (USE_MXU && B_PROLOGUE != GemmBPrologueKind::FullPrecision)+#define GEMM_TGA_ELEMENTS \+ ((USE_MXU) ? 1 : (gemm_tiling_block_m(GEMM_TILING) * (gemm_tiling_block_k(GEMM_TILING) + 16 / int(sizeof(AT)))))+#define GEMM_TGB_ELEMENTS \+ ((USE_MXU) ? (GEMM_MXU_QUANT ? (gemm_tiling_block_n(GEMM_TILING) * (int(GROUP_SIZE) + 16 / int(sizeof(BT)))) : 1) \+ : (gemm_tiling_block_n(GEMM_TILING) * (gemm_tiling_block_k(GEMM_TILING) + 16 / int(sizeof(BT)))))template <typename AT,@@ -117,51 +111,37 @@ KERNEL(Gemm)((void)thread_z;if constexpr (USE_MXU) {- MxuMmaCore<- AT,- BT,- DT,- GEMM_TILING,- TRANSPOSE_B,- B_PROLOGUE,- BITS,- GROUP_SIZE>::- run(a,- b,- d,- params,- alignment,- output_transform,- scales,- biases,- zero_points,- output_bias,- rht_factors,- b_shared,- thread_context);+ MxuMmaCore<AT, BT, DT, GEMM_TILING, TRANSPOSE_B, B_PROLOGUE, BITS, GROUP_SIZE>::run(+ a,+ b,+ d,+ params,+ alignment,+ output_transform,+ scales,+ biases,+ zero_points,+ output_bias,+ rht_factors,+ b_shared,+ thread_context+ );} else {- SimdgroupMmaCore<- AT,- BT,- DT,- GEMM_TILING,- TRANSPOSE_B,- B_PROLOGUE,- BITS,- GROUP_SIZE>::- run(a,- b,- d,- params,- alignment,- output_transform,- scales,- biases,- zero_points,- output_bias,- rht_factors,- a_shared,- b_shared,- thread_context);+ SimdgroupMmaCore<AT, BT, DT, GEMM_TILING, TRANSPOSE_B, B_PROLOGUE, BITS, GROUP_SIZE>::run(+ a,+ b,+ d,+ params,+ alignment,+ output_transform,+ scales,+ biases,+ zero_points,+ output_bias,+ rht_factors,+ a_shared,+ b_shared,+ thread_context+ );}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm_split_k_reduce.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm_split_k_reduce.metalindex cc579546d..ef20f04a3 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm_split_k_reduce.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm_split_k_reduce.metal@@ -26,28 +26,21 @@ KERNEL(GemmSplitKReduce)((void)threadgroup_index;(void)thread_index_in_threadgroup;- const uint threads_per_threadgroup =- thread_context.simdgroups_per_threadgroup * thread_context.simdgroup_size;+ const uint threads_per_threadgroup = thread_context.simdgroups_per_threadgroup * thread_context.simdgroup_size;const uint local_thread_index =- thread_context.simdgroup_index * thread_context.simdgroup_size +- thread_context.simd_lane_id;+ thread_context.simdgroup_index * thread_context.simdgroup_size + thread_context.simd_lane_id;const uint vector_count = element_count / 4u;- const uint vector_index =- thread_context.threadgroup_position.x * threads_per_threadgroup +- local_thread_index;+ const uint vector_index = thread_context.threadgroup_position.x * threads_per_threadgroup + local_thread_index;if (vector_index >= vector_count) {return;}- device vec<T, 4>* output_vectors =- reinterpret_cast<device vec<T, 4>*>(output);- const device vec<T, 4>* partial_sum_vectors =- reinterpret_cast<const device vec<T, 4>*>(partial_sums);+ device vec<T, 4>* output_vectors = reinterpret_cast<device vec<T, 4>*>(output);+ const device vec<T, 4>* partial_sum_vectors = reinterpret_cast<const device vec<T, 4>*>(partial_sums);float4 accumulator = float4(0.0f);for (uint partition = 0u; partition < partition_count; ++partition) {- accumulator +=- float4(partial_sum_vectors[partition * vector_count + vector_index]);+ accumulator += float4(partial_sum_vectors[partition * vector_count + vector_index]);}if (output_transform.contains(GemmDTransform::SCALE)) {@@ -58,9 +51,7 @@ KERNEL(GemmSplitKReduce)(}if (output_transform.contains(GemmDTransform::BIAS)) {const uint column = (vector_index * 4u) % column_count;- accumulator += float4(- *reinterpret_cast<const device vec<T, 4>*>(output_bias + column)- );+ accumulator += float4(*reinterpret_cast<const device vec<T, 4>*>(output_bias + column));}output_vectors[vector_index] = vec<T, 4>(accumulator);diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/b_source.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/b_source.hindex a9fa82495..070ccaf06 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/b_source.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/b_source.h@@ -40,42 +40,24 @@ struct BSource {const uint k_start = k_slice * block_size;const device BT* w = reinterpret_cast<const device BT*>(b);w += out_row * in_vec_size + simd_lane * values_per_thread + k_start;- const device AT* in =- a + batch_idx * in_vec_size + simd_lane * values_per_thread + k_start;+ const device AT* in = a + batch_idx * in_vec_size + simd_lane * values_per_thread + k_start;uint k = k_start;for (; k + block_size <= in_vec_size; k += k_stride) {- float4 xv =- static_cast<float4>(*reinterpret_cast<const device I4*>(in));- result[0] +=- dot(static_cast<float4>(*reinterpret_cast<const device W4*>(w)),- xv);- result[1] +=- dot(static_cast<float4>(- *reinterpret_cast<const device W4*>(w + in_vec_size)- ),- xv);- result[2] +=- dot(static_cast<float4>(- *reinterpret_cast<const device W4*>(w + 2 * in_vec_size)- ),- xv);- result[3] +=- dot(static_cast<float4>(- *reinterpret_cast<const device W4*>(w + 3 * in_vec_size)- ),- xv);+ float4 xv = static_cast<float4>(*reinterpret_cast<const device I4*>(in));+ result[0] += dot(static_cast<float4>(*reinterpret_cast<const device W4*>(w)), xv);+ result[1] += dot(static_cast<float4>(*reinterpret_cast<const device W4*>(w + in_vec_size)), xv);+ result[2] += dot(static_cast<float4>(*reinterpret_cast<const device W4*>(w + 2 * in_vec_size)), xv);+ result[3] += dot(static_cast<float4>(*reinterpret_cast<const device W4*>(w + 3 * in_vec_size)), xv);w += k_stride;in += k_stride;}if constexpr (K_SPLIT == 1 && !INPUT_ALIGNED) {const uint thread_offset = simd_lane * values_per_thread;- const int remaining =- (k + thread_offset < in_vec_size)- ? min(static_cast<int>(in_vec_size - k - thread_offset),- static_cast<int>(values_per_thread))- : 0;+ const int remaining = (k + thread_offset < in_vec_size) ? min(static_cast<int>(in_vec_size - k - thread_offset),+ static_cast<int>(values_per_thread))+ : 0;if (remaining > 0) {const device BT* w0 = w;const device BT* w1 = w + in_vec_size;@@ -102,8 +84,7 @@ struct BSource {const uint in_vec_size_w = in_vec_size * bytes_per_pack / pack_factor;const uint in_vec_size_g = (in_vec_size + GROUP_SIZE - 1) / GROUP_SIZE;- ws += out_row * in_vec_size_w +- simd_lane * packs_per_thread * bytes_per_pack;+ ws += out_row * in_vec_size_w + simd_lane * packs_per_thread * bytes_per_pack;QuantRowOffsets<BT, U, B_PROLOGUE, BITS> prep;prep.group_stride = in_vec_size_g;@@ -111,9 +92,7 @@ struct BSource {prep.scales = scales + out_row * in_vec_size_g + g_offset;if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant) {prep.biases = biases + out_row * in_vec_size_g + g_offset;- } else if constexpr (- B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant- ) {+ } else if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant) {if (BITS == 4) {prep.zp_stride = (in_vec_size_g + 1) / 2;prep.zps = zero_points + out_row * prep.zp_stride + g_offset / 2;@@ -124,8 +103,7 @@ struct BSource {}}- const device AT* in =- a + batch_idx * in_vec_size + simd_lane * values_per_thread;+ const device AT* in = a + batch_idx * in_vec_size + simd_lane * values_per_thread;uint k = 0;for (; k + block_size <= in_vec_size; k += block_size) {@@ -139,34 +117,10 @@ struct BSource {U scale[4];U offset[4];prep.load(scale, offset);- result[0] += qdot<U, values_per_thread, BITS>(- wl0,- x_thread,- scale[0],- offset[0],- sum- );- result[1] += qdot<U, values_per_thread, BITS>(- wl1,- x_thread,- scale[1],- offset[1],- sum- );- result[2] += qdot<U, values_per_thread, BITS>(- wl2,- x_thread,- scale[2],- offset[2],- sum- );- result[3] += qdot<U, values_per_thread, BITS>(- wl3,- x_thread,- scale[3],- offset[3],- sum- );+ result[0] += qdot<U, values_per_thread, BITS>(wl0, x_thread, scale[0], offset[0], sum);+ result[1] += qdot<U, values_per_thread, BITS>(wl1, x_thread, scale[1], offset[1], sum);+ result[2] += qdot<U, values_per_thread, BITS>(wl2, x_thread, scale[2], offset[2], sum);+ result[3] += qdot<U, values_per_thread, BITS>(wl3, x_thread, scale[3], offset[3], sum);ws += block_size * bytes_per_pack / pack_factor;prep.advance(block_size / GROUP_SIZE);@@ -175,17 +129,11 @@ struct BSource {if constexpr (!INPUT_ALIGNED) {const uint thread_offset = simd_lane * values_per_thread;- const int remaining =- (k + thread_offset < in_vec_size)- ? min(static_cast<int>(in_vec_size - k - thread_offset),- static_cast<int>(values_per_thread))- : 0;+ const int remaining = (k + thread_offset < in_vec_size) ? min(static_cast<int>(in_vec_size - k - thread_offset),+ static_cast<int>(values_per_thread))+ : 0;if (remaining > 0) {- U sum = load_vector_safe<AT, U, values_per_thread>(- in,- x_thread,- remaining- );+ U sum = load_vector_safe<AT, U, values_per_thread>(in, x_thread, remaining);const device uint8_t* wl0 = ws;const device uint8_t* wl1 = ws + in_vec_size_w;@@ -195,38 +143,10 @@ struct BSource {U scale[4];U offset[4];prep.load(scale, offset);- result[0] += qdot_safe<U, values_per_thread, BITS>(- wl0,- x_thread,- scale[0],- offset[0],- sum,- remaining- );- result[1] += qdot_safe<U, values_per_thread, BITS>(- wl1,- x_thread,- scale[1],- offset[1],- sum,- remaining- );- result[2] += qdot_safe<U, values_per_thread, BITS>(- wl2,- x_thread,- scale[2],- offset[2],- sum,- remaining- );- result[3] += qdot_safe<U, values_per_thread, BITS>(- wl3,- x_thread,- scale[3],- offset[3],- sum,- remaining- );+ result[0] += qdot_safe<U, values_per_thread, BITS>(wl0, x_thread, scale[0], offset[0], sum, remaining);+ result[1] += qdot_safe<U, values_per_thread, BITS>(wl1, x_thread, scale[1], offset[1], sum, remaining);+ result[2] += qdot_safe<U, values_per_thread, BITS>(wl2, x_thread, scale[2], offset[2], sum, remaining);+ result[3] += qdot_safe<U, values_per_thread, BITS>(wl3, x_thread, scale[3], offset[3], sum, remaining);}}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/epilogue.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/epilogue.hindex 1babf6c5a..72707af54 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/epilogue.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/epilogue.h@@ -24,8 +24,7 @@ struct Epilogue {bool writer) {const bool is_scale = output_transform.contains(GemmDTransform::SCALE);- const bool is_accumulate =- output_transform.contains(GemmDTransform::ACCUMULATE);+ const bool is_accumulate = output_transform.contains(GemmDTransform::ACCUMULATE);const bool is_bias = output_transform.contains(GemmDTransform::BIAS);const bool use_hadamard = output_transform.contains(GemmDTransform::RHT);@@ -51,8 +50,7 @@ struct Epilogue {if (simd_lane == 0) {METAL_PRAGMA_UNROLLfor (uint row = 0; row < RESULTS_PER_SIMDGROUP; row++) {- shared_results[simd_group * RESULTS_PER_SIMDGROUP + row] =- result[row];+ shared_results[simd_group * RESULTS_PER_SIMDGROUP + row] = result[row];}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/output_tile.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/output_tile.hindex 6a28cbbb5..6fba89d29 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/output_tile.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/output_tile.h@@ -12,17 +12,12 @@ struct OutputTile {uint k_slice;bool writer;- static METAL_FUNC OutputTile- make(uint out_block_idx, uint simd_group, uint out_vec_size) {- constexpr uint rows_per_threadgroup =- (NUM_SIMDGROUPS / K_SPLIT) * RESULTS_PER_SIMDGROUP;+ static METAL_FUNC OutputTile make(uint out_block_idx, uint simd_group, uint out_vec_size) {+ constexpr uint rows_per_threadgroup = (NUM_SIMDGROUPS / K_SPLIT) * RESULTS_PER_SIMDGROUP;const uint row_group = simd_group / K_SPLIT;const uint k_slice = simd_group % K_SPLIT;- uint out_row = out_block_idx * rows_per_threadgroup +- row_group * RESULTS_PER_SIMDGROUP;- out_row = out_vec_size > RESULTS_PER_SIMDGROUP- ? min(out_row, out_vec_size - RESULTS_PER_SIMDGROUP)- : 0u;+ uint out_row = out_block_idx * rows_per_threadgroup + row_group * RESULTS_PER_SIMDGROUP;+ out_row = out_vec_size > RESULTS_PER_SIMDGROUP ? min(out_row, out_vec_size - RESULTS_PER_SIMDGROUP) : 0u;OutputTile tile;tile.out_row = out_row;tile.row_group = row_group;diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/quant_row_offsets.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/quant_row_offsets.hindex 0965132c0..d68cd99ca 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/quant_row_offsets.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/quant_row_offsets.h@@ -8,10 +8,7 @@ namespace gemm {template <typename BT, typename U, GemmBPrologueKind B_PROLOGUE, uint BITS>struct QuantRowOffsets {- static_assert(- BITS == 4 || BITS == 8,- "QuantRowOffsets supports 4- and 8-bit only"- );+ static_assert(BITS == 4 || BITS == 8, "QuantRowOffsets supports 4- and 8-bit only");const device BT* scales = nullptr;const device BT* biases = nullptr;const device uint8_t* zps = nullptr;@@ -30,15 +27,8 @@ struct QuantRowOffsets {offset[1] = static_cast<U>(biases[group_stride]);offset[2] = static_cast<U>(biases[2 * group_stride]);offset[3] = static_cast<U>(biases[3 * group_stride]);- } else if constexpr (- B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant- ) {- uchar4 zp_bytes = uchar4(- zps[0],- zps[zp_stride],- zps[2 * zp_stride],- zps[3 * zp_stride]- );+ } else if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant) {+ uchar4 zp_bytes = uchar4(zps[0], zps[zp_stride], zps[2 * zp_stride], zps[3 * zp_stride]);uchar4 zp_nibbles;if constexpr (BITS == 4) {const uint8_t shift = high_nibble ? 4u : 0u;@@ -63,9 +53,7 @@ struct QuantRowOffsets {scales += groups;if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant) {biases += groups;- } else if constexpr (- B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant- ) {+ } else if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant) {constexpr uint zps_per_byte = get_pack_factor<BITS, 8>();zps += groups / zps_per_byte;}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/reduce.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/reduce.hindex d08ad70bb..ac3947b3a 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/reduce.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/common/reduce.h@@ -24,8 +24,7 @@ struct Reduce {if (simd_lane == 0) {METAL_PRAGMA_UNROLLfor (uint row = 0; row < RESULTS_PER_SIMDGROUP; row++) {- shared_results[simd_group * RESULTS_PER_SIMDGROUP + row] =- result[row];+ shared_results[simd_group * RESULTS_PER_SIMDGROUP + row] = result[row];}}threadgroup_barrier(mem_flags::mem_threadgroup);@@ -35,8 +34,7 @@ struct Reduce {U acc = 0;METAL_PRAGMA_UNROLLfor (uint s = 0; s < K_SPLIT; s++) {- acc += shared_results- [(row_group * K_SPLIT + s) * RESULTS_PER_SIMDGROUP + row];+ acc += shared_results[(row_group * K_SPLIT + s) * RESULTS_PER_SIMDGROUP + row];}result[row] = acc;}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metalindex 0aa6c8cf5..e69aa5cc2 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal@@ -68,37 +68,25 @@ KERNEL(Gemv)(thread U result[RESULTS_PER_SIMDGROUP] = {0};OutputTile<K_SPLIT, NUM_SIMDGROUPS> tile =- OutputTile<K_SPLIT, NUM_SIMDGROUPS>::make(- out_block_idx,- simd_group,- out_vec_size- );+ OutputTile<K_SPLIT, NUM_SIMDGROUPS>::make(out_block_idx, simd_group, out_vec_size);d += batch_idx * out_vec_size + tile.out_row;- BSource<BT, AT, U, B_PROLOGUE, GROUP_SIZE, BITS, K_SPLIT, INPUT_ALIGNED>::- accumulate(- result,- b,- scales,- zero_points,- biases,- a,- in_vec_size,- tile.out_row,- batch_idx,- simd_lane,- tile.k_slice- );-- Reduce<U, K_SPLIT, NUM_SIMDGROUPS>::run(+ BSource<BT, AT, U, B_PROLOGUE, GROUP_SIZE, BITS, K_SPLIT, INPUT_ALIGNED>::accumulate(result,- shared_results,- simd_group,+ b,+ scales,+ zero_points,+ biases,+ a,+ in_vec_size,+ tile.out_row,+ batch_idx,simd_lane,- tile.row_group,tile.k_slice);+ Reduce<U, K_SPLIT, NUM_SIMDGROUPS>::run(result, shared_results, simd_group, simd_lane, tile.row_group, tile.k_slice);+Epilogue<BT, DT, U>::store(result,d,diff --git a/crates/backend-uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal b/crates/backend-uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metalindex 8b10d91f3..b82f05de8 100644--- a/crates/backend-uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal@@ -6,11 +6,7 @@#include "../common/threadgroup_reduce.h"template <ushort BLOCK_SIZE, typename T>-static T threadgroup_raking_prefix_exclusive_sum(- T value,- threadgroup T* shared,- const ushort lid-) {+static T threadgroup_raking_prefix_exclusive_sum(T value, threadgroup T* shared, const ushort lid) {shared[lid] = value;threadgroup_barrier(mem_flags::mem_threadgroup);@@ -48,16 +44,14 @@ PUBLIC KERNEL(MoeCountsOffsetsFused)(device const int* topk_ids,device uint* offsets, // output: exclusive scan [E+1]device uint* sum_k_out, // output: total count [1]- device uint*- partials, // output: partials [num_tiles * TILE_E] (for block_bases)+ device uint* partials, // output: partials [num_tiles * TILE_E] (for block_bases)constant uint& t_input,constant uint& e_input,constant uint& k_input,threadgroup _atomic<uint> tg_hist[TILE_E],threadgroup uint scan_shared[BLOCK_SIZE],threadgroup uint reduce_shared[BLOCK_SIZE],- threadgroup uint- counts_shared[BLOCK_SIZE], // Cache counts in threadgroup memory+ threadgroup uint counts_shared[BLOCK_SIZE], // Cache counts in threadgroup memorythreadgroup uint& carry,const ThreadContext thread_context,const uint lid THREADS(128)@@ -102,8 +96,7 @@ PUBLIC KERNEL(MoeCountsOffsetsFused)(// Write final counts and partials for this tilefor (uint e = lid; e < tile_e; e += BLOCK_SIZE) {- const uint count_val =- atomic_load_explicit(&tg_hist[e], memory_order_relaxed);+ const uint count_val = atomic_load_explicit(&tg_hist[e], memory_order_relaxed);counts_shared[e0 + e] = count_val;partials[e0 + e] = count_val;}@@ -124,22 +117,13 @@ PUBLIC KERNEL(MoeCountsOffsetsFused)(uint v = (lid < chunk_n) ? counts_shared[lid] : 0u;- uint prefix_local = threadgroup_raking_prefix_exclusive_sum<BLOCK_SIZE>(- v,- scan_shared,- (ushort)lid- );+ uint prefix_local = threadgroup_raking_prefix_exclusive_sum<BLOCK_SIZE>(v, scan_shared, (ushort)lid);uint prefix_global = prefix_local + carry;if (lid < chunk_n) {offsets[base + lid] = prefix_global;}- uint block_sum =- threadgroup_cooperative_reduce<SimdReduceSum<uint>, BLOCK_SIZE>(- v,- reduce_shared,- thread_context- );+ uint block_sum = threadgroup_cooperative_reduce<SimdReduceSum<uint>, BLOCK_SIZE>(v, reduce_shared, thread_context);if (lid == 0) {carry += block_sum;}diff --git a/crates/backend-uzu/src/backends/metal/kernel/moe/experts_single.metal b/crates/backend-uzu/src/backends/metal/kernel/moe/experts_single.metalindex 75e029e9b..9f4022cee 100644--- a/crates/backend-uzu/src/backends/metal/kernel/moe/experts_single.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/moe/experts_single.metal@@ -46,8 +46,7 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassA)(const ulong bias_base = (ulong)expert_u * (ulong)(2 * d_ff);device const T* w_up_row = w13_all + w13_base + (ulong)h_idx * (ulong)d_model;- device const T* w_gate_row =- w13_all + w13_base + (ulong)(d_ff + h_idx) * (ulong)d_model;+ device const T* w_gate_row = w13_all + w13_base + (ulong)(d_ff + h_idx) * (ulong)d_model;float acc_up = 0.0f;float acc_gate = 0.0f;@@ -58,16 +57,14 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassA)(uint base_idx = i * 128 + thread_context.simd_lane_id * 4;device const T* x_vec2 = reinterpret_cast<device const T*>(x + base_idx);- device const T* w_up_vec2 =- reinterpret_cast<device const T*>(w_up_row + base_idx);+ device const T* w_up_vec2 = reinterpret_cast<device const T*>(w_up_row + base_idx);acc_up += float(x_vec2[0]) * float(w_up_vec2[0]);acc_up += float(x_vec2[1]) * float(w_up_vec2[1]);acc_up += float(x_vec2[2]) * float(w_up_vec2[2]);acc_up += float(x_vec2[3]) * float(w_up_vec2[3]);if (gating_sel > 1) {- device const T* w_gate_vec2 =- reinterpret_cast<device const T*>(w_gate_row + base_idx);+ device const T* w_gate_vec2 = reinterpret_cast<device const T*>(w_gate_row + base_idx);acc_gate += float(x_vec2[0]) * float(w_gate_vec2[0]);acc_gate += float(x_vec2[1]) * float(w_gate_vec2[1]);acc_gate += float(x_vec2[2]) * float(w_gate_vec2[2]);@@ -90,25 +87,14 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassA)(}if (thread_context.simd_lane_id == 0) {- float up_val = clamp(- acc_up + float(biases[bias_base + h_idx]),- up_clip_min,- up_clip_max- );+ float up_val = clamp(acc_up + float(biases[bias_base + h_idx]), up_clip_min, up_clip_max);float activated;if (gating_sel <= 1) {- activated = (gating_sel == 0) ? activate_gelu(up_val)- : activate_silu_alpha(up_val, silu_alpha);+ activated = (gating_sel == 0) ? activate_gelu(up_val) : activate_silu_alpha(up_val, silu_alpha);} else {- float gate_val = clamp(- acc_gate + float(biases[bias_base + d_ff + h_idx]),- gate_clip_min,- gate_clip_max- );- float gate_act = (gating_sel == 2)- ? activate_silu_alpha(gate_val, silu_alpha)- : activate_gelu(gate_val);+ float gate_val = clamp(acc_gate + float(biases[bias_base + d_ff + h_idx]), gate_clip_min, gate_clip_max);+ float gate_act = (gating_sel == 2) ? activate_silu_alpha(gate_val, silu_alpha) : activate_gelu(gate_val);activated = gate_act * up_val;}@@ -153,8 +139,7 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassB)(const float prob = float(topk_probs[k]);device const float* hidden_ptr = hidden + (ulong)k * (ulong)d_ff;- device const T* w2_ptr = w2_all + (ulong)expert_u * w2_expert_stride +- (ulong)my_col * (ulong)d_ff;+ device const T* w2_ptr = w2_all + (ulong)expert_u * w2_expert_stride + (ulong)my_col * (ulong)d_ff;float acc = 0.0f;@@ -162,10 +147,8 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassB)(for (uint i = 0; i < vec_iters; ++i) {uint base_idx = i * 128 + thread_context.simd_lane_id * 4;- float4 h_vec =- *reinterpret_cast<device const float4*>(hidden_ptr + base_idx);- device const T* w_vec2 =- reinterpret_cast<device const T*>(w2_ptr + base_idx);+ float4 h_vec = *reinterpret_cast<device const float4*>(hidden_ptr + base_idx);+ device const T* w_vec2 = reinterpret_cast<device const T*>(w2_ptr + base_idx);acc += h_vec.x * float(w_vec2[0]);acc += h_vec.y * float(w_vec2[1]);acc += h_vec.z * float(w_vec2[2]);@@ -173,8 +156,7 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassB)(}// Remainder- for (uint idx = vec_iters * 128 + thread_context.simd_lane_id; idx < d_ff;- idx += 32) {+ for (uint idx = vec_iters * 128 + thread_context.simd_lane_id; idx < d_ff; idx += 32) {acc += hidden_ptr[idx] * float(w2_ptr[idx]);}diff --git a/crates/backend-uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal b/crates/backend-uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metalindex 4a46fe2a0..dabc1a252 100644--- a/crates/backend-uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal@@ -56,8 +56,7 @@ PUBLIC KERNEL(MoeExpertsDecodePassA)(device const T* x_ptr = x_perm + x_row_base;device const T* w_up_row = w13_all + w13_base + (ulong)h_idx * (ulong)d_model;- device const T* w_gate_row =- w13_all + w13_base + (ulong)(d_ff + h_idx) * (ulong)d_model;+ device const T* w_gate_row = w13_all + w13_base + (ulong)(d_ff + h_idx) * (ulong)d_model;float acc_up = 0.0f;float acc_gate = 0.0f;@@ -69,16 +68,14 @@ PUBLIC KERNEL(MoeExpertsDecodePassA)(uint base_idx = i * 128 + thread_context.simd_lane_id * 4;device const T* x_vec = reinterpret_cast<device const T*>(x_ptr + base_idx);- device const T* w_up_vec =- reinterpret_cast<device const T*>(w_up_row + base_idx);+ device const T* w_up_vec = reinterpret_cast<device const T*>(w_up_row + base_idx);acc_up += float(x_vec[0]) * float(w_up_vec[0]);acc_up += float(x_vec[1]) * float(w_up_vec[1]);acc_up += float(x_vec[2]) * float(w_up_vec[2]);acc_up += float(x_vec[3]) * float(w_up_vec[3]);if (gating_sel > 1) {- device const T* w_gate_vec =- reinterpret_cast<device const T*>(w_gate_row + base_idx);+ device const T* w_gate_vec = reinterpret_cast<device const T*>(w_gate_row + base_idx);acc_gate += float(x_vec[0]) * float(w_gate_vec[0]);acc_gate += float(x_vec[1]) * float(w_gate_vec[1]);acc_gate += float(x_vec[2]) * float(w_gate_vec[2]);@@ -109,14 +106,11 @@ PUBLIC KERNEL(MoeExpertsDecodePassA)(float activated;if (gating_sel <= 1) {- activated = (gating_sel == 0) ? activate_gelu(up_val)- : activate_silu_alpha(up_val, silu_alpha);+ activated = (gating_sel == 0) ? activate_gelu(up_val) : activate_silu_alpha(up_val, silu_alpha);} else {float gate_val = acc_gate + float(up_biases[bias_base + d_ff + h_idx]);gate_val = clamp(gate_val, gate_clip_min, gate_clip_max);- float gate_act = (gating_sel == 2)- ? activate_silu_alpha(gate_val, silu_alpha)- : activate_gelu(gate_val);+ float gate_act = (gating_sel == 2) ? activate_silu_alpha(gate_val, silu_alpha) : activate_gelu(gate_val);activated = gate_act * up_val;}@@ -151,8 +145,7 @@ PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(const uint row_idx = tgpig_y;// Each simdgroup computes one output column- const uint my_col =- tgpig_x * SIMDGROUPS_PER_TG + thread_context.simdgroup_index;+ const uint my_col = tgpig_x * SIMDGROUPS_PER_TG + thread_context.simdgroup_index;if (my_col >= d_model)return;@@ -160,8 +153,7 @@ PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(// Base addresses for this output columnconst ulong hidden_base = (ulong)row_idx * (ulong)d_ff;- const ulong w2_col_base = (ulong)expert_idx * (ulong)d_model * (ulong)d_ff +- (ulong)my_col * (ulong)d_ff;+ const ulong w2_col_base = (ulong)expert_idx * (ulong)d_model * (ulong)d_ff + (ulong)my_col * (ulong)d_ff;// Dual accumulators for ILP: breaks FMA dependency chainsAccumT acc0 = AccumT(0.0);@@ -173,8 +165,7 @@ PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(const uint k_vec_iters = k_iters / 8;for (uint iter = 0; iter < k_vec_iters; ++iter) {- const uint k_base =- iter * (8 * THREADS_PER_SIMD) + thread_context.simd_lane_id;+ const uint k_base = iter * (8 * THREADS_PER_SIMD) + thread_context.simd_lane_id;// hidden: stride-32 per lane, already f32 from Pass Aconst AccumT h0 = hidden[hidden_base + k_base + 0 * THREADS_PER_SIMD];@@ -187,22 +178,14 @@ PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(const AccumT h7 = hidden[hidden_base + k_base + 7 * THREADS_PER_SIMD];// W2: lane-coalesced- const AccumT w0 =- AccumT(w2_all[w2_col_base + k_base + 0 * THREADS_PER_SIMD]);- const AccumT w1 =- AccumT(w2_all[w2_col_base + k_base + 1 * THREADS_PER_SIMD]);- const AccumT w2 =- AccumT(w2_all[w2_col_base + k_base + 2 * THREADS_PER_SIMD]);- const AccumT w3 =- AccumT(w2_all[w2_col_base + k_base + 3 * THREADS_PER_SIMD]);- const AccumT w4 =- AccumT(w2_all[w2_col_base + k_base + 4 * THREADS_PER_SIMD]);- const AccumT w5 =- AccumT(w2_all[w2_col_base + k_base + 5 * THREADS_PER_SIMD]);- const AccumT w6 =- AccumT(w2_all[w2_col_base + k_base + 6 * THREADS_PER_SIMD]);- const AccumT w7 =- AccumT(w2_all[w2_col_base + k_base + 7 * THREADS_PER_SIMD]);+ const AccumT w0 = AccumT(w2_all[w2_col_base + k_base + 0 * THREADS_PER_SIMD]);+ const AccumT w1 = AccumT(w2_all[w2_col_base + k_base + 1 * THREADS_PER_SIMD]);+ const AccumT w2 = AccumT(w2_all[w2_col_base + k_base + 2 * THREADS_PER_SIMD]);+ const AccumT w3 = AccumT(w2_all[w2_col_base + k_base + 3 * THREADS_PER_SIMD]);+ const AccumT w4 = AccumT(w2_all[w2_col_base + k_base + 4 * THREADS_PER_SIMD]);+ const AccumT w5 = AccumT(w2_all[w2_col_base + k_base + 5 * THREADS_PER_SIMD]);+ const AccumT w6 = AccumT(w2_all[w2_col_base + k_base + 6 * THREADS_PER_SIMD]);+ const AccumT w7 = AccumT(w2_all[w2_col_base + k_base + 7 * THREADS_PER_SIMD]);// dual trees for ILPacc0 = fma(h0, w0, acc0);@@ -221,20 +204,14 @@ PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(// Handle remaining full iterationsfor (uint iter = k_vec_iters * 8; iter < k_iters; ++iter) {const uint k = iter * THREADS_PER_SIMD + thread_context.simd_lane_id;- acc =- fma(AccumT(hidden[hidden_base + k]),- AccumT(w2_all[w2_col_base + k]),- acc);+ acc = fma(AccumT(hidden[hidden_base + k]), AccumT(w2_all[w2_col_base + k]), acc);}// Handle leftover elements (d_ff % 32)const uint leftover_start = k_iters * THREADS_PER_SIMD;if (leftover_start + thread_context.simd_lane_id < d_ff) {const uint k = leftover_start + thread_context.simd_lane_id;- acc =- fma(AccumT(hidden[hidden_base + k]),- AccumT(w2_all[w2_col_base + k]),- acc);+ acc = fma(AccumT(hidden[hidden_base + k]), AccumT(w2_all[w2_col_base + k]), acc);}// Simdgroup reductiondiff --git a/crates/backend-uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal b/crates/backend-uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metalindex bccc6aab4..0d7ab3bbc 100644--- a/crates/backend-uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal@@ -7,9 +7,7 @@// ------------------------ helpers ------------------------static inline uint ceil_div(uint a, uint b) { return (a + b - 1u) / b; }-static inline uint linear_tid(uint3 tid, uint3 tpg) {- return tid.z * (tpg.x * tpg.y) + tid.y * tpg.x + tid.x;-}+static inline uint linear_tid(uint3 tid, uint3 tpg) { return tid.z * (tpg.x * tpg.y) + tid.y * tpg.x + tid.x; }template <typename T>inline void store_vec4(device T* dst, ulong base, float4 vals) {@@ -98,10 +96,8 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(const uint m_rows = min(Bm, seg_len - row_tg_off);const uint n_cols = min(Bn, d_ff - col_tg_off);- const ulong x_base =- ((ulong)tile_seg_start + (ulong)row_tg_off) * (ulong)d_model;- const ulong w13_expert_base =- (ulong)expert_idx * (ulong)(2 * d_ff) * (ulong)d_model;+ const ulong x_base = ((ulong)tile_seg_start + (ulong)row_tg_off) * (ulong)d_model;+ const ulong w13_expert_base = (ulong)expert_idx * (ulong)(2 * d_ff) * (ulong)d_model;const ulong bias_base = (ulong)expert_idx * (ulong)(2 * d_ff);// simdgroup tile mapping@@ -128,8 +124,7 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(}}- const uint threads_total =- threads_per_tg.x * threads_per_tg.y * threads_per_tg.z;+ const uint threads_total = threads_per_tg.x * threads_per_tg.y * threads_per_tg.z;const uint lin = linear_tid(local_tid, threads_per_tg);// K-loop (tile along D)@@ -157,13 +152,10 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(if (r < m_rows && col_base < valid_k) {// Safe to read - at least first element is valid- const ulong base =- x_base + (ulong)r * (ulong)d_model + (ulong)(k_off + col_base);+ const ulong base = x_base + (ulong)r * (ulong)d_model + (ulong)(k_off + col_base);METAL_PRAGMA_UNROLLfor (uint j = 0; j < vec_size; j++) {- dst[j] = (col_base + j < valid_k)- ? static_cast<T>(float(x_perm[base + j]))- : static_cast<T>(0.0f);+ dst[j] = (col_base + j < valid_k) ? static_cast<T>(float(x_perm[base + j])) : static_cast<T>(0.0f);}} else {// Row out of bounds or entire vec8 chunk is out of K bounds@@ -198,21 +190,14 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(if (n < n_cols && col_base < valid_k) {// Safe to read - at least first element is validconst uint gcol = col_tg_off + n;- const ulong up_base = w13_expert_base +- (ulong)gcol * (ulong)d_model +- (ulong)(k_off + col_base);- const ulong gate_base = w13_expert_base +- (ulong)(d_ff + gcol) * (ulong)d_model +- (ulong)(k_off + col_base);+ const ulong up_base = w13_expert_base + (ulong)gcol * (ulong)d_model + (ulong)(k_off + col_base);+ const ulong gate_base = w13_expert_base + (ulong)(d_ff + gcol) * (ulong)d_model + (ulong)(k_off + col_base);METAL_PRAGMA_UNROLLfor (uint j = 0; j < vec_size; j++) {- up_dst[j] = (col_base + j < valid_k)- ? static_cast<T>(float(w13_all[up_base + j]))- : static_cast<T>(0.0f);+ up_dst[j] = (col_base + j < valid_k) ? static_cast<T>(float(w13_all[up_base + j])) : static_cast<T>(0.0f);if (gating_sel > 1u) {- gt_dst[j] = (col_base + j < valid_k)- ? static_cast<T>(float(w13_all[gate_base + j]))- : static_cast<T>(0.0f);+ gt_dst[j] =+ (col_base + j < valid_k) ? static_cast<T>(float(w13_all[gate_base + j])) : static_cast<T>(0.0f);}}} else {@@ -244,35 +229,13 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(const uint c_idx = n_sub / SG_TILE;const uint tile = r_idx * (SgBn / SG_TILE) + c_idx;metal::simdgroup_matrix<T, 8, 8> rhs_up;- simdgroup_load(- rhs_up,- Wk_up,- W_LD,- ulong2(kk, col_sg_off + n_sub),- true- );- simdgroup_multiply_accumulate(- OutUp[tile],- lhs_frag,- rhs_up,- OutUp[tile]- );+ simdgroup_load(rhs_up, Wk_up, W_LD, ulong2(kk, col_sg_off + n_sub), true);+ simdgroup_multiply_accumulate(OutUp[tile], lhs_frag, rhs_up, OutUp[tile]);if (gating_sel > 1u) {metal::simdgroup_matrix<T, 8, 8> rhs_gate;- simdgroup_load(- rhs_gate,- Wk_gate,- W_LD,- ulong2(kk, col_sg_off + n_sub),- true- );- simdgroup_multiply_accumulate(- OutGate[tile],- lhs_frag,- rhs_gate,- OutGate[tile]- );+ simdgroup_load(rhs_gate, Wk_gate, W_LD, ulong2(kk, col_sg_off + n_sub), true);+ simdgroup_multiply_accumulate(OutGate[tile], lhs_frag, rhs_gate, OutGate[tile]);}}}@@ -283,12 +246,9 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(// Note: bias layout is [up | gate] in contiguous FF chunks.for (uint c_local = lin; c_local < Bn; c_local += threads_total) {const uint c_global = col_tg_off + c_local;- bias_up[c_local] =- (c_global < d_ff) ? float(up_biases[bias_base + c_global]) : 0.0f;+ bias_up[c_local] = (c_global < d_ff) ? float(up_biases[bias_base + c_global]) : 0.0f;if (gating_sel > 1u) {- bias_gate[c_local] = (c_global < d_ff)- ? float(up_biases[bias_base + d_ff + c_global])- : 0.0f;+ bias_gate[c_local] = (c_global < d_ff) ? float(up_biases[bias_base + d_ff + c_global]) : 0.0f;}}@@ -335,21 +295,13 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(const ulong out_row = seg_start + row_tg_off + local_row;if (col0 < n_cols) {- float up_v =- clamp(up_frag[0] + bias_up[col0], up_clip_min, up_clip_max);+ float up_v = clamp(up_frag[0] + bias_up[col0], up_clip_min, up_clip_max);float out_val;if (gating_sel <= 1u) {- out_val = (gating_sel == 0u) ? activate_gelu(up_v)- : activate_silu_alpha(up_v, silu_alpha);+ out_val = (gating_sel == 0u) ? activate_gelu(up_v) : activate_silu_alpha(up_v, silu_alpha);} else {- float gate_v = clamp(- gate_frag_0 + bias_gate[col0],- gate_clip_min,- gate_clip_max- );- const float gate_act = (gating_sel == 2u)- ? activate_silu_alpha(gate_v, silu_alpha)- : activate_gelu(gate_v);+ float gate_v = clamp(gate_frag_0 + bias_gate[col0], gate_clip_min, gate_clip_max);+ const float gate_act = (gating_sel == 2u) ? activate_silu_alpha(gate_v, silu_alpha) : activate_gelu(gate_v);out_val = gate_act * up_v;}const ulong out_col = col_tg_off + col0;@@ -357,21 +309,13 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(}if (col1 < n_cols) {- float up_v =- clamp(up_frag[1] + bias_up[col1], up_clip_min, up_clip_max);+ float up_v = clamp(up_frag[1] + bias_up[col1], up_clip_min, up_clip_max);float out_val;if (gating_sel <= 1u) {- out_val = (gating_sel == 0u) ? activate_gelu(up_v)- : activate_silu_alpha(up_v, silu_alpha);+ out_val = (gating_sel == 0u) ? activate_gelu(up_v) : activate_silu_alpha(up_v, silu_alpha);} else {- float gate_v = clamp(- gate_frag_1 + bias_gate[col1],- gate_clip_min,- gate_clip_max- );- const float gate_act = (gating_sel == 2u)- ? activate_silu_alpha(gate_v, silu_alpha)- : activate_gelu(gate_v);+ float gate_v = clamp(gate_frag_1 + bias_gate[col1], gate_clip_min, gate_clip_max);+ const float gate_act = (gating_sel == 2u) ? activate_silu_alpha(gate_v, silu_alpha) : activate_gelu(gate_v);out_val = gate_act * up_v;}const ulong out_col = col_tg_off + col1;@@ -489,8 +433,7 @@ PUBLIC KERNEL(MoeExpertsPrefillPassB)(threadgroup float* my_dst = Hs + h_bi * H_LD + h_bj;if (h_bi < m_rows && h_bj < valid_k) {// Safe to read - at least first element is valid- device const float* my_src =- hidden + h_base + (ulong)h_bi * (ulong)d_ff + (ulong)(k_off + h_bj);+ device const float* my_src = hidden + h_base + (ulong)h_bi * (ulong)d_ff + (ulong)(k_off + h_bj);METAL_PRAGMA_UNROLLfor (uint j = 0; j < H_VEC; j++) {my_dst[j] = (h_bj + j < valid_k) ? my_src[j] : 0.0f;@@ -512,13 +455,10 @@ PUBLIC KERNEL(MoeExpertsPrefillPassB)(if (row < n_cols && w_bj < valid_k) {// Safe to read - at least first element is validconst uint gcol = col_tg_off + row;- device const T* my_src = w2_all + w2_expert_base +- (ulong)gcol * (ulong)d_ff +- (ulong)(k_off + w_bj);+ device const T* my_src = w2_all + w2_expert_base + (ulong)gcol * (ulong)d_ff + (ulong)(k_off + w_bj);METAL_PRAGMA_UNROLLfor (uint j = 0; j < W_VEC; j++) {- my_dst[j] = (w_bj + j < valid_k) ? static_cast<T>(float(my_src[j]))- : static_cast<T>(0.0f);+ my_dst[j] = (w_bj + j < valid_k) ? static_cast<T>(float(my_src[j])) : static_cast<T>(0.0f);}} else {METAL_PRAGMA_UNROLL@@ -556,18 +496,15 @@ PUBLIC KERNEL(MoeExpertsPrefillPassB)(// ---- Bias tile (cooperative load) ----for (uint c_local = lin; c_local < Bn; c_local += TGP_SIZE) {const uint c_global = col_tg_off + c_local;- bias_tile[c_local] =- (c_global < d_model) ? float(down_biases[bias_base + c_global]) : 0.0f;+ bias_tile[c_local] = (c_global < d_model) ? float(down_biases[bias_base + c_global]) : 0.0f;}threadgroup_barrier(mem_flags::mem_threadgroup);// ---- Writeout with fragment extraction ----const uint lane_qid = thread_context.simd_lane_id >> 2;- const uint lane_row =- (lane_qid & 4u) + ((thread_context.simd_lane_id >> 1) & 3u);- const uint lane_col_base =- ((lane_qid & 2u) * 2u) + ((thread_context.simd_lane_id & 1u) * 2u);+ const uint lane_row = (lane_qid & 4u) + ((thread_context.simd_lane_id >> 1) & 3u);+ const uint lane_col_base = ((lane_qid & 2u) * 2u) + ((thread_context.simd_lane_id & 1u) * 2u);const uint local_row = row_sg_off + lane_row;if (local_row >= m_rows)diff --git a/crates/backend-uzu/src/backends/metal/kernel/moe/router_topk.metal b/crates/backend-uzu/src/backends/metal/kernel/moe/router_topk.metalindex 78eca3add..7215c157f 100644--- a/crates/backend-uzu/src/backends/metal/kernel/moe/router_topk.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/moe/router_topk.metal@@ -43,27 +43,16 @@ PUBLIC KERNEL(MoeRouterTopK)(const device ScalarT* x_vec = input + (ulong)token_idx * (ulong)vecs * 4;for (uint c = lid; c < vecs; c += THREADS_PER_TG) {- x_cache[c] = float4(- x_vec[c * 4 + 0],- x_vec[c * 4 + 1],- x_vec[c * 4 + 2],- x_vec[c * 4 + 3]- );+ x_cache[c] = float4(x_vec[c * 4 + 0], x_vec[c * 4 + 1], x_vec[c * 4 + 2], x_vec[c * 4 + 3]);}threadgroup_barrier(mem_flags::mem_threadgroup);- for (uint row = thread_context.simdgroup_index; row < e;- row += thread_context.simdgroups_per_threadgroup) {+ for (uint row = thread_context.simdgroup_index; row < e; row += thread_context.simdgroups_per_threadgroup) {const device ScalarT* w_vec = weight + (ulong)row * (ulong)vecs * 4;float4 accum4 = float4(0.0f);for (uint c = thread_context.simd_lane_id; c < vecs; c += 32u) {- const float4 wv = float4(- w_vec[c * 4 + 0],- w_vec[c * 4 + 1],- w_vec[c * 4 + 2],- w_vec[c * 4 + 3]- );+ const float4 wv = float4(w_vec[c * 4 + 0], w_vec[c * 4 + 1], w_vec[c * 4 + 2], w_vec[c * 4 + 3]);const float4 xv = x_cache[c];accum4 = fma(wv, xv, accum4);}@@ -89,27 +78,18 @@ PUBLIC KERNEL(MoeRouterTopK)(uint local_idx = 0xFFFFFFFFu;for (uint row = lid; row < e; row += THREADS_PER_TG) {float candidate = logits_shared[row];- if (candidate > local_best ||- (candidate == local_best && row < local_idx)) {+ if (candidate > local_best || (candidate == local_best && row < local_idx)) {local_best = candidate;local_idx = row;}}float max_val =- threadgroup_cooperative_reduce<SimdReduceMax<float>, THREADS_PER_TG>(- local_best,- reduce_tmp,- thread_context- );+ threadgroup_cooperative_reduce<SimdReduceMax<float>, THREADS_PER_TG>(local_best, reduce_tmp, thread_context);uint candidate_id = (local_best == max_val) ? local_idx : 0xFFFFFFFFu;uint best_idx =- threadgroup_cooperative_reduce<SimdReduceMin<uint>, THREADS_PER_TG>(- candidate_id,- reduce_tmp_u,- thread_context- );+ threadgroup_cooperative_reduce<SimdReduceMin<uint>, THREADS_PER_TG>(candidate_id, reduce_tmp_u, thread_context);if (lid == 0) {shared_best_idx[0] = best_idx;@@ -143,9 +123,7 @@ PUBLIC KERNEL(MoeRouterTopK)(float default_prob = 1.0f / float(effective_k);float inv_sum = (sum_exp > 0.0f) ? (1.0f / sum_exp) : default_prob;for (uint i = 0; i < effective_k; ++i) {- float prob = (sum_exp > 0.0f)- ? exp(float(out_probs[i]) - max_logit) * inv_sum- : default_prob;+ float prob = (sum_exp > 0.0f) ? exp(float(out_probs[i]) - max_logit) * inv_sum : default_prob;out_probs[i] = static_cast<ScalarT>(prob);}for (uint i = effective_k; i < k; ++i) {diff --git a/crates/backend-uzu/src/backends/metal/kernel/moe/scatter_buckets.metal b/crates/backend-uzu/src/backends/metal/kernel/moe/scatter_buckets.metalindex 860fb1332..1b7a3269a 100644--- a/crates/backend-uzu/src/backends/metal/kernel/moe/scatter_buckets.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/moe/scatter_buckets.metal@@ -45,11 +45,7 @@ inline void moe_scatter_buckets_impl(// Zero counts and bases for active tilefor (uint te = lid; te < tile_e; te += BLOCK_SIZE) {for (uint sg = 0; sg < NUM_SG; ++sg) {- atomic_store_explicit(- &sg_counts[sg * TILE_E + te],- 0u,- memory_order_relaxed- );+ atomic_store_explicit(&sg_counts[sg * TILE_E + te], 0u, memory_order_relaxed);sg_base[sg * TILE_E + te] = 0u;}}@@ -78,14 +74,12 @@ inline void moe_scatter_buckets_impl(// Phase 2: compute per-sg bases via prefix across sgs with capacity clampfor (uint te = lid; te < tile_e; te += BLOCK_SIZE) {const uint tile_id = e0 / TILE_E;- const uint base_idx_block =- (block_id * num_tiles + tile_id) * TILE_E + te;+ const uint base_idx_block = (block_id * num_tiles + tile_id) * TILE_E + te;const uint alloc_total = block_alloc[base_idx_block];uint prefix = 0u;for (uint sg = 0; sg < NUM_SG; ++sg) {const uint idx = sg * TILE_E + te;- const uint c =- atomic_load_explicit(&sg_counts[idx], memory_order_relaxed);+ const uint c = atomic_load_explicit(&sg_counts[idx], memory_order_relaxed);const uint room = (prefix < alloc_total) ? (alloc_total - prefix) : 0u;const uint take = min(c, room);sg_base[idx] = prefix;@@ -93,11 +87,7 @@ inline void moe_scatter_buckets_impl(}// Reuse sg_counts as write indices: reset to zerofor (uint sg = 0; sg < NUM_SG; ++sg) {- atomic_store_explicit(- &sg_counts[sg * TILE_E + te],- 0u,- memory_order_relaxed- );+ atomic_store_explicit(&sg_counts[sg * TILE_E + te], 0u, memory_order_relaxed);}}threadgroup_barrier(mem_flags::mem_threadgroup);@@ -115,19 +105,12 @@ inline void moe_scatter_buckets_impl(if (ue >= e0 && ue < e0 + tile_e) {const uint te = ue - e0;const uint tile_id = e0 / TILE_E;- const uint base_idx_block =- (block_id * num_tiles + tile_id) * TILE_E + te;- const uint idx_sg_te =- thread_context.simdgroup_index * TILE_E + te;- const uint local = atomic_fetch_add_explicit(- &sg_counts[idx_sg_te],- 1u,- memory_order_relaxed- );+ const uint base_idx_block = (block_id * num_tiles + tile_id) * TILE_E + te;+ const uint idx_sg_te = thread_context.simdgroup_index * TILE_E + te;+ const uint local = atomic_fetch_add_explicit(&sg_counts[idx_sg_te], 1u, memory_order_relaxed);const uint base_s = sg_base[idx_sg_te];- const uint slot =- offsets[ue] + block_bases[base_idx_block] + base_s + local;+ const uint slot = offsets[ue] + block_bases[base_idx_block] + base_s + local;out_ids[slot] = (int)t;out_probs[slot] = topk_probs[base + k];if (tok2row_opt != nullptr) {@@ -167,10 +150,7 @@ PUBLIC KERNEL(MoeBlockBasesFromPartials)(for (uint b = 0; b < num_blocks; ++b) {const uint idx = (b * num_tiles + tile_id) * TILE_E + te;const uint p = partials[idx];- const uint alloc = clamp ? (running < capacity_per_expert- ? min(p, capacity_per_expert - running)- : 0u)- : p;+ const uint alloc = clamp ? (running < capacity_per_expert ? min(p, capacity_per_expert - running) : 0u) : p;block_bases[idx] = running;block_alloc[idx] = alloc;running += alloc;diff --git a/crates/backend-uzu/src/backends/metal/kernel/moe/tiles_map.metal b/crates/backend-uzu/src/backends/metal/kernel/moe/tiles_map.metalindex c9ee593df..d49745802 100644--- a/crates/backend-uzu/src/backends/metal/kernel/moe/tiles_map.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/moe/tiles_map.metal@@ -95,10 +95,9 @@ PUBLIC KERNEL(MoeBuildTileMap)(// Write MTLDispatchThreadgroupsIndirectArguments {x, y, z} where:// x = num_tiles_n (computed on CPU and passed in), y = total_tiles, z = 1PUBLIC KERNEL(MoeWriteDispatchArgs)(- device const uint*- total_tiles_buf, // [>=1], total_tiles_buf[0] = total_rows- device uint* dispatch_args, // [3] u32- constant uint& num_tiles_n, // x dimension+ device const uint* total_tiles_buf, // [>=1], total_tiles_buf[0] = total_rows+ device uint* dispatch_args, // [3] u32+ constant uint& num_tiles_n, // x dimensionuint gid AXIS(1, 1)) {const uint total_rows = total_tiles_buf[0];diff --git a/crates/backend-uzu/src/backends/metal/kernel/moe/tiles_pass_a.metal b/crates/backend-uzu/src/backends/metal/kernel/moe/tiles_pass_a.metalindex 899c6c168..1e566a1ea 100644--- a/crates/backend-uzu/src/backends/metal/kernel/moe/tiles_pass_a.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/moe/tiles_pass_a.metal@@ -108,8 +108,7 @@ PUBLIC KERNEL(MoePassABuildTileMap)(const uint expert_idx = row_expert_map[row_idx];const uint row_start = expert_offsets[expert_idx];const uint row_in_expert = row_idx - row_start;- const uint tile_base =- tile_offsets[expert_idx] + row_in_expert * h_blocks + h_block;+ const uint tile_base = tile_offsets[expert_idx] + row_in_expert * h_blocks + h_block;tile_map[tile_base * 3 + 0] = h_block;tile_map[tile_base * 3 + 1] = expert_idx;@@ -119,9 +118,8 @@ PUBLIC KERNEL(MoePassABuildTileMap)(// Write dispatch args for indirect dispatch (reusable from tiled version)PUBLIC KERNEL(MoePassAWriteDispatchArgs)(device const uint* total_tiles, // [1]- device uint*- dispatch_args, // [3] - MTLDispatchThreadgroupsIndirectArguments- constant uint& num_tiles_y, // usually 1 for Pass A+ device uint* dispatch_args, // [3] - MTLDispatchThreadgroupsIndirectArguments+ constant uint& num_tiles_y, // usually 1 for Pass Auint tid AXIS(1, 1)) {dispatch_args[0] = total_tiles[0]; // x dimension = total tilesdiff --git a/crates/backend-uzu/src/backends/metal/kernel/pooling/pooling.metal b/crates/backend-uzu/src/backends/metal/kernel/pooling/pooling.metalindex ba4c7a47f..f366c77e0 100644--- a/crates/backend-uzu/src/backends/metal/kernel/pooling/pooling.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/pooling/pooling.metal@@ -14,8 +14,7 @@ PUBLIC KERNEL(PoolingCls) (uint dim_idx AXIS(hidden_dim, 16),uint batch_idx AXIS(batch_size, 16)) {- output[batch_idx * hidden_dim + dim_idx] =- input[batch_idx * seq_len * hidden_dim + dim_idx];+ output[batch_idx * hidden_dim + dim_idx] = input[batch_idx * seq_len * hidden_dim + dim_idx];}// Mean pooling: Average across sequence [batch, seq_len, hidden_dim] → [batch,@@ -33,9 +32,7 @@ PUBLIC KERNEL(PoolingMean) () {float sum = 0.0f;for (uint seq_idx = 0; seq_idx < seq_len; seq_idx++) {- sum += float(- input[batch_idx * seq_len * hidden_dim + seq_idx * hidden_dim + dim_idx]- );+ sum += float(input[batch_idx * seq_len * hidden_dim + seq_idx * hidden_dim + dim_idx]);}output[batch_idx * hidden_dim + dim_idx] = T(sum / float(seq_len));}diff --git a/crates/backend-uzu/src/backends/metal/kernel/rms_norm/qkv_norm.metal b/crates/backend-uzu/src/backends/metal/kernel/rms_norm/qkv_norm.metalindex d0d78d8aa..2cd29ea4c 100644--- a/crates/backend-uzu/src/backends/metal/kernel/rms_norm/qkv_norm.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/rms_norm/qkv_norm.metal@@ -47,8 +47,7 @@ PUBLIC KERNEL(QKVNorm)(return;const ulong slice_offset =- (ulong)batch_idx * (ulong)total_heads_in_buffer * (ulong)head_dim +- (ulong)logical_head_idx * (ulong)head_dim;+ (ulong)batch_idx * (ulong)total_heads_in_buffer * (ulong)head_dim + (ulong)logical_head_idx * (ulong)head_dim;const device InputT* input_data = qkv_input + slice_offset;const device ScaleT* scales_data = scales;@@ -58,8 +57,7 @@ PUBLIC KERNEL(QKVNorm)(AccumT partial_sum = static_cast<AccumT>(0.0f);// Sum of squares: each lane processes GRAIN_SIZE elements per iteration.- for (uint base_i = lane_id * GRAIN_SIZE; base_i < element_count;- base_i += METAL_SIMD_SIZE * GRAIN_SIZE) {+ for (uint base_i = lane_id * GRAIN_SIZE; base_i < element_count; base_i += METAL_SIMD_SIZE * GRAIN_SIZE) {AccumT vals[GRAIN_SIZE];for (uint j = 0; j < GRAIN_SIZE; ++j) {uint i = base_i + j;@@ -74,13 +72,11 @@ PUBLIC KERNEL(QKVNorm)(AccumT total_sum = simd_sum(partial_sum);// RMS factor.- AccumT mean_square =- static_cast<AccumT>(total_sum) / static_cast<AccumT>(element_count);+ AccumT mean_square = static_cast<AccumT>(total_sum) / static_cast<AccumT>(element_count);AccumT rms_norm = rsqrt(mean_square + static_cast<AccumT>(epsilon));// Normalize + scale.- for (uint base_i = lane_id * GRAIN_SIZE; base_i < element_count;- base_i += METAL_SIMD_SIZE * GRAIN_SIZE) {+ for (uint base_i = lane_id * GRAIN_SIZE; base_i < element_count; base_i += METAL_SIMD_SIZE * GRAIN_SIZE) {AccumT vals[GRAIN_SIZE];for (uint j = 0; j < GRAIN_SIZE; ++j) {uint i = base_i + j;@@ -97,16 +93,12 @@ PUBLIC KERNEL(QKVNorm)(if (scale_free) {output_data[i] = static_cast<OutputT>(normalized_high);} else if (full_layer) {- AccumT scale_value_high = static_cast<AccumT>(scales_data[i]) +- static_cast<AccumT>(scale_offset);- output_data[i] =- static_cast<OutputT>(normalized_high * scale_value_high);+ AccumT scale_value_high = static_cast<AccumT>(scales_data[i]) + static_cast<AccumT>(scale_offset);+ output_data[i] = static_cast<OutputT>(normalized_high * scale_value_high);} else {OutputT normalized_low = static_cast<OutputT>(normalized_high);- OutputT scale_value_low = static_cast<OutputT>(- static_cast<AccumT>(scales_data[i]) +- static_cast<AccumT>(scale_offset)- );+ OutputT scale_value_low =+ static_cast<OutputT>(static_cast<AccumT>(scales_data[i]) + static_cast<AccumT>(scale_offset));OutputT product_low = normalized_low * scale_value_low;output_data[i] = static_cast<OutputT>(product_low);}diff --git a/crates/backend-uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal b/crates/backend-uzu/src/backends/metal/kernel/rms_norm/rms_norm.metalindex 9915a5c20..af3911905 100644--- a/crates/backend-uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal@@ -61,8 +61,7 @@ PUBLIC KERNEL(RMSNorm)(if (residual_add) {val += shortcut[i];if (scale_residual_sum) {- val =- static_cast<InputT>(static_cast<float>(val) * post_layer_scalar);+ val = static_cast<InputT>(static_cast<float>(val) * post_layer_scalar);}}shortcut[i] = val;@@ -72,18 +71,14 @@ PUBLIC KERNEL(RMSNorm)(}// Step 2 - threads reduce their partial sums of squares- AccumT total_sum_of_squares =- threadgroup_cooperative_reduce<SimdReduceSum<AccumT>, BLOCK_SIZE>(- thread_sum_of_squares,- shared_sum,- thread_context- );+ AccumT total_sum_of_squares = threadgroup_cooperative_reduce<SimdReduceSum<AccumT>, BLOCK_SIZE>(+ thread_sum_of_squares,+ shared_sum,+ thread_context+ );// And pre-calculate rms_inv- AccumT rms_inv = rsqrt(- total_sum_of_squares / static_cast<AccumT>(element_count) +- static_cast<AccumT>(epsilon)- );+ AccumT rms_inv = rsqrt(total_sum_of_squares / static_cast<AccumT>(element_count) + static_cast<AccumT>(epsilon));// Step 3 - elementwise normalizationfor (uint i = thread_in_row; i < element_count; i += BLOCK_SIZE) {@@ -96,8 +91,7 @@ PUBLIC KERNEL(RMSNorm)(x = static_cast<AccumT>(input[i]);}- AccumT scale =- static_cast<AccumT>(scales[i]) + static_cast<AccumT>(scale_offset);+ AccumT scale = static_cast<AccumT>(scales[i]) + static_cast<AccumT>(scale_offset);// If full_layer, normalize and scale in AccumT, cast to OutputT at the end// If not, cast to OutputT after normalize, scale in OutputTdiff --git a/crates/backend-uzu/src/backends/metal/kernel/rng.h b/crates/backend-uzu/src/backends/metal/kernel/rng.hindex 8715ebc7d..bc887762d 100644--- a/crates/backend-uzu/src/backends/metal/kernel/rng.h+++ b/crates/backend-uzu/src/backends/metal/kernel/rng.h@@ -43,11 +43,7 @@ inline void philox4x32bumpkey(thread uint* key) {key[1] += PHILOX_W32_1;}-inline void curand_philox4x32_10(- thread uint* ctr,- thread uint* key,- thread uint* out-) {+inline void curand_philox4x32_10(thread uint* ctr, thread uint* key, thread uint* out) {uint local_ctr[4] = {ctr[0], ctr[1], ctr[2], ctr[3]};uint local_key[2] = {key[0], key[1]};@@ -90,11 +86,7 @@ inline void philox_state_incr(thread PhiloxState* s) {++s->ctr[3];}-inline void philox_init(- thread PhiloxState* state,- uint64_t seed,- uint64_t offset-) {+inline void philox_init(thread PhiloxState* state, uint64_t seed, uint64_t offset) {// Initialize key from seedstate->key[0] = uint(seed);state->key[1] = uint(seed >> 32);diff --git a/crates/backend-uzu/src/backends/metal/kernel/rope/rope.metal b/crates/backend-uzu/src/backends/metal/kernel/rope/rope.metalindex 670208e68..933f90b0b 100644--- a/crates/backend-uzu/src/backends/metal/kernel/rope/rope.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/rope/rope.metal@@ -13,10 +13,8 @@ inline ElementT applyRopeTransform(float sin_val) {float inputVal = float(qkv_tensor_view(token_idx, head_idx, dim_idx));- float pairedVal =- (dim_idx < half_dim)- ? -float(qkv_tensor_view(token_idx, head_idx, dim_idx + half_dim))- : float(qkv_tensor_view(token_idx, head_idx, dim_idx - half_dim));+ float pairedVal = (dim_idx < half_dim) ? -float(qkv_tensor_view(token_idx, head_idx, dim_idx + half_dim))+ : float(qkv_tensor_view(token_idx, head_idx, dim_idx - half_dim));return static_cast<ElementT>(inputVal * cos_val + pairedVal * sin_val);}@@ -39,8 +37,7 @@ PUBLIC KERNEL(Rope)(const uint token_index AXIS(suffix_length, 1),const uint dimension_index AXIS(head_dim, 128)) {- if (head_index >= num_heads || token_index >= suffix_length ||- dimension_index >= head_dim)+ if (head_index >= num_heads || token_index >= suffix_length || dimension_index >= head_dim)return;if (head_dim & 1)return;@@ -55,31 +52,17 @@ PUBLIC KERNEL(Rope)(const uint half_rope_dim = rope_dim / 2;TensorView3D<const ElementT> qkv_tensor_view =- TensorView3D<const ElementT>(qkv)- .shaped(suffix_length, total_heads, head_dim);- TensorView2D<const RopeT> cosines_tensor_view =- TensorView2D<const RopeT>(cosines).shaped(suffix_length, rope_dim);- TensorView2D<const RopeT> sines_tensor_view =- TensorView2D<const RopeT>(sines).shaped(suffix_length, rope_dim);+ TensorView3D<const ElementT>(qkv).shaped(suffix_length, total_heads, head_dim);+ TensorView2D<const RopeT> cosines_tensor_view = TensorView2D<const RopeT>(cosines).shaped(suffix_length, rope_dim);+ TensorView2D<const RopeT> sines_tensor_view = TensorView2D<const RopeT>(sines).shaped(suffix_length, rope_dim);TensorView3D<ElementT> rotated_queries_tensor_view =- TensorView3D<ElementT>(rotated_queries)- .shaped(num_heads, suffix_length, head_dim);+ TensorView3D<ElementT>(rotated_queries).shaped(num_heads, suffix_length, head_dim);if (dimension_index < rope_dim) {- const float cos_val =- float(cosines_tensor_view(token_index, dimension_index));- const float sin_val =- float(sines_tensor_view(token_index, dimension_index));+ const float cos_val = float(cosines_tensor_view(token_index, dimension_index));+ const float sin_val = float(sines_tensor_view(token_index, dimension_index));rotated_queries_tensor_view(head_index, token_index, dimension_index) =- applyRopeTransform(- qkv_tensor_view,- token_index,- head_index,- dimension_index,- half_rope_dim,- cos_val,- sin_val- );+ applyRopeTransform(qkv_tensor_view, token_index, head_index, dimension_index, half_rope_dim, cos_val, sin_val);} else {rotated_queries_tensor_view(head_index, token_index, dimension_index) =qkv_tensor_view(token_index, head_index, dimension_index);@@ -94,24 +77,20 @@ PUBLIC KERNEL(Rope)(return;TensorView3D<ElementT> rotated_keys_tensor_view =- TensorView3D<ElementT>(rotated_keys)- .shaped(num_groups, suffix_length, head_dim);+ TensorView3D<ElementT>(rotated_keys).shaped(num_groups, suffix_length, head_dim);const uint key_head_index = num_heads + group_index;if (dimension_index < rope_dim) {- const float cos_val =- float(cosines_tensor_view(token_index, dimension_index));- const float sin_val =- float(sines_tensor_view(token_index, dimension_index));- rotated_keys_tensor_view(group_index, token_index, dimension_index) =- applyRopeTransform(- qkv_tensor_view,- token_index,- key_head_index,- dimension_index,- half_rope_dim,- cos_val,- sin_val- );+ const float cos_val = float(cosines_tensor_view(token_index, dimension_index));+ const float sin_val = float(sines_tensor_view(token_index, dimension_index));+ rotated_keys_tensor_view(group_index, token_index, dimension_index) = applyRopeTransform(+ qkv_tensor_view,+ token_index,+ key_head_index,+ dimension_index,+ half_rope_dim,+ cos_val,+ sin_val+ );} else {rotated_keys_tensor_view(group_index, token_index, dimension_index) =qkv_tensor_view(token_index, key_head_index, dimension_index);diff --git a/crates/backend-uzu/src/backends/metal/kernel/sampling/argmax.metal b/crates/backend-uzu/src/backends/metal/kernel/sampling/argmax.metalindex 852a43700..38a96c209 100644--- a/crates/backend-uzu/src/backends/metal/kernel/sampling/argmax.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/sampling/argmax.metal@@ -17,19 +17,14 @@ bool argmax_is_better(ArgmaxPair a, ArgmaxPair b) {return a.value > b.value || (a.value == b.value && a.index < b.index);}-ArgmaxPair argmax_combine(ArgmaxPair a, ArgmaxPair b) {- return argmax_is_better(a, b) ? a : b;-}+ArgmaxPair argmax_combine(ArgmaxPair a, ArgmaxPair b) { return argmax_is_better(a, b) ? a : b; }struct SimdReduceArgmax {using value_type = ArgmaxPair;static constant constexpr ArgmaxPair identity = ARGMAX_INIT;static ArgmaxPair simd_reduce(ArgmaxPair x) {for (ushort offset = 16; offset > 0; offset /= 2) {- ArgmaxPair other = {- simd_shuffle_down(x.value, offset),- simd_shuffle_down(x.index, offset)- };+ ArgmaxPair other = {simd_shuffle_down(x.value, offset), simd_shuffle_down(x.index, offset)};x = argmax_combine(x, other);}return x;@@ -37,9 +32,7 @@ struct SimdReduceArgmax {};template <ushort GRAIN_SIZE_PARAM>-static inline ArgmaxPair thread_argmax_reduce(- thread ArgmaxPair (&values)[GRAIN_SIZE_PARAM]-) {+static inline ArgmaxPair thread_argmax_reduce(thread ArgmaxPair (&values)[GRAIN_SIZE_PARAM]) {ArgmaxPair result = values[0];for (ushort i = 1; i < GRAIN_SIZE_PARAM; i++) {result = argmax_combine(result, values[i]);@@ -70,11 +63,7 @@ static void load_blocked_argmax_batched(}template <ushort BLOCK_SIZE_PARAM>-static ArgmaxPair threadgroup_raking_argmax(- ArgmaxPair value,- threadgroup ArgmaxPair* shared,- const ushort lid-) {+static ArgmaxPair threadgroup_raking_argmax(ArgmaxPair value, threadgroup ArgmaxPair* shared, const ushort lid) {shared[lid] = value;threadgroup_barrier(mem_flags::mem_threadgroup);@@ -90,17 +79,11 @@ static ArgmaxPair threadgroup_raking_argmax(ArgmaxPair simd_result = thread_result;for (ushort offset = 16; offset > 0; offset /= 2) {- ArgmaxPair other = {- simd_shuffle_down(simd_result.value, offset),- simd_shuffle_down(simd_result.index, offset)- };+ ArgmaxPair other = {simd_shuffle_down(simd_result.value, offset), simd_shuffle_down(simd_result.index, offset)};simd_result = argmax_combine(simd_result, other);}- ArgmaxPair final_result = {- simd_broadcast(simd_result.value, 0),- simd_broadcast(simd_result.index, 0)- };+ ArgmaxPair final_result = {simd_broadcast(simd_result.value, 0), simd_broadcast(simd_result.index, 0)};for (ushort i = first_index; i < first_index + values_per_thread; i++) {shared[i] = final_result;@@ -132,23 +115,14 @@ PUBLIC KERNEL(ArgmaxSingle)(const uint elements_per_tile = BLOCK_SIZE * GRAIN_SIZE;ArgmaxPair best = ARGMAX_INIT;- for (uint vocab_offset = 0; vocab_offset < vocab_size;- vocab_offset += elements_per_tile) {+ for (uint vocab_offset = 0; vocab_offset < vocab_size; vocab_offset += elements_per_tile) {ArgmaxPair buf[GRAIN_SIZE];- load_blocked_argmax_batched<GRAIN_SIZE>(- buf,- logits_data,- local_id,- batch_idx,- vocab_offset,- vocab_size- );+ load_blocked_argmax_batched<GRAIN_SIZE>(buf, logits_data, local_id, batch_idx, vocab_offset, vocab_size);ArgmaxPair tile_best = thread_argmax_reduce<GRAIN_SIZE>(buf);best = argmax_combine(best, tile_best);}- ArgmaxPair group_best =- threadgroup_raking_argmax<BLOCK_SIZE>(best, shared, local_id);+ ArgmaxPair group_best = threadgroup_raking_argmax<BLOCK_SIZE>(best, shared, local_id);if (local_id == 0) {final_tokens[batch_idx] = group_best.index;@@ -178,26 +152,14 @@ PUBLIC KERNEL(ArgmaxMain)(const uint vocab_offset = vocab_group_idx * elements_per_group;ArgmaxPair values[GRAIN_SIZE];- load_blocked_argmax_batched<GRAIN_SIZE>(- values,- logits_data,- local_id,- batch_idx,- vocab_offset,- vocab_size- );+ load_blocked_argmax_batched<GRAIN_SIZE>(values, logits_data, local_id, batch_idx, vocab_offset, vocab_size);ArgmaxPair thread_result = thread_argmax_reduce<GRAIN_SIZE>(values);ArgmaxPair group_result =- threadgroup_cooperative_reduce<SimdReduceArgmax, BLOCK_SIZE>(- thread_result,- shared,- thread_context- );+ threadgroup_cooperative_reduce<SimdReduceArgmax, BLOCK_SIZE>(thread_result, shared, thread_context);if (local_id == 0) {- uint vocab_groups_per_batch =- (vocab_size + elements_per_group - 1) / elements_per_group;+ uint vocab_groups_per_batch = (vocab_size + elements_per_group - 1) / elements_per_group;uint partial_idx = batch_idx * vocab_groups_per_batch + vocab_group_idx;partial_results[partial_idx] = group_result;}@@ -218,23 +180,17 @@ PUBLIC KERNEL(ArgmaxFinal)(return;const uint elements_per_group = BLOCK_SIZE * GRAIN_SIZE;- const uint vocab_groups_per_batch =- (vocab_size + elements_per_group - 1) / elements_per_group;+ const uint vocab_groups_per_batch = (vocab_size + elements_per_group - 1) / elements_per_group;const uint partial_offset = batch_idx * vocab_groups_per_batch;ArgmaxPair thread_result = ARGMAX_INIT;for (uint i = local_id; i < vocab_groups_per_batch; i += BLOCK_SIZE) {- thread_result =- argmax_combine(thread_result, partial_results[partial_offset + i]);+ thread_result = argmax_combine(thread_result, partial_results[partial_offset + i]);}ArgmaxPair result =- threadgroup_cooperative_reduce<SimdReduceArgmax, BLOCK_SIZE>(- thread_result,- shared,- thread_context- );+ threadgroup_cooperative_reduce<SimdReduceArgmax, BLOCK_SIZE>(thread_result, shared, thread_context);if (local_id == 0) {final_tokens[batch_idx] = result.index;diff --git a/crates/backend-uzu/src/backends/metal/kernel/sampling/bitmask.metal b/crates/backend-uzu/src/backends/metal/kernel/sampling/bitmask.metalindex 3dd8b2011..e5f776520 100644--- a/crates/backend-uzu/src/backends/metal/kernel/sampling/bitmask.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/sampling/bitmask.metal@@ -23,8 +23,7 @@ PUBLIC KERNEL(Bitmask) (}uint bitmask_size = (vocab_size + (BITS_IN_U32 - 1)) / BITS_IN_U32;- uint base_idx =- batch_idx * vocab_size + group_idx * BLOCK_SIZE * GRAIN_SIZE + thread_idx;+ uint base_idx = batch_idx * vocab_size + group_idx * BLOCK_SIZE * GRAIN_SIZE + thread_idx;uint batch_end = batch_idx * vocab_size + vocab_size;#pragma unroll(4)@@ -34,8 +33,7 @@ PUBLIC KERNEL(Bitmask) (uint token_idx = global_idx - batch_idx * vocab_size;uint bitmask_idx = batch_idx * bitmask_size + (token_idx / BITS_IN_U32);bool mask = (bitmask[bitmask_idx] >> (token_idx % BITS_IN_U32)) & 0b1;- processed_logits[global_idx] =- select(T(-INFINITY), logits[global_idx], mask);+ processed_logits[global_idx] = select(T(-INFINITY), logits[global_idx], mask);}}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/sampling/gumbel.metal b/crates/backend-uzu/src/backends/metal/kernel/sampling/gumbel.metalindex 2625bd27e..5fb7b19aa 100644--- a/crates/backend-uzu/src/backends/metal/kernel/sampling/gumbel.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/sampling/gumbel.metal@@ -32,8 +32,7 @@ PUBLIC KERNEL(Gumbel) (uint grain_offset = batch_start + grain_offset_in_batch;uint64_t rng_seed = batch_seeds[batch_seeds_offset + batch_idx];- uint64_t rng_offset = (group_idx * BLOCK_SIZE + thread_idx) *- (GRAIN_SIZE + WORDS_PER_OFFSET - 1) / WORDS_PER_OFFSET;+ uint64_t rng_offset = (group_idx * BLOCK_SIZE + thread_idx) * (GRAIN_SIZE + WORDS_PER_OFFSET - 1) / WORDS_PER_OFFSET;PhiloxState rng;philox_init(&rng, rng_seed, rng_offset);@@ -41,8 +40,7 @@ PUBLIC KERNEL(Gumbel) (for (uint i = 0; i < GRAIN_SIZE; i++) {uint global_idx = grain_offset + i * BLOCK_SIZE;if (global_idx < batch_end) {- processed_logits[global_idx] =- logits[global_idx] + T(-fast::log(-fast::log(uniform_float(&rng))));+ processed_logits[global_idx] = logits[global_idx] + T(-fast::log(-fast::log(uniform_float(&rng))));}}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/sampling/min_p.metal b/crates/backend-uzu/src/backends/metal/kernel/sampling/min_p.metalindex 36b988cec..79355c6fe 100644--- a/crates/backend-uzu/src/backends/metal/kernel/sampling/min_p.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/sampling/min_p.metal@@ -34,11 +34,7 @@ PUBLIC KERNEL(MinP) (local_max = fmax(local_max, logit_value);}float max_logit =- threadgroup_cooperative_reduce<SimdReduceMax<float>, BLOCK_SIZE>(- local_max,- shared_reduce_buffer,- thread_context- );+ threadgroup_cooperative_reduce<SimdReduceMax<float>, BLOCK_SIZE>(local_max, shared_reduce_buffer, thread_context);// Then the threshold is just max_logit + log(min_p), mask everything strictly// below it@@ -46,7 +42,6 @@ PUBLIC KERNEL(MinP) (#pragma unroll(4)for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {T logit_value = logits[batch_start + i];- processed_logits[batch_start + i] =- select(T(-INFINITY), logit_value, logit_value >= t_threshold);+ processed_logits[batch_start + i] = select(T(-INFINITY), logit_value, logit_value >= t_threshold);}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/sampling/temperature.metal b/crates/backend-uzu/src/backends/metal/kernel/sampling/temperature.metalindex 73c7bd3e4..57b63dbb1 100644--- a/crates/backend-uzu/src/backends/metal/kernel/sampling/temperature.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/sampling/temperature.metal@@ -21,8 +21,7 @@ PUBLIC KERNEL(Temperature) (logits = processed_logits;}- uint base_idx =- batch_idx * vocab_size + group_idx * BLOCK_SIZE * GRAIN_SIZE + thread_idx;+ uint base_idx = batch_idx * vocab_size + group_idx * BLOCK_SIZE * GRAIN_SIZE + thread_idx;uint batch_end = batch_idx * vocab_size + vocab_size;#pragma unroll(4)diff --git a/crates/backend-uzu/src/backends/metal/kernel/sampling/top_k.metal b/crates/backend-uzu/src/backends/metal/kernel/sampling/top_k.metalindex 7a27876aa..b88dc4092 100644--- a/crates/backend-uzu/src/backends/metal/kernel/sampling/top_k.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/sampling/top_k.metal@@ -34,24 +34,12 @@ PUBLIC KERNEL(TopK)(for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {float logit_value = float(logits[batch_start + i]);local_max = fmax(local_max, logit_value);- local_min = select(- local_min,- fmin(local_min, logit_value),- logit_value > -INFINITY- );+ local_min = select(local_min, fmin(local_min, logit_value), logit_value > -INFINITY);}float max_logit =- threadgroup_cooperative_reduce<SimdReduceMax<float>, BLOCK_SIZE>(- local_max,- shared_reduce_buffer,- thread_context- );+ threadgroup_cooperative_reduce<SimdReduceMax<float>, BLOCK_SIZE>(local_max, shared_reduce_buffer, thread_context);float min_logit =- threadgroup_cooperative_reduce<SimdReduceMin<float>, BLOCK_SIZE>(- local_min,- shared_reduce_buffer,- thread_context- );+ threadgroup_cooperative_reduce<SimdReduceMin<float>, BLOCK_SIZE>(local_min, shared_reduce_buffer, thread_context);// Do the binary search on the thresholdfloat low = min_logit;float high = max_logit;@@ -65,12 +53,11 @@ PUBLIC KERNEL(TopK)(float logit_value = float(logits[batch_start + i]);local_num_above_threshold += select(0, 1, logit_value >= threshold);}- uint num_above_threshold =- threadgroup_cooperative_reduce<SimdReduceSum<uint>, BLOCK_SIZE>(- local_num_above_threshold,- (threadgroup uint*)shared_reduce_buffer,- thread_context- );+ uint num_above_threshold = threadgroup_cooperative_reduce<SimdReduceSum<uint>, BLOCK_SIZE>(+ local_num_above_threshold,+ (threadgroup uint*)shared_reduce_buffer,+ thread_context+ );// Update binary searchif (num_above_threshold == top_k) {@@ -89,7 +76,6 @@ PUBLIC KERNEL(TopK)(#pragma unroll(4)for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {T logit_value = logits[batch_start + i];- processed_logits[batch_start + i] =- select(T(-INFINITY), logit_value, logit_value >= t_threshold);+ processed_logits[batch_start + i] = select(T(-INFINITY), logit_value, logit_value >= t_threshold);}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/sampling/top_p.metal b/crates/backend-uzu/src/backends/metal/kernel/sampling/top_p.metalindex a4b9ea0d8..7a6303feb 100644--- a/crates/backend-uzu/src/backends/metal/kernel/sampling/top_p.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/sampling/top_p.metal@@ -35,42 +35,22 @@ PUBLIC KERNEL(TopP) (for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {float logit_value = float(logits[batch_start + i]);local_max = fmax(local_max, logit_value);- local_min = select(- local_min,- fmin(local_min, logit_value),- logit_value > -INFINITY- );+ local_min = select(local_min, fmin(local_min, logit_value), logit_value > -INFINITY);}float max_logit =- threadgroup_cooperative_reduce<SimdReduceMax<float>, BLOCK_SIZE>(- local_max,- shared_reduce_buffer,- thread_context- );+ threadgroup_cooperative_reduce<SimdReduceMax<float>, BLOCK_SIZE>(local_max, shared_reduce_buffer, thread_context);float min_logit =- threadgroup_cooperative_reduce<SimdReduceMin<float>, BLOCK_SIZE>(- local_min,- shared_reduce_buffer,- thread_context- );+ threadgroup_cooperative_reduce<SimdReduceMin<float>, BLOCK_SIZE>(local_min, shared_reduce_buffer, thread_context);// Find denominator for softmaxfloat local_sum = 0.0f;#pragma unroll(4)for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {float logit_value = float(logits[batch_start + i]);- local_sum += select(- 0.0,- fast::exp(logit_value - max_logit),- logit_value > -INFINITY- );+ local_sum += select(0.0, fast::exp(logit_value - max_logit), logit_value > -INFINITY);}float total_sum =- threadgroup_cooperative_reduce<SimdReduceSum<float>, BLOCK_SIZE>(- local_sum,- shared_reduce_buffer,- thread_context- );+ threadgroup_cooperative_reduce<SimdReduceSum<float>, BLOCK_SIZE>(local_sum, shared_reduce_buffer, thread_context);// Do the binary search on the thresholdfloat target_mass = top_p * total_sum;@@ -87,29 +67,23 @@ PUBLIC KERNEL(TopP) (for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {float logit_value = float(logits[batch_start + i]);float logit_mass = fast::exp(logit_value - max_logit);- local_sum_above_threshold +=- select(0.0, logit_mass, logit_value >= threshold);- local_min_above_threshold = fmin(- local_min_above_threshold,- select(INFINITY, logit_mass, logit_value >= threshold)- );+ local_sum_above_threshold += select(0.0, logit_mass, logit_value >= threshold);+ local_min_above_threshold =+ fmin(local_min_above_threshold, select(INFINITY, logit_mass, logit_value >= threshold));}- float sum_above_threshold =- threadgroup_cooperative_reduce<SimdReduceSum<float>, BLOCK_SIZE>(- local_sum_above_threshold,- shared_reduce_buffer,- thread_context- );- float min_above_threshold =- threadgroup_cooperative_reduce<SimdReduceMin<float>, BLOCK_SIZE>(- local_min_above_threshold,- shared_reduce_buffer,- thread_context- );+ float sum_above_threshold = threadgroup_cooperative_reduce<SimdReduceSum<float>, BLOCK_SIZE>(+ local_sum_above_threshold,+ shared_reduce_buffer,+ thread_context+ );+ float min_above_threshold = threadgroup_cooperative_reduce<SimdReduceMin<float>, BLOCK_SIZE>(+ local_min_above_threshold,+ shared_reduce_buffer,+ thread_context+ );// Early exit- if (sum_above_threshold >= target_mass &&- sum_above_threshold - min_above_threshold < target_mass) {+ if (sum_above_threshold >= target_mass && sum_above_threshold - min_above_threshold < target_mass) {break;}// Update binary search@@ -127,7 +101,6 @@ PUBLIC KERNEL(TopP) (#pragma unroll(4)for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {T logit_value = logits[batch_start + i];- processed_logits[batch_start + i] =- select(T(-INFINITY), logit_value, logit_value >= t_threshold);+ processed_logits[batch_start + i] = select(T(-INFINITY), logit_value, logit_value >= t_threshold);}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/short_conv/short_conv.metal b/crates/backend-uzu/src/backends/metal/kernel/short_conv/short_conv.metalindex b8b61860a..4b81d6450 100644--- a/crates/backend-uzu/src/backends/metal/kernel/short_conv/short_conv.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/short_conv/short_conv.metal@@ -187,10 +187,8 @@ PUBLIC KERNEL(ShortConvTrie)(// Select parent state (root uses base_state)const int parent = parents[node];- const device T* parent_state =- (parent < 0) ? (base_state + base_state_offset)- : (suffix_state +- (parent * model_dim + channel_idx) * state_stride);+ const device T* parent_state = (parent < 0) ? (base_state + base_state_offset)+ : (suffix_state + (parent * model_dim + channel_idx) * state_stride);float acc = 0.0f;if (has_bias) {@@ -212,8 +210,7 @@ PUBLIC KERNEL(ShortConvTrie)(// Write post-state for this nodeif (tap_count > 0) {- device T* dst_state =- suffix_state + (node * model_dim + channel_idx) * state_stride;+ device T* dst_state = suffix_state + (node * model_dim + channel_idx) * state_stride;for (uint tap = 0; tap < tap_count - 1; ++tap) {dst_state[tap] = parent_state[tap + 1];}diff --git a/crates/backend-uzu/src/backends/metal/kernel/softmax/softmax.metal b/crates/backend-uzu/src/backends/metal/kernel/softmax/softmax.metalindex a716e7c92..3cb83c001 100644--- a/crates/backend-uzu/src/backends/metal/kernel/softmax/softmax.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/softmax/softmax.metal@@ -42,8 +42,7 @@ PUBLIC KERNEL(Softmax)(threadgroup_barrier(mem_flags::mem_threadgroup);// Pass 1: online (max, normalizer) accumulation over the row.- for (uint base = tid * SOFTMAX_ELEMENTS_PER_THREAD; base < row_length;- base += stride) {+ for (uint base = tid * SOFTMAX_ELEMENTS_PER_THREAD; base < row_length; base += stride) {float vals[SOFTMAX_ELEMENTS_PER_THREAD];for (uint i = 0; i < SOFTMAX_ELEMENTS_PER_THREAD; i++) {vals[i] = (base + i < row_length) ? float(row[base + i]) : -FLT_MAX;@@ -86,12 +85,10 @@ PUBLIC KERNEL(Softmax)(const float inv_norm = 1.0f / shared_norm[0];// Pass 2: normalize.- for (uint base = tid * SOFTMAX_ELEMENTS_PER_THREAD; base < row_length;- base += stride) {+ for (uint base = tid * SOFTMAX_ELEMENTS_PER_THREAD; base < row_length; base += stride) {for (uint i = 0; i < SOFTMAX_ELEMENTS_PER_THREAD; i++) {if (base + i < row_length) {- row[base + i] =- T(fast::exp(float(row[base + i]) - final_max) * inv_norm);+ row[base + i] = T(fast::exp(float(row[base + i]) - final_max) * inv_norm);}}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/ssm/conv1d.metal b/crates/backend-uzu/src/backends/metal/kernel/ssm/conv1d.metalindex e84becbdd..9d997c001 100644--- a/crates/backend-uzu/src/backends/metal/kernel/ssm/conv1d.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/ssm/conv1d.metal@@ -87,8 +87,7 @@ PUBLIC KERNEL(Conv1dDecode)(const uint dst = token_idx * proj_dim + (channel_idx - inner_dim);b_out[dst] = activated;} else if (channel_idx < inner_dim + 2 * proj_dim) {- const uint dst =- token_idx * proj_dim + (channel_idx - inner_dim - proj_dim);+ const uint dst = token_idx * proj_dim + (channel_idx - inner_dim - proj_dim);c_out[dst] = activated;}@@ -153,8 +152,7 @@ PUBLIC KERNEL(Conv1dScan)(const uint dst = token_idx * proj_dim + (channel_idx - inner_dim);b_out[dst] = activated;} else if (channel_idx < inner_dim + 2 * proj_dim) {- const uint dst =- token_idx * proj_dim + (channel_idx - inner_dim - proj_dim);+ const uint dst = token_idx * proj_dim + (channel_idx - inner_dim - proj_dim);c_out[dst] = activated;}} else if (tap_count > 0) {diff --git a/crates/backend-uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal b/crates/backend-uzu/src/backends/metal/kernel/ssm/ssd_prefill.metalindex a49389ef8..d351752fd 100644--- a/crates/backend-uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal@@ -71,21 +71,18 @@ PUBLIC KERNEL(SSDPrefill64)(const uint dt_idx = token * dt_token_stride + dt_base;const float x_val = float(x[x_idx]);- const float decay_val =- fast::exp(-float(activate_softplus(float(dt_raw[dt_idx]))));+ const float decay_val = fast::exp(-float(activate_softplus(float(dt_raw[dt_idx]))));const float gate = float(activate_silu(z[x_idx]));const float skip = d_scalar * x_val;const float dt_scaled_input = x_val;float contrib = 0.0f;- const float new_state0 =- decay_val * state0 + dt_scaled_input * float(b[cb_idx0]);+ const float new_state0 = decay_val * state0 + dt_scaled_input * float(b[cb_idx0]);state0 = new_state0;contrib += new_state0 * float(c[cb_idx0]);cb_idx0 += cb_token_stride;- const float new_state1 =- decay_val * state1 + dt_scaled_input * float(b[cb_idx1]);+ const float new_state1 = decay_val * state1 + dt_scaled_input * float(b[cb_idx1]);state1 = new_state1;contrib += new_state1 * float(c[cb_idx1]);cb_idx1 += cb_token_stride;@@ -149,8 +146,7 @@ PUBLIC KERNEL(SSDPrefill)(const float d_scalar = float(d[h_idx]);const int max_chunks = SSM_PREFILL_MAX_STATE / thread_context.simdgroup_size;- const int chunk_count = (state_dim + int(thread_context.simdgroup_size) - 1) /- int(thread_context.simdgroup_size);+ const int chunk_count = (state_dim + int(thread_context.simdgroup_size) - 1) / int(thread_context.simdgroup_size);if (state_dim <= 0 || chunk_count > max_chunks) {return;}@@ -173,8 +169,7 @@ PUBLIC KERNEL(SSDPrefill)(const uint dt_idx = token * dt_token_stride + dt_base;const float x_val = float(x[x_idx]);- const float decay_val =- fast::exp(-float(activate_softplus(float(dt_raw[dt_idx]))));+ const float decay_val = fast::exp(-float(activate_softplus(float(dt_raw[dt_idx]))));const float gate = float(activate_silu(z[x_idx]));const float skip = d_scalar * x_val;const float dt_scaled_input = x_val;@@ -182,13 +177,10 @@ PUBLIC KERNEL(SSDPrefill)(float contrib_sum = 0.0f;#pragma unrollfor (int chunk = 0; chunk < chunk_count; ++chunk) {- const int idx =- chunk * int(thread_context.simdgroup_size) + int(lane_idx);+ const int idx = chunk * int(thread_context.simdgroup_size) + int(lane_idx);if (idx < state_dim) {- const uint cb_idx = cb_group_base + uint(idx) * cb_state_stride +- token * cb_token_stride;- const float new_state =- decay_val * lane_states[chunk] + dt_scaled_input * float(b[cb_idx]);+ const uint cb_idx = cb_group_base + uint(idx) * cb_state_stride + token * cb_token_stride;+ const float new_state = decay_val * lane_states[chunk] + dt_scaled_input * float(b[cb_idx]);lane_states[chunk] = new_state;contrib_sum += new_state * float(c[cb_idx]);}@@ -235,12 +227,10 @@ PUBLIC KERNEL(SSDPrefillSequential)() {const uint safe_group = max(group_size, 1u);const uint group_idx = h_idx / safe_group;- device T* state_row = state + size_t(h_idx) * state_strides[0] +- size_t(dh_idx) * state_strides[1];+ device T* state_row = state + size_t(h_idx) * state_strides[0] + size_t(dh_idx) * state_strides[1];for (uint token = 0; token < suffix_len; ++token) {- const uint x_idx =- token * x_strides[0] + h_idx * x_strides[1] + dh_idx * x_strides[2];+ const uint x_idx = token * x_strides[0] + h_idx * x_strides[1] + dh_idx * x_strides[2];const uint dt_idx = token * dt_strides[0] + h_idx * dt_strides[1];const uint cb_base = token * cb_strides[0] + group_idx * cb_strides[1];@@ -258,8 +248,7 @@ PUBLIC KERNEL(SSDPrefillSequential)(for (; s < vec_bound; s += 4) {const uint state_idx = s * state_strides[2];const uint cb_idx = cb_base + s * cb_strides[2];- auto prev_state =- *reinterpret_cast<device vec<T, 4>*>(state_row + state_idx);+ auto prev_state = *reinterpret_cast<device vec<T, 4>*>(state_row + state_idx);auto b_vec = *reinterpret_cast<device const vec<T, 4>*>(b + cb_idx);auto c_vec = *reinterpret_cast<device const vec<T, 4>*>(c + cb_idx);vec<T, 4> new_state = prev_state * this_decay + b_vec * dt_scaled_input;diff --git a/crates/backend-uzu/src/backends/metal/kernel/ssm/ssd_update.metal b/crates/backend-uzu/src/backends/metal/kernel/ssm/ssd_update.metalindex c4564d601..3e6156b6a 100644--- a/crates/backend-uzu/src/backends/metal/kernel/ssm/ssd_update.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/ssm/ssd_update.metal@@ -39,14 +39,10 @@ PUBLIC KERNEL(SSDUpdate)(state = next_state;}- const uint cb_start_idx =- b_idx * cb_strides[0] + (h_idx / group_size) * cb_strides[1];- const uint x_idx =- b_idx * x_strides[0] + h_idx * x_strides[1] + dh_idx * x_strides[2];+ const uint cb_start_idx = b_idx * cb_strides[0] + (h_idx / group_size) * cb_strides[1];+ const uint x_idx = b_idx * x_strides[0] + h_idx * x_strides[1] + dh_idx * x_strides[2];const uint dt_idx = b_idx * dt_strides[0] + h_idx * dt_strides[1];- const uint state_start_idx = b_idx * state_strides[0] +- h_idx * state_strides[1] +- dh_idx * state_strides[2];+ const uint state_start_idx = b_idx * state_strides[0] + h_idx * state_strides[1] + dh_idx * state_strides[2];// load dataT this_x = x[x_idx];@@ -63,8 +59,7 @@ PUBLIC KERNEL(SSDUpdate)(for (uint i = 0; i < state_size; ++i) {uint cb_idx = cb_start_idx + i;uint state_idx = state_start_idx + i;- T this_new_state =- state[state_idx] * this_decay + b[cb_idx] * dt_scaled_input;+ T this_new_state = state[state_idx] * this_decay + b[cb_idx] * dt_scaled_input;next_state[state_idx] = this_new_state;temp = temp + this_new_state * c[cb_idx];}diff --git a/crates/backend-uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal b/crates/backend-uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metalindex 3809cb350..8c2541c5d 100644--- a/crates/backend-uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal@@ -18,7 +18,5 @@ PUBLIC KERNEL(TensorAddBias)(}uint col = position % num_cols;- output[position] = static_cast<T>(- static_cast<float>(input[position]) + static_cast<float>(bias[col])- );+ output[position] = static_cast<T>(static_cast<float>(input[position]) + static_cast<float>(bias[col]));}diff --git a/crates/backend-uzu/src/backends/metal/kernel/token_copy/token_copy.metal b/crates/backend-uzu/src/backends/metal/kernel/token_copy/token_copy.metalindex e5cc942f6..72d0e3c0e 100644--- a/crates/backend-uzu/src/backends/metal/kernel/token_copy/token_copy.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/token_copy/token_copy.metal@@ -3,18 +3,10 @@/// Copies a single sampled token (u32) to the token_ids buffer (u64)./// Used in async pipeline to pass sampled token to next forward pass.-PUBLIC KERNEL(TokenCopySampled)(- device const uint32_t* src,- device uint64_t* dst-) {+PUBLIC KERNEL(TokenCopySampled)(device const uint32_t* src, device uint64_t* dst) {dst[0] = static_cast<uint64_t>(src[0]);}/// Copies a single sampled token (u32) to an indexed results buffer (u32)./// Used in async pipeline to store result for callback access.-PUBLIC KERNEL(TokenCopyToResults)(- device const uint32_t* src,- device uint32_t* dst-) {- dst[0] = src[0];-}+PUBLIC KERNEL(TokenCopyToResults)(device const uint32_t* src, device uint32_t* dst) { dst[0] = src[0]; }