A high-performance inference engine for AI models
Dsl simd (#215)
Aleksandr Golokozebed7db2ade8
diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metalindex 06ed9897c..22853c4b5 100644--- a/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal@@ -32,7 +32,6 @@ METAL_FUNC T row_reduce_sum(T v) {}#define BQ 32-#define SIMD_WIDTH 32#define WM 4#define WN 1@@ -58,17 +57,12 @@ KERNEL(AttentionGemm)(const bool has_sinks SPECIALIZE,threadgroup T q_smem[BQ * (BD + 16 / sizeof(T))],threadgroup T kv_smem[BK * (BD + 16 / sizeof(T))],+ const Simd simd,const uint tgid_x GROUPS(suffix_length.div_ceil(BQ)),const uint tgid_y GROUPS(num_heads),const uint tgid_z GROUPS(1),const uint lid THREADS(128)) {- const uint simd_group_id = lid / SIMD_WIDTH;- const uint simd_lane_id = lid % SIMD_WIDTH;-- // Pacifying compiler- (void)lid;-// -------------------------------------------------------------------------// Pointer setup (all strides are in elements)// tgid_x: query tile index (BQ rows)@@ -140,9 +134,9 @@ KERNEL(AttentionGemm)(const int k_src_ld = int(params.k_strides[2]);const int v_src_ld = int(params.v_strides[2]);- thread QBlockLoader loader_q(q, q_src_ld, Qs, simd_group_id, simd_lane_id);- thread KBlockLoader loader_k(k, k_src_ld, Ks, simd_group_id, simd_lane_id);- thread VBlockLoader loader_v(v, v_src_ld, Vs, simd_group_id, simd_lane_id);+ thread QBlockLoader loader_q(q, q_src_ld, Qs, simd.group_idx, simd.lane_idx);+ thread KBlockLoader loader_k(k, k_src_ld, Ks, simd.group_idx, simd.lane_idx);+ thread VBlockLoader loader_v(v, v_src_ld, Vs, simd.group_idx, simd.lane_idx);TransformScale<T> ts(static_cast<T>(params.scale * M_LOG2E_F));@@ -177,11 +171,11 @@ KERNEL(AttentionGemm)(// -------------------------------------------------------------------------// Lane coordinates and pointer offsets- const short2 simd_coord = MMAFrag_acc_t::get_coord(simd_lane_id);+ const short2 simd_coord = MMAFrag_acc_t::get_coord(simd.lane_idx);const short sm = simd_coord.y;const short sn = simd_coord.x;- const short tm = kFragSize * TQ * short(simd_group_id);+ const short tm = kFragSize * TQ * short(simd.group_idx);// Qs is row-major [BQ, BD]const short Qs_offset = (tm + sm) * LDQ_tgp + sn;diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metalindex b3ef95a54..e151c7d38 100644--- a/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal@@ -2,7 +2,6 @@#include <metal_simdgroup>#include "../definitions.metal"-#define SIMD_WIDTH 32#define SEQUENCE_BLOCK_SIZE 32#define HEAD_BLOCK_SIZE 32@@ -35,14 +34,12 @@ KERNEL(AttentionSinglePass)(const bool has_mask SPECIALIZE,const bool has_sinks SPECIALIZE,const bool do_causal SPECIALIZE,+ const Simd simd,const uint head_idx GROUPS(num_heads),const uint q_seq_idx GROUPS(suffix_length),const uint tid THREADS(1024)) {const uint3 tpg = {num_heads, suffix_length, 1};- const uint simd_gid = tid / SIMD_WIDTH;- const uint simd_lid = tid % SIMD_WIDTH;- constexpr bool bool_mask = false;constexpr bool query_transposed = false;constexpr uint value_dim = head_dim;@@ -62,22 +59,18 @@ KERNEL(AttentionSinglePass)(const uint q_offset = query_transposed ? tpg.x * q_seq_idx + head_idx: head_idx * tpg.y + q_seq_idx;- queries += q_offset * head_dim + simd_lid * qk_elements_per_thread;- keys += kv_head_idx * k_head_stride + simd_gid * k_seq_stride +- simd_lid * qk_elements_per_thread;- values += kv_head_idx * v_head_stride + simd_gid * v_seq_stride +- simd_lid * value_elements_per_thread;+ queries += q_offset * head_dim + simd.lane_idx * qk_elements_per_thread;+ keys += kv_head_idx * k_head_stride + simd.group_idx * k_seq_stride ++ simd.lane_idx * qk_elements_per_thread;+ values += kv_head_idx * v_head_stride + simd.group_idx * v_seq_stride ++ simd.lane_idx * value_elements_per_thread;- if (bool_mask) {- // bmask += head_idx * mask_head_stride + simd_gid * mask_kv_seq_stride +- // q_seq_idx * mask_q_seq_stride;- }if (float_mask) {- fmask += head_idx * mask_head_stride + simd_gid * mask_kv_seq_stride ++ fmask += head_idx * mask_head_stride + simd.group_idx * mask_kv_seq_stride +q_seq_idx * mask_q_seq_stride;}- out += o_offset * value_dim + simd_gid * value_elements_per_thread;+ out += o_offset * value_dim + simd.group_idx * value_elements_per_thread;// Read the query and 0 the output accumulatorfor (int i = 0; i < qk_elements_per_thread; i++) {@@ -89,7 +82,7 @@ KERNEL(AttentionSinglePass)(U max_score = -INFINITY;U sum_exp_score = 0;- if (has_sinks && simd_gid == 0) {+ if (has_sinks && simd.group_idx == 0) {const int num_q_heads = static_cast<int>(tpg.x);int q_head_idx = head_idx % num_q_heads;max_score = static_cast<U>(sinks[q_head_idx]);@@ -97,12 +90,10 @@ KERNEL(AttentionSinglePass)(}// For each key- for (uint i = simd_gid; i < sequence_length; i += SEQUENCE_BLOCK_SIZE) {+ for (uint i = simd.group_idx; i < sequence_length; i += SEQUENCE_BLOCK_SIZE) {bool use_key = true;if (do_causal) {use_key = i <= (sequence_length - tpg.y + q_seq_idx);- } else if (bool_mask) {- // use_key = bmask[0];}if (use_key) {@@ -138,38 +129,37 @@ KERNEL(AttentionSinglePass)(// Move the pointers to the next kvkeys += inner_k_stride;values += inner_v_stride;- if (bool_mask) {- // bmask += SEQUENCE_BLOCK_SIZE * mask_kv_seq_stride;- }if (float_mask) {fmask += SEQUENCE_BLOCK_SIZE * mask_kv_seq_stride;}}// Each thread has a partial part of the output so we need to combine them.- if (simd_lid == 0) {- shared_max_scores[simd_gid] = max_score;- shared_sum_exp_scores[simd_gid] = sum_exp_score;+ if (simd.lane_idx == 0) {+ shared_max_scores[simd.group_idx] = max_score;+ shared_sum_exp_scores[simd.group_idx] = sum_exp_score;}threadgroup_barrier(mem_flags::mem_threadgroup);- max_score = shared_max_scores[simd_lid];+ max_score = shared_max_scores[simd.lane_idx];U new_max = simd_max(max_score);U factor = fast::exp(max_score - new_max);- sum_exp_score = simd_sum(shared_sum_exp_scores[simd_lid] * factor);+ sum_exp_score = simd_sum(shared_sum_exp_scores[simd.lane_idx] * factor);// Now we need to aggregate all the outputsfor (int i = 0; i < value_elements_per_thread; i++) {- shared_outputs[simd_lid * HEAD_BLOCK_SIZE + simd_gid] = o[i];+ shared_outputs[simd.lane_idx * HEAD_BLOCK_SIZE + simd.group_idx] = o[i];threadgroup_barrier(mem_flags::mem_threadgroup);- o[i] = simd_sum(- shared_outputs[simd_gid * HEAD_BLOCK_SIZE + simd_lid] * factor- ) /- sum_exp_score;+ o[i] =+ simd_sum(+ shared_outputs[simd.group_idx * HEAD_BLOCK_SIZE + simd.lane_idx] *+ factor+ ) /+ sum_exp_score;threadgroup_barrier(mem_flags::mem_threadgroup);}// And write the output- if (simd_lid == 0) {+ if (simd.lane_idx == 0) {for (int i = 0; i < value_elements_per_thread; i++) {out[i] = static_cast<T>(o[i]);}diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metalindex 3c0295233..bb322b0fe 100644--- a/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal@@ -5,7 +5,6 @@#define HEAD_BLOCK_SIZE 32#define SEQUENCE_BLOCK_SIZE_1 8#define SEQUENCE_BLOCK_SIZE_2 32-#define SIMD_WIDTH 32#define TOTAL_BLOCKS_COUNT 32template <typename T, uint head_dim>@@ -39,14 +38,13 @@ KERNEL(AttentionTwoPass1)(threadgroup float shared_max_scores[SEQUENCE_BLOCK_SIZE_1],threadgroup float shared_sum_exp_scores[SEQUENCE_BLOCK_SIZE_1],threadgroup float shared_outputs[SEQUENCE_BLOCK_SIZE_1 * HEAD_BLOCK_SIZE],+ const Simd simd,const uint head_idx GROUPS(num_heads),const uint q_seq_idx GROUPS(suffix_length),const uint block_idx GROUPS(TOTAL_BLOCKS_COUNT),const uint tid THREADS(256)) {const uint3 tpg = {num_heads, suffix_length, TOTAL_BLOCKS_COUNT};- const uint simd_gid = tid / SIMD_WIDTH;- const uint simd_lid = tid % SIMD_WIDTH;constexpr bool query_transposed = false;constexpr uint value_dim = head_dim;@@ -68,20 +66,21 @@ KERNEL(AttentionTwoPass1)(: head_idx * tpg.y + q_seq_idx; // Consistent with single-passconst uint kv_head_idx = head_idx / gqa_factor;- queries += q_offset * head_dim + simd_lid * qk_elements_per_thread;+ queries += q_offset * head_dim + simd.lane_idx * qk_elements_per_thread;keys += kv_head_idx * k_head_stride +- (block_idx * SEQUENCE_BLOCK_SIZE_1 + simd_gid) * k_seq_stride +- simd_lid * qk_elements_per_thread;- values += kv_head_idx * v_head_stride +- (block_idx * SEQUENCE_BLOCK_SIZE_1 + simd_gid) * v_seq_stride +- simd_lid * value_elements_per_thread;+ (block_idx * SEQUENCE_BLOCK_SIZE_1 + simd.group_idx) * k_seq_stride ++ simd.lane_idx * qk_elements_per_thread;+ values +=+ kv_head_idx * v_head_stride ++ (block_idx * SEQUENCE_BLOCK_SIZE_1 + simd.group_idx) * v_seq_stride ++ simd.lane_idx * value_elements_per_thread;out += o_offset * TOTAL_BLOCKS_COUNT * value_dim + block_idx * value_dim +- simd_lid * value_elements_per_thread;+ simd.lane_idx * value_elements_per_thread;if (float_mask) {- fmask +=- head_idx * mask_head_stride +- (block_idx * SEQUENCE_BLOCK_SIZE_1 + simd_gid) * mask_kv_seq_stride +- q_seq_idx * mask_q_seq_stride;+ fmask += head_idx * mask_head_stride ++ (block_idx * SEQUENCE_BLOCK_SIZE_1 + simd.group_idx) *+ mask_kv_seq_stride ++ q_seq_idx * mask_q_seq_stride;}sums += o_offset * TOTAL_BLOCKS_COUNT + block_idx;maxs += o_offset * TOTAL_BLOCKS_COUNT + block_idx;@@ -96,7 +95,7 @@ KERNEL(AttentionTwoPass1)(U max_score = -1e9;U sum_exp_score = 0;- if (has_sinks && block_idx == 0 && simd_gid == 0) {+ if (has_sinks && block_idx == 0 && simd.group_idx == 0) {const uint num_q_heads = tpg.x;int q_head_idx = head_idx % num_q_heads;max_score = static_cast<U>(sinks[q_head_idx]);@@ -104,7 +103,7 @@ KERNEL(AttentionTwoPass1)(}// For each key- for (uint i = block_idx * SEQUENCE_BLOCK_SIZE_1 + simd_gid;+ for (uint i = block_idx * SEQUENCE_BLOCK_SIZE_1 + simd.group_idx;i < sequence_length;i += TOTAL_BLOCKS_COUNT * SEQUENCE_BLOCK_SIZE_1) {bool use_key = true;@@ -151,36 +150,38 @@ KERNEL(AttentionTwoPass1)(}// Each thread has a partial part of the output so we need to combine them.- if (simd_lid == 0) {- shared_max_scores[simd_gid] = max_score;- shared_sum_exp_scores[simd_gid] = sum_exp_score;+ if (simd.lane_idx == 0) {+ shared_max_scores[simd.group_idx] = max_score;+ shared_sum_exp_scores[simd.group_idx] = sum_exp_score;}threadgroup_barrier(mem_flags::mem_threadgroup);- max_score =- (simd_lid < SEQUENCE_BLOCK_SIZE_1) ? shared_max_scores[simd_lid] : -1e9;+ max_score = (simd.lane_idx < SEQUENCE_BLOCK_SIZE_1)+ ? shared_max_scores[simd.lane_idx]+ : -1e9;U new_max = simd_max(max_score);U factor = fast::exp(max_score - new_max);- sum_exp_score =- (simd_lid < SEQUENCE_BLOCK_SIZE_1) ? shared_sum_exp_scores[simd_lid] : 0;+ sum_exp_score = (simd.lane_idx < SEQUENCE_BLOCK_SIZE_1)+ ? shared_sum_exp_scores[simd.lane_idx]+ : 0;sum_exp_score = simd_sum(sum_exp_score * factor);// Write the sum and new max- if (simd_gid == 0) {+ if (simd.group_idx == 0) {sums[0] = sum_exp_score;maxs[0] = new_max;}// Now we need to aggregate all the outputsfor (uint i = 0; i < value_elements_per_thread; i++) {- shared_outputs[simd_lid * SEQUENCE_BLOCK_SIZE_1 + simd_gid] =- o[i] * fast::exp(shared_max_scores[simd_gid] - new_max);+ shared_outputs[simd.lane_idx * SEQUENCE_BLOCK_SIZE_1 + simd.group_idx] =+ o[i] * fast::exp(shared_max_scores[simd.group_idx] - new_max);threadgroup_barrier(mem_flags::mem_threadgroup);// And write the output- if (simd_gid == 0) {- U output = shared_outputs[simd_lid * SEQUENCE_BLOCK_SIZE_1];+ if (simd.group_idx == 0) {+ U output = shared_outputs[simd.lane_idx * SEQUENCE_BLOCK_SIZE_1];for (uint j = 1; j < SEQUENCE_BLOCK_SIZE_1; j++) {- output += shared_outputs[simd_lid * SEQUENCE_BLOCK_SIZE_1 + j];+ output += shared_outputs[simd.lane_idx * SEQUENCE_BLOCK_SIZE_1 + j];}out[i] = static_cast<T>(output);}@@ -199,14 +200,12 @@ KERNEL(AttentionTwoPass2)(const constant uint& num_heads,const constant uint& suffix_length,threadgroup float shared_outputs[SEQUENCE_BLOCK_SIZE_2 * HEAD_BLOCK_SIZE],+ const Simd simd,const uint head_idx GROUPS(num_heads),const uint q_seq_idx GROUPS(suffix_length),const uint tid THREADS(1024)) {const uint3 tpg = {num_heads, suffix_length, 1};- const uint simd_gid = tid / SIMD_WIDTH;- const uint simd_lid = tid % SIMD_WIDTH;-constexpr uint elements_per_thread = head_dim / HEAD_BLOCK_SIZE;typedef float U;@@ -214,18 +213,18 @@ KERNEL(AttentionTwoPass2)(const uint o_offset = q_seq_idx * tpg.x + head_idx; // Our custom layout- partials += o_offset * TOTAL_BLOCKS_COUNT * head_dim + simd_gid * head_dim +- simd_lid * elements_per_thread;+ partials += o_offset * TOTAL_BLOCKS_COUNT * head_dim ++ simd.group_idx * head_dim + simd.lane_idx * elements_per_thread;sums += o_offset * TOTAL_BLOCKS_COUNT;maxs += o_offset * TOTAL_BLOCKS_COUNT;out += o_offset * head_dim +- simd_gid * elements_per_thread; // Our custom output layout+ simd.group_idx * elements_per_thread; // Our custom output layout// First everybody reads the max and sum_exp- U max_score = maxs[simd_lid];+ U max_score = maxs[simd.lane_idx];U new_max = simd_max(max_score);U factor = fast::exp(max_score - new_max);- U sum_exp_score = simd_sum(sums[simd_lid] * factor);+ U sum_exp_score = simd_sum(sums[simd.lane_idx] * factor);// Now read the block into registers and then use shared memory to transpose// it@@ -233,17 +232,19 @@ KERNEL(AttentionTwoPass2)(o[i] = partials[i];}for (uint i = 0; i < elements_per_thread; i++) {- shared_outputs[simd_lid * HEAD_BLOCK_SIZE + simd_gid] = o[i];+ shared_outputs[simd.lane_idx * HEAD_BLOCK_SIZE + simd.group_idx] = o[i];threadgroup_barrier(mem_flags::mem_threadgroup);- o[i] = simd_sum(- shared_outputs[simd_gid * HEAD_BLOCK_SIZE + simd_lid] * factor- ) /- sum_exp_score;+ o[i] =+ simd_sum(+ shared_outputs[simd.group_idx * HEAD_BLOCK_SIZE + simd.lane_idx] *+ factor+ ) /+ sum_exp_score;threadgroup_barrier(mem_flags::mem_threadgroup);}// And write the output- if (simd_lid == 0) {+ if (simd.lane_idx == 0) {for (uint i = 0; i < elements_per_thread; i++) {out[i] = static_cast<T>(o[i]);}diff --git a/crates/uzu/src/backends/metal/kernel/definitions.metal b/crates/uzu/src/backends/metal/kernel/definitions.metalindex 8f2e4e150..5f48dde72 100644--- a/crates/uzu/src/backends/metal/kernel/definitions.metal+++ b/crates/uzu/src/backends/metal/kernel/definitions.metal@@ -126,6 +126,17 @@ struct TensorView4D {}};+struct Simd {+ uint lane_idx;+ uint group_idx;+ uint group_size;+ uint groups_per_threadgroup;++ uint groups_count(const uint threads_count) const {+ return (threads_count + this->group_size - 1) / this->group_size;+ }+};+///////////////////////////////////////////////////////////////////////////////// MARK: - Thread Functions///////////////////////////////////////////////////////////////////////////////@@ -351,17 +362,15 @@ template <ushort BLOCK_SIZE, typename T>static T threadgroup_cooperative_reduce_sum(T value,threadgroup T* shared,- const ushort lid+ const ushort lid,+ const thread Simd& simd) {- const ushort simd_group_id = lid / 32;- const ushort simd_lane_id = lid % 32;-// Reduce within simdgroupT local_sum = simd_sum(value);// First thread in each simdgroup writes to shared memory- if (simd_lane_id == 0) {- shared[simd_group_id] = local_sum;+ if (simd.lane_idx == 0) {+ shared[simd.group_idx] = local_sum;}// Synchronize across the threadgroup@@ -369,7 +378,7 @@ static T threadgroup_cooperative_reduce_sum(// Reduce across simdgroupsT total_sum = T(0);- const ushort num_simd_groups = (BLOCK_SIZE + 31) / 32;+ const ushort num_simd_groups = simd.groups_count(BLOCK_SIZE);if (lid < num_simd_groups) {total_sum = shared[lid];}@@ -392,24 +401,23 @@ template <ushort BLOCK_SIZE, typename T>static T threadgroup_cooperative_reduce_max(T value,threadgroup T* shared,- const ushort lid+ const ushort lid,+ const thread Simd& simd) {- const ushort simd_group_id = lid / 32;- const ushort simd_lane_id = lid % 32;-// Reduce within simdgroupT local_max = simd_max(value);// First thread in each simdgroup writes to shared memory- if (simd_lane_id == 0) {- shared[simd_group_id] = local_max;+ if (simd.lane_idx == 0) {+ shared[simd.group_idx] = local_max;}// Synchronize across the threadgroupthreadgroup_barrier(mem_flags::mem_threadgroup);// Reduce across simdgroups- T total_max = (lid < ((BLOCK_SIZE + 31) / 32)) ? shared[lid] : T(-INFINITY);+ T total_max =+ lid < simd.groups_count(BLOCK_SIZE) ? shared[lid] : T(-INFINITY);total_max = simd_max(total_max);// Broadcast the result to all threads@@ -429,24 +437,22 @@ template <ushort BLOCK_SIZE, typename T>static T threadgroup_cooperative_reduce_min(T value,threadgroup T* shared,- const ushort lid+ const ushort lid,+ const thread Simd& simd) {- const ushort simd_group_id = lid / 32;- const ushort simd_lane_id = lid % 32;-// Reduce within simdgroupT local_min = simd_min(value);// First thread in each simdgroup writes to shared memory- if (simd_lane_id == 0) {- shared[simd_group_id] = local_min;+ if (simd.lane_idx == 0) {+ shared[simd.group_idx] = local_min;}// Synchronize across the threadgroupthreadgroup_barrier(mem_flags::mem_threadgroup);// Reduce across simdgroups- T total_min = (lid < ((BLOCK_SIZE + 31) / 32)) ? shared[lid] : T(INFINITY);+ T total_min = lid < simd.groups_count(BLOCK_SIZE) ? shared[lid] : T(INFINITY);total_min = simd_min(total_min);// Broadcast the result to all threads@@ -489,13 +495,6 @@ static T threadgroup_cooperative_reduce_min(#define GROUPS(EXPR) DSL_META("dsl.groups", DSL_XSTR(EXPR))#define THREADS(EXPR) DSL_META("dsl.threads", DSL_XSTR(EXPR))-struct Simd {- uint lane_idx;- uint group_idx;- uint group_size;- uint groups_per_threadgroup;-};-// MARK: - Generate Template Kernels#define generateKernel( \diff --git a/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal b/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metalindex a033243f4..1593a3931 100644--- a/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal+++ b/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal@@ -17,7 +17,8 @@ void layer_norm_core(threadgroup AccumT* shared_mean,threadgroup AccumT* shared_variance,uint thread_in_row,- bool full_layer+ bool full_layer,+ const thread Simd& simd) {AccumT partial_sum = static_cast<AccumT>(0.0f);@@ -36,7 +37,8 @@ void layer_norm_core(AccumT total_sum = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(partial_sum,shared_mean,- thread_in_row+ thread_in_row,+ simd);AccumT mean = total_sum / static_cast<AccumT>(element_count);@@ -57,7 +59,8 @@ void layer_norm_core(AccumT total_var = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(partial_var_sum,shared_variance,- thread_in_row+ thread_in_row,+ simd);AccumT variance = total_var / static_cast<AccumT>(element_count);AccumT inv_std = rsqrt(variance + static_cast<AccumT>(epsilon));@@ -126,6 +129,7 @@ KERNEL(LayerNorm) (constant uint& full_layer,threadgroup ACC shared_mean[SIMD_SIZE],threadgroup ACC shared_variance[SIMD_SIZE],+ const Simd simd,uint batch_idx GROUPS(batch_size),uint thread_in_row THREADS(BLOCK_SIZE)) {@@ -140,6 +144,7 @@ KERNEL(LayerNorm) (shared_mean,shared_variance,thread_in_row,- full_layer+ full_layer,+ simd);}diff --git a/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal b/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metalindex 743dddc44..3d44d6b05 100644--- a/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal@@ -23,6 +23,7 @@ KERNEL(MoeCountsOffsetsFused)(threadgroup uintcounts_shared[BLOCK_SIZE], // Cache counts in threadgroup memorythreadgroup uint& carry,+ const Simd simd,const uint lid THREADS(128)) {if (e_input == 0) {@@ -100,7 +101,8 @@ KERNEL(MoeCountsOffsetsFused)(uint block_sum = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(v,reduce_shared,- (ushort)lid+ (ushort)lid,+ simd);if (lid == 0) {carry += block_sum;diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_single.metalindex 6c1abb418..bab241e36 100644--- a/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal@@ -3,8 +3,6 @@#include "../definitions.metal"#include "moe_commons.h"-#define SIMD_SIZE 32-// ============================================================================// Pass A: x @ W13[expert] → hidden[k]// Each threadgroup: 4 simdgroups, outputs 4 elements (1 per simdgroup)@@ -28,19 +26,17 @@ KERNEL(MoeExpertsDecodeSinglePassA)(constant float& up_clip_min,constant float& up_clip_max,const uint gating_sel SPECIALIZE, // 0=GELU, 1=SiLU, 2=SwiGLU, 3=GEGLU+ const Simd simd,const uint h_block_idx GROUPS(d_ff.div_ceil(4)),const uint k_slot GROUPS(k),const uint tid THREADS(128)) {- const uint simd_gid = tid / SIMD_SIZE;- const uint simd_lid = tid % SIMD_SIZE;-const int expert_id = topk_ids[k_slot];if (expert_id < 0)return;const uint expert_u = uint(expert_id);- const uint h_idx = h_block_idx * 4 + simd_gid;+ const uint h_idx = h_block_idx * 4 + simd.group_idx;if (h_idx >= d_ff)return;@@ -58,7 +54,7 @@ KERNEL(MoeExpertsDecodeSinglePassA)(const uint vec_iters = d_model / 128;for (uint i = 0; i < vec_iters; ++i) {- uint base_idx = i * 128 + simd_lid * 4;+ uint base_idx = i * 128 + simd.lane_idx * 4;device const T* x_vec2 = reinterpret_cast<device const T*>(x + base_idx);device const T* w_up_vec2 =@@ -78,7 +74,7 @@ KERNEL(MoeExpertsDecodeSinglePassA)(}}- uint leftover_start = vec_iters * 128 + simd_lid;+ uint leftover_start = vec_iters * 128 + simd.lane_idx;for (uint idx = leftover_start; idx < d_model; idx += 32) {float xv = float(x[idx]);acc_up += xv * float(w_up_row[idx]);@@ -92,7 +88,7 @@ KERNEL(MoeExpertsDecodeSinglePassA)(acc_gate = simd_sum(acc_gate);}- if (simd_lid == 0) {+ if (simd.lane_idx == 0) {float up_val = clamp(acc_up + float(biases[bias_base + h_idx]),up_clip_min,@@ -136,13 +132,11 @@ KERNEL(MoeExpertsDecodeSinglePassB)(constant uint& d_model,constant uint& d_ff,constant uint& k_input,+ const Simd simd,const uint d_block GROUPS(d_model.div_ceil(8)),const uint tid THREADS(256)) {- const uint simd_gid = tid / SIMD_SIZE;- const uint simd_lid = tid % SIMD_SIZE;-- const uint my_col = d_block * 8 + simd_gid;+ const uint my_col = d_block * 8 + simd.group_idx;if (my_col >= d_model)return;@@ -164,7 +158,7 @@ KERNEL(MoeExpertsDecodeSinglePassB)(// Vectorized reductionfor (uint i = 0; i < vec_iters; ++i) {- uint base_idx = i * 128 + simd_lid * 4;+ uint base_idx = i * 128 + simd.lane_idx * 4;float4 h_vec =*reinterpret_cast<device const float4*>(hidden_ptr + base_idx);@@ -177,19 +171,19 @@ KERNEL(MoeExpertsDecodeSinglePassB)(}// Remainder- for (uint idx = vec_iters * 128 + simd_lid; idx < d_ff; idx += 32) {+ for (uint idx = vec_iters * 128 + simd.lane_idx; idx < d_ff; idx += 32) {acc += hidden_ptr[idx] * float(w2_ptr[idx]);}// Simdgroup reduction and accumulatefloat result = simd_sum(acc);- if (simd_lid == 0) {+ if (simd.lane_idx == 0) {result += float(biases[(ulong)expert_u * (ulong)d_model + (ulong)my_col]);final_acc += prob * result;}}- if (simd_lid == 0) {+ if (simd.lane_idx == 0) {y[my_col] = T(final_acc);}}\ No newline at end of filediff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metalindex f220d43aa..9ac862664 100644--- a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal@@ -8,8 +8,6 @@ using namespace metal;// Structure: 4 simdgroups (128 threads), each outputs 1 hidden element// Each simdgroup: 32 threads reduce d_model using float4 vectorized loads// Grid: (h_blocks, rows, 1) where h_blocks = ceil(d_ff/4)-#define SIMD_SIZE 32-template <typename T>VARIANTS(T, float, half, bfloat)KERNEL(MoeExpertsDecodePassA)(@@ -28,12 +26,10 @@ KERNEL(MoeExpertsDecodePassA)(constant float& silu_alpha,device const uint* tile_map,const uint gating_sel SPECIALIZE,+ const Simd simd,const uint tile_idx GROUPS(INDIRECT),const uint tid THREADS(128)) {- const uint simd_gid = tid / SIMD_SIZE;- const uint simd_lid = tid % SIMD_SIZE;-// Read tile descriptorconst uint h_block_idx = tile_map[tile_idx * 3 + 0];const uint expert_idx = tile_map[tile_idx * 3 + 1];@@ -47,7 +43,7 @@ KERNEL(MoeExpertsDecodePassA)(return;// Each simdgroup outputs one hidden element- const uint h_idx = h_block_idx * 4 + simd_gid;+ const uint h_idx = h_block_idx * 4 + simd.group_idx;if (h_idx >= d_ff)return;@@ -69,7 +65,7 @@ KERNEL(MoeExpertsDecodePassA)(const uint vec_iters = d_model / 128;for (uint i = 0; i < vec_iters; ++i) {- uint base_idx = i * 128 + simd_lid * 4;+ uint base_idx = i * 128 + simd.lane_idx * 4;device const T* x_vec = reinterpret_cast<device const T*>(x_ptr + base_idx);device const T* w_up_vec =@@ -90,7 +86,7 @@ KERNEL(MoeExpertsDecodePassA)(}// Handle leftover elements- uint leftover_start = vec_iters * 128 + simd_lid;+ uint leftover_start = vec_iters * 128 + simd.lane_idx;for (uint idx = leftover_start; idx < d_model; idx += 32) {float xv = float(x_ptr[idx]);acc_up += xv * float(w_up_row[idx]);@@ -106,7 +102,7 @@ KERNEL(MoeExpertsDecodePassA)(}// Lane 0 applies activation and writes result- if (simd_lid == 0) {+ if (simd.lane_idx == 0) {float up_val = acc_up + float(up_biases[bias_base + h_idx]);up_val = clamp(up_val, up_clip_min, up_clip_max);@@ -145,16 +141,15 @@ KERNEL(MoeExpertsDecodeDownFused2D)(constant uint& d_model,constant uint& d_ff,constant uint& e,+ const Simd simd,const uint tgpig_x GROUPS(d_model.div_ceil(SIMDGROUPS_PER_TG)),const uint tgpig_y GROUPS(total_rows),const uint tid THREADS(256)) {- const uint simd_gid = tid / SIMD_SIZE;- const uint simd_lid = tid % SIMD_SIZE;const uint row_idx = tgpig_y;// Each simdgroup computes one output column- const uint my_col = tgpig_x * SIMDGROUPS_PER_TG + simd_gid;+ const uint my_col = tgpig_x * SIMDGROUPS_PER_TG + simd.group_idx;if (my_col >= d_model)return;@@ -175,7 +170,7 @@ KERNEL(MoeExpertsDecodeDownFused2D)(const uint k_vec_iters = k_iters / 8;for (uint iter = 0; iter < k_vec_iters; ++iter) {- const uint k_base = iter * (8 * THREADS_PER_SIMD) + simd_lid;+ const uint k_base = iter * (8 * THREADS_PER_SIMD) + simd.lane_idx;// hidden: stride-32 per lane, already f32 from Pass Aconst AccumT h0 = hidden[hidden_base + k_base + 0 * THREADS_PER_SIMD];@@ -221,7 +216,7 @@ KERNEL(MoeExpertsDecodeDownFused2D)(// Handle remaining full iterationsfor (uint iter = k_vec_iters * 8; iter < k_iters; ++iter) {- const uint k = iter * THREADS_PER_SIMD + simd_lid;+ const uint k = iter * THREADS_PER_SIMD + simd.lane_idx;acc =fma(AccumT(hidden[hidden_base + k]),AccumT(w2_all[w2_col_base + k]),@@ -230,8 +225,8 @@ KERNEL(MoeExpertsDecodeDownFused2D)(// Handle leftover elements (d_ff % 32)const uint leftover_start = k_iters * THREADS_PER_SIMD;- if (leftover_start + simd_lid < d_ff) {- const uint k = leftover_start + simd_lid;+ if (leftover_start + simd.lane_idx < d_ff) {+ const uint k = leftover_start + simd.lane_idx;acc =fma(AccumT(hidden[hidden_base + k]),AccumT(w2_all[w2_col_base + k]),@@ -242,7 +237,7 @@ KERNEL(MoeExpertsDecodeDownFused2D)(AccumT result = simd_sum(acc);// Lane 0 writes result- if (simd_lid == 0) {+ if (simd.lane_idx == 0) {const ulong bias_idx = (ulong)expert_idx * (ulong)d_model + (ulong)my_col;result += AccumT(down_biases[bias_idx]);diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metalindex 6ff575bdc..6b5d54b9b 100644--- a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal@@ -68,7 +68,6 @@ KERNEL(MoeExpertsPrefillPassA)(const uint3 threads_per_tg = {128, 1, 1};const uint3 local_tid = {local_tid_x, 0, 0};- const uint sg_id = simd.group_idx;constexpr uint ROW_TILE = PASSA_BM;const uint tile_seg_start = tile_map[base + 1u];@@ -106,15 +105,15 @@ KERNEL(MoeExpertsPrefillPassA)(// simdgroup tile mappingconst uint sg_col_count = Bn / SgBn;- const uint row_sg = sg_id / sg_col_count;- const uint col_sg = sg_id % sg_col_count;+ const uint row_sg = simd.group_idx / sg_col_count;+ const uint col_sg = simd.group_idx % sg_col_count;const uint row_sg_off = row_sg * SgBm;const uint col_sg_off = col_sg * SgBn;// Guard against misconfigured TG sizesconstexpr uint SG_TILE = 8; // simdgroup fragment dimension (8x8)constexpr uint SG_EXPECTED = (Bm / SG_TILE) * (Bn / SG_TILE);- if (sg_id >= SG_EXPECTED)+ if (simd.group_idx >= SG_EXPECTED)return;// partial accumulators (8x8 tiles per simdgroup)@@ -295,7 +294,7 @@ KERNEL(MoeExpertsPrefillPassA)(threadgroup_barrier(mem_flags::mem_threadgroup);// ---- Epilogue: activation/gating → hidden_out (f32) ----- const uint sg_lane_start = sg_id * 32;+ const uint sg_lane_start = simd.group_idx * 32;const uint sg_lane_end = sg_lane_start + 32;if (lin < sg_lane_start || lin >= sg_lane_end)return;@@ -414,9 +413,6 @@ KERNEL(MoeExpertsPrefillPassB)(if (expert_idx >= e)return;- const uint simd_lid = simd.lane_idx;- const uint sg_id = simd.group_idx;-constexpr uint ROW_TILE = PASSB_BM;const uint row_off_elems = tile_map[base + 2u];const uint tile_m = row_off_elems / ROW_TILE;@@ -449,14 +445,14 @@ KERNEL(MoeExpertsPrefillPassB)(const ulong bias_base = (ulong)expert_idx * (ulong)d_model;// 2×2 simdgroup layout for 16×64 output (each sg handles 8×32)- const uint row_sg = sg_id / 2; // 0-1 → rows- const uint col_sg = sg_id % 2; // 0-1 → cols+ const uint row_sg = simd.group_idx / 2; // 0-1 → rows+ const uint col_sg = simd.group_idx % 2; // 0-1 → colsconst uint row_sg_off = row_sg * SgBm;const uint col_sg_off = col_sg * SgBn;// Guard against excessive simdgroupsconstexpr uint SG_EXPECTED = (PASSB_BM / SgBm) * (PASSB_BN / SgBn);- if (sg_id >= SG_EXPECTED)+ if (simd.group_idx >= SG_EXPECTED)return;// 4 accumulators per simdgroup: 1×4 layout of 8×8 tiles@@ -564,9 +560,10 @@ KERNEL(MoeExpertsPrefillPassB)(threadgroup_barrier(mem_flags::mem_threadgroup);// ---- Writeout with fragment extraction ----- const uint lane_qid = simd_lid >> 2;- const uint lane_row = (lane_qid & 4u) + ((simd_lid >> 1) & 3u);- const uint lane_col_base = ((lane_qid & 2u) << 1) + ((simd_lid & 1u) << 1);+ const uint lane_qid = simd.lane_idx >> 2;+ const uint lane_row = (lane_qid & 4u) + ((simd.lane_idx >> 1) & 3u);+ const uint lane_col_base =+ ((lane_qid & 2u) << 1) + ((simd.lane_idx & 1u) << 1);const uint local_row = row_sg_off + lane_row;if (local_row >= m_rows)diff --git a/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal b/crates/uzu/src/backends/metal/kernel/moe/router_topk.metalindex 4e01cedbd..a4fcca27e 100644--- a/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal@@ -37,10 +37,6 @@ KERNEL(MoeRouterTopK)(return;}- const uint simd_lane = simd.lane_idx;- const uint simdgroup_idx = simd.group_idx;- const uint simdgroups_per_tg = simd.groups_per_threadgroup;-const uint vecs = d_model / 4u;const device ScalarT* x_vec = input + (ulong)token_idx * (ulong)vecs * 4;@@ -54,11 +50,11 @@ KERNEL(MoeRouterTopK)(}threadgroup_barrier(mem_flags::mem_threadgroup);- for (uint row = simdgroup_idx; row < e; row += simdgroups_per_tg) {+ for (uint row = simd.group_idx; row < e; row += simd.groups_per_threadgroup) {const device ScalarT* w_vec = weight + (ulong)row * (ulong)vecs * 4;float4 accum4 = float4(0.0f);- for (uint c = simd_lane; c < vecs; c += 32u) {+ for (uint c = simd.lane_idx; c < vecs; c += 32u) {const float4 wv = float4(w_vec[c * 4 + 0],w_vec[c * 4 + 1],@@ -100,14 +96,16 @@ KERNEL(MoeRouterTopK)(float max_val = threadgroup_cooperative_reduce_max<THREADS_PER_TG>(local_best,reduce_tmp,- lid+ lid,+ simd);uint candidate_id = (local_best == max_val) ? local_idx : 0xFFFFFFFFu;uint best_idx = threadgroup_cooperative_reduce_min<THREADS_PER_TG>(candidate_id,reduce_tmp_u,- lid+ lid,+ simd);if (lid == 0) {diff --git a/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal b/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metalindex 3248e0a30..ea6d92a79 100644--- a/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal@@ -28,15 +28,14 @@ inline void moe_scatter_buckets_impl(uint lid,uint3 tgpig,threadgroup _atomic<uint>* sg_counts,- threadgroup uint* sg_base+ threadgroup uint* sg_base,+ const thread Simd& simd) {if (E == 0 || T == 0 || K == 0)return;const uint block_id = tgpig.x;const uint t_start = block_id * BLOCK_SIZE;const uint t_end = min(t_start + BLOCK_SIZE, T);- const uint sg_id = lid / SIMD_WIDTH;- const uint lane = lid % SIMD_WIDTH;// Per-tile processingfor (uint e0 = 0; e0 < E; e0 += TILE_E) {@@ -65,7 +64,7 @@ inline void moe_scatter_buckets_impl(if (ue >= e0 && ue < e0 + tile_e) {const uint te = ue - e0;atomic_fetch_add_explicit(- &sg_counts[sg_id * TILE_E + te],+ &sg_counts[simd.group_idx * TILE_E + te],1u,memory_order_relaxed);@@ -103,8 +102,8 @@ inline void moe_scatter_buckets_impl(threadgroup_barrier(mem_flags::mem_threadgroup);// Phase 3: deterministic write with 32-lane sequencing, no device atomics- for (uint step = 0; step < SIMD_WIDTH; ++step) {- if (lane == step) {+ for (uint step = 0; step < simd.group_size; ++step) {+ if (simd.lane_idx == step) {const uint t = t_start + lid;if (t < t_end) {const uint base = t * K;@@ -117,7 +116,7 @@ inline void moe_scatter_buckets_impl(const uint tile_id = e0 / TILE_E;const uint base_idx_block =(block_id * num_tiles + tile_id) * TILE_E + te;- const uint idx_sg_te = sg_id * TILE_E + te;+ const uint idx_sg_te = simd.group_idx * TILE_E + te;const uint local = atomic_fetch_add_explicit(&sg_counts[idx_sg_te],1u,@@ -202,6 +201,7 @@ KERNEL(MoeScatterBuckets)(constant uint& num_tiles,threadgroup _atomic<uint> sg_counts[NUM_SG * TILE_E],threadgroup uint sg_base[NUM_SG * TILE_E],+ const Simd simd,const uint tgpig_x GROUPS(num_blocks),const uint lid THREADS(256)) {@@ -226,7 +226,8 @@ KERNEL(MoeScatterBuckets)(lid,tgpig,sg_counts,- sg_base+ sg_base,+ simd);}@@ -249,6 +250,7 @@ KERNEL(MoeScatterBucketsMap)(device int* tok2row,threadgroup _atomic<uint> sg_counts[NUM_SG * TILE_E],threadgroup uint sg_base[NUM_SG * TILE_E],+ const Simd simd,const uint tgpig_x GROUPS(num_blocks),const uint lid THREADS(256)) {@@ -273,6 +275,7 @@ KERNEL(MoeScatterBucketsMap)(lid,tgpig,sg_counts,- sg_base+ sg_base,+ simd);}\ No newline at end of filediff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metalindex a744e065f..1da5b65bb 100644--- a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal+++ b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal@@ -22,6 +22,7 @@ KERNEL(RMSNorm)(constant float& scale_offset,constant bool& full_layer,threadgroup AccumT shared_sum[SIMD_SIZE],+ const Simd simd,const uint batch_idx GROUPS(batch_size),const uint thread_in_row THREADS(1024)) {@@ -51,7 +52,8 @@ KERNEL(RMSNorm)(AccumT total_sum = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(partial_sum,shared_sum,- thread_in_row+ thread_in_row,+ simd);// Compute RMS norm factordiff --git a/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal b/crates/uzu/src/backends/metal/kernel/sampling/argmax.metalindex 2de31b445..3cff521c7 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal+++ b/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal@@ -26,11 +26,9 @@ template <ushort BLOCK_SIZE_PARAM>static ArgmaxPair threadgroup_cooperative_argmax(ArgmaxPair value,threadgroup ArgmaxPair* shared,- const ushort lid+ const ushort lid,+ const thread Simd& simd) {- const ushort simd_group_id = lid / 32;- const ushort simd_lane_id = lid % 32;-// Reduce within simdgroup using manual shuffle operationsArgmaxPair local_result = value;for (ushort offset = 16; offset > 0; offset /= 2) {@@ -42,8 +40,8 @@ static ArgmaxPair threadgroup_cooperative_argmax(}// First thread in each simdgroup writes to shared memory- if (simd_lane_id == 0) {- shared[simd_group_id] = local_result;+ if (simd.lane_idx == 0) {+ shared[simd.group_idx] = local_result;}// Synchronize across the threadgroup@@ -205,6 +203,7 @@ KERNEL(ArgmaxMain)(constant uint& batch_size,constant uint& vocab_size,threadgroup ArgmaxPair shared[BLOCK_SIZE],+ const Simd simd,uint batch_idx GROUPS(batch_size),uint vocab_group_idx GROUPS(vocab_size.div_ceil(BLOCK_SIZE * GRAIN_SIZE)),ushort local_id THREADS(BLOCK_SIZE)@@ -230,7 +229,8 @@ KERNEL(ArgmaxMain)(ArgmaxPair group_result = threadgroup_cooperative_argmax<BLOCK_SIZE>(thread_result,shared,- local_id+ local_id,+ simd);if (local_id == 0) {@@ -247,6 +247,7 @@ KERNEL(ArgmaxFinal)(constant uint& batch_size,constant uint& vocab_size,threadgroup ArgmaxPair shared[BLOCK_SIZE],+ const Simd simd,uint batch_idx GROUPS(batch_size),ushort local_id THREADS(BLOCK_SIZE)) {@@ -269,7 +270,8 @@ KERNEL(ArgmaxFinal)(ArgmaxPair result = threadgroup_cooperative_argmax<BLOCK_SIZE>(thread_result,shared,- local_id+ local_id,+ simd);if (local_id == 0) {diff --git a/crates/uzu/src/backends/metal/kernel/sampling/min_p.metal b/crates/uzu/src/backends/metal/kernel/sampling/min_p.metalindex aca8878fb..889dfae91 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/min_p.metal+++ b/crates/uzu/src/backends/metal/kernel/sampling/min_p.metal@@ -13,6 +13,7 @@ KERNEL(MinP) (constant uint& batch_size,constant uint& vocab_size,constant float& min_p,+ const Simd simd,uint batch_idx GROUPS(batch_size),uint thread_idx THREADS(BLOCK_SIZE)) {@@ -28,7 +29,8 @@ KERNEL(MinP) (float max_logit = threadgroup_cooperative_reduce_max<BLOCK_SIZE>(local_max,shared_reduce_buffer,- thread_idx+ thread_idx,+ simd);// Then the threshold is just max_logit + log(min_p), mask everything strictlydiff --git a/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal b/crates/uzu/src/backends/metal/kernel/sampling/top_k.metalindex bfefe0380..d040f3f0d 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal+++ b/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal@@ -7,13 +7,14 @@template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(TopK) (+KERNEL(TopK)(device const T* logits,device T* processed_logits,threadgroup float shared_reduce_buffer[BLOCK_SIZE_IN_SIMDS],constant uint& batch_size,constant uint& vocab_size,constant uint& top_k,+ const Simd simd,uint batch_idx GROUPS(batch_size),uint thread_idx THREADS(BLOCK_SIZE)) {@@ -35,12 +36,14 @@ KERNEL(TopK) (float max_logit = threadgroup_cooperative_reduce_max<BLOCK_SIZE>(local_max,shared_reduce_buffer,- thread_idx+ thread_idx,+ simd);float min_logit = threadgroup_cooperative_reduce_min<BLOCK_SIZE>(local_min,shared_reduce_buffer,- thread_idx+ thread_idx,+ simd);// Do the binary search on the thresholdfloat low = min_logit;@@ -58,7 +61,8 @@ KERNEL(TopK) (uint num_above_threshold = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(local_num_above_threshold,(threadgroup uint*)shared_reduce_buffer,- thread_idx+ thread_idx,+ simd);// Update binary searchdiff --git a/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal b/crates/uzu/src/backends/metal/kernel/sampling/top_p.metalindex b1c3bc481..b22c3eee5 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal+++ b/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal@@ -15,6 +15,7 @@ KERNEL(TopP) (constant uint& batch_size,constant uint& vocab_size,constant float& top_p,+ const Simd simd,uint batch_idx GROUPS(batch_size),uint thread_idx THREADS(BLOCK_SIZE)) {@@ -36,12 +37,14 @@ KERNEL(TopP) (float max_logit = threadgroup_cooperative_reduce_max<BLOCK_SIZE>(local_max,shared_reduce_buffer,- thread_idx+ thread_idx,+ simd);float min_logit = threadgroup_cooperative_reduce_min<BLOCK_SIZE>(local_min,shared_reduce_buffer,- thread_idx+ thread_idx,+ simd);// Find denominator for softmax@@ -58,7 +61,8 @@ KERNEL(TopP) (float total_sum = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(local_sum,shared_reduce_buffer,- thread_idx+ thread_idx,+ simd);// Do the binary search on the threshold@@ -86,12 +90,14 @@ KERNEL(TopP) (float sum_above_threshold = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(local_sum_above_threshold,shared_reduce_buffer,- thread_idx+ thread_idx,+ simd);float min_above_threshold = threadgroup_cooperative_reduce_min<BLOCK_SIZE>(local_min_above_threshold,shared_reduce_buffer,- thread_idx+ thread_idx,+ simd);// Early exitdiff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal b/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metalindex f8517a6ba..904574cb5 100644--- a/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal+++ b/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal@@ -27,10 +27,10 @@ KERNEL(SSDPrefill64)(constant const uint* state_strides,constant const uint& num_heads,constant const uint& head_dim,+ const Simd simd,const uint pair_idx GROUPS(num_heads * head_dim),const uint lane_idx THREADS(32)) {- const uint simd_width = 32;const uint h_idx = pair_idx / head_dim;const uint dh_idx = pair_idx % head_dim;const uint safe_group = uint(max(group_size, 1));@@ -55,7 +55,7 @@ KERNEL(SSDPrefill64)(const float d_scalar = float(d[h_idx]);const uint idx0 = lane_idx;- const uint idx1 = lane_idx + simd_width;+ const uint idx1 = lane_idx + simd.group_size;float state0 = 0.0f;float state1 = 0.0f;@@ -125,10 +125,10 @@ KERNEL(SSDPrefill)(constant const uint* state_strides,constant const uint& num_heads,constant const uint& head_dim,+ const Simd simd,const uint pair_idx GROUPS(num_heads * head_dim),const uint lane_idx THREADS(32)) {- const uint simd_width = 32;const int state_dim = state_size;const uint h_idx = pair_idx / head_dim;const uint dh_idx = pair_idx % head_dim;@@ -153,8 +153,9 @@ KERNEL(SSDPrefill)(const uint cb_group_base = group_idx * cb_group_stride;const float d_scalar = float(d[h_idx]);- const int max_chunks = SSM_PREFILL_MAX_STATE / simd_width;- const int chunk_count = (state_dim + int(simd_width) - 1) / int(simd_width);+ const int max_chunks = SSM_PREFILL_MAX_STATE / simd.group_size;+ const int chunk_count =+ (state_dim + int(simd.group_size) - 1) / int(simd.group_size);if (state_dim <= 0 || chunk_count > max_chunks) {return;}@@ -163,7 +164,7 @@ KERNEL(SSDPrefill)(#pragma unrollfor (int chunk = 0; chunk < chunk_count; ++chunk) {- const int idx = chunk * int(simd_width) + int(lane_idx);+ const int idx = chunk * int(simd.group_size) + int(lane_idx);const uint state_idx = state_base + idx * state_inner_stride;lane_states[chunk] = float(state[state_idx]);}@@ -181,7 +182,7 @@ KERNEL(SSDPrefill)(float contrib_sum = 0.0f;#pragma unrollfor (int chunk = 0; chunk < chunk_count; ++chunk) {- const int idx = chunk * int(simd_width) + int(lane_idx);+ const int idx = chunk * int(simd.group_size) + int(lane_idx);const uint cb_idx =cb_group_base + uint(idx) * cb_state_stride + token * cb_token_stride;const float new_state =@@ -198,7 +199,7 @@ KERNEL(SSDPrefill)(#pragma unrollfor (int chunk = 0; chunk < chunk_count; ++chunk) {- const int idx = chunk * int(simd_width) + int(lane_idx);+ const int idx = chunk * int(simd.group_size) + int(lane_idx);const uint state_idx = state_base + uint(idx) * state_inner_stride;state[state_idx] = static_cast<T>(lane_states[chunk]);}