Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Dsl simd (#215)

Aleksandr Golokozebed7db2ade8
diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal
index 06ed9897c..22853c4b5 100644
--- a/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal
+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal
@@ -32,7 +32,6 @@ METAL_FUNC T row_reduce_sum(T v) {
}
#define BQ 32
-#define SIMD_WIDTH 32
#define WM 4
#define WN 1
@@ -58,17 +57,12 @@ KERNEL(AttentionGemm)(
const bool has_sinks SPECIALIZE,
threadgroup T q_smem[BQ * (BD + 16 / sizeof(T))],
threadgroup T kv_smem[BK * (BD + 16 / sizeof(T))],
+ const Simd simd,
const uint tgid_x GROUPS(suffix_length.div_ceil(BQ)),
const uint tgid_y GROUPS(num_heads),
const uint tgid_z GROUPS(1),
const uint lid THREADS(128)
) {
- const uint simd_group_id = lid / SIMD_WIDTH;
- const uint simd_lane_id = lid % SIMD_WIDTH;
-
- // Pacifying compiler
- (void)lid;
-
// -------------------------------------------------------------------------
// Pointer setup (all strides are in elements)
// tgid_x: query tile index (BQ rows)
@@ -140,9 +134,9 @@ KERNEL(AttentionGemm)(
const int k_src_ld = int(params.k_strides[2]);
const int v_src_ld = int(params.v_strides[2]);
- thread QBlockLoader loader_q(q, q_src_ld, Qs, simd_group_id, simd_lane_id);
- thread KBlockLoader loader_k(k, k_src_ld, Ks, simd_group_id, simd_lane_id);
- thread VBlockLoader loader_v(v, v_src_ld, Vs, simd_group_id, simd_lane_id);
+ thread QBlockLoader loader_q(q, q_src_ld, Qs, simd.group_idx, simd.lane_idx);
+ thread KBlockLoader loader_k(k, k_src_ld, Ks, simd.group_idx, simd.lane_idx);
+ thread VBlockLoader loader_v(v, v_src_ld, Vs, simd.group_idx, simd.lane_idx);
TransformScale<T> ts(static_cast<T>(params.scale * M_LOG2E_F));
@@ -177,11 +171,11 @@ KERNEL(AttentionGemm)(
// -------------------------------------------------------------------------
// Lane coordinates and pointer offsets
- const short2 simd_coord = MMAFrag_acc_t::get_coord(simd_lane_id);
+ const short2 simd_coord = MMAFrag_acc_t::get_coord(simd.lane_idx);
const short sm = simd_coord.y;
const short sn = simd_coord.x;
- const short tm = kFragSize * TQ * short(simd_group_id);
+ const short tm = kFragSize * TQ * short(simd.group_idx);
// Qs is row-major [BQ, BD]
const short Qs_offset = (tm + sm) * LDQ_tgp + sn;
diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal
index b3ef95a54..e151c7d38 100644
--- a/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal
+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal
@@ -2,7 +2,6 @@
#include <metal_simdgroup>
#include "../definitions.metal"
-#define SIMD_WIDTH 32
#define SEQUENCE_BLOCK_SIZE 32
#define HEAD_BLOCK_SIZE 32
@@ -35,14 +34,12 @@ KERNEL(AttentionSinglePass)(
const bool has_mask SPECIALIZE,
const bool has_sinks SPECIALIZE,
const bool do_causal SPECIALIZE,
+ const Simd simd,
const uint head_idx GROUPS(num_heads),
const uint q_seq_idx GROUPS(suffix_length),
const uint tid THREADS(1024)
) {
const uint3 tpg = {num_heads, suffix_length, 1};
- const uint simd_gid = tid / SIMD_WIDTH;
- const uint simd_lid = tid % SIMD_WIDTH;
- constexpr bool bool_mask = false;
constexpr bool query_transposed = false;
constexpr uint value_dim = head_dim;
@@ -62,22 +59,18 @@ KERNEL(AttentionSinglePass)(
const uint q_offset = query_transposed ? tpg.x * q_seq_idx + head_idx
: head_idx * tpg.y + q_seq_idx;
- queries += q_offset * head_dim + simd_lid * qk_elements_per_thread;
- keys += kv_head_idx * k_head_stride + simd_gid * k_seq_stride +
- simd_lid * qk_elements_per_thread;
- values += kv_head_idx * v_head_stride + simd_gid * v_seq_stride +
- simd_lid * value_elements_per_thread;
+ queries += q_offset * head_dim + simd.lane_idx * qk_elements_per_thread;
+ keys += kv_head_idx * k_head_stride + simd.group_idx * k_seq_stride +
+ simd.lane_idx * qk_elements_per_thread;
+ values += kv_head_idx * v_head_stride + simd.group_idx * v_seq_stride +
+ simd.lane_idx * value_elements_per_thread;
- if (bool_mask) {
- // bmask += head_idx * mask_head_stride + simd_gid * mask_kv_seq_stride +
- // q_seq_idx * mask_q_seq_stride;
- }
if (float_mask) {
- fmask += head_idx * mask_head_stride + simd_gid * mask_kv_seq_stride +
+ fmask += head_idx * mask_head_stride + simd.group_idx * mask_kv_seq_stride +
q_seq_idx * mask_q_seq_stride;
}
- out += o_offset * value_dim + simd_gid * value_elements_per_thread;
+ out += o_offset * value_dim + simd.group_idx * value_elements_per_thread;
// Read the query and 0 the output accumulator
for (int i = 0; i < qk_elements_per_thread; i++) {
@@ -89,7 +82,7 @@ KERNEL(AttentionSinglePass)(
U max_score = -INFINITY;
U sum_exp_score = 0;
- if (has_sinks && simd_gid == 0) {
+ if (has_sinks && simd.group_idx == 0) {
const int num_q_heads = static_cast<int>(tpg.x);
int q_head_idx = head_idx % num_q_heads;
max_score = static_cast<U>(sinks[q_head_idx]);
@@ -97,12 +90,10 @@ KERNEL(AttentionSinglePass)(
}
// For each key
- for (uint i = simd_gid; i < sequence_length; i += SEQUENCE_BLOCK_SIZE) {
+ for (uint i = simd.group_idx; i < sequence_length; i += SEQUENCE_BLOCK_SIZE) {
bool use_key = true;
if (do_causal) {
use_key = i <= (sequence_length - tpg.y + q_seq_idx);
- } else if (bool_mask) {
- // use_key = bmask[0];
}
if (use_key) {
@@ -138,38 +129,37 @@ KERNEL(AttentionSinglePass)(
// Move the pointers to the next kv
keys += inner_k_stride;
values += inner_v_stride;
- if (bool_mask) {
- // bmask += SEQUENCE_BLOCK_SIZE * mask_kv_seq_stride;
- }
if (float_mask) {
fmask += SEQUENCE_BLOCK_SIZE * mask_kv_seq_stride;
}
}
// Each thread has a partial part of the output so we need to combine them.
- if (simd_lid == 0) {
- shared_max_scores[simd_gid] = max_score;
- shared_sum_exp_scores[simd_gid] = sum_exp_score;
+ if (simd.lane_idx == 0) {
+ shared_max_scores[simd.group_idx] = max_score;
+ shared_sum_exp_scores[simd.group_idx] = sum_exp_score;
}
threadgroup_barrier(mem_flags::mem_threadgroup);
- max_score = shared_max_scores[simd_lid];
+ max_score = shared_max_scores[simd.lane_idx];
U new_max = simd_max(max_score);
U factor = fast::exp(max_score - new_max);
- sum_exp_score = simd_sum(shared_sum_exp_scores[simd_lid] * factor);
+ sum_exp_score = simd_sum(shared_sum_exp_scores[simd.lane_idx] * factor);
// Now we need to aggregate all the outputs
for (int i = 0; i < value_elements_per_thread; i++) {
- shared_outputs[simd_lid * HEAD_BLOCK_SIZE + simd_gid] = o[i];
+ shared_outputs[simd.lane_idx * HEAD_BLOCK_SIZE + simd.group_idx] = o[i];
threadgroup_barrier(mem_flags::mem_threadgroup);
- o[i] = simd_sum(
- shared_outputs[simd_gid * HEAD_BLOCK_SIZE + simd_lid] * factor
- ) /
- sum_exp_score;
+ o[i] =
+ simd_sum(
+ shared_outputs[simd.group_idx * HEAD_BLOCK_SIZE + simd.lane_idx] *
+ factor
+ ) /
+ sum_exp_score;
threadgroup_barrier(mem_flags::mem_threadgroup);
}
// And write the output
- if (simd_lid == 0) {
+ if (simd.lane_idx == 0) {
for (int i = 0; i < value_elements_per_thread; i++) {
out[i] = static_cast<T>(o[i]);
}
diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal
index 3c0295233..bb322b0fe 100644
--- a/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal
+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal
@@ -5,7 +5,6 @@
#define HEAD_BLOCK_SIZE 32
#define SEQUENCE_BLOCK_SIZE_1 8
#define SEQUENCE_BLOCK_SIZE_2 32
-#define SIMD_WIDTH 32
#define TOTAL_BLOCKS_COUNT 32
template <typename T, uint head_dim>
@@ -39,14 +38,13 @@ KERNEL(AttentionTwoPass1)(
threadgroup float shared_max_scores[SEQUENCE_BLOCK_SIZE_1],
threadgroup float shared_sum_exp_scores[SEQUENCE_BLOCK_SIZE_1],
threadgroup float shared_outputs[SEQUENCE_BLOCK_SIZE_1 * HEAD_BLOCK_SIZE],
+ const Simd simd,
const uint head_idx GROUPS(num_heads),
const uint q_seq_idx GROUPS(suffix_length),
const uint block_idx GROUPS(TOTAL_BLOCKS_COUNT),
const uint tid THREADS(256)
) {
const uint3 tpg = {num_heads, suffix_length, TOTAL_BLOCKS_COUNT};
- const uint simd_gid = tid / SIMD_WIDTH;
- const uint simd_lid = tid % SIMD_WIDTH;
constexpr bool query_transposed = false;
constexpr uint value_dim = head_dim;
@@ -68,20 +66,21 @@ KERNEL(AttentionTwoPass1)(
: head_idx * tpg.y + q_seq_idx; // Consistent with single-pass
const uint kv_head_idx = head_idx / gqa_factor;
- queries += q_offset * head_dim + simd_lid * qk_elements_per_thread;
+ queries += q_offset * head_dim + simd.lane_idx * qk_elements_per_thread;
keys += kv_head_idx * k_head_stride +
- (block_idx * SEQUENCE_BLOCK_SIZE_1 + simd_gid) * k_seq_stride +
- simd_lid * qk_elements_per_thread;
- values += kv_head_idx * v_head_stride +
- (block_idx * SEQUENCE_BLOCK_SIZE_1 + simd_gid) * v_seq_stride +
- simd_lid * value_elements_per_thread;
+ (block_idx * SEQUENCE_BLOCK_SIZE_1 + simd.group_idx) * k_seq_stride +
+ simd.lane_idx * qk_elements_per_thread;
+ values +=
+ kv_head_idx * v_head_stride +
+ (block_idx * SEQUENCE_BLOCK_SIZE_1 + simd.group_idx) * v_seq_stride +
+ simd.lane_idx * value_elements_per_thread;
out += o_offset * TOTAL_BLOCKS_COUNT * value_dim + block_idx * value_dim +
- simd_lid * value_elements_per_thread;
+ simd.lane_idx * value_elements_per_thread;
if (float_mask) {
- fmask +=
- head_idx * mask_head_stride +
- (block_idx * SEQUENCE_BLOCK_SIZE_1 + simd_gid) * mask_kv_seq_stride +
- q_seq_idx * mask_q_seq_stride;
+ fmask += head_idx * mask_head_stride +
+ (block_idx * SEQUENCE_BLOCK_SIZE_1 + simd.group_idx) *
+ mask_kv_seq_stride +
+ q_seq_idx * mask_q_seq_stride;
}
sums += o_offset * TOTAL_BLOCKS_COUNT + block_idx;
maxs += o_offset * TOTAL_BLOCKS_COUNT + block_idx;
@@ -96,7 +95,7 @@ KERNEL(AttentionTwoPass1)(
U max_score = -1e9;
U sum_exp_score = 0;
- if (has_sinks && block_idx == 0 && simd_gid == 0) {
+ if (has_sinks && block_idx == 0 && simd.group_idx == 0) {
const uint num_q_heads = tpg.x;
int q_head_idx = head_idx % num_q_heads;
max_score = static_cast<U>(sinks[q_head_idx]);
@@ -104,7 +103,7 @@ KERNEL(AttentionTwoPass1)(
}
// For each key
- for (uint i = block_idx * SEQUENCE_BLOCK_SIZE_1 + simd_gid;
+ for (uint i = block_idx * SEQUENCE_BLOCK_SIZE_1 + simd.group_idx;
i < sequence_length;
i += TOTAL_BLOCKS_COUNT * SEQUENCE_BLOCK_SIZE_1) {
bool use_key = true;
@@ -151,36 +150,38 @@ KERNEL(AttentionTwoPass1)(
}
// Each thread has a partial part of the output so we need to combine them.
- if (simd_lid == 0) {
- shared_max_scores[simd_gid] = max_score;
- shared_sum_exp_scores[simd_gid] = sum_exp_score;
+ if (simd.lane_idx == 0) {
+ shared_max_scores[simd.group_idx] = max_score;
+ shared_sum_exp_scores[simd.group_idx] = sum_exp_score;
}
threadgroup_barrier(mem_flags::mem_threadgroup);
- max_score =
- (simd_lid < SEQUENCE_BLOCK_SIZE_1) ? shared_max_scores[simd_lid] : -1e9;
+ max_score = (simd.lane_idx < SEQUENCE_BLOCK_SIZE_1)
+ ? shared_max_scores[simd.lane_idx]
+ : -1e9;
U new_max = simd_max(max_score);
U factor = fast::exp(max_score - new_max);
- sum_exp_score =
- (simd_lid < SEQUENCE_BLOCK_SIZE_1) ? shared_sum_exp_scores[simd_lid] : 0;
+ sum_exp_score = (simd.lane_idx < SEQUENCE_BLOCK_SIZE_1)
+ ? shared_sum_exp_scores[simd.lane_idx]
+ : 0;
sum_exp_score = simd_sum(sum_exp_score * factor);
// Write the sum and new max
- if (simd_gid == 0) {
+ if (simd.group_idx == 0) {
sums[0] = sum_exp_score;
maxs[0] = new_max;
}
// Now we need to aggregate all the outputs
for (uint i = 0; i < value_elements_per_thread; i++) {
- shared_outputs[simd_lid * SEQUENCE_BLOCK_SIZE_1 + simd_gid] =
- o[i] * fast::exp(shared_max_scores[simd_gid] - new_max);
+ shared_outputs[simd.lane_idx * SEQUENCE_BLOCK_SIZE_1 + simd.group_idx] =
+ o[i] * fast::exp(shared_max_scores[simd.group_idx] - new_max);
threadgroup_barrier(mem_flags::mem_threadgroup);
// And write the output
- if (simd_gid == 0) {
- U output = shared_outputs[simd_lid * SEQUENCE_BLOCK_SIZE_1];
+ if (simd.group_idx == 0) {
+ U output = shared_outputs[simd.lane_idx * SEQUENCE_BLOCK_SIZE_1];
for (uint j = 1; j < SEQUENCE_BLOCK_SIZE_1; j++) {
- output += shared_outputs[simd_lid * SEQUENCE_BLOCK_SIZE_1 + j];
+ output += shared_outputs[simd.lane_idx * SEQUENCE_BLOCK_SIZE_1 + j];
}
out[i] = static_cast<T>(output);
}
@@ -199,14 +200,12 @@ KERNEL(AttentionTwoPass2)(
const constant uint& num_heads,
const constant uint& suffix_length,
threadgroup float shared_outputs[SEQUENCE_BLOCK_SIZE_2 * HEAD_BLOCK_SIZE],
+ const Simd simd,
const uint head_idx GROUPS(num_heads),
const uint q_seq_idx GROUPS(suffix_length),
const uint tid THREADS(1024)
) {
const uint3 tpg = {num_heads, suffix_length, 1};
- const uint simd_gid = tid / SIMD_WIDTH;
- const uint simd_lid = tid % SIMD_WIDTH;
-
constexpr uint elements_per_thread = head_dim / HEAD_BLOCK_SIZE;
typedef float U;
@@ -214,18 +213,18 @@ KERNEL(AttentionTwoPass2)(
const uint o_offset = q_seq_idx * tpg.x + head_idx; // Our custom layout
- partials += o_offset * TOTAL_BLOCKS_COUNT * head_dim + simd_gid * head_dim +
- simd_lid * elements_per_thread;
+ partials += o_offset * TOTAL_BLOCKS_COUNT * head_dim +
+ simd.group_idx * head_dim + simd.lane_idx * elements_per_thread;
sums += o_offset * TOTAL_BLOCKS_COUNT;
maxs += o_offset * TOTAL_BLOCKS_COUNT;
out += o_offset * head_dim +
- simd_gid * elements_per_thread; // Our custom output layout
+ simd.group_idx * elements_per_thread; // Our custom output layout
// First everybody reads the max and sum_exp
- U max_score = maxs[simd_lid];
+ U max_score = maxs[simd.lane_idx];
U new_max = simd_max(max_score);
U factor = fast::exp(max_score - new_max);
- U sum_exp_score = simd_sum(sums[simd_lid] * factor);
+ U sum_exp_score = simd_sum(sums[simd.lane_idx] * factor);
// Now read the block into registers and then use shared memory to transpose
// it
@@ -233,17 +232,19 @@ KERNEL(AttentionTwoPass2)(
o[i] = partials[i];
}
for (uint i = 0; i < elements_per_thread; i++) {
- shared_outputs[simd_lid * HEAD_BLOCK_SIZE + simd_gid] = o[i];
+ shared_outputs[simd.lane_idx * HEAD_BLOCK_SIZE + simd.group_idx] = o[i];
threadgroup_barrier(mem_flags::mem_threadgroup);
- o[i] = simd_sum(
- shared_outputs[simd_gid * HEAD_BLOCK_SIZE + simd_lid] * factor
- ) /
- sum_exp_score;
+ o[i] =
+ simd_sum(
+ shared_outputs[simd.group_idx * HEAD_BLOCK_SIZE + simd.lane_idx] *
+ factor
+ ) /
+ sum_exp_score;
threadgroup_barrier(mem_flags::mem_threadgroup);
}
// And write the output
- if (simd_lid == 0) {
+ if (simd.lane_idx == 0) {
for (uint i = 0; i < elements_per_thread; i++) {
out[i] = static_cast<T>(o[i]);
}
diff --git a/crates/uzu/src/backends/metal/kernel/definitions.metal b/crates/uzu/src/backends/metal/kernel/definitions.metal
index 8f2e4e150..5f48dde72 100644
--- a/crates/uzu/src/backends/metal/kernel/definitions.metal
+++ b/crates/uzu/src/backends/metal/kernel/definitions.metal
@@ -126,6 +126,17 @@ struct TensorView4D {
}
};
+struct Simd {
+ uint lane_idx;
+ uint group_idx;
+ uint group_size;
+ uint groups_per_threadgroup;
+
+ uint groups_count(const uint threads_count) const {
+ return (threads_count + this->group_size - 1) / this->group_size;
+ }
+};
+
///////////////////////////////////////////////////////////////////////////////
// MARK: - Thread Functions
///////////////////////////////////////////////////////////////////////////////
@@ -351,17 +362,15 @@ template <ushort BLOCK_SIZE, typename T>
static T threadgroup_cooperative_reduce_sum(
T value,
threadgroup T* shared,
- const ushort lid
+ const ushort lid,
+ const thread Simd& simd
) {
- const ushort simd_group_id = lid / 32;
- const ushort simd_lane_id = lid % 32;
-
// Reduce within simdgroup
T local_sum = simd_sum(value);
// First thread in each simdgroup writes to shared memory
- if (simd_lane_id == 0) {
- shared[simd_group_id] = local_sum;
+ if (simd.lane_idx == 0) {
+ shared[simd.group_idx] = local_sum;
}
// Synchronize across the threadgroup
@@ -369,7 +378,7 @@ static T threadgroup_cooperative_reduce_sum(
// Reduce across simdgroups
T total_sum = T(0);
- const ushort num_simd_groups = (BLOCK_SIZE + 31) / 32;
+ const ushort num_simd_groups = simd.groups_count(BLOCK_SIZE);
if (lid < num_simd_groups) {
total_sum = shared[lid];
}
@@ -392,24 +401,23 @@ template <ushort BLOCK_SIZE, typename T>
static T threadgroup_cooperative_reduce_max(
T value,
threadgroup T* shared,
- const ushort lid
+ const ushort lid,
+ const thread Simd& simd
) {
- const ushort simd_group_id = lid / 32;
- const ushort simd_lane_id = lid % 32;
-
// Reduce within simdgroup
T local_max = simd_max(value);
// First thread in each simdgroup writes to shared memory
- if (simd_lane_id == 0) {
- shared[simd_group_id] = local_max;
+ if (simd.lane_idx == 0) {
+ shared[simd.group_idx] = local_max;
}
// Synchronize across the threadgroup
threadgroup_barrier(mem_flags::mem_threadgroup);
// Reduce across simdgroups
- T total_max = (lid < ((BLOCK_SIZE + 31) / 32)) ? shared[lid] : T(-INFINITY);
+ T total_max =
+ lid < simd.groups_count(BLOCK_SIZE) ? shared[lid] : T(-INFINITY);
total_max = simd_max(total_max);
// Broadcast the result to all threads
@@ -429,24 +437,22 @@ template <ushort BLOCK_SIZE, typename T>
static T threadgroup_cooperative_reduce_min(
T value,
threadgroup T* shared,
- const ushort lid
+ const ushort lid,
+ const thread Simd& simd
) {
- const ushort simd_group_id = lid / 32;
- const ushort simd_lane_id = lid % 32;
-
// Reduce within simdgroup
T local_min = simd_min(value);
// First thread in each simdgroup writes to shared memory
- if (simd_lane_id == 0) {
- shared[simd_group_id] = local_min;
+ if (simd.lane_idx == 0) {
+ shared[simd.group_idx] = local_min;
}
// Synchronize across the threadgroup
threadgroup_barrier(mem_flags::mem_threadgroup);
// Reduce across simdgroups
- T total_min = (lid < ((BLOCK_SIZE + 31) / 32)) ? shared[lid] : T(INFINITY);
+ T total_min = lid < simd.groups_count(BLOCK_SIZE) ? shared[lid] : T(INFINITY);
total_min = simd_min(total_min);
// Broadcast the result to all threads
@@ -489,13 +495,6 @@ static T threadgroup_cooperative_reduce_min(
#define GROUPS(EXPR) DSL_META("dsl.groups", DSL_XSTR(EXPR))
#define THREADS(EXPR) DSL_META("dsl.threads", DSL_XSTR(EXPR))
-struct Simd {
- uint lane_idx;
- uint group_idx;
- uint group_size;
- uint groups_per_threadgroup;
-};
-
// MARK: - Generate Template Kernels
#define generateKernel( \
diff --git a/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal b/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal
index a033243f4..1593a3931 100644
--- a/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal
+++ b/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal
@@ -17,7 +17,8 @@ void layer_norm_core(
threadgroup AccumT* shared_mean,
threadgroup AccumT* shared_variance,
uint thread_in_row,
- bool full_layer
+ bool full_layer,
+ const thread Simd& simd
) {
AccumT partial_sum = static_cast<AccumT>(0.0f);
@@ -36,7 +37,8 @@ void layer_norm_core(
AccumT total_sum = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(
partial_sum,
shared_mean,
- thread_in_row
+ thread_in_row,
+ simd
);
AccumT mean = total_sum / static_cast<AccumT>(element_count);
@@ -57,7 +59,8 @@ void layer_norm_core(
AccumT total_var = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(
partial_var_sum,
shared_variance,
- thread_in_row
+ thread_in_row,
+ simd
);
AccumT variance = total_var / static_cast<AccumT>(element_count);
AccumT inv_std = rsqrt(variance + static_cast<AccumT>(epsilon));
@@ -126,6 +129,7 @@ KERNEL(LayerNorm) (
constant uint& full_layer,
threadgroup ACC shared_mean[SIMD_SIZE],
threadgroup ACC shared_variance[SIMD_SIZE],
+ const Simd simd,
uint batch_idx GROUPS(batch_size),
uint thread_in_row THREADS(BLOCK_SIZE)
) {
@@ -140,6 +144,7 @@ KERNEL(LayerNorm) (
shared_mean,
shared_variance,
thread_in_row,
- full_layer
+ full_layer,
+ simd
);
}
diff --git a/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal b/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal
index 743dddc44..3d44d6b05 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal
@@ -23,6 +23,7 @@ KERNEL(MoeCountsOffsetsFused)(
threadgroup uint
counts_shared[BLOCK_SIZE], // Cache counts in threadgroup memory
threadgroup uint& carry,
+ const Simd simd,
const uint lid THREADS(128)
) {
if (e_input == 0) {
@@ -100,7 +101,8 @@ KERNEL(MoeCountsOffsetsFused)(
uint block_sum = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(
v,
reduce_shared,
- (ushort)lid
+ (ushort)lid,
+ simd
);
if (lid == 0) {
carry += block_sum;
diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal
index 6c1abb418..bab241e36 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal
@@ -3,8 +3,6 @@
#include "../definitions.metal"
#include "moe_commons.h"
-#define SIMD_SIZE 32
-
// ============================================================================
// Pass A: x @ W13[expert] → hidden[k]
// Each threadgroup: 4 simdgroups, outputs 4 elements (1 per simdgroup)
@@ -28,19 +26,17 @@ KERNEL(MoeExpertsDecodeSinglePassA)(
constant float& up_clip_min,
constant float& up_clip_max,
const uint gating_sel SPECIALIZE, // 0=GELU, 1=SiLU, 2=SwiGLU, 3=GEGLU
+ const Simd simd,
const uint h_block_idx GROUPS(d_ff.div_ceil(4)),
const uint k_slot GROUPS(k),
const uint tid THREADS(128)
) {
- const uint simd_gid = tid / SIMD_SIZE;
- const uint simd_lid = tid % SIMD_SIZE;
-
const int expert_id = topk_ids[k_slot];
if (expert_id < 0)
return;
const uint expert_u = uint(expert_id);
- const uint h_idx = h_block_idx * 4 + simd_gid;
+ const uint h_idx = h_block_idx * 4 + simd.group_idx;
if (h_idx >= d_ff)
return;
@@ -58,7 +54,7 @@ KERNEL(MoeExpertsDecodeSinglePassA)(
const uint vec_iters = d_model / 128;
for (uint i = 0; i < vec_iters; ++i) {
- uint base_idx = i * 128 + simd_lid * 4;
+ uint base_idx = i * 128 + simd.lane_idx * 4;
device const T* x_vec2 = reinterpret_cast<device const T*>(x + base_idx);
device const T* w_up_vec2 =
@@ -78,7 +74,7 @@ KERNEL(MoeExpertsDecodeSinglePassA)(
}
}
- uint leftover_start = vec_iters * 128 + simd_lid;
+ uint leftover_start = vec_iters * 128 + simd.lane_idx;
for (uint idx = leftover_start; idx < d_model; idx += 32) {
float xv = float(x[idx]);
acc_up += xv * float(w_up_row[idx]);
@@ -92,7 +88,7 @@ KERNEL(MoeExpertsDecodeSinglePassA)(
acc_gate = simd_sum(acc_gate);
}
- if (simd_lid == 0) {
+ if (simd.lane_idx == 0) {
float up_val = clamp(
acc_up + float(biases[bias_base + h_idx]),
up_clip_min,
@@ -136,13 +132,11 @@ KERNEL(MoeExpertsDecodeSinglePassB)(
constant uint& d_model,
constant uint& d_ff,
constant uint& k_input,
+ const Simd simd,
const uint d_block GROUPS(d_model.div_ceil(8)),
const uint tid THREADS(256)
) {
- const uint simd_gid = tid / SIMD_SIZE;
- const uint simd_lid = tid % SIMD_SIZE;
-
- const uint my_col = d_block * 8 + simd_gid;
+ const uint my_col = d_block * 8 + simd.group_idx;
if (my_col >= d_model)
return;
@@ -164,7 +158,7 @@ KERNEL(MoeExpertsDecodeSinglePassB)(
// Vectorized reduction
for (uint i = 0; i < vec_iters; ++i) {
- uint base_idx = i * 128 + simd_lid * 4;
+ uint base_idx = i * 128 + simd.lane_idx * 4;
float4 h_vec =
*reinterpret_cast<device const float4*>(hidden_ptr + base_idx);
@@ -177,19 +171,19 @@ KERNEL(MoeExpertsDecodeSinglePassB)(
}
// Remainder
- for (uint idx = vec_iters * 128 + simd_lid; idx < d_ff; idx += 32) {
+ for (uint idx = vec_iters * 128 + simd.lane_idx; idx < d_ff; idx += 32) {
acc += hidden_ptr[idx] * float(w2_ptr[idx]);
}
// Simdgroup reduction and accumulate
float result = simd_sum(acc);
- if (simd_lid == 0) {
+ if (simd.lane_idx == 0) {
result += float(biases[(ulong)expert_u * (ulong)d_model + (ulong)my_col]);
final_acc += prob * result;
}
}
- if (simd_lid == 0) {
+ if (simd.lane_idx == 0) {
y[my_col] = T(final_acc);
}
}
\ No newline at end of file
diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal
index f220d43aa..9ac862664 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal
@@ -8,8 +8,6 @@ using namespace metal;
// Structure: 4 simdgroups (128 threads), each outputs 1 hidden element
// Each simdgroup: 32 threads reduce d_model using float4 vectorized loads
// Grid: (h_blocks, rows, 1) where h_blocks = ceil(d_ff/4)
-#define SIMD_SIZE 32
-
template <typename T>
VARIANTS(T, float, half, bfloat)
KERNEL(MoeExpertsDecodePassA)(
@@ -28,12 +26,10 @@ KERNEL(MoeExpertsDecodePassA)(
constant float& silu_alpha,
device const uint* tile_map,
const uint gating_sel SPECIALIZE,
+ const Simd simd,
const uint tile_idx GROUPS(INDIRECT),
const uint tid THREADS(128)
) {
- const uint simd_gid = tid / SIMD_SIZE;
- const uint simd_lid = tid % SIMD_SIZE;
-
// Read tile descriptor
const uint h_block_idx = tile_map[tile_idx * 3 + 0];
const uint expert_idx = tile_map[tile_idx * 3 + 1];
@@ -47,7 +43,7 @@ KERNEL(MoeExpertsDecodePassA)(
return;
// Each simdgroup outputs one hidden element
- const uint h_idx = h_block_idx * 4 + simd_gid;
+ const uint h_idx = h_block_idx * 4 + simd.group_idx;
if (h_idx >= d_ff)
return;
@@ -69,7 +65,7 @@ KERNEL(MoeExpertsDecodePassA)(
const uint vec_iters = d_model / 128;
for (uint i = 0; i < vec_iters; ++i) {
- uint base_idx = i * 128 + simd_lid * 4;
+ uint base_idx = i * 128 + simd.lane_idx * 4;
device const T* x_vec = reinterpret_cast<device const T*>(x_ptr + base_idx);
device const T* w_up_vec =
@@ -90,7 +86,7 @@ KERNEL(MoeExpertsDecodePassA)(
}
// Handle leftover elements
- uint leftover_start = vec_iters * 128 + simd_lid;
+ uint leftover_start = vec_iters * 128 + simd.lane_idx;
for (uint idx = leftover_start; idx < d_model; idx += 32) {
float xv = float(x_ptr[idx]);
acc_up += xv * float(w_up_row[idx]);
@@ -106,7 +102,7 @@ KERNEL(MoeExpertsDecodePassA)(
}
// Lane 0 applies activation and writes result
- if (simd_lid == 0) {
+ if (simd.lane_idx == 0) {
float up_val = acc_up + float(up_biases[bias_base + h_idx]);
up_val = clamp(up_val, up_clip_min, up_clip_max);
@@ -145,16 +141,15 @@ KERNEL(MoeExpertsDecodeDownFused2D)(
constant uint& d_model,
constant uint& d_ff,
constant uint& e,
+ const Simd simd,
const uint tgpig_x GROUPS(d_model.div_ceil(SIMDGROUPS_PER_TG)),
const uint tgpig_y GROUPS(total_rows),
const uint tid THREADS(256)
) {
- const uint simd_gid = tid / SIMD_SIZE;
- const uint simd_lid = tid % SIMD_SIZE;
const uint row_idx = tgpig_y;
// Each simdgroup computes one output column
- const uint my_col = tgpig_x * SIMDGROUPS_PER_TG + simd_gid;
+ const uint my_col = tgpig_x * SIMDGROUPS_PER_TG + simd.group_idx;
if (my_col >= d_model)
return;
@@ -175,7 +170,7 @@ KERNEL(MoeExpertsDecodeDownFused2D)(
const uint k_vec_iters = k_iters / 8;
for (uint iter = 0; iter < k_vec_iters; ++iter) {
- const uint k_base = iter * (8 * THREADS_PER_SIMD) + simd_lid;
+ const uint k_base = iter * (8 * THREADS_PER_SIMD) + simd.lane_idx;
// hidden: stride-32 per lane, already f32 from Pass A
const AccumT h0 = hidden[hidden_base + k_base + 0 * THREADS_PER_SIMD];
@@ -221,7 +216,7 @@ KERNEL(MoeExpertsDecodeDownFused2D)(
// Handle remaining full iterations
for (uint iter = k_vec_iters * 8; iter < k_iters; ++iter) {
- const uint k = iter * THREADS_PER_SIMD + simd_lid;
+ const uint k = iter * THREADS_PER_SIMD + simd.lane_idx;
acc =
fma(AccumT(hidden[hidden_base + k]),
AccumT(w2_all[w2_col_base + k]),
@@ -230,8 +225,8 @@ KERNEL(MoeExpertsDecodeDownFused2D)(
// Handle leftover elements (d_ff % 32)
const uint leftover_start = k_iters * THREADS_PER_SIMD;
- if (leftover_start + simd_lid < d_ff) {
- const uint k = leftover_start + simd_lid;
+ if (leftover_start + simd.lane_idx < d_ff) {
+ const uint k = leftover_start + simd.lane_idx;
acc =
fma(AccumT(hidden[hidden_base + k]),
AccumT(w2_all[w2_col_base + k]),
@@ -242,7 +237,7 @@ KERNEL(MoeExpertsDecodeDownFused2D)(
AccumT result = simd_sum(acc);
// Lane 0 writes result
- if (simd_lid == 0) {
+ if (simd.lane_idx == 0) {
const ulong bias_idx = (ulong)expert_idx * (ulong)d_model + (ulong)my_col;
result += AccumT(down_biases[bias_idx]);
diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal
index 6ff575bdc..6b5d54b9b 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal
@@ -68,7 +68,6 @@ KERNEL(MoeExpertsPrefillPassA)(
const uint3 threads_per_tg = {128, 1, 1};
const uint3 local_tid = {local_tid_x, 0, 0};
- const uint sg_id = simd.group_idx;
constexpr uint ROW_TILE = PASSA_BM;
const uint tile_seg_start = tile_map[base + 1u];
@@ -106,15 +105,15 @@ KERNEL(MoeExpertsPrefillPassA)(
// simdgroup tile mapping
const uint sg_col_count = Bn / SgBn;
- const uint row_sg = sg_id / sg_col_count;
- const uint col_sg = sg_id % sg_col_count;
+ const uint row_sg = simd.group_idx / sg_col_count;
+ const uint col_sg = simd.group_idx % sg_col_count;
const uint row_sg_off = row_sg * SgBm;
const uint col_sg_off = col_sg * SgBn;
// Guard against misconfigured TG sizes
constexpr uint SG_TILE = 8; // simdgroup fragment dimension (8x8)
constexpr uint SG_EXPECTED = (Bm / SG_TILE) * (Bn / SG_TILE);
- if (sg_id >= SG_EXPECTED)
+ if (simd.group_idx >= SG_EXPECTED)
return;
// partial accumulators (8x8 tiles per simdgroup)
@@ -295,7 +294,7 @@ KERNEL(MoeExpertsPrefillPassA)(
threadgroup_barrier(mem_flags::mem_threadgroup);
// ---- Epilogue: activation/gating → hidden_out (f32) ----
- const uint sg_lane_start = sg_id * 32;
+ const uint sg_lane_start = simd.group_idx * 32;
const uint sg_lane_end = sg_lane_start + 32;
if (lin < sg_lane_start || lin >= sg_lane_end)
return;
@@ -414,9 +413,6 @@ KERNEL(MoeExpertsPrefillPassB)(
if (expert_idx >= e)
return;
- const uint simd_lid = simd.lane_idx;
- const uint sg_id = simd.group_idx;
-
constexpr uint ROW_TILE = PASSB_BM;
const uint row_off_elems = tile_map[base + 2u];
const uint tile_m = row_off_elems / ROW_TILE;
@@ -449,14 +445,14 @@ KERNEL(MoeExpertsPrefillPassB)(
const ulong bias_base = (ulong)expert_idx * (ulong)d_model;
// 2×2 simdgroup layout for 16×64 output (each sg handles 8×32)
- const uint row_sg = sg_id / 2; // 0-1 → rows
- const uint col_sg = sg_id % 2; // 0-1 → cols
+ const uint row_sg = simd.group_idx / 2; // 0-1 → rows
+ const uint col_sg = simd.group_idx % 2; // 0-1 → cols
const uint row_sg_off = row_sg * SgBm;
const uint col_sg_off = col_sg * SgBn;
// Guard against excessive simdgroups
constexpr uint SG_EXPECTED = (PASSB_BM / SgBm) * (PASSB_BN / SgBn);
- if (sg_id >= SG_EXPECTED)
+ if (simd.group_idx >= SG_EXPECTED)
return;
// 4 accumulators per simdgroup: 1×4 layout of 8×8 tiles
@@ -564,9 +560,10 @@ KERNEL(MoeExpertsPrefillPassB)(
threadgroup_barrier(mem_flags::mem_threadgroup);
// ---- Writeout with fragment extraction ----
- const uint lane_qid = simd_lid >> 2;
- const uint lane_row = (lane_qid & 4u) + ((simd_lid >> 1) & 3u);
- const uint lane_col_base = ((lane_qid & 2u) << 1) + ((simd_lid & 1u) << 1);
+ const uint lane_qid = simd.lane_idx >> 2;
+ const uint lane_row = (lane_qid & 4u) + ((simd.lane_idx >> 1) & 3u);
+ const uint lane_col_base =
+ ((lane_qid & 2u) << 1) + ((simd.lane_idx & 1u) << 1);
const uint local_row = row_sg_off + lane_row;
if (local_row >= m_rows)
diff --git a/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal b/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal
index 4e01cedbd..a4fcca27e 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal
@@ -37,10 +37,6 @@ KERNEL(MoeRouterTopK)(
return;
}
- const uint simd_lane = simd.lane_idx;
- const uint simdgroup_idx = simd.group_idx;
- const uint simdgroups_per_tg = simd.groups_per_threadgroup;
-
const uint vecs = d_model / 4u;
const device ScalarT* x_vec = input + (ulong)token_idx * (ulong)vecs * 4;
@@ -54,11 +50,11 @@ KERNEL(MoeRouterTopK)(
}
threadgroup_barrier(mem_flags::mem_threadgroup);
- for (uint row = simdgroup_idx; row < e; row += simdgroups_per_tg) {
+ for (uint row = simd.group_idx; row < e; row += simd.groups_per_threadgroup) {
const device ScalarT* w_vec = weight + (ulong)row * (ulong)vecs * 4;
float4 accum4 = float4(0.0f);
- for (uint c = simd_lane; c < vecs; c += 32u) {
+ for (uint c = simd.lane_idx; c < vecs; c += 32u) {
const float4 wv = float4(
w_vec[c * 4 + 0],
w_vec[c * 4 + 1],
@@ -100,14 +96,16 @@ KERNEL(MoeRouterTopK)(
float max_val = threadgroup_cooperative_reduce_max<THREADS_PER_TG>(
local_best,
reduce_tmp,
- lid
+ lid,
+ simd
);
uint candidate_id = (local_best == max_val) ? local_idx : 0xFFFFFFFFu;
uint best_idx = threadgroup_cooperative_reduce_min<THREADS_PER_TG>(
candidate_id,
reduce_tmp_u,
- lid
+ lid,
+ simd
);
if (lid == 0) {
diff --git a/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal b/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal
index 3248e0a30..ea6d92a79 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal
@@ -28,15 +28,14 @@ inline void moe_scatter_buckets_impl(
uint lid,
uint3 tgpig,
threadgroup _atomic<uint>* sg_counts,
- threadgroup uint* sg_base
+ threadgroup uint* sg_base,
+ const thread Simd& simd
) {
if (E == 0 || T == 0 || K == 0)
return;
const uint block_id = tgpig.x;
const uint t_start = block_id * BLOCK_SIZE;
const uint t_end = min(t_start + BLOCK_SIZE, T);
- const uint sg_id = lid / SIMD_WIDTH;
- const uint lane = lid % SIMD_WIDTH;
// Per-tile processing
for (uint e0 = 0; e0 < E; e0 += TILE_E) {
@@ -65,7 +64,7 @@ inline void moe_scatter_buckets_impl(
if (ue >= e0 && ue < e0 + tile_e) {
const uint te = ue - e0;
atomic_fetch_add_explicit(
- &sg_counts[sg_id * TILE_E + te],
+ &sg_counts[simd.group_idx * TILE_E + te],
1u,
memory_order_relaxed
);
@@ -103,8 +102,8 @@ inline void moe_scatter_buckets_impl(
threadgroup_barrier(mem_flags::mem_threadgroup);
// Phase 3: deterministic write with 32-lane sequencing, no device atomics
- for (uint step = 0; step < SIMD_WIDTH; ++step) {
- if (lane == step) {
+ for (uint step = 0; step < simd.group_size; ++step) {
+ if (simd.lane_idx == step) {
const uint t = t_start + lid;
if (t < t_end) {
const uint base = t * K;
@@ -117,7 +116,7 @@ inline void moe_scatter_buckets_impl(
const uint tile_id = e0 / TILE_E;
const uint base_idx_block =
(block_id * num_tiles + tile_id) * TILE_E + te;
- const uint idx_sg_te = sg_id * TILE_E + te;
+ const uint idx_sg_te = simd.group_idx * TILE_E + te;
const uint local = atomic_fetch_add_explicit(
&sg_counts[idx_sg_te],
1u,
@@ -202,6 +201,7 @@ KERNEL(MoeScatterBuckets)(
constant uint& num_tiles,
threadgroup _atomic<uint> sg_counts[NUM_SG * TILE_E],
threadgroup uint sg_base[NUM_SG * TILE_E],
+ const Simd simd,
const uint tgpig_x GROUPS(num_blocks),
const uint lid THREADS(256)
) {
@@ -226,7 +226,8 @@ KERNEL(MoeScatterBuckets)(
lid,
tgpig,
sg_counts,
- sg_base
+ sg_base,
+ simd
);
}
@@ -249,6 +250,7 @@ KERNEL(MoeScatterBucketsMap)(
device int* tok2row,
threadgroup _atomic<uint> sg_counts[NUM_SG * TILE_E],
threadgroup uint sg_base[NUM_SG * TILE_E],
+ const Simd simd,
const uint tgpig_x GROUPS(num_blocks),
const uint lid THREADS(256)
) {
@@ -273,6 +275,7 @@ KERNEL(MoeScatterBucketsMap)(
lid,
tgpig,
sg_counts,
- sg_base
+ sg_base,
+ simd
);
}
\ No newline at end of file
diff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
index a744e065f..1da5b65bb 100644
--- a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
+++ b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
@@ -22,6 +22,7 @@ KERNEL(RMSNorm)(
constant float& scale_offset,
constant bool& full_layer,
threadgroup AccumT shared_sum[SIMD_SIZE],
+ const Simd simd,
const uint batch_idx GROUPS(batch_size),
const uint thread_in_row THREADS(1024)
) {
@@ -51,7 +52,8 @@ KERNEL(RMSNorm)(
AccumT total_sum = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(
partial_sum,
shared_sum,
- thread_in_row
+ thread_in_row,
+ simd
);
// Compute RMS norm factor
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal b/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal
index 2de31b445..3cff521c7 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal
@@ -26,11 +26,9 @@ template <ushort BLOCK_SIZE_PARAM>
static ArgmaxPair threadgroup_cooperative_argmax(
ArgmaxPair value,
threadgroup ArgmaxPair* shared,
- const ushort lid
+ const ushort lid,
+ const thread Simd& simd
) {
- const ushort simd_group_id = lid / 32;
- const ushort simd_lane_id = lid % 32;
-
// Reduce within simdgroup using manual shuffle operations
ArgmaxPair local_result = value;
for (ushort offset = 16; offset > 0; offset /= 2) {
@@ -42,8 +40,8 @@ static ArgmaxPair threadgroup_cooperative_argmax(
}
// First thread in each simdgroup writes to shared memory
- if (simd_lane_id == 0) {
- shared[simd_group_id] = local_result;
+ if (simd.lane_idx == 0) {
+ shared[simd.group_idx] = local_result;
}
// Synchronize across the threadgroup
@@ -205,6 +203,7 @@ KERNEL(ArgmaxMain)(
constant uint& batch_size,
constant uint& vocab_size,
threadgroup ArgmaxPair shared[BLOCK_SIZE],
+ const Simd simd,
uint batch_idx GROUPS(batch_size),
uint vocab_group_idx GROUPS(vocab_size.div_ceil(BLOCK_SIZE * GRAIN_SIZE)),
ushort local_id THREADS(BLOCK_SIZE)
@@ -230,7 +229,8 @@ KERNEL(ArgmaxMain)(
ArgmaxPair group_result = threadgroup_cooperative_argmax<BLOCK_SIZE>(
thread_result,
shared,
- local_id
+ local_id,
+ simd
);
if (local_id == 0) {
@@ -247,6 +247,7 @@ KERNEL(ArgmaxFinal)(
constant uint& batch_size,
constant uint& vocab_size,
threadgroup ArgmaxPair shared[BLOCK_SIZE],
+ const Simd simd,
uint batch_idx GROUPS(batch_size),
ushort local_id THREADS(BLOCK_SIZE)
) {
@@ -269,7 +270,8 @@ KERNEL(ArgmaxFinal)(
ArgmaxPair result = threadgroup_cooperative_argmax<BLOCK_SIZE>(
thread_result,
shared,
- local_id
+ local_id,
+ simd
);
if (local_id == 0) {
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/min_p.metal b/crates/uzu/src/backends/metal/kernel/sampling/min_p.metal
index aca8878fb..889dfae91 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/min_p.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/min_p.metal
@@ -13,6 +13,7 @@ KERNEL(MinP) (
constant uint& batch_size,
constant uint& vocab_size,
constant float& min_p,
+ const Simd simd,
uint batch_idx GROUPS(batch_size),
uint thread_idx THREADS(BLOCK_SIZE)
) {
@@ -28,7 +29,8 @@ KERNEL(MinP) (
float max_logit = threadgroup_cooperative_reduce_max<BLOCK_SIZE>(
local_max,
shared_reduce_buffer,
- thread_idx
+ thread_idx,
+ simd
);
// Then the threshold is just max_logit + log(min_p), mask everything strictly
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal b/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal
index bfefe0380..d040f3f0d 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal
@@ -7,13 +7,14 @@
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(TopK) (
+KERNEL(TopK)(
device const T* logits,
device T* processed_logits,
threadgroup float shared_reduce_buffer[BLOCK_SIZE_IN_SIMDS],
constant uint& batch_size,
constant uint& vocab_size,
constant uint& top_k,
+ const Simd simd,
uint batch_idx GROUPS(batch_size),
uint thread_idx THREADS(BLOCK_SIZE)
) {
@@ -35,12 +36,14 @@ KERNEL(TopK) (
float max_logit = threadgroup_cooperative_reduce_max<BLOCK_SIZE>(
local_max,
shared_reduce_buffer,
- thread_idx
+ thread_idx,
+ simd
);
float min_logit = threadgroup_cooperative_reduce_min<BLOCK_SIZE>(
local_min,
shared_reduce_buffer,
- thread_idx
+ thread_idx,
+ simd
);
// Do the binary search on the threshold
float low = min_logit;
@@ -58,7 +61,8 @@ KERNEL(TopK) (
uint num_above_threshold = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(
local_num_above_threshold,
(threadgroup uint*)shared_reduce_buffer,
- thread_idx
+ thread_idx,
+ simd
);
// Update binary search
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal b/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal
index b1c3bc481..b22c3eee5 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal
@@ -15,6 +15,7 @@ KERNEL(TopP) (
constant uint& batch_size,
constant uint& vocab_size,
constant float& top_p,
+ const Simd simd,
uint batch_idx GROUPS(batch_size),
uint thread_idx THREADS(BLOCK_SIZE)
) {
@@ -36,12 +37,14 @@ KERNEL(TopP) (
float max_logit = threadgroup_cooperative_reduce_max<BLOCK_SIZE>(
local_max,
shared_reduce_buffer,
- thread_idx
+ thread_idx,
+ simd
);
float min_logit = threadgroup_cooperative_reduce_min<BLOCK_SIZE>(
local_min,
shared_reduce_buffer,
- thread_idx
+ thread_idx,
+ simd
);
// Find denominator for softmax
@@ -58,7 +61,8 @@ KERNEL(TopP) (
float total_sum = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(
local_sum,
shared_reduce_buffer,
- thread_idx
+ thread_idx,
+ simd
);
// Do the binary search on the threshold
@@ -86,12 +90,14 @@ KERNEL(TopP) (
float sum_above_threshold = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(
local_sum_above_threshold,
shared_reduce_buffer,
- thread_idx
+ thread_idx,
+ simd
);
float min_above_threshold = threadgroup_cooperative_reduce_min<BLOCK_SIZE>(
local_min_above_threshold,
shared_reduce_buffer,
- thread_idx
+ thread_idx,
+ simd
);
// Early exit
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal b/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal
index f8517a6ba..904574cb5 100644
--- a/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal
+++ b/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal
@@ -27,10 +27,10 @@ KERNEL(SSDPrefill64)(
constant const uint* state_strides,
constant const uint& num_heads,
constant const uint& head_dim,
+ const Simd simd,
const uint pair_idx GROUPS(num_heads * head_dim),
const uint lane_idx THREADS(32)
) {
- const uint simd_width = 32;
const uint h_idx = pair_idx / head_dim;
const uint dh_idx = pair_idx % head_dim;
const uint safe_group = uint(max(group_size, 1));
@@ -55,7 +55,7 @@ KERNEL(SSDPrefill64)(
const float d_scalar = float(d[h_idx]);
const uint idx0 = lane_idx;
- const uint idx1 = lane_idx + simd_width;
+ const uint idx1 = lane_idx + simd.group_size;
float state0 = 0.0f;
float state1 = 0.0f;
@@ -125,10 +125,10 @@ KERNEL(SSDPrefill)(
constant const uint* state_strides,
constant const uint& num_heads,
constant const uint& head_dim,
+ const Simd simd,
const uint pair_idx GROUPS(num_heads * head_dim),
const uint lane_idx THREADS(32)
) {
- const uint simd_width = 32;
const int state_dim = state_size;
const uint h_idx = pair_idx / head_dim;
const uint dh_idx = pair_idx % head_dim;
@@ -153,8 +153,9 @@ KERNEL(SSDPrefill)(
const uint cb_group_base = group_idx * cb_group_stride;
const float d_scalar = float(d[h_idx]);
- const int max_chunks = SSM_PREFILL_MAX_STATE / simd_width;
- const int chunk_count = (state_dim + int(simd_width) - 1) / int(simd_width);
+ const int max_chunks = SSM_PREFILL_MAX_STATE / simd.group_size;
+ const int chunk_count =
+ (state_dim + int(simd.group_size) - 1) / int(simd.group_size);
if (state_dim <= 0 || chunk_count > max_chunks) {
return;
}
@@ -163,7 +164,7 @@ KERNEL(SSDPrefill)(
#pragma unroll
for (int chunk = 0; chunk < chunk_count; ++chunk) {
- const int idx = chunk * int(simd_width) + int(lane_idx);
+ const int idx = chunk * int(simd.group_size) + int(lane_idx);
const uint state_idx = state_base + idx * state_inner_stride;
lane_states[chunk] = float(state[state_idx]);
}
@@ -181,7 +182,7 @@ KERNEL(SSDPrefill)(
float contrib_sum = 0.0f;
#pragma unroll
for (int chunk = 0; chunk < chunk_count; ++chunk) {
- const int idx = chunk * int(simd_width) + int(lane_idx);
+ const int idx = chunk * int(simd.group_size) + int(lane_idx);
const uint cb_idx =
cb_group_base + uint(idx) * cb_state_stride + token * cb_token_stride;
const float new_state =
@@ -198,7 +199,7 @@ KERNEL(SSDPrefill)(
#pragma unroll
for (int chunk = 0; chunk < chunk_count; ++chunk) {
- const int idx = chunk * int(simd_width) + int(lane_idx);
+ const int idx = chunk * int(simd.group_size) + int(lane_idx);
const uint state_idx = state_base + uint(idx) * state_inner_stride;
state[state_idx] = static_cast<T>(lane_states[chunk]);
}