A high-performance inference engine for AI models
format
eugeneaac41e1875de
diff --git a/.clang-format b/.clang-formatnew file mode 100644index 000000000..ad70f0798--- /dev/null+++ b/.clang-format@@ -0,0 +1,26 @@+BasedOnStyle: LLVM+IndentWidth: 2+ContinuationIndentWidth: 4+ColumnLimit: 80++# Keep return type with function name+AlwaysBreakAfterReturnType: None+PenaltyReturnTypeOnItsOwnLine: 1000000++# One parameter/argument per line; indent as block under '('+BinPackParameters: false+BinPackArguments: false+AllowAllParametersOfDeclarationOnNextLine: false+AllowAllArgumentsOnNextLine: false+AlignAfterOpenBracket: BlockIndent++# Template closing bracket on own line+BreakTemplateDeclarations: Yes+Cpp11BracedListStyle: true++# Brace and pointer/reference style+BreakBeforeBraces: Attach+DerivePointerAlignment: false+PointerAlignment: Left++SortIncludes: falsediff --git a/.vscode/extensions.json b/.vscode/extensions.jsonindex 2d402c4e5..69837c6b5 100644--- a/.vscode/extensions.json+++ b/.vscode/extensions.json@@ -3,6 +3,7 @@"vadimcn.vscode-lldb","tamasfe.even-better-toml","doublebuffer.metal-shader",- "rust-lang.rust-analyzer"+ "rust-lang.rust-analyzer",+ "xaver.clang-format"]}\ No newline at end of filediff --git a/.vscode/settings.json b/.vscode/settings.jsonindex 953b78551..d767a8a34 100644--- a/.vscode/settings.json+++ b/.vscode/settings.json@@ -19,4 +19,23 @@"[toml]": {"editor.defaultFormatter": "tamasfe.even-better-toml"}+ // Treat Metal headers/sources as C++ for clang-format+ "files.associations": {+ "*.metal": "cpp",+ "*.h": "cpp",+ "*.hpp": "cpp"+ },+ // Use repository .clang-format+ "clang-format.style": "file",+ "C_Cpp.formatting": "clangFormat",+ "C_Cpp.clang_format_style": "file",+ // Default to MS C/C++ extension formatter for C/C++ (clang-format backend)+ "[cpp]": {+ "editor.defaultFormatter": "ms-vscode.cpptools",+ "editor.formatOnSave": true+ },+ "[c]": {+ "editor.defaultFormatter": "ms-vscode.cpptools",+ "editor.formatOnSave": true+ }}\ No newline at end of filediff --git a/crates/uzu/src/backends/metal/kernel/attention/attention.metal b/crates/uzu/src/backends/metal/kernel/attention/attention.metalindex 1b2aac913..271b97341 100644--- a/crates/uzu/src/backends/metal/kernel/attention/attention.metal+++ b/crates/uzu/src/backends/metal/kernel/attention/attention.metal@@ -30,138 +30,146 @@ void attention_single_pass_impl(const constant int& mask_q_seq_stride,const constant int& mask_head_stride,const device float* sinks,- uint3 tid, // threadgroup position in grid- uint3 tpg, // threadgroups per grid+ uint3 tid, // threadgroup position in grid+ uint3 tpg, // threadgroups per griduint simd_gid, // simdgroup index in threadgroupuint simd_lid, // thread index in simdgroupthreadgroup float* shared_max_scores,threadgroup float* shared_sum_exp_scores,threadgroup float* shared_outputs) {- constexpr int sequence_block_size = 32;- constexpr int head_block_size = 32;- constexpr int qk_elements_per_thread = head_dim / head_block_size;- constexpr int value_elements_per_thread = value_dim / head_block_size;- int inner_k_stride = sequence_block_size * int(k_seq_stride);- int inner_v_stride = sequence_block_size * int(v_seq_stride);-- typedef float U;-- thread U q[qk_elements_per_thread];- thread U k[qk_elements_per_thread];- thread U o[value_elements_per_thread];-- const int head_idx = tid.x;- const int q_seq_idx = tid.y;- const int kv_head_idx = head_idx / gqa_factor;- const int o_offset = q_seq_idx * tpg.x + head_idx;- const int q_offset = query_transposed ? tpg.x * q_seq_idx + head_idx : head_idx * tpg.y + q_seq_idx;-- queries += q_offset * head_dim + simd_lid * qk_elements_per_thread;- keys += kv_head_idx * k_head_stride + simd_gid * k_seq_stride + simd_lid * qk_elements_per_thread;- values += kv_head_idx * v_head_stride + simd_gid * v_seq_stride + simd_lid * value_elements_per_thread;-- if (bool_mask) {- bmask += head_idx * mask_head_stride + simd_gid * mask_kv_seq_stride + q_seq_idx * mask_q_seq_stride;- }- if (float_mask) {- fmask += head_idx * mask_head_stride + simd_gid * mask_kv_seq_stride + q_seq_idx * mask_q_seq_stride;+ constexpr int sequence_block_size = 32;+ constexpr int head_block_size = 32;+ constexpr int qk_elements_per_thread = head_dim / head_block_size;+ constexpr int value_elements_per_thread = value_dim / head_block_size;+ int inner_k_stride = sequence_block_size * int(k_seq_stride);+ int inner_v_stride = sequence_block_size * int(v_seq_stride);++ typedef float U;++ thread U q[qk_elements_per_thread];+ thread U k[qk_elements_per_thread];+ thread U o[value_elements_per_thread];++ const int head_idx = tid.x;+ const int q_seq_idx = tid.y;+ const int kv_head_idx = head_idx / gqa_factor;+ const int o_offset = q_seq_idx * tpg.x + head_idx;+ const int q_offset = query_transposed ? tpg.x * q_seq_idx + head_idx+ : head_idx * tpg.y + q_seq_idx;++ queries += q_offset * head_dim + simd_lid * qk_elements_per_thread;+ keys += kv_head_idx * k_head_stride + simd_gid * k_seq_stride ++ simd_lid * qk_elements_per_thread;+ values += kv_head_idx * v_head_stride + simd_gid * v_seq_stride ++ simd_lid * value_elements_per_thread;++ if (bool_mask) {+ bmask += head_idx * mask_head_stride + simd_gid * mask_kv_seq_stride ++ q_seq_idx * mask_q_seq_stride;+ }+ if (float_mask) {+ fmask += head_idx * mask_head_stride + simd_gid * mask_kv_seq_stride ++ q_seq_idx * mask_q_seq_stride;+ }++ out += o_offset * value_dim + simd_gid * value_elements_per_thread;++ // Read the query and 0 the output accumulator+ for (int i = 0; i < qk_elements_per_thread; i++) {+ q[i] = static_cast<U>(scale) * queries[i];+ }+ for (int i = 0; i < value_elements_per_thread; i++) {+ o[i] = 0;+ }++ U max_score = -INFINITY;+ U sum_exp_score = 0;+ if (has_sinks && simd_gid == 0) {+ const int num_q_heads = static_cast<int>(tpg.x);+ int q_head_idx = head_idx % num_q_heads;+ max_score = static_cast<U>(sinks[q_head_idx]);+ sum_exp_score = 1;+ }++ // For each key+ for (int i = simd_gid; i < sequence_length; i += sequence_block_size) {+ bool use_key = true;+ if (do_causal) {+ use_key = i <= (sequence_length - int(tpg.y) + int(q_seq_idx));+ } else if (bool_mask) {+ use_key = bmask[0];}- out += o_offset * value_dim + simd_gid * value_elements_per_thread;-- // Read the query and 0 the output accumulator- for (int i = 0; i < qk_elements_per_thread; i++) {- q[i] = static_cast<U>(scale) * queries[i];- }- for (int i = 0; i < value_elements_per_thread; i++) {- o[i] = 0;+ if (use_key) {+ // Read the key+ for (int j = 0; j < qk_elements_per_thread; j++) {+ k[j] = keys[j];+ }++ // Compute the i-th score+ U score = 0;+ for (int j = 0; j < qk_elements_per_thread; j++) {+ score += q[j] * k[j];+ }+ score = simd_sum(score);+ if (float_mask) {+ score += max(-1e9f, static_cast<U>(fmask[0]));+ }++ // Update the accumulators+ U new_max = max(max_score, score);+ U factor = fast::exp(max_score - new_max);+ U exp_score = fast::exp(score - new_max);++ max_score = new_max;+ sum_exp_score = sum_exp_score * factor + exp_score;++ // Update the output accumulator+ for (int j = 0; j < value_elements_per_thread; j++) {+ o[j] = o[j] * factor + exp_score * values[j];+ }}- U max_score = -INFINITY;- U sum_exp_score = 0;- if (has_sinks && simd_gid == 0) {- const int num_q_heads = static_cast<int>(tpg.x);- int q_head_idx = head_idx % num_q_heads;- max_score = static_cast<U>(sinks[q_head_idx]);- sum_exp_score = 1;- }-- // For each key- for (int i = simd_gid; i < sequence_length; i += sequence_block_size) {- bool use_key = true;- if (do_causal) {- use_key = i <= (sequence_length - int(tpg.y) + int(q_seq_idx));- } else if (bool_mask) {- use_key = bmask[0];- }-- if (use_key) {- // Read the key- for (int j = 0; j < qk_elements_per_thread; j++) {- k[j] = keys[j];- }-- // Compute the i-th score- U score = 0;- for (int j = 0; j < qk_elements_per_thread; j++) {- score += q[j] * k[j];- }- score = simd_sum(score);- if (float_mask) {- score += max(-1e9f, static_cast<U>(fmask[0]));- }-- // Update the accumulators- U new_max = max(max_score, score);- U factor = fast::exp(max_score - new_max);- U exp_score = fast::exp(score - new_max);-- max_score = new_max;- sum_exp_score = sum_exp_score * factor + exp_score;-- // Update the output accumulator- for (int j = 0; j < value_elements_per_thread; j++) {- o[j] = o[j] * factor + exp_score * values[j];- }- }-- // Move the pointers to the next kv- keys += inner_k_stride;- values += inner_v_stride;- if (bool_mask) {- bmask += sequence_block_size * mask_kv_seq_stride;- }- if (float_mask) {- fmask += sequence_block_size * mask_kv_seq_stride;- }+ // Move the pointers to the next kv+ keys += inner_k_stride;+ values += inner_v_stride;+ if (bool_mask) {+ bmask += sequence_block_size * mask_kv_seq_stride;}-- // Each thread has a partial part of the output so we need to combine them.- if (simd_lid == 0) {- shared_max_scores[simd_gid] = max_score;- shared_sum_exp_scores[simd_gid] = sum_exp_score;+ if (float_mask) {+ fmask += sequence_block_size * mask_kv_seq_stride;}+ }++ // Each thread has a partial part of the output so we need to combine them.+ if (simd_lid == 0) {+ shared_max_scores[simd_gid] = max_score;+ shared_sum_exp_scores[simd_gid] = sum_exp_score;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ max_score = shared_max_scores[simd_lid];+ U new_max = simd_max(max_score);+ U factor = fast::exp(max_score - new_max);+ sum_exp_score = simd_sum(shared_sum_exp_scores[simd_lid] * factor);++ // Now we need to aggregate all the outputs+ for (int i = 0; i < value_elements_per_thread; i++) {+ shared_outputs[simd_lid * head_block_size + simd_gid] = o[i];threadgroup_barrier(mem_flags::mem_threadgroup);- max_score = shared_max_scores[simd_lid];- U new_max = simd_max(max_score);- U factor = fast::exp(max_score - new_max);- sum_exp_score = simd_sum(shared_sum_exp_scores[simd_lid] * factor);+ o[i] = simd_sum(+ shared_outputs[simd_gid * head_block_size + simd_lid] * factor+ ) /+ sum_exp_score;+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }- // Now we need to aggregate all the outputs+ // And write the output+ if (simd_lid == 0) {for (int i = 0; i < value_elements_per_thread; i++) {- shared_outputs[simd_lid * head_block_size + simd_gid] = o[i];- threadgroup_barrier(mem_flags::mem_threadgroup);- o[i] = simd_sum(shared_outputs[simd_gid * head_block_size + simd_lid] * factor) / sum_exp_score;- threadgroup_barrier(mem_flags::mem_threadgroup);- }-- // And write the output- if (simd_lid == 0) {- for (int i = 0; i < value_elements_per_thread; i++) {- out[i] = static_cast<T>(o[i]);- }+ out[i] = static_cast<T>(o[i]);}+ }}template <typename T, int head_dim, int value_dim = head_dim>@@ -185,149 +193,165 @@ void attention_2pass_1_impl(const constant int& mask_q_seq_stride,const constant int& mask_head_stride,const device float* sinks,- uint3 tid, // threadgroup position in grid- uint3 tpg, // threadgroups per grid+ uint3 tid, // threadgroup position in grid+ uint3 tpg, // threadgroups per griduint simd_gid, // simdgroup index in threadgroupuint simd_lid, // thread index in simdgroupthreadgroup float* shared_max_scores,threadgroup float* shared_sum_exp_scores,threadgroup float* shared_outputs) {- constexpr int sequence_block_size = 8;- constexpr int head_block_size = 32;- constexpr int qk_elements_per_thread = head_dim / head_block_size;- constexpr int value_elements_per_thread = value_dim / head_block_size;- int inner_k_stride = sequence_block_size * int(k_seq_stride);- int inner_v_stride = sequence_block_size * int(v_seq_stride);- constexpr int total_blocks_count = 32;-- typedef float U;-- thread U q[qk_elements_per_thread];- thread U k[qk_elements_per_thread];- thread U o[value_elements_per_thread];-- const int block_idx = tid.z;- const int head_idx = tid.x;- const int q_seq_idx = tid.y;- const int o_offset = q_seq_idx * tpg.x + head_idx; // Our custom layout- const int q_offset = query_transposed ? tpg.x * q_seq_idx + head_idx : head_idx * tpg.y + q_seq_idx; // Consistent with single-pass- const int kv_head_idx = head_idx / gqa_factor;-- queries += q_offset * head_dim + simd_lid * qk_elements_per_thread;- keys += kv_head_idx * k_head_stride + (block_idx * sequence_block_size + simd_gid) * k_seq_stride + simd_lid * qk_elements_per_thread;- values += kv_head_idx * v_head_stride + (block_idx * sequence_block_size + simd_gid) * v_seq_stride + simd_lid * value_elements_per_thread;- out += o_offset * total_blocks_count * value_dim + block_idx * value_dim + simd_lid * value_elements_per_thread;-- if (bool_mask) {- bmask += head_idx * mask_head_stride + (block_idx * sequence_block_size + simd_gid) * mask_kv_seq_stride + q_seq_idx * mask_q_seq_stride;- }- if (float_mask) {- fmask += head_idx * mask_head_stride + (block_idx * sequence_block_size + simd_gid) * mask_kv_seq_stride + q_seq_idx * mask_q_seq_stride;- }- sums += o_offset * total_blocks_count + block_idx;- maxs += o_offset * total_blocks_count + block_idx;-- // Read the query and 0 the output accumulator- for (int i = 0; i < qk_elements_per_thread; i++) {- q[i] = static_cast<U>(scale) * queries[i];- }- for (int i = 0; i < value_elements_per_thread; i++) {- o[i] = 0;+ constexpr int sequence_block_size = 8;+ constexpr int head_block_size = 32;+ constexpr int qk_elements_per_thread = head_dim / head_block_size;+ constexpr int value_elements_per_thread = value_dim / head_block_size;+ int inner_k_stride = sequence_block_size * int(k_seq_stride);+ int inner_v_stride = sequence_block_size * int(v_seq_stride);+ constexpr int total_blocks_count = 32;++ typedef float U;++ thread U q[qk_elements_per_thread];+ thread U k[qk_elements_per_thread];+ thread U o[value_elements_per_thread];++ const int block_idx = tid.z;+ const int head_idx = tid.x;+ const int q_seq_idx = tid.y;+ const int o_offset = q_seq_idx * tpg.x + head_idx; // Our custom layout+ const int q_offset =+ query_transposed+ ? tpg.x * q_seq_idx + head_idx+ : head_idx * tpg.y + q_seq_idx; // Consistent with single-pass+ const int kv_head_idx = head_idx / gqa_factor;++ queries += q_offset * head_dim + simd_lid * qk_elements_per_thread;+ keys += kv_head_idx * k_head_stride ++ (block_idx * sequence_block_size + simd_gid) * k_seq_stride ++ simd_lid * qk_elements_per_thread;+ values += kv_head_idx * v_head_stride ++ (block_idx * sequence_block_size + simd_gid) * v_seq_stride ++ simd_lid * value_elements_per_thread;+ out += o_offset * total_blocks_count * value_dim + block_idx * value_dim ++ simd_lid * value_elements_per_thread;++ if (bool_mask) {+ bmask += head_idx * mask_head_stride ++ (block_idx * sequence_block_size + simd_gid) * mask_kv_seq_stride ++ q_seq_idx * mask_q_seq_stride;+ }+ if (float_mask) {+ fmask += head_idx * mask_head_stride ++ (block_idx * sequence_block_size + simd_gid) * mask_kv_seq_stride ++ q_seq_idx * mask_q_seq_stride;+ }+ sums += o_offset * total_blocks_count + block_idx;+ maxs += o_offset * total_blocks_count + block_idx;++ // Read the query and 0 the output accumulator+ for (int i = 0; i < qk_elements_per_thread; i++) {+ q[i] = static_cast<U>(scale) * queries[i];+ }+ for (int i = 0; i < value_elements_per_thread; i++) {+ o[i] = 0;+ }++ U max_score = -1e9;+ U sum_exp_score = 0;+ if (has_sinks && block_idx == 0 && simd_gid == 0) {+ const int num_q_heads = static_cast<int>(tpg.x);+ int q_head_idx = head_idx % num_q_heads;+ max_score = static_cast<U>(sinks[q_head_idx]);+ sum_exp_score = 1;+ }++ // For each key+ for (int i = block_idx * sequence_block_size + simd_gid; i < sequence_length;+ i += total_blocks_count * sequence_block_size) {+ bool use_key = true;+ if (do_causal) {+ use_key = i <= (sequence_length - int(tpg.y) + int(q_seq_idx));+ } else if (bool_mask) {+ use_key = bmask[0];}- U max_score = -1e9;- U sum_exp_score = 0;- if (has_sinks && block_idx == 0 && simd_gid == 0) {- const int num_q_heads = static_cast<int>(tpg.x);- int q_head_idx = head_idx % num_q_heads;- max_score = static_cast<U>(sinks[q_head_idx]);- sum_exp_score = 1;+ if (use_key) {+ // Read the key+ for (int j = 0; j < qk_elements_per_thread; j++) {+ k[j] = keys[j];+ }++ // Compute the i-th score+ U score = 0;+ for (int j = 0; j < qk_elements_per_thread; j++) {+ score += q[j] * k[j];+ }+ score = simd_sum(score);+ if (float_mask) {+ score += max(-1e9f, static_cast<U>(fmask[0]));+ }++ // Update the accumulators+ U new_max = max(max_score, score);+ U factor = fast::exp(max_score - new_max);+ U exp_score = fast::exp(score - new_max);++ max_score = new_max;+ sum_exp_score = sum_exp_score * factor + exp_score;++ // Update the output accumulator+ for (int j = 0; j < value_elements_per_thread; j++) {+ o[j] = o[j] * factor + exp_score * values[j];+ }}- // For each key- for (int i = block_idx * sequence_block_size + simd_gid; i < sequence_length; i += total_blocks_count * sequence_block_size) {- bool use_key = true;- if (do_causal) {- use_key = i <= (sequence_length - int(tpg.y) + int(q_seq_idx));- } else if (bool_mask) {- use_key = bmask[0];- }-- if (use_key) {- // Read the key- for (int j = 0; j < qk_elements_per_thread; j++) {- k[j] = keys[j];- }-- // Compute the i-th score- U score = 0;- for (int j = 0; j < qk_elements_per_thread; j++) {- score += q[j] * k[j];- }- score = simd_sum(score);- if (float_mask) {- score += max(-1e9f, static_cast<U>(fmask[0]));- }-- // Update the accumulators- U new_max = max(max_score, score);- U factor = fast::exp(max_score - new_max);- U exp_score = fast::exp(score - new_max);-- max_score = new_max;- sum_exp_score = sum_exp_score * factor + exp_score;-- // Update the output accumulator- for (int j = 0; j < value_elements_per_thread; j++) {- o[j] = o[j] * factor + exp_score * values[j];- }- }-- // Move the pointers to the next kv- keys += total_blocks_count * inner_k_stride;- values += total_blocks_count * inner_v_stride;- if (bool_mask) {- bmask += sequence_block_size * total_blocks_count * mask_kv_seq_stride;- }- if (float_mask) {- fmask += sequence_block_size * total_blocks_count * mask_kv_seq_stride;- }+ // Move the pointers to the next kv+ keys += total_blocks_count * inner_k_stride;+ values += total_blocks_count * inner_v_stride;+ if (bool_mask) {+ bmask += sequence_block_size * total_blocks_count * mask_kv_seq_stride;}-- // Each thread has a partial part of the output so we need to combine them.- if (simd_lid == 0) {- shared_max_scores[simd_gid] = max_score;- shared_sum_exp_scores[simd_gid] = sum_exp_score;+ if (float_mask) {+ fmask += sequence_block_size * total_blocks_count * mask_kv_seq_stride;}+ }++ // Each thread has a partial part of the output so we need to combine them.+ if (simd_lid == 0) {+ shared_max_scores[simd_gid] = max_score;+ shared_sum_exp_scores[simd_gid] = sum_exp_score;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ max_score =+ (simd_lid < sequence_block_size) ? shared_max_scores[simd_lid] : -1e9;+ U new_max = simd_max(max_score);+ U factor = fast::exp(max_score - new_max);+ sum_exp_score =+ (simd_lid < sequence_block_size) ? shared_sum_exp_scores[simd_lid] : 0;+ sum_exp_score = simd_sum(sum_exp_score * factor);++ // Write the sum and new max+ if (simd_gid == 0) {+ sums[0] = sum_exp_score;+ maxs[0] = new_max;+ }++ // Now we need to aggregate all the outputs+ for (int i = 0; i < value_elements_per_thread; i++) {+ shared_outputs[simd_lid * sequence_block_size + simd_gid] =+ o[i] * fast::exp(shared_max_scores[simd_gid] - new_max);threadgroup_barrier(mem_flags::mem_threadgroup);- max_score = (simd_lid < sequence_block_size) ? shared_max_scores[simd_lid] : -1e9;- U new_max = simd_max(max_score);- U factor = fast::exp(max_score - new_max);- sum_exp_score = (simd_lid < sequence_block_size) ? shared_sum_exp_scores[simd_lid] : 0;- sum_exp_score = simd_sum(sum_exp_score * factor);- // Write the sum and new max+ // And write the outputif (simd_gid == 0) {- sums[0] = sum_exp_score;- maxs[0] = new_max;- }-- // Now we need to aggregate all the outputs- for (int i = 0; i < value_elements_per_thread; i++) {- shared_outputs[simd_lid * sequence_block_size + simd_gid] = o[i] * fast::exp(shared_max_scores[simd_gid] - new_max);- threadgroup_barrier(mem_flags::mem_threadgroup);-- // And write the output- if (simd_gid == 0) {- U output = shared_outputs[simd_lid * sequence_block_size];- for (int j = 1; j < sequence_block_size; j++) {- output += shared_outputs[simd_lid * sequence_block_size + j];- }- out[i] = static_cast<T>(output);- }- threadgroup_barrier(mem_flags::mem_threadgroup);+ U output = shared_outputs[simd_lid * sequence_block_size];+ for (int j = 1; j < sequence_block_size; j++) {+ output += shared_outputs[simd_lid * sequence_block_size + j];+ }+ out[i] = static_cast<T>(output);}+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }}template <typename T, int head_dim>@@ -342,48 +366,57 @@ void attention_2pass_2_impl(uint simd_lid,threadgroup float* shared_outputs) {- constexpr int sequence_block_size = 32;- constexpr int head_block_size = 32;- constexpr int elements_per_thread = head_dim / head_block_size;- constexpr int total_blocks_count = 32;-- typedef float U;-- thread U o[elements_per_thread];-- const int head_idx = tid.x;- const int q_seq_idx = tid.y;- const int o_offset = q_seq_idx * tpg.x + head_idx; // Our custom layout- const int q_offset = query_transposed ? tpg.x * q_seq_idx + head_idx : head_idx * tpg.y + q_seq_idx; // Consistent with single-pass-- partials += o_offset * total_blocks_count * head_dim + simd_gid * head_dim + simd_lid * elements_per_thread;- sums += o_offset * total_blocks_count;- maxs += o_offset * total_blocks_count;- out += o_offset * head_dim + simd_gid * elements_per_thread; // Our custom output layout-- // First everybody reads the max and sum_exp- U max_score = maxs[simd_lid];- U new_max = simd_max(max_score);- U factor = fast::exp(max_score - new_max);- U sum_exp_score = simd_sum(sums[simd_lid] * factor);-- // Now read the block into registers and then use shared memory to transpose it- for (int i = 0; i < elements_per_thread; i++) {- o[i] = partials[i];- }- for (int i = 0; i < elements_per_thread; i++) {- shared_outputs[simd_lid * head_block_size + simd_gid] = o[i];- threadgroup_barrier(mem_flags::mem_threadgroup);- o[i] = simd_sum(shared_outputs[simd_gid * head_block_size + simd_lid] * factor) / sum_exp_score;- threadgroup_barrier(mem_flags::mem_threadgroup);- }+ constexpr int sequence_block_size = 32;+ constexpr int head_block_size = 32;+ constexpr int elements_per_thread = head_dim / head_block_size;+ constexpr int total_blocks_count = 32;++ typedef float U;++ thread U o[elements_per_thread];++ const int head_idx = tid.x;+ const int q_seq_idx = tid.y;+ const int o_offset = q_seq_idx * tpg.x + head_idx; // Our custom layout+ const int q_offset =+ query_transposed+ ? tpg.x * q_seq_idx + head_idx+ : head_idx * tpg.y + q_seq_idx; // Consistent with single-pass++ partials += o_offset * total_blocks_count * head_dim + simd_gid * head_dim ++ simd_lid * elements_per_thread;+ sums += o_offset * total_blocks_count;+ maxs += o_offset * total_blocks_count;+ out += o_offset * head_dim ++ simd_gid * elements_per_thread; // Our custom output layout++ // First everybody reads the max and sum_exp+ U max_score = maxs[simd_lid];+ U new_max = simd_max(max_score);+ U factor = fast::exp(max_score - new_max);+ U sum_exp_score = simd_sum(sums[simd_lid] * factor);++ // Now read the block into registers and then use shared memory to transpose+ // it+ for (int i = 0; i < elements_per_thread; i++) {+ o[i] = partials[i];+ }+ for (int i = 0; i < elements_per_thread; i++) {+ shared_outputs[simd_lid * head_block_size + simd_gid] = o[i];+ threadgroup_barrier(mem_flags::mem_threadgroup);+ o[i] = simd_sum(+ shared_outputs[simd_gid * head_block_size + simd_lid] * factor+ ) /+ sum_exp_score;+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }- // And write the output- if (simd_lid == 0) {- for (int i = 0; i < elements_per_thread; i++) {- out[i] = static_cast<T>(o[i]);- }+ // And write the output+ if (simd_lid == 0) {+ for (int i = 0; i < elements_per_thread; i++) {+ out[i] = static_cast<T>(o[i]);}+ }}template <typename T, int head_dim, int value_dim = head_dim>@@ -413,14 +446,32 @@ void attention_single_pass(threadgroup float* shared_sum_exp_scores,threadgroup float* shared_outputs) {- attention_single_pass_impl<T, head_dim, value_dim>(- queries, keys, values, out, gqa_factor, sequence_length,- k_head_stride, k_seq_stride, v_head_stride, v_seq_stride,- scale, bmask, fmask, mask_kv_seq_stride, mask_q_seq_stride, mask_head_stride,- sinks,- tid, tpg, simd_gid, simd_lid,- shared_max_scores, shared_sum_exp_scores, shared_outputs- );+ attention_single_pass_impl<T, head_dim, value_dim>(+ queries,+ keys,+ values,+ out,+ gqa_factor,+ sequence_length,+ k_head_stride,+ k_seq_stride,+ v_head_stride,+ v_seq_stride,+ scale,+ bmask,+ fmask,+ mask_kv_seq_stride,+ mask_q_seq_stride,+ mask_head_stride,+ sinks,+ tid,+ tpg,+ simd_gid,+ simd_lid,+ shared_max_scores,+ shared_sum_exp_scores,+ shared_outputs+ );}template <typename T, int head_dim, int value_dim = head_dim>@@ -452,14 +503,34 @@ void attention_2pass_1(threadgroup float* shared_sum_exp_scores,threadgroup float* shared_outputs) {- attention_2pass_1_impl<T, head_dim, value_dim>(- queries, keys, values, out, sums, maxs, gqa_factor, sequence_length,- k_head_stride, k_seq_stride, v_head_stride, v_seq_stride,- scale, bmask, fmask, mask_kv_seq_stride, mask_q_seq_stride, mask_head_stride,- sinks,- tid, tpg, simd_gid, simd_lid,- shared_max_scores, shared_sum_exp_scores, shared_outputs- );+ attention_2pass_1_impl<T, head_dim, value_dim>(+ queries,+ keys,+ values,+ out,+ sums,+ maxs,+ gqa_factor,+ sequence_length,+ k_head_stride,+ k_seq_stride,+ v_head_stride,+ v_seq_stride,+ scale,+ bmask,+ fmask,+ mask_kv_seq_stride,+ mask_q_seq_stride,+ mask_head_stride,+ sinks,+ tid,+ tpg,+ simd_gid,+ simd_lid,+ shared_max_scores,+ shared_sum_exp_scores,+ shared_outputs+ );}template <typename T, int head_dim>@@ -474,66 +545,106 @@ void attention_2pass_2(uint simd_lid,threadgroup float* shared_outputs) {- attention_2pass_2_impl<T, head_dim>(- partials, sums, maxs, out, tid, tpg, simd_gid, simd_lid, shared_outputs- );+ attention_2pass_2_impl<T, head_dim>(+ partials,+ sums,+ maxs,+ out,+ tid,+ tpg,+ simd_gid,+ simd_lid,+ shared_outputs+ );}-#define outerArguments(T) \-(const device T* queries [[ buffer(0) ]], \- const device T* keys [[ buffer(1) ]], \- const device T* values [[ buffer(2) ]], \- device T* out [[ buffer(3) ]], \- const constant int& gqa_factor [[ buffer(4) ]], \- const constant int& sequence_length [[ buffer(5) ]], \- const constant int& k_head_stride [[ buffer(6) ]], \- const constant int& k_seq_stride [[ buffer(7) ]], \- const constant int& v_head_stride [[ buffer(8) ]], \- const constant int& v_seq_stride [[ buffer(9) ]], \- const constant float& scale [[ buffer(10) ]], \- const device bool* bmask [[ buffer(11), function_constant(bool_mask) ]], \- const device T* fmask [[ buffer(12), function_constant(float_mask) ]], \- const constant int& mask_kv_seq_stride [[ buffer(13), function_constant(has_mask) ]], \- const constant int& mask_q_seq_stride [[ buffer(14), function_constant(has_mask) ]], \- const constant int& mask_head_stride [[ buffer(15), function_constant(has_mask) ]], \- const device float* sinks [[ buffer(16), function_constant(has_sinks) ]], \- uint3 tid [[ threadgroup_position_in_grid ]], \- uint3 tpg [[ threadgroups_per_grid ]], \- uint simd_gid [[ simdgroup_index_in_threadgroup ]], \- uint simd_lid [[ thread_index_in_simdgroup ]]) \--#define innerArguments \-(queries, keys, values, out, gqa_factor, sequence_length, k_head_stride, k_seq_stride, \- v_head_stride, v_seq_stride, scale, bmask, fmask, mask_kv_seq_stride, \- mask_q_seq_stride, mask_head_stride, sinks, tid, tpg, simd_gid, simd_lid, \- shared_max_scores, shared_sum_exp_scores, shared_outputs) \+#define outerArguments(T) \+ (const device T* queries [[buffer(0)]], \+ const device T* keys [[buffer(1)]], \+ const device T* values [[buffer(2)]], \+ device T* out [[buffer(3)]], \+ const constant int& gqa_factor [[buffer(4)]], \+ const constant int& sequence_length [[buffer(5)]], \+ const constant int& k_head_stride [[buffer(6)]], \+ const constant int& k_seq_stride [[buffer(7)]], \+ const constant int& v_head_stride [[buffer(8)]], \+ const constant int& v_seq_stride [[buffer(9)]], \+ const constant float& scale [[buffer(10)]], \+ const device bool* bmask [[buffer(11), function_constant(bool_mask)]], \+ const device T* fmask [[buffer(12), function_constant(float_mask)]], \+ const constant int& mask_kv_seq_stride \+ [[buffer(13), function_constant(has_mask)]], \+ const constant int& mask_q_seq_stride \+ [[buffer(14), function_constant(has_mask)]], \+ const constant int& mask_head_stride \+ [[buffer(15), function_constant(has_mask)]], \+ const device float* sinks [[buffer(16), function_constant(has_sinks)]], \+ uint3 tid [[threadgroup_position_in_grid]], \+ uint3 tpg [[threadgroups_per_grid]], \+ uint simd_gid [[simdgroup_index_in_threadgroup]], \+ uint simd_lid [[thread_index_in_simdgroup]])++#define innerArguments \+ (queries, \+ keys, \+ values, \+ out, \+ gqa_factor, \+ sequence_length, \+ k_head_stride, \+ k_seq_stride, \+ v_head_stride, \+ v_seq_stride, \+ scale, \+ bmask, \+ fmask, \+ mask_kv_seq_stride, \+ mask_q_seq_stride, \+ mask_head_stride, \+ sinks, \+ tid, \+ tpg, \+ simd_gid, \+ simd_lid, \+ shared_max_scores, \+ shared_sum_exp_scores, \+ shared_outputs)// Generate single-pass kernels for different head dimensions-#define GENERATE_SINGLE_PASS_KERNELS(head_dim_value) \-[[max_total_threads_per_threadgroup(1024)]] kernel void attention_single_pass_half_##head_dim_value outerArguments(half) { \- constexpr int sequence_block_size = 32; \- constexpr int head_block_size = 32; \- threadgroup float shared_max_scores[sequence_block_size]; \- threadgroup float shared_sum_exp_scores[sequence_block_size]; \- threadgroup float shared_outputs[sequence_block_size * head_block_size]; \- attention_single_pass<half, head_dim_value> innerArguments; \-} \-[[max_total_threads_per_threadgroup(1024)]] kernel void attention_single_pass_bfloat_##head_dim_value outerArguments(bfloat) { \- constexpr int sequence_block_size = 32; \- constexpr int head_block_size = 32; \- threadgroup float shared_max_scores[sequence_block_size]; \- threadgroup float shared_sum_exp_scores[sequence_block_size]; \- threadgroup float shared_outputs[sequence_block_size * head_block_size]; \- attention_single_pass<bfloat, head_dim_value> innerArguments; \-} \-[[max_total_threads_per_threadgroup(1024)]] kernel void attention_single_pass_float_##head_dim_value outerArguments(float) { \- constexpr int sequence_block_size = 32; \- constexpr int head_block_size = 32; \- threadgroup float shared_max_scores[sequence_block_size]; \- threadgroup float shared_sum_exp_scores[sequence_block_size]; \- threadgroup float shared_outputs[sequence_block_size * head_block_size]; \- attention_single_pass<float, head_dim_value> innerArguments; \-}+#define GENERATE_SINGLE_PASS_KERNELS(head_dim_value) \+ [[max_total_threads_per_threadgroup( \+ 1024 \+ )]] kernel void attention_single_pass_half_##head_dim_value \+ outerArguments(half) { \+ constexpr int sequence_block_size = 32; \+ constexpr int head_block_size = 32; \+ threadgroup float shared_max_scores[sequence_block_size]; \+ threadgroup float shared_sum_exp_scores[sequence_block_size]; \+ threadgroup float shared_outputs[sequence_block_size * head_block_size]; \+ attention_single_pass<half, head_dim_value> innerArguments; \+ } \+ [[max_total_threads_per_threadgroup( \+ 1024 \+ )]] kernel void attention_single_pass_bfloat_##head_dim_value \+ outerArguments(bfloat) { \+ constexpr int sequence_block_size = 32; \+ constexpr int head_block_size = 32; \+ threadgroup float shared_max_scores[sequence_block_size]; \+ threadgroup float shared_sum_exp_scores[sequence_block_size]; \+ threadgroup float shared_outputs[sequence_block_size * head_block_size]; \+ attention_single_pass<bfloat, head_dim_value> innerArguments; \+ } \+ [[max_total_threads_per_threadgroup( \+ 1024 \+ )]] kernel void attention_single_pass_float_##head_dim_value \+ outerArguments(float) { \+ constexpr int sequence_block_size = 32; \+ constexpr int head_block_size = 32; \+ threadgroup float shared_max_scores[sequence_block_size]; \+ threadgroup float shared_sum_exp_scores[sequence_block_size]; \+ threadgroup float shared_outputs[sequence_block_size * head_block_size]; \+ attention_single_pass<float, head_dim_value> innerArguments; \+ }GENERATE_SINGLE_PASS_KERNELS(64)GENERATE_SINGLE_PASS_KERNELS(128)@@ -542,63 +653,97 @@ GENERATE_SINGLE_PASS_KERNELS(256)#undef outerArguments#undef innerArguments-#define outerArguments(T) \-(const device T* queries [[ buffer(0) ]], \- const device T* keys [[ buffer(1) ]], \- const device T* values [[ buffer(2) ]], \- device float* out [[ buffer(3) ]], \- device float* sums [[ buffer(4) ]], \- device float* maxs [[ buffer(5) ]], \- const constant int& gqa_factor [[ buffer(6) ]], \- const constant int& sequence_length [[ buffer(7) ]], \- const constant int& k_head_stride [[ buffer(8) ]], \- const constant int& k_seq_stride [[ buffer(9) ]], \- const constant int& v_head_stride [[ buffer(10) ]], \- const constant int& v_seq_stride [[ buffer(11) ]], \- const constant float& scale [[ buffer(12) ]], \- const device bool* bmask [[ buffer(13), function_constant(bool_mask) ]], \- const device T* fmask [[ buffer(14), function_constant(float_mask) ]], \- const constant int& mask_kv_seq_stride [[ buffer(15), function_constant(has_mask) ]], \- const constant int& mask_q_seq_stride [[ buffer(16), function_constant(has_mask) ]], \- const constant int& mask_head_stride [[ buffer(17), function_constant(has_mask) ]], \- const device float* sinks [[ buffer(18), function_constant(has_sinks) ]], \- uint3 tid [[ threadgroup_position_in_grid ]], \- uint3 tpg [[ threadgroups_per_grid ]], \- uint simd_gid [[ simdgroup_index_in_threadgroup ]], \- uint simd_lid [[ thread_index_in_simdgroup ]]) \--#define innerArguments \-(queries, keys, values, out, sums, maxs, gqa_factor, sequence_length, k_head_stride, k_seq_stride, \- v_head_stride, v_seq_stride, scale, bmask, fmask, mask_kv_seq_stride, \- mask_q_seq_stride, mask_head_stride, sinks, tid, tpg, simd_gid, simd_lid, \- shared_max_scores, shared_sum_exp_scores, shared_outputs) \+#define outerArguments(T) \+ (const device T* queries [[buffer(0)]], \+ const device T* keys [[buffer(1)]], \+ const device T* values [[buffer(2)]], \+ device float* out [[buffer(3)]], \+ device float* sums [[buffer(4)]], \+ device float* maxs [[buffer(5)]], \+ const constant int& gqa_factor [[buffer(6)]], \+ const constant int& sequence_length [[buffer(7)]], \+ const constant int& k_head_stride [[buffer(8)]], \+ const constant int& k_seq_stride [[buffer(9)]], \+ const constant int& v_head_stride [[buffer(10)]], \+ const constant int& v_seq_stride [[buffer(11)]], \+ const constant float& scale [[buffer(12)]], \+ const device bool* bmask [[buffer(13), function_constant(bool_mask)]], \+ const device T* fmask [[buffer(14), function_constant(float_mask)]], \+ const constant int& mask_kv_seq_stride \+ [[buffer(15), function_constant(has_mask)]], \+ const constant int& mask_q_seq_stride \+ [[buffer(16), function_constant(has_mask)]], \+ const constant int& mask_head_stride \+ [[buffer(17), function_constant(has_mask)]], \+ const device float* sinks [[buffer(18), function_constant(has_sinks)]], \+ uint3 tid [[threadgroup_position_in_grid]], \+ uint3 tpg [[threadgroups_per_grid]], \+ uint simd_gid [[simdgroup_index_in_threadgroup]], \+ uint simd_lid [[thread_index_in_simdgroup]])++#define innerArguments \+ (queries, \+ keys, \+ values, \+ out, \+ sums, \+ maxs, \+ gqa_factor, \+ sequence_length, \+ k_head_stride, \+ k_seq_stride, \+ v_head_stride, \+ v_seq_stride, \+ scale, \+ bmask, \+ fmask, \+ mask_kv_seq_stride, \+ mask_q_seq_stride, \+ mask_head_stride, \+ sinks, \+ tid, \+ tpg, \+ simd_gid, \+ simd_lid, \+ shared_max_scores, \+ shared_sum_exp_scores, \+ shared_outputs)// Generate 2-pass Pass 1 kernels for different head dimensions-#define GENERATE_2PASS_1_KERNELS(head_dim_value) \-[[max_total_threads_per_threadgroup(256)]] kernel void attention_2pass_1_half_##head_dim_value outerArguments(half) { \- constexpr int sequence_block_size = 8; \- constexpr int head_block_size = 32; \- threadgroup float shared_max_scores[sequence_block_size]; \- threadgroup float shared_sum_exp_scores[sequence_block_size]; \- threadgroup float shared_outputs[sequence_block_size * head_block_size]; \- attention_2pass_1<half, head_dim_value> innerArguments; \-} \-[[max_total_threads_per_threadgroup(256)]] kernel void attention_2pass_1_bfloat_##head_dim_value outerArguments(bfloat) { \- constexpr int sequence_block_size = 8; \- constexpr int head_block_size = 32; \- threadgroup float shared_max_scores[sequence_block_size]; \- threadgroup float shared_sum_exp_scores[sequence_block_size]; \- threadgroup float shared_outputs[sequence_block_size * head_block_size]; \- attention_2pass_1<bfloat, head_dim_value> innerArguments; \-} \-[[max_total_threads_per_threadgroup(256)]] kernel void attention_2pass_1_float_##head_dim_value outerArguments(float) { \- constexpr int sequence_block_size = 8; \- constexpr int head_block_size = 32; \- threadgroup float shared_max_scores[sequence_block_size]; \- threadgroup float shared_sum_exp_scores[sequence_block_size]; \- threadgroup float shared_outputs[sequence_block_size * head_block_size]; \- attention_2pass_1<float, head_dim_value> innerArguments; \-}+#define GENERATE_2PASS_1_KERNELS(head_dim_value) \+ [[max_total_threads_per_threadgroup( \+ 256 \+ )]] kernel void attention_2pass_1_half_##head_dim_value \+ outerArguments(half) { \+ constexpr int sequence_block_size = 8; \+ constexpr int head_block_size = 32; \+ threadgroup float shared_max_scores[sequence_block_size]; \+ threadgroup float shared_sum_exp_scores[sequence_block_size]; \+ threadgroup float shared_outputs[sequence_block_size * head_block_size]; \+ attention_2pass_1<half, head_dim_value> innerArguments; \+ } \+ [[max_total_threads_per_threadgroup( \+ 256 \+ )]] kernel void attention_2pass_1_bfloat_##head_dim_value \+ outerArguments(bfloat) { \+ constexpr int sequence_block_size = 8; \+ constexpr int head_block_size = 32; \+ threadgroup float shared_max_scores[sequence_block_size]; \+ threadgroup float shared_sum_exp_scores[sequence_block_size]; \+ threadgroup float shared_outputs[sequence_block_size * head_block_size]; \+ attention_2pass_1<bfloat, head_dim_value> innerArguments; \+ } \+ [[max_total_threads_per_threadgroup( \+ 256 \+ )]] kernel void attention_2pass_1_float_##head_dim_value \+ outerArguments(float) { \+ constexpr int sequence_block_size = 8; \+ constexpr int head_block_size = 32; \+ threadgroup float shared_max_scores[sequence_block_size]; \+ threadgroup float shared_sum_exp_scores[sequence_block_size]; \+ threadgroup float shared_outputs[sequence_block_size * head_block_size]; \+ attention_2pass_1<float, head_dim_value> innerArguments; \+ }GENERATE_2PASS_1_KERNELS(64)GENERATE_2PASS_1_KERNELS(128)@@ -607,33 +752,39 @@ GENERATE_2PASS_1_KERNELS(256)#undef outerArguments#undef innerArguments-#define outerArguments(T) \-(const device float* partials [[ buffer(0) ]], \- const device float* sums [[ buffer(1) ]], \- const device float* maxs [[ buffer(2) ]], \- device T* out [[ buffer(3) ]], \- uint3 tid [[ threadgroup_position_in_grid ]], \- uint3 tpg [[ threadgroups_per_grid ]], \- uint simd_gid [[ simdgroup_index_in_threadgroup ]], \- uint simd_lid [[ thread_index_in_simdgroup ]]) \+#define outerArguments(T) \+ (const device float* partials [[buffer(0)]], \+ const device float* sums [[buffer(1)]], \+ const device float* maxs [[buffer(2)]], \+ device T* out [[buffer(3)]], \+ uint3 tid [[threadgroup_position_in_grid]], \+ uint3 tpg [[threadgroups_per_grid]], \+ uint simd_gid [[simdgroup_index_in_threadgroup]], \+ uint simd_lid [[thread_index_in_simdgroup]])-#define innerArguments \-(partials, sums, maxs, out, tid, tpg, simd_gid, simd_lid, shared_outputs) \+#define innerArguments \+ (partials, sums, maxs, out, tid, tpg, simd_gid, simd_lid, shared_outputs)// Generate 2-pass Pass 2 kernels for different head dimensions-#define GENERATE_2PASS_2_KERNELS(head_dim_value) \-[[max_total_threads_per_threadgroup(1024)]] kernel void attention_2pass_2_half_##head_dim_value outerArguments(half) { \- constexpr int sequence_block_size = 32; \- constexpr int head_block_size = 32; \- threadgroup float shared_outputs[sequence_block_size * head_block_size]; \- attention_2pass_2<half, head_dim_value> innerArguments; \-} \-[[max_total_threads_per_threadgroup(1024)]] kernel void attention_2pass_2_float_##head_dim_value outerArguments(float) { \- constexpr int sequence_block_size = 32; \- constexpr int head_block_size = 32; \- threadgroup float shared_outputs[sequence_block_size * head_block_size]; \- attention_2pass_2<float, head_dim_value> innerArguments; \-}+#define GENERATE_2PASS_2_KERNELS(head_dim_value) \+ [[max_total_threads_per_threadgroup( \+ 1024 \+ )]] kernel void attention_2pass_2_half_##head_dim_value \+ outerArguments(half) { \+ constexpr int sequence_block_size = 32; \+ constexpr int head_block_size = 32; \+ threadgroup float shared_outputs[sequence_block_size * head_block_size]; \+ attention_2pass_2<half, head_dim_value> innerArguments; \+ } \+ [[max_total_threads_per_threadgroup( \+ 1024 \+ )]] kernel void attention_2pass_2_float_##head_dim_value \+ outerArguments(float) { \+ constexpr int sequence_block_size = 32; \+ constexpr int head_block_size = 32; \+ threadgroup float shared_outputs[sequence_block_size * head_block_size]; \+ attention_2pass_2<float, head_dim_value> innerArguments; \+ }GENERATE_2PASS_2_KERNELS(64)GENERATE_2PASS_2_KERNELS(128)@@ -644,66 +795,85 @@ GENERATE_2PASS_2_KERNELS(256)template <typename T>void update_kv_cache(- const device T* rotated_keys, // [num_groups, suffix_length, head_dim]- const device T* qkv, // [suffix_length, (num_heads + 2*num_groups) * head_dim]- device T* key_cache, // [num_groups, max_sequence_length, head_dim]- device T* value_cache, // [num_groups, max_sequence_length, head_dim]+ const device T* rotated_keys, // [num_groups, suffix_length, head_dim]+ const device T*+ qkv, // [suffix_length, (num_heads + 2*num_groups) * head_dim]+ device T* key_cache, // [num_groups, max_sequence_length, head_dim]+ device T* value_cache, // [num_groups, max_sequence_length, head_dim]const constant int& num_groups,const constant int& num_heads,const constant int& head_dim,const constant int& suffix_length,const constant int& prefix_segment_length,const constant int& max_sequence_length,- uint3 position // x: group_idx, y: token_idx, z: dim_idx+ uint3 position // x: group_idx, y: token_idx, z: dim_idx) {- const uint groupIndex = position.x;- const uint tokenIndex = position.y;- const uint dimIndex = position.z;-- if (groupIndex >= num_groups || tokenIndex >= suffix_length || dimIndex >= head_dim) {- return;- }-- TensorView3D<const T> rotatedKeysTensorView = TensorView3D<const T>(rotated_keys, num_groups, suffix_length, head_dim);- TensorView3D<T> keyCacheTensorView = TensorView3D<T>(key_cache, num_groups, max_sequence_length, head_dim);- TensorView3D<T> valueCacheTensorView = TensorView3D<T>(value_cache, num_groups, max_sequence_length, head_dim);-- // Copy rotated key to cache- keyCacheTensorView(groupIndex, prefix_segment_length + tokenIndex, dimIndex) = rotatedKeysTensorView(groupIndex, tokenIndex, dimIndex);-- // Update value cache (only first thread in each token processes values to avoid redundant work)- if (dimIndex == 0) {- const uint totalQueryDim = num_heads * head_dim;- const uint totalKeyValueDim = num_groups * head_dim;-- const int qkvStride = totalQueryDim + 2 * totalKeyValueDim;- TensorView2D<const T> qkvTensorView = TensorView2D<const T>(qkv, suffix_length, qkvStride);-- // Extract values from QKV tensor- // Values start at offset: total_query_dim + total_key_value_dim- for (uint d = 0; d < head_dim; d++) {- const uint valueOffset = totalQueryDim + totalKeyValueDim + groupIndex * head_dim + d;- valueCacheTensorView(groupIndex, prefix_segment_length + tokenIndex, d) = qkvTensorView(tokenIndex, valueOffset);- }+ const uint groupIndex = position.x;+ const uint tokenIndex = position.y;+ const uint dimIndex = position.z;++ if (groupIndex >= num_groups || tokenIndex >= suffix_length ||+ dimIndex >= head_dim) {+ return;+ }++ TensorView3D<const T> rotatedKeysTensorView =+ TensorView3D<const T>(rotated_keys, num_groups, suffix_length, head_dim);+ TensorView3D<T> keyCacheTensorView =+ TensorView3D<T>(key_cache, num_groups, max_sequence_length, head_dim);+ TensorView3D<T> valueCacheTensorView =+ TensorView3D<T>(value_cache, num_groups, max_sequence_length, head_dim);++ // Copy rotated key to cache+ keyCacheTensorView(groupIndex, prefix_segment_length + tokenIndex, dimIndex) =+ rotatedKeysTensorView(groupIndex, tokenIndex, dimIndex);++ // Update value cache (only first thread in each token processes values to+ // avoid redundant work)+ if (dimIndex == 0) {+ const uint totalQueryDim = num_heads * head_dim;+ const uint totalKeyValueDim = num_groups * head_dim;++ const int qkvStride = totalQueryDim + 2 * totalKeyValueDim;+ TensorView2D<const T> qkvTensorView =+ TensorView2D<const T>(qkv, suffix_length, qkvStride);++ // Extract values from QKV tensor+ // Values start at offset: total_query_dim + total_key_value_dim+ for (uint d = 0; d < head_dim; d++) {+ const uint valueOffset =+ totalQueryDim + totalKeyValueDim + groupIndex * head_dim + d;+ valueCacheTensorView(groupIndex, prefix_segment_length + tokenIndex, d) =+ qkvTensorView(tokenIndex, valueOffset);}+ }}-#define outerArguments(T) \-(const device T* rotated_keys [[ buffer(0) ]], \- const device T* qkv [[ buffer(1) ]], \- device T* key_cache [[ buffer(2) ]], \- device T* value_cache [[ buffer(3) ]], \- const constant int& num_groups [[ buffer(4) ]], \- const constant int& num_heads [[ buffer(5) ]], \- const constant int& head_dim [[ buffer(6) ]], \- const constant int& suffix_length [[ buffer(7) ]], \- const constant int& prefix_segment_length [[ buffer(8) ]], \- const constant int& max_sequence_length [[ buffer(9) ]], \- uint3 position [[ thread_position_in_grid ]]) \--#define innerArguments \-(rotated_keys, qkv, key_cache, value_cache, num_groups, num_heads, \- head_dim, suffix_length, prefix_segment_length, max_sequence_length, position) \+#define outerArguments(T) \+ (const device T* rotated_keys [[buffer(0)]], \+ const device T* qkv [[buffer(1)]], \+ device T* key_cache [[buffer(2)]], \+ device T* value_cache [[buffer(3)]], \+ const constant int& num_groups [[buffer(4)]], \+ const constant int& num_heads [[buffer(5)]], \+ const constant int& head_dim [[buffer(6)]], \+ const constant int& suffix_length [[buffer(7)]], \+ const constant int& prefix_segment_length [[buffer(8)]], \+ const constant int& max_sequence_length [[buffer(9)]], \+ uint3 position [[thread_position_in_grid]])++#define innerArguments \+ (rotated_keys, \+ qkv, \+ key_cache, \+ value_cache, \+ num_groups, \+ num_heads, \+ head_dim, \+ suffix_length, \+ prefix_segment_length, \+ max_sequence_length, \+ position)generateKernels(update_kv_cache)diff --git a/crates/uzu/src/backends/metal/kernel/definitions.metal b/crates/uzu/src/backends/metal/kernel/definitions.metalindex 399c0ae51..972ccd1c7 100644--- a/crates/uzu/src/backends/metal/kernel/definitions.metal+++ b/crates/uzu/src/backends/metal/kernel/definitions.metal@@ -7,426 +7,458 @@ using namespace metal;// MARK: - Tensor Access Templates-template<typename T>+template <typename T>struct TensorView2D {- device T* data;- uint dim0, dim1;- uint stride0, stride1;-- TensorView2D(device T* ptr) : data(ptr) {}-- TensorView2D(device T* ptr, int d0, int d1)- : data(ptr), dim0(d0), dim1(d1) {- stride0 = d1;- stride1 = 1;- }-- // Custom stride constructor- TensorView2D(device T* ptr, int d0, int d1, int s0, int s1)- : data(ptr), dim0(d0), dim1(d1), stride0(s0), stride1(s1) {}-- thread TensorView2D& shaped(int d0, int d1) {- dim0 = d0;- dim1 = d1;- stride0 = d1;- stride1 = 1;- return *this;- }-- device T& at(uint i, uint j) const {- return data[i * stride0 + j * stride1];- }-- device T& operator()(uint i, uint j) const {- return at(i, j);- }+ device T* data;+ uint dim0, dim1;+ uint stride0, stride1;++ TensorView2D(device T* ptr) : data(ptr) {}++ TensorView2D(device T* ptr, int d0, int d1) : data(ptr), dim0(d0), dim1(d1) {+ stride0 = d1;+ stride1 = 1;+ }++ // Custom stride constructor+ TensorView2D(device T* ptr, int d0, int d1, int s0, int s1)+ : data(ptr), dim0(d0), dim1(d1), stride0(s0), stride1(s1) {}++ thread TensorView2D& shaped(int d0, int d1) {+ dim0 = d0;+ dim1 = d1;+ stride0 = d1;+ stride1 = 1;+ return *this;+ }++ device T& at(uint i, uint j) const { return data[i * stride0 + j * stride1]; }++ device T& operator()(uint i, uint j) const { return at(i, j); }};-template<typename T>+template <typename T>struct TensorView3D {- device T* data;- uint dim0, dim1, dim2;- uint stride0, stride1, stride2;-- TensorView3D(device T* ptr) : data(ptr) {}-- TensorView3D(device T* ptr, int d0, int d1, int d2)- : data(ptr), dim0(d0), dim1(d1), dim2(d2) {- stride0 = d1 * d2;- stride1 = d2;- stride2 = 1;- }-- // Custom stride constructor- TensorView3D(device T* ptr, int d0, int d1, int d2, int s0, int s1, int s2)- : data(ptr), dim0(d0), dim1(d1), dim2(d2), stride0(s0), stride1(s1), stride2(s2) {}-- thread TensorView3D& shaped(int d0, int d1, int d2) {- dim0 = d0;- dim1 = d1;- dim2 = d2;- stride0 = d1 * d2;- stride1 = d2;- stride2 = 1;- return *this;- }-- device T& at(uint i, uint j, uint k) const {- return data[i * stride0 + j * stride1 + k * stride2];- }-- device T& operator()(uint i, uint j, uint k) const {- return at(i, j, k);- }+ device T* data;+ uint dim0, dim1, dim2;+ uint stride0, stride1, stride2;++ TensorView3D(device T* ptr) : data(ptr) {}++ TensorView3D(device T* ptr, int d0, int d1, int d2)+ : data(ptr), dim0(d0), dim1(d1), dim2(d2) {+ stride0 = d1 * d2;+ stride1 = d2;+ stride2 = 1;+ }++ // Custom stride constructor+ TensorView3D(device T* ptr, int d0, int d1, int d2, int s0, int s1, int s2)+ : data(ptr), dim0(d0), dim1(d1), dim2(d2), stride0(s0), stride1(s1),+ stride2(s2) {}++ thread TensorView3D& shaped(int d0, int d1, int d2) {+ dim0 = d0;+ dim1 = d1;+ dim2 = d2;+ stride0 = d1 * d2;+ stride1 = d2;+ stride2 = 1;+ return *this;+ }++ device T& at(uint i, uint j, uint k) const {+ return data[i * stride0 + j * stride1 + k * stride2];+ }++ device T& operator()(uint i, uint j, uint k) const { return at(i, j, k); }};-template<typename T>+template <typename T>struct TensorView4D {- device T* data;- uint dim0, dim1, dim2, dim3;- uint stride0, stride1, stride2, stride3;-- TensorView4D(device T* ptr) : data(ptr) {}-- TensorView4D(device T* ptr, int d0, int d1, int d2, int d3)- : data(ptr), dim0(d0), dim1(d1), dim2(d2), dim3(d3) {- stride0 = d1 * d2 * d3;- stride1 = d2 * d3;- stride2 = d3;- stride3 = 1;- }-- // Custom stride constructor- TensorView4D(device T* ptr, int d0, int d1, int d2, int d3, int s0, int s1, int s2, int s3)- : data(ptr), dim0(d0), dim1(d1), dim2(d2), dim3(d3), stride0(s0), stride1(s1), stride2(s2), stride3(s3) {}-- thread TensorView4D& shaped(int d0, int d1, int d2, int d3) {- dim0 = d0;- dim1 = d1;- dim2 = d2;- dim3 = d3;- stride0 = d1 * d2 * d3;- stride1 = d2 * d3;- stride2 = d3;- stride3 = 1;- return *this;- }-- device T& at(uint i, uint j, uint k, uint l) const {- return data[i * stride0 + j * stride1 + k * stride2 + l * stride3];- }-- device T& operator()(uint i, uint j, uint k, uint l) const {- return at(i, j, k, l);- }+ device T* data;+ uint dim0, dim1, dim2, dim3;+ uint stride0, stride1, stride2, stride3;++ TensorView4D(device T* ptr) : data(ptr) {}++ TensorView4D(device T* ptr, int d0, int d1, int d2, int d3)+ : data(ptr), dim0(d0), dim1(d1), dim2(d2), dim3(d3) {+ stride0 = d1 * d2 * d3;+ stride1 = d2 * d3;+ stride2 = d3;+ stride3 = 1;+ }++ // Custom stride constructor+ TensorView4D(+ device T* ptr,+ int d0,+ int d1,+ int d2,+ int d3,+ int s0,+ int s1,+ int s2,+ int s3+ )+ : data(ptr), dim0(d0), dim1(d1), dim2(d2), dim3(d3), stride0(s0),+ stride1(s1), stride2(s2), stride3(s3) {}++ thread TensorView4D& shaped(int d0, int d1, int d2, int d3) {+ dim0 = d0;+ dim1 = d1;+ dim2 = d2;+ dim3 = d3;+ stride0 = d1 * d2 * d3;+ stride1 = d2 * d3;+ stride2 = d3;+ stride3 = 1;+ return *this;+ }++ device T& at(uint i, uint j, uint k, uint l) const {+ return data[i * stride0 + j * stride1 + k * stride2 + l * stride3];+ }++ device T& operator()(uint i, uint j, uint k, uint l) const {+ return at(i, j, k, l);+ }};///////////////////////////////////////////////////////////////////////////////// MARK: - Thread Functions///////////////////////////////////////////////////////////////////////////////-template<ushort LENGTH, typename T>-static inline T thread_prefix_inclusive_sum(thread T (&values)[LENGTH]){- for (ushort i = 1; i < LENGTH; i++){- values[i] += values[i - 1];- }- return values[LENGTH - 1];+template <ushort LENGTH, typename T>+static inline T thread_prefix_inclusive_sum(thread T (&values)[LENGTH]) {+ for (ushort i = 1; i < LENGTH; i++) {+ values[i] += values[i - 1];+ }+ return values[LENGTH - 1];}-template<ushort LENGTH, typename T>-static inline T thread_prefix_inclusive_sum(threadgroup T* values){- for (ushort i = 1; i < LENGTH; i++){- values[i] += values[i - 1];- }- return values[LENGTH - 1];+template <ushort LENGTH, typename T>+static inline T thread_prefix_inclusive_sum(threadgroup T* values) {+ for (ushort i = 1; i < LENGTH; i++) {+ values[i] += values[i - 1];+ }+ return values[LENGTH - 1];}--template<ushort LENGTH, typename T>-static inline T thread_prefix_exclusive_sum(thread T (&values)[LENGTH]){- // do as an inclusive scan first- T inclusive_prefix = thread_prefix_inclusive_sum<LENGTH>(values);- // convert to an exclusive scan in the reverse direction- for (ushort i = LENGTH - 1; i > 0; i--){- values[i] = values[i - 1];- }- values[0] = 0;- return inclusive_prefix;+template <ushort LENGTH, typename T>+static inline T thread_prefix_exclusive_sum(thread T (&values)[LENGTH]) {+ // do as an inclusive scan first+ T inclusive_prefix = thread_prefix_inclusive_sum<LENGTH>(values);+ // convert to an exclusive scan in the reverse direction+ for (ushort i = LENGTH - 1; i > 0; i--) {+ values[i] = values[i - 1];+ }+ values[0] = 0;+ return inclusive_prefix;}-template<ushort LENGTH, typename T>-static inline T thread_prefix_exclusive_sum(threadgroup T* values){- // do as an inclusive scan first- T inclusive_prefix = thread_prefix_inclusive_sum<LENGTH>(values);- // convert to an exclusive scan in the reverse direction- for (ushort i = LENGTH - 1; i > 0; i--){- values[i] = values[i - 1];- }- values[0] = 0;- return inclusive_prefix;+template <ushort LENGTH, typename T>+static inline T thread_prefix_exclusive_sum(threadgroup T* values) {+ // do as an inclusive scan first+ T inclusive_prefix = thread_prefix_inclusive_sum<LENGTH>(values);+ // convert to an exclusive scan in the reverse direction+ for (ushort i = LENGTH - 1; i > 0; i--) {+ values[i] = values[i - 1];+ }+ values[0] = 0;+ return inclusive_prefix;}--template<ushort LENGTH, typename T> static inline void-thread_uniform_add(thread T (&values)[LENGTH], T uni){- for (ushort i = 0; i < LENGTH; i++){- values[i] += uni;- }+template <ushort LENGTH, typename T>+static inline void thread_uniform_add(thread T (&values)[LENGTH], T uni) {+ for (ushort i = 0; i < LENGTH; i++) {+ values[i] += uni;+ }}-template<ushort LENGTH, typename T> static inline void-thread_uniform_add(threadgroup T* values, T uni){- for (ushort i = 0; i < LENGTH; i++){- values[i] += uni;- }+template <ushort LENGTH, typename T>+static inline void thread_uniform_add(threadgroup T* values, T uni) {+ for (ushort i = 0; i < LENGTH; i++) {+ values[i] += uni;+ }}-///////////////////////////////////////////////////////////////////////////////// MARK: - Threadgroup Functions/////////////////////////////////////////////////////////////////////////////////------------------------------------------------------------------------------------------------//// Raking threadgroup scan-template<ushort BLOCK_SIZE, typename T> static T-threadgroup_raking_prefix_exclusive_sum(T value, threadgroup T* shared, const ushort lid) {- // load values into shared memory- shared[lid] = value;- threadgroup_barrier(mem_flags::mem_threadgroup);-- // only the first 32 threads form the rake- if (lid < 32) {- // scan by thread in shared mem- const short values_per_thread = BLOCK_SIZE / 32;- const short first_index = lid * values_per_thread;- for (short i = first_index + 1; i < first_index + values_per_thread; i++){- shared[i] += shared[i - 1];- }- T partial_sum = shared[first_index + values_per_thread - 1];- for (short i = first_index + values_per_thread - 1; i > first_index; i--){- shared[i] = shared[i - 1];- }- shared[first_index] = 0;-- // scan the partial sums- T prefix = simd_prefix_exclusive_sum(partial_sum);-- // add back the prefix- for (short i = first_index; i < first_index + values_per_thread; i++){- shared[i] += prefix;- }-+template <ushort BLOCK_SIZE, typename T>+static T threadgroup_raking_prefix_exclusive_sum(+ T value,+ threadgroup T* shared,+ const ushort lid+) {+ // load values into shared memory+ shared[lid] = value;+ threadgroup_barrier(mem_flags::mem_threadgroup);++ // only the first 32 threads form the rake+ if (lid < 32) {+ // scan by thread in shared mem+ const short values_per_thread = BLOCK_SIZE / 32;+ const short first_index = lid * values_per_thread;+ for (short i = first_index + 1; i < first_index + values_per_thread; i++) {+ shared[i] += shared[i - 1];+ }+ T partial_sum = shared[first_index + values_per_thread - 1];+ for (short i = first_index + values_per_thread - 1; i > first_index; i--) {+ shared[i] = shared[i - 1];}- threadgroup_barrier(mem_flags::mem_threadgroup);+ shared[first_index] = 0;- return shared[lid];+ // scan the partial sums+ T prefix = simd_prefix_exclusive_sum(partial_sum);++ // add back the prefix+ for (short i = first_index; i < first_index + values_per_thread; i++) {+ shared[i] += prefix;+ }+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);++ return shared[lid];}//------------------------------------------------------------------------------------------------//// Raking threadgroup sum reduction-template<ushort BLOCK_SIZE, typename T> static T-threadgroup_raking_reduce_sum(T value, threadgroup T* shared, const ushort lid) {- // load values into shared memory- shared[lid] = value;- threadgroup_barrier(mem_flags::mem_threadgroup);-- // only the first 32 threads form the rake- if (lid < 32) {- // reduce by thread in shared mem- const short values_per_thread = BLOCK_SIZE / 32;- const short first_index = lid * values_per_thread;-- T thread_sum = shared[first_index];- for (short i = first_index + 1; i < first_index + values_per_thread; i++){- thread_sum += shared[i];- }-- // reduce the partial sums using SIMD- T total_sum = simd_sum(thread_sum);-- // broadcast result back to shared memory- if (lid == 0) {- shared[0] = total_sum;- }+template <ushort BLOCK_SIZE, typename T>+static T threadgroup_raking_reduce_sum(+ T value,+ threadgroup T* shared,+ const ushort lid+) {+ // load values into shared memory+ shared[lid] = value;+ threadgroup_barrier(mem_flags::mem_threadgroup);++ // only the first 32 threads form the rake+ if (lid < 32) {+ // reduce by thread in shared mem+ const short values_per_thread = BLOCK_SIZE / 32;+ const short first_index = lid * values_per_thread;++ T thread_sum = shared[first_index];+ for (short i = first_index + 1; i < first_index + values_per_thread; i++) {+ thread_sum += shared[i];+ }++ // reduce the partial sums using SIMD+ T total_sum = simd_sum(thread_sum);++ // broadcast result back to shared memory+ if (lid == 0) {+ shared[0] = total_sum;}+ }- threadgroup_barrier(mem_flags::mem_threadgroup);+ threadgroup_barrier(mem_flags::mem_threadgroup);- return shared[0];+ return shared[0];}//------------------------------------------------------------------------------------------------//-// Raking threadgroup max reduction-template<ushort BLOCK_SIZE, typename T> static T-threadgroup_raking_reduce_max(T value, threadgroup T* shared, const ushort lid) {- // load values into shared memory- shared[lid] = value;- threadgroup_barrier(mem_flags::mem_threadgroup);-- // only the first 32 threads form the rake- if (lid < 32) {- // reduce by thread in shared mem- const short values_per_thread = BLOCK_SIZE / 32;- const short first_index = lid * values_per_thread;-- T thread_max = shared[first_index];- for (short i = first_index + 1; i < first_index + values_per_thread; i++){- thread_max = max(thread_max, shared[i]);- }-- // reduce the partial maxes using SIMD- T total_max = simd_max(thread_max);-- // broadcast result back to shared memory- if (lid == 0) {- shared[0] = total_max;- }+// Raking threadgroup max reduction+template <ushort BLOCK_SIZE, typename T>+static T threadgroup_raking_reduce_max(+ T value,+ threadgroup T* shared,+ const ushort lid+) {+ // load values into shared memory+ shared[lid] = value;+ threadgroup_barrier(mem_flags::mem_threadgroup);++ // only the first 32 threads form the rake+ if (lid < 32) {+ // reduce by thread in shared mem+ const short values_per_thread = BLOCK_SIZE / 32;+ const short first_index = lid * values_per_thread;++ T thread_max = shared[first_index];+ for (short i = first_index + 1; i < first_index + values_per_thread; i++) {+ thread_max = max(thread_max, shared[i]);}- threadgroup_barrier(mem_flags::mem_threadgroup);+ // reduce the partial maxes using SIMD+ T total_max = simd_max(thread_max);- return shared[0];+ // broadcast result back to shared memory+ if (lid == 0) {+ shared[0] = total_max;+ }+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ return shared[0];}//------------------------------------------------------------------------------------------------//-// Raking threadgroup min reduction-template<ushort BLOCK_SIZE, typename T> static T-threadgroup_raking_reduce_min(T value, threadgroup T* shared, const ushort lid) {- // load values into shared memory- shared[lid] = value;- threadgroup_barrier(mem_flags::mem_threadgroup);-- // only the first 32 threads form the rake- if (lid < 32) {- // reduce by thread in shared mem- const short values_per_thread = BLOCK_SIZE / 32;- const short first_index = lid * values_per_thread;-- T thread_min = shared[first_index];- for (short i = first_index + 1; i < first_index + values_per_thread; i++){- thread_min = min(thread_min, shared[i]);- }-- // reduce the partial mins using SIMD- T total_min = simd_min(thread_min);-- // broadcast result back to shared memory- if (lid == 0) {- shared[0] = total_min;- }+// Raking threadgroup min reduction+template <ushort BLOCK_SIZE, typename T>+static T threadgroup_raking_reduce_min(+ T value,+ threadgroup T* shared,+ const ushort lid+) {+ // load values into shared memory+ shared[lid] = value;+ threadgroup_barrier(mem_flags::mem_threadgroup);++ // only the first 32 threads form the rake+ if (lid < 32) {+ // reduce by thread in shared mem+ const short values_per_thread = BLOCK_SIZE / 32;+ const short first_index = lid * values_per_thread;++ T thread_min = shared[first_index];+ for (short i = first_index + 1; i < first_index + values_per_thread; i++) {+ thread_min = min(thread_min, shared[i]);}- threadgroup_barrier(mem_flags::mem_threadgroup);- return shared[0];+ // reduce the partial mins using SIMD+ T total_min = simd_min(thread_min);++ // broadcast result back to shared memory+ if (lid == 0) {+ shared[0] = total_min;+ }+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);++ return shared[0];}//------------------------------------------------------------------------------------------------//// Cooperative threadgroup sum reduction (2-level hierarchical)-template<ushort BLOCK_SIZE, typename T> static T-threadgroup_cooperative_reduce_sum(T value, threadgroup T* shared, const ushort lid) {- const ushort simd_group_id = lid / 32;- const ushort simd_lane_id = lid % 32;-- // Reduce within simdgroup- T local_sum = simd_sum(value);-- // First thread in each simdgroup writes to shared memory- if (simd_lane_id == 0) {- shared[simd_group_id] = local_sum;- }-- // Synchronize across the threadgroup- threadgroup_barrier(mem_flags::mem_threadgroup);-- // Reduce across simdgroups- T total_sum = T(0);- const ushort num_simd_groups = (BLOCK_SIZE + 31) / 32;- if (lid < num_simd_groups) {- total_sum = shared[lid];- }- total_sum = simd_sum(total_sum);-- // Broadcast the result to all threads- if (lid == 0) {- shared[0] = total_sum;- }- threadgroup_barrier(mem_flags::mem_threadgroup);-- return shared[0];+template <ushort BLOCK_SIZE, typename T>+static T threadgroup_cooperative_reduce_sum(+ T value,+ threadgroup T* shared,+ const ushort lid+) {+ const ushort simd_group_id = lid / 32;+ const ushort simd_lane_id = lid % 32;++ // Reduce within simdgroup+ T local_sum = simd_sum(value);++ // First thread in each simdgroup writes to shared memory+ if (simd_lane_id == 0) {+ shared[simd_group_id] = local_sum;+ }++ // Synchronize across the threadgroup+ threadgroup_barrier(mem_flags::mem_threadgroup);++ // Reduce across simdgroups+ T total_sum = T(0);+ const ushort num_simd_groups = (BLOCK_SIZE + 31) / 32;+ if (lid < num_simd_groups) {+ total_sum = shared[lid];+ }+ total_sum = simd_sum(total_sum);++ // Broadcast the result to all threads+ if (lid == 0) {+ shared[0] = total_sum;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);++ return shared[0];}//------------------------------------------------------------------------------------------------//// Cooperative threadgroup max reduction (2-level hierarchical)-template<ushort BLOCK_SIZE, typename T> static T-threadgroup_cooperative_reduce_max(T value, threadgroup T* shared, const ushort lid) {- const ushort simd_group_id = lid / 32;- const ushort simd_lane_id = lid % 32;-- // Reduce within simdgroup- T local_max = simd_max(value);-- // First thread in each simdgroup writes to shared memory- if (simd_lane_id == 0) {- shared[simd_group_id] = local_max;- }-- // Synchronize across the threadgroup- threadgroup_barrier(mem_flags::mem_threadgroup);-- // Reduce across simdgroups- T total_max = (lid < ((BLOCK_SIZE + 31) / 32)) ? shared[lid] : T(-INFINITY);- total_max = simd_max(total_max);-- // Broadcast the result to all threads- if (lid == 0) {- shared[0] = total_max;- }- threadgroup_barrier(mem_flags::mem_threadgroup);-- return shared[0];+template <ushort BLOCK_SIZE, typename T>+static T threadgroup_cooperative_reduce_max(+ T value,+ threadgroup T* shared,+ const ushort lid+) {+ const ushort simd_group_id = lid / 32;+ const ushort simd_lane_id = lid % 32;++ // Reduce within simdgroup+ T local_max = simd_max(value);++ // First thread in each simdgroup writes to shared memory+ if (simd_lane_id == 0) {+ shared[simd_group_id] = local_max;+ }++ // Synchronize across the threadgroup+ threadgroup_barrier(mem_flags::mem_threadgroup);++ // Reduce across simdgroups+ T total_max = (lid < ((BLOCK_SIZE + 31) / 32)) ? shared[lid] : T(-INFINITY);+ total_max = simd_max(total_max);++ // Broadcast the result to all threads+ if (lid == 0) {+ shared[0] = total_max;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);++ return shared[0];}//------------------------------------------------------------------------------------------------//// Cooperative threadgroup min reduction (2-level hierarchical)-template<ushort BLOCK_SIZE, typename T> static T-threadgroup_cooperative_reduce_min(T value, threadgroup T* shared, const ushort lid) {- const ushort simd_group_id = lid / 32;- const ushort simd_lane_id = lid % 32;-- // Reduce within simdgroup- T local_min = simd_min(value);-- // First thread in each simdgroup writes to shared memory- if (simd_lane_id == 0) {- shared[simd_group_id] = local_min;- }-- // Synchronize across the threadgroup- threadgroup_barrier(mem_flags::mem_threadgroup);-- // Reduce across simdgroups- T total_min = (lid < ((BLOCK_SIZE + 31) / 32)) ? shared[lid] : T(INFINITY);- total_min = simd_min(total_min);-- // Broadcast the result to all threads- if (lid == 0) {- shared[0] = total_min;- }- threadgroup_barrier(mem_flags::mem_threadgroup);-- return shared[0];+template <ushort BLOCK_SIZE, typename T>+static T threadgroup_cooperative_reduce_min(+ T value,+ threadgroup T* shared,+ const ushort lid+) {+ const ushort simd_group_id = lid / 32;+ const ushort simd_lane_id = lid % 32;++ // Reduce within simdgroup+ T local_min = simd_min(value);++ // First thread in each simdgroup writes to shared memory+ if (simd_lane_id == 0) {+ shared[simd_group_id] = local_min;+ }++ // Synchronize across the threadgroup+ threadgroup_barrier(mem_flags::mem_threadgroup);++ // Reduce across simdgroups+ T total_min = (lid < ((BLOCK_SIZE + 31) / 32)) ? shared[lid] : T(INFINITY);+ total_min = simd_min(total_min);++ // Broadcast the result to all threads+ if (lid == 0) {+ shared[0] = total_min;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);++ return shared[0];}// MARK: - Generate Template Kernels-#define generateKernel(functionName, scalarType, outerArgs, innerArgs) \-kernel void functionName##_##scalarType outerArgs { \- functionName innerArgs; \-}--#define generateKernels(functionName) \-generateKernel(functionName, float, outerArguments(float), innerArguments); \-generateKernel(functionName, bfloat, outerArguments(bfloat), innerArguments); \-generateKernel(functionName, half, outerArguments(half), innerArguments);+#define generateKernel(functionName, scalarType, outerArgs, innerArgs) \+ kernel void functionName##_##scalarType outerArgs { functionName innerArgs; }++#define generateKernels(functionName) \+ generateKernel(functionName, float, outerArguments(float), innerArguments); \+ generateKernel( \+ functionName, \+ bfloat, \+ outerArguments(bfloat), \+ innerArguments \+ ); \+ generateKernel(functionName, half, outerArguments(half), innerArguments);#endif /* definitions_metal */diff --git a/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal b/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metalindex 76b5b3612..a1e2f9e8a 100644--- a/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal+++ b/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal@@ -1,63 +1,63 @@#include <metal_stdlib>using namespace metal;-template<typename T, uint PACKING_DIVISOR, bool SIGNED_STORAGE>+template <typename T, uint PACKING_DIVISOR, bool SIGNED_STORAGE>[[kernel]] void quantized_embedding_lookup(- const device uint64_t* token_ids [[buffer(0)]], // [batch_size]- const device uint8_t* weights [[buffer(1)]], // [vocab_size, model_dim/packing_divisor] packed- const device T* scales [[buffer(2)]], // [vocab_size, num_groups]- const device T* biases [[buffer(3)]], // [vocab_size, num_groups]- device T* output [[buffer(4)]], // [batch_size, model_dim]+ const device uint64_t* token_ids [[buffer(0)]], // [batch_size]+ const device uint8_t* weights+ [[buffer(1)]], // [vocab_size, model_dim/packing_divisor] packed+ const device T* scales [[buffer(2)]], // [vocab_size, num_groups]+ const device T* biases [[buffer(3)]], // [vocab_size, num_groups]+ device T* output [[buffer(4)]], // [batch_size, model_dim]constant uint32_t& batch_size [[buffer(5)]],constant uint32_t& vocab_size [[buffer(6)]],constant uint32_t& model_dim [[buffer(7)]],constant uint32_t& group_size [[buffer(8)]],uint thread_position_in_grid [[thread_position_in_grid]]) {- const uint batch_idx = thread_position_in_grid / model_dim;- const uint dim_idx = thread_position_in_grid % model_dim;-- if (batch_idx >= batch_size) return;-- const uint64_t token_id = token_ids[batch_idx];- if (token_id >= vocab_size) {- output[thread_position_in_grid] = T(0);- return;- }-- const uint group_idx = dim_idx / group_size;- const uint num_groups = (model_dim + group_size - 1) / group_size;-- const T scale = scales[token_id * num_groups + group_idx];- const T bias = biases[token_id * num_groups + group_idx];-- const uint weights_stride = model_dim / PACKING_DIVISOR;- const device uint8_t* weights_u8 = weights;- const device int8_t* weights_i8 =- reinterpret_cast<const device int8_t*>(weights);-- int quantized_value = 0;- if (PACKING_DIVISOR == 2) {- const uint byte_idx =- token_id * weights_stride + (dim_idx / 2);- const uint8_t packed = weights_u8[byte_idx];- if ((dim_idx & 1) == 0) {- quantized_value = int(packed & 0x0F);- } else {- quantized_value = int((packed >> 4) & 0x0F);- }+ const uint batch_idx = thread_position_in_grid / model_dim;+ const uint dim_idx = thread_position_in_grid % model_dim;++ if (batch_idx >= batch_size)+ return;++ const uint64_t token_id = token_ids[batch_idx];+ if (token_id >= vocab_size) {+ output[thread_position_in_grid] = T(0);+ return;+ }++ const uint group_idx = dim_idx / group_size;+ const uint num_groups = (model_dim + group_size - 1) / group_size;++ const T scale = scales[token_id * num_groups + group_idx];+ const T bias = biases[token_id * num_groups + group_idx];++ const uint weights_stride = model_dim / PACKING_DIVISOR;+ const device uint8_t* weights_u8 = weights;+ const device int8_t* weights_i8 =+ reinterpret_cast<const device int8_t*>(weights);++ int quantized_value = 0;+ if (PACKING_DIVISOR == 2) {+ const uint byte_idx = token_id * weights_stride + (dim_idx / 2);+ const uint8_t packed = weights_u8[byte_idx];+ if ((dim_idx & 1) == 0) {+ quantized_value = int(packed & 0x0F);} else {- const uint elem_idx = token_id * weights_stride + dim_idx;- quantized_value =- SIGNED_STORAGE ? int(weights_i8[elem_idx])- : int(weights_u8[elem_idx]);+ quantized_value = int((packed >> 4) & 0x0F);}+ } else {+ const uint elem_idx = token_id * weights_stride + dim_idx;+ quantized_value =+ SIGNED_STORAGE ? int(weights_i8[elem_idx]) : int(weights_u8[elem_idx]);+ }- output[thread_position_in_grid] = scale * T(quantized_value) + bias;+ output[thread_position_in_grid] = scale * T(quantized_value) + bias;}-template [[host_name("quantized_embedding_lookup_f32_uint4")]]-[[kernel]] void quantized_embedding_lookup<float, 2, false>(+template [[host_name("quantized_embedding_lookup_f32_uint4")]] [[kernel]] void+quantized_embedding_lookup<float, 2, false>(const device uint64_t* token_ids [[buffer(0)]],const device uint8_t* weights [[buffer(1)]],const device float* scales [[buffer(2)]],@@ -70,8 +70,8 @@ template [[host_name("quantized_embedding_lookup_f32_uint4")]]uint thread_position_in_grid [[thread_position_in_grid]]);-template [[host_name("quantized_embedding_lookup_f16_uint4")]]-[[kernel]] void quantized_embedding_lookup<half, 2, false>(+template [[host_name("quantized_embedding_lookup_f16_uint4")]] [[kernel]] void+quantized_embedding_lookup<half, 2, false>(const device uint64_t* token_ids [[buffer(0)]],const device uint8_t* weights [[buffer(1)]],const device half* scales [[buffer(2)]],@@ -84,8 +84,8 @@ template [[host_name("quantized_embedding_lookup_f16_uint4")]]uint thread_position_in_grid [[thread_position_in_grid]]);-template [[host_name("quantized_embedding_lookup_bf16_uint4")]]-[[kernel]] void quantized_embedding_lookup<bfloat, 2, false>(+template [[host_name("quantized_embedding_lookup_bf16_uint4")]] [[kernel]] void+quantized_embedding_lookup<bfloat, 2, false>(const device uint64_t* token_ids [[buffer(0)]],const device uint8_t* weights [[buffer(1)]],const device bfloat* scales [[buffer(2)]],@@ -98,8 +98,8 @@ template [[host_name("quantized_embedding_lookup_bf16_uint4")]]uint thread_position_in_grid [[thread_position_in_grid]]);-template [[host_name("quantized_embedding_lookup_f32_int8")]]-[[kernel]] void quantized_embedding_lookup<float, 1, true>(+template [[host_name("quantized_embedding_lookup_f32_int8")]] [[kernel]] void+quantized_embedding_lookup<float, 1, true>(const device uint64_t* token_ids [[buffer(0)]],const device uint8_t* weights [[buffer(1)]],const device float* scales [[buffer(2)]],@@ -112,8 +112,8 @@ template [[host_name("quantized_embedding_lookup_f32_int8")]]uint thread_position_in_grid [[thread_position_in_grid]]);-template [[host_name("quantized_embedding_lookup_f16_int8")]]-[[kernel]] void quantized_embedding_lookup<half, 1, true>(+template [[host_name("quantized_embedding_lookup_f16_int8")]] [[kernel]] void+quantized_embedding_lookup<half, 1, true>(const device uint64_t* token_ids [[buffer(0)]],const device uint8_t* weights [[buffer(1)]],const device half* scales [[buffer(2)]],@@ -126,8 +126,8 @@ template [[host_name("quantized_embedding_lookup_f16_int8")]]uint thread_position_in_grid [[thread_position_in_grid]]);-template [[host_name("quantized_embedding_lookup_bf16_int8")]]-[[kernel]] void quantized_embedding_lookup<bfloat, 1, true>(+template [[host_name("quantized_embedding_lookup_bf16_int8")]] [[kernel]] void+quantized_embedding_lookup<bfloat, 1, true>(const device uint64_t* token_ids [[buffer(0)]],const device uint8_t* weights [[buffer(1)]],const device bfloat* scales [[buffer(2)]],@@ -140,8 +140,8 @@ template [[host_name("quantized_embedding_lookup_bf16_int8")]]uint thread_position_in_grid [[thread_position_in_grid]]);-template [[host_name("quantized_embedding_lookup_f32_uint8")]]-[[kernel]] void quantized_embedding_lookup<float, 1, false>(+template [[host_name("quantized_embedding_lookup_f32_uint8")]] [[kernel]] void+quantized_embedding_lookup<float, 1, false>(const device uint64_t* token_ids [[buffer(0)]],const device uint8_t* weights [[buffer(1)]],const device float* scales [[buffer(2)]],@@ -154,8 +154,8 @@ template [[host_name("quantized_embedding_lookup_f32_uint8")]]uint thread_position_in_grid [[thread_position_in_grid]]);-template [[host_name("quantized_embedding_lookup_f16_uint8")]]-[[kernel]] void quantized_embedding_lookup<half, 1, false>(+template [[host_name("quantized_embedding_lookup_f16_uint8")]] [[kernel]] void+quantized_embedding_lookup<half, 1, false>(const device uint64_t* token_ids [[buffer(0)]],const device uint8_t* weights [[buffer(1)]],const device half* scales [[buffer(2)]],@@ -168,8 +168,8 @@ template [[host_name("quantized_embedding_lookup_f16_uint8")]]uint thread_position_in_grid [[thread_position_in_grid]]);-template [[host_name("quantized_embedding_lookup_bf16_uint8")]]-[[kernel]] void quantized_embedding_lookup<bfloat, 1, false>(+template [[host_name("quantized_embedding_lookup_bf16_uint8")]] [[kernel]] void+quantized_embedding_lookup<bfloat, 1, false>(const device uint64_t* token_ids [[buffer(0)]],const device uint8_t* weights [[buffer(1)]],const device bfloat* scales [[buffer(2)]],diff --git a/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal b/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metalindex 110921510..c4b0b52f6 100644--- a/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal+++ b/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal@@ -2,14 +2,10 @@#include "../definitions.metal"template <typename T>-void swap(- device T* buffer,- const int sourceIdx,- const int destIdx-) {- const T temp = buffer[sourceIdx];- buffer[sourceIdx] = buffer[destIdx];- buffer[destIdx] = temp;+void swap(device T* buffer, const int sourceIdx, const int destIdx) {+ const T temp = buffer[sourceIdx];+ buffer[sourceIdx] = buffer[destIdx];+ buffer[destIdx] = temp;}template <typename T>@@ -23,37 +19,38 @@ void updateKVCache(constant int& headDim,const uint2 position) {- for (int i = 0; i < swapCount; ++i) {- // [headIdx: 0..numHeads, tokenIdx: 0..maxSequenceLength, channelIdx: 0..headDim]- const int headOffset = position.x * maxSequenceLength * headDim;- const int channelOffset = position.y;- const int sourceIdx = headOffset + swaps[i].x * headDim + channelOffset;- const int destIdx = headOffset + swaps[i].y * headDim + channelOffset;+ for (int i = 0; i < swapCount; ++i) {+ // [headIdx: 0..numHeads, tokenIdx: 0..maxSequenceLength, channelIdx:+ // 0..headDim]+ const int headOffset = position.x * maxSequenceLength * headDim;+ const int channelOffset = position.y;+ const int sourceIdx = headOffset + swaps[i].x * headDim + channelOffset;+ const int destIdx = headOffset + swaps[i].y * headDim + channelOffset;- swap(inPlaceKeys, sourceIdx, destIdx);- swap(inPlaceValues, sourceIdx, destIdx);- }+ swap(inPlaceKeys, sourceIdx, destIdx);+ swap(inPlaceValues, sourceIdx, destIdx);+ }}-#define outerArguments(T) \-(device T* inPlaceKeys [[ buffer(0) ]], \-device T* inPlaceValues [[ buffer(1) ]], \-device int2* swaps [[ buffer(2) ]], \-constant int& swapCount [[ buffer(3) ]], \-constant int& numHeads [[ buffer(4) ]], \-constant int& maxSequenceLength [[ buffer(5) ]], \-constant int& headDim [[ buffer(6) ]], \-const uint2 position [[ thread_position_in_grid ]]) \+#define outerArguments(T) \+ (device T * inPlaceKeys [[buffer(0)]], \+ device T * inPlaceValues [[buffer(1)]], \+ device int2 * swaps [[buffer(2)]], \+ constant int& swapCount [[buffer(3)]], \+ constant int& numHeads [[buffer(4)]], \+ constant int& maxSequenceLength [[buffer(5)]], \+ constant int& headDim [[buffer(6)]], \+ const uint2 position [[thread_position_in_grid]])-#define innerArguments \-(inPlaceKeys, \-inPlaceValues, \-swaps, \-swapCount, \-numHeads, \-maxSequenceLength, \-headDim, \-position) \+#define innerArguments \+ (inPlaceKeys, \+ inPlaceValues, \+ swaps, \+ swapCount, \+ numHeads, \+ maxSequenceLength, \+ headDim, \+ position)generateKernels(updateKVCache)diff --git a/crates/uzu/src/backends/metal/kernel/media_kernels/image_ops.metal b/crates/uzu/src/backends/metal/kernel/media_kernels/image_ops.metalindex db94fe047..1ef155c1c 100644--- a/crates/uzu/src/backends/metal/kernel/media_kernels/image_ops.metal+++ b/crates/uzu/src/backends/metal/kernel/media_kernels/image_ops.metal@@ -4,110 +4,126 @@ using namespace metal;constexpr sampler bicubicSampler(filter::linear, address::clamp_to_edge);struct ImageParameters {- uint2 inputDimensions;- float3 imageMean;- float3 imageStd;- float paddingValueRgb;+ uint2 inputDimensions;+ float3 imageMean;+ float3 imageStd;+ float paddingValueRgb;};struct PatchParameters {- uint2 paddedDimensions;- uint patchSize;- uint numChannels;- uint temporalSlices;+ uint2 paddedDimensions;+ uint patchSize;+ uint numChannels;+ uint temporalSlices;};kernel void scalePadNormalizeImage(texture2d<float, access::sample> inputTexture [[texture(0)]],texture2d<float, access::write> outputTexture [[texture(1)]],constant ImageParameters& params [[buffer(0)]],- uint2 gid [[thread_position_in_grid]]) {-- uint2 outputDims = uint2(outputTexture.get_width(), outputTexture.get_height());-- if (gid.x >= outputDims.x || gid.y >= outputDims.y) {- return;- }-- float inputAspectRatio = float(params.inputDimensions.x) / float(params.inputDimensions.y);- float outputAspectRatio = float(outputDims.x) / float(outputDims.y);-- float scaleFactor;- if (inputAspectRatio > outputAspectRatio) {- scaleFactor = float(outputDims.x) / float(params.inputDimensions.x);- } else {- scaleFactor = float(outputDims.y) / float(params.inputDimensions.y);- }-- float scaledInputWidth = float(params.inputDimensions.x) * scaleFactor;- float scaledInputHeight = float(params.inputDimensions.y) * scaleFactor;-- float offsetX = (float(outputDims.x) - scaledInputWidth) / 2.0f;- float offsetY = (float(outputDims.y) - scaledInputHeight) / 2.0f;-- float targetXInScaledArea = float(gid.x) - offsetX;- float targetYInScaledArea = float(gid.y) - offsetY;-- float4 pixelColor;-- if (targetXInScaledArea >= 0.0f && targetXInScaledArea < scaledInputWidth &&- targetYInScaledArea >= 0.0f && targetYInScaledArea < scaledInputHeight) {-- float uNorm = targetXInScaledArea / scaledInputWidth;- float vNorm = targetYInScaledArea / scaledInputHeight;-- pixelColor = inputTexture.sample(bicubicSampler, float2(uNorm, vNorm));- } else {- pixelColor = float4(params.paddingValueRgb, params.paddingValueRgb, params.paddingValueRgb, 1.0f);- }-- float3 normalizedRgb;- float epsilon = 1e-6f;- normalizedRgb.r = (pixelColor.r - params.imageMean.r) / (params.imageStd.r + epsilon);- normalizedRgb.g = (pixelColor.g - params.imageMean.g) / (params.imageStd.g + epsilon);- normalizedRgb.b = (pixelColor.b - params.imageMean.b) / (params.imageStd.b + epsilon);-- outputTexture.write(float4(normalizedRgb, 1.0f), gid);+ uint2 gid [[thread_position_in_grid]]+) {++ uint2 outputDims =+ uint2(outputTexture.get_width(), outputTexture.get_height());++ if (gid.x >= outputDims.x || gid.y >= outputDims.y) {+ return;+ }++ float inputAspectRatio =+ float(params.inputDimensions.x) / float(params.inputDimensions.y);+ float outputAspectRatio = float(outputDims.x) / float(outputDims.y);++ float scaleFactor;+ if (inputAspectRatio > outputAspectRatio) {+ scaleFactor = float(outputDims.x) / float(params.inputDimensions.x);+ } else {+ scaleFactor = float(outputDims.y) / float(params.inputDimensions.y);+ }++ float scaledInputWidth = float(params.inputDimensions.x) * scaleFactor;+ float scaledInputHeight = float(params.inputDimensions.y) * scaleFactor;++ float offsetX = (float(outputDims.x) - scaledInputWidth) / 2.0f;+ float offsetY = (float(outputDims.y) - scaledInputHeight) / 2.0f;++ float targetXInScaledArea = float(gid.x) - offsetX;+ float targetYInScaledArea = float(gid.y) - offsetY;++ float4 pixelColor;++ if (targetXInScaledArea >= 0.0f && targetXInScaledArea < scaledInputWidth &&+ targetYInScaledArea >= 0.0f && targetYInScaledArea < scaledInputHeight) {++ float uNorm = targetXInScaledArea / scaledInputWidth;+ float vNorm = targetYInScaledArea / scaledInputHeight;++ pixelColor = inputTexture.sample(bicubicSampler, float2(uNorm, vNorm));+ } else {+ pixelColor = float4(+ params.paddingValueRgb,+ params.paddingValueRgb,+ params.paddingValueRgb,+ 1.0f+ );+ }++ float3 normalizedRgb;+ float epsilon = 1e-6f;+ normalizedRgb.r =+ (pixelColor.r - params.imageMean.r) / (params.imageStd.r + epsilon);+ normalizedRgb.g =+ (pixelColor.g - params.imageMean.g) / (params.imageStd.g + epsilon);+ normalizedRgb.b =+ (pixelColor.b - params.imageMean.b) / (params.imageStd.b + epsilon);++ outputTexture.write(float4(normalizedRgb, 1.0f), gid);}kernel void extractImagePatches(texture2d<float, access::read> paddedNormalizedTexture [[texture(0)]],device float* outputBuffer [[buffer(0)]],constant PatchParameters& params [[buffer(1)]],- uint2 gid [[thread_position_in_grid]]) {+ uint2 gid [[thread_position_in_grid]]+) {- if (gid.x >= params.paddedDimensions.x || gid.y >= params.paddedDimensions.y) {- return;- }+ if (gid.x >= params.paddedDimensions.x ||+ gid.y >= params.paddedDimensions.y) {+ return;+ }++ uint numPatchesX = params.paddedDimensions.x / params.patchSize;++ uint patchCol = gid.x / params.patchSize;+ uint patchRow = gid.y / params.patchSize;++ uint spatialPatchIdxFlat = patchRow * numPatchesX + patchCol;- uint numPatchesX = params.paddedDimensions.x / params.patchSize;+ uint pxInPatch = gid.x % params.patchSize;+ uint pyInPatch = gid.y % params.patchSize;- uint patchCol = gid.x / params.patchSize;- uint patchRow = gid.y / params.patchSize;+ float4 normalizedPixelRgba = paddedNormalizedTexture.read(gid);- uint spatialPatchIdxFlat = patchRow * numPatchesX + patchCol;+ uint patchPixelFlatIdx = pyInPatch * params.patchSize + pxInPatch;+ uint singlePatchChannelTemporalSliceSize =+ params.patchSize * params.patchSize;+ uint singlePatchChannelSize =+ params.temporalSlices * singlePatchChannelTemporalSliceSize;+ uint singlePatchSize = params.numChannels * singlePatchChannelSize;- uint pxInPatch = gid.x % params.patchSize;- uint pyInPatch = gid.y % params.patchSize;+ uint outputPatchBaseIdx = spatialPatchIdxFlat * singlePatchSize;- float4 normalizedPixelRgba = paddedNormalizedTexture.read(gid);+ for (uint c = 0; c < params.numChannels; ++c) {+ float valueToStore = normalizedPixelRgba[c];+ for (uint t = 0; t < params.temporalSlices; ++t) {+ uint channelBlockOffset = c * singlePatchChannelSize;+ uint temporalBlockOffset = t * singlePatchChannelTemporalSliceSize;- uint patchPixelFlatIdx = pyInPatch * params.patchSize + pxInPatch;- uint singlePatchChannelTemporalSliceSize = params.patchSize * params.patchSize;- uint singlePatchChannelSize = params.temporalSlices * singlePatchChannelTemporalSliceSize;- uint singlePatchSize = params.numChannels * singlePatchChannelSize;-- uint outputPatchBaseIdx = spatialPatchIdxFlat * singlePatchSize;+ uint outputBufferIdx = outputPatchBaseIdx + channelBlockOffset ++ temporalBlockOffset + patchPixelFlatIdx;- for (uint c = 0; c < params.numChannels; ++c) {- float valueToStore = normalizedPixelRgba[c];- for (uint t = 0; t < params.temporalSlices; ++t) {- uint channelBlockOffset = c * singlePatchChannelSize;- uint temporalBlockOffset = t * singlePatchChannelTemporalSliceSize;-- uint outputBufferIdx = outputPatchBaseIdx + channelBlockOffset + temporalBlockOffset + patchPixelFlatIdx;-- outputBuffer[outputBufferIdx] = valueToStore;- }+ outputBuffer[outputBufferIdx] = valueToStore;}-}\ No newline at end of file+ }+}\ No newline at end of filediff --git a/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal b/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metalindex 8d08af8fc..54048bbfc 100644--- a/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal+++ b/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal@@ -25,7 +25,8 @@ inline T gelu_approx(T x) {template <typename T>inline T activate(T x, ushort act) {- if (act == ACT_SILU) return silu(x);+ if (act == ACT_SILU)+ return silu(x);return gelu_approx(x);}@@ -36,10 +37,12 @@ template <typename T>const constant int& H [[buffer(2)]],const constant int& M [[buffer(3)]],const constant ushort& act_type [[buffer(4)]],- uint2 tid [[thread_position_in_grid]]) {+ uint2 tid [[thread_position_in_grid]]+) {int row = tid.y;int j = tid.x;- if (row >= M || j >= H) return;+ if (row >= M || j >= H)+ return;int base = row * (2 * H);T up = fused_up[base + j];T gate = fused_up[base + H + j];@@ -48,31 +51,32 @@ template <typename T>}// Explicit instantiations with stable host names-template [[host_name("mlp_activation_mul_f16")]]-[[kernel]] void mlp_activation_mul<half>(+template [[host_name("mlp_activation_mul_f16")]] [[kernel]] void+mlp_activation_mul<half>(const device half* fused_up [[buffer(0)]],device half* hidden [[buffer(1)]],const constant int& H [[buffer(2)]],const constant int& M [[buffer(3)]],const constant ushort& act_type [[buffer(4)]],- uint2 tid [[thread_position_in_grid]]);+ uint2 tid [[thread_position_in_grid]]+);-template [[host_name("mlp_activation_mul_f32")]]-[[kernel]] void mlp_activation_mul<float>(+template [[host_name("mlp_activation_mul_f32")]] [[kernel]] void+mlp_activation_mul<float>(const device float* fused_up [[buffer(0)]],device float* hidden [[buffer(1)]],const constant int& H [[buffer(2)]],const constant int& M [[buffer(3)]],const constant ushort& act_type [[buffer(4)]],- uint2 tid [[thread_position_in_grid]]);+ uint2 tid [[thread_position_in_grid]]+);-template [[host_name("mlp_activation_mul_bf16")]]-[[kernel]] void mlp_activation_mul<bfloat>(+template [[host_name("mlp_activation_mul_bf16")]] [[kernel]] void+mlp_activation_mul<bfloat>(const device bfloat* fused_up [[buffer(0)]],device bfloat* hidden [[buffer(1)]],const constant int& H [[buffer(2)]],const constant int& M [[buffer(3)]],const constant ushort& act_type [[buffer(4)]],- uint2 tid [[thread_position_in_grid]]);--+ uint2 tid [[thread_position_in_grid]]+);diff --git a/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal b/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metalindex d8d5a9aca..21ec0fccc 100644--- a/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal@@ -11,93 +11,108 @@ using namespace metal;// This kernel is launched with SINGLE threadgroupkernel void moe_counts_offsets_fused(device const int* topk_ids [[buffer(0)]],- device uint* offsets [[buffer(1)]], // output: exclusive scan [E+1]- device uint* sum_k_out [[buffer(2)]], // output: total count [1]- device uint* partials [[buffer(3)]], // output: partials [num_tiles * TILE_E] (for block_bases)+ device uint* offsets [[buffer(1)]], // output: exclusive scan [E+1]+ device uint* sum_k_out [[buffer(2)]], // output: total count [1]+ device uint* partials+ [[buffer(3)]], // output: partials [num_tiles * TILE_E] (for block_bases)constant uint& T [[buffer(4)]],constant uint& E [[buffer(5)]],constant uint& K [[buffer(6)]],- uint lid [[thread_index_in_threadgroup]])-{- if (E == 0) {- if (lid == 0) {- offsets[0] = 0u;- sum_k_out[0] = 0u;- }- return;+ uint lid [[thread_index_in_threadgroup]]+) {+ if (E == 0) {+ if (lid == 0) {+ offsets[0] = 0u;+ sum_k_out[0] = 0u;}+ return;+ }- threadgroup _atomic<uint> tg_hist[TILE_E];- threadgroup uint scan_shared[BLOCK_SIZE];- threadgroup uint reduce_shared[BLOCK_SIZE];- threadgroup uint counts_shared[BLOCK_SIZE]; // Cache counts in threadgroup memory-- // ═══════════════════════════════════════════════════════════- // PHASE 1: Count tokens per expert using tiled histogram- // ═══════════════════════════════════════════════════════════- // Tile over E dimension- for (uint e0 = 0; e0 < E; e0 += TILE_E) {- const uint tile_e = (e0 + TILE_E <= E) ? TILE_E : (E - e0);+ threadgroup _atomic<uint> tg_hist[TILE_E];+ threadgroup uint scan_shared[BLOCK_SIZE];+ threadgroup uint reduce_shared[BLOCK_SIZE];+ threadgroup uint+ counts_shared[BLOCK_SIZE]; // Cache counts in threadgroup memory- // Zero threadgroup histogram- for (uint e = lid; e < tile_e; e += BLOCK_SIZE) {- atomic_store_explicit(&tg_hist[e], 0u, memory_order_relaxed);- }- threadgroup_barrier(mem_flags::mem_threadgroup);+ // ═══════════════════════════════════════════════════════════+ // PHASE 1: Count tokens per expert using tiled histogram+ // ═══════════════════════════════════════════════════════════+ // Tile over E dimension+ for (uint e0 = 0; e0 < E; e0 += TILE_E) {+ const uint tile_e = (e0 + TILE_E <= E) ? TILE_E : (E - e0);- // Process all tokens in strided fashion- // Since we have only 1 threadgroup, we need to cover all T tokens- for (uint t = lid; t < T; t += BLOCK_SIZE) {- const uint base = t * K;- for (uint k = 0; k < K; ++k) {- int eid = topk_ids[base + k];- if (eid >= 0) {- uint ue = uint(eid);- if (ue >= e0 && ue < e0 + tile_e) {- uint te = ue - e0;- atomic_fetch_add_explicit(&tg_hist[te], 1u, memory_order_relaxed);- }- }- }- }- threadgroup_barrier(mem_flags::mem_threadgroup);+ // Zero threadgroup histogram+ for (uint e = lid; e < tile_e; e += BLOCK_SIZE) {+ atomic_store_explicit(&tg_hist[e], 0u, memory_order_relaxed);+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);- // Write final counts and partials for this tile- for (uint e = lid; e < tile_e; e += BLOCK_SIZE) {- const uint count_val = atomic_load_explicit(&tg_hist[e], memory_order_relaxed);- counts_shared[e0 + e] = count_val;- partials[e0 + e] = count_val;+ // Process all tokens in strided fashion+ // Since we have only 1 threadgroup, we need to cover all T tokens+ for (uint t = lid; t < T; t += BLOCK_SIZE) {+ const uint base = t * K;+ for (uint k = 0; k < K; ++k) {+ int eid = topk_ids[base + k];+ if (eid >= 0) {+ uint ue = uint(eid);+ if (ue >= e0 && ue < e0 + tile_e) {+ uint te = ue - e0;+ atomic_fetch_add_explicit(&tg_hist[te], 1u, memory_order_relaxed);+ }}- threadgroup_barrier(mem_flags::mem_threadgroup);+ }}+ threadgroup_barrier(mem_flags::mem_threadgroup);- // ═══════════════════════════════════════════════════════════- // PHASE 2: Compute exclusive prefix scan on counts- // ═══════════════════════════════════════════════════════════- threadgroup uint carry;- if (lid == 0) { carry = 0u; }+ // Write final counts and partials for this tile+ for (uint e = lid; e < tile_e; e += BLOCK_SIZE) {+ const uint count_val =+ atomic_load_explicit(&tg_hist[e], memory_order_relaxed);+ counts_shared[e0 + e] = count_val;+ partials[e0 + e] = count_val;+ }threadgroup_barrier(mem_flags::mem_threadgroup);+ }- for (uint base = 0; base < E; base += BLOCK_SIZE) {- uint remaining = E - base;- uint chunk_n = remaining < BLOCK_SIZE ? remaining : BLOCK_SIZE;+ // ═══════════════════════════════════════════════════════════+ // PHASE 2: Compute exclusive prefix scan on counts+ // ═══════════════════════════════════════════════════════════+ threadgroup uint carry;+ if (lid == 0) {+ carry = 0u;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);- uint v = (lid < chunk_n) ? counts_shared[lid] : 0u;+ for (uint base = 0; base < E; base += BLOCK_SIZE) {+ uint remaining = E - base;+ uint chunk_n = remaining < BLOCK_SIZE ? remaining : BLOCK_SIZE;- uint prefix_local = threadgroup_raking_prefix_exclusive_sum<BLOCK_SIZE>(v, scan_shared, (ushort)lid);- uint prefix_global = prefix_local + carry;- if (lid < chunk_n) {- offsets[base + lid] = prefix_global;- }+ uint v = (lid < chunk_n) ? counts_shared[lid] : 0u;- uint block_sum = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(v, reduce_shared, (ushort)lid);- if (lid == 0) { carry += block_sum; }- threadgroup_barrier(mem_flags::mem_threadgroup);+ uint prefix_local = threadgroup_raking_prefix_exclusive_sum<BLOCK_SIZE>(+ v,+ scan_shared,+ (ushort)lid+ );+ uint prefix_global = prefix_local + carry;+ if (lid < chunk_n) {+ offsets[base + lid] = prefix_global;}- // Write final offset and total+ uint block_sum = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(+ v,+ reduce_shared,+ (ushort)lid+ );if (lid == 0) {- offsets[E] = carry;- sum_k_out[0] = carry;+ carry += block_sum;}+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }++ // Write final offset and total+ if (lid == 0) {+ offsets[E] = carry;+ sum_k_out[0] = carry;+ }}diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metalindex edf485ceb..67d95608e 100644--- a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal@@ -2,31 +2,36 @@#include <metal_simdgroup>using namespace metal;-constant uint GATING_SEL [[function_constant(30)]]; // 0=GELU,1=SiLU,2=SwiGLU,3=GEGLU-constant uint TILE_H [[function_constant(32)]]; // tile size for hidden dimension in Pass A+constant uint GATING_SEL+ [[function_constant(30)]]; // 0=GELU,1=SiLU,2=SwiGLU,3=GEGLU+constant uint TILE_H+ [[function_constant(32)]]; // tile size for hidden dimension in Pass Astatic inline float gelu_approx(float x) {- const float k0 = 0.7978845608f;- const float k1 = 0.044715f;- if (x > 10.0f) return x;- if (x < -10.0f) return 0.0f;- return 0.5f * x * (1.0f + tanh(clamp(k0 * (x + k1 * x * x * x), -10.0f, 10.0f)));+ const float k0 = 0.7978845608f;+ const float k1 = 0.044715f;+ if (x > 10.0f)+ return x;+ if (x < -10.0f)+ return 0.0f;+ return 0.5f * x *+ (1.0f + tanh(clamp(k0 * (x + k1 * x * x * x), -10.0f, 10.0f)));}static inline float silu(float x, float alpha) {- return x / (1.0f + exp(-alpha * x));+ return x / (1.0f + exp(-alpha * x));}#define MTL_CONST static constant constexpr const#define MTL_PRAGMA_UNROLL _Pragma("clang loop unroll(full)")// Tiling configuration (tunable)-MTL_CONST uint BM = 1; // Simdgroups in M dimension within threadgroup-MTL_CONST uint BN = 4; // Simdgroups in N dimension within threadgroup-MTL_CONST uint SM = 1; // Thread rows per simdgroup-MTL_CONST uint SN = 32; // Thread cols per simdgroup (full warp width)-MTL_CONST uint TM = 4; // Output elements per thread in M-MTL_CONST uint TN = 4; // Input elements per thread in N (per iteration)+MTL_CONST uint BM = 1; // Simdgroups in M dimension within threadgroup+MTL_CONST uint BN = 4; // Simdgroups in N dimension within threadgroup+MTL_CONST uint SM = 1; // Thread rows per simdgroup+MTL_CONST uint SN = 32; // Thread cols per simdgroup (full warp width)+MTL_CONST uint TM = 4; // Output elements per thread in M+MTL_CONST uint TN = 4; // Input elements per thread in N (per iteration)MTL_CONST uint THREADS_M = BM * SM;MTL_CONST uint THREADS_N = BN * SN;@@ -34,18 +39,22 @@ MTL_CONST uint BLOCK_M = THREADS_M * TM;MTL_CONST uint BLOCK_N = THREADS_N * TN;static_assert(SM * SN == 32, "simdgroup must have 32 threads");-static_assert(SN == 4 || SN == 8 || SN == 16 || SN == 32, "SN must be 4, 8, 16, or 32");+static_assert(+ SN == 4 || SN == 8 || SN == 16 || SN == 32,+ "SN must be 4, 8, 16, or 32"+);// === Pass A: Optimized GEMV with hierarchical tiling ===// Computes: hidden[row, h] = activation(x[row, d] @ W13[d, 2*h])// AccumT is used for internal accumulations, output still stored as T-template<typename T, typename AccumT>+template <typename T, typename AccumT>void moe_experts_decode_pass_a_impl(- device const T* X_perm, // [total_rows, d_model]- device const uint* expert_offsets, // [E + 1]- device const T* W13_all, // weights in layout [E, 2*d_ff, d_model]- device const T* up_biases, // [E, 2*d_ff]- device float* hidden_out, // [total_rows, d_ff] - f32 for activation precision+ device const T* X_perm, // [total_rows, d_model]+ device const uint* expert_offsets, // [E + 1]+ device const T* W13_all, // weights in layout [E, 2*d_ff, d_model]+ device const T* up_biases, // [E, 2*d_ff]+ device float*+ hidden_out, // [total_rows, d_ff] - f32 for activation precisionuint d_model,uint d_ff,uint E,@@ -61,227 +70,254 @@ void moe_experts_decode_pass_a_impl(uint simd_lid,threadgroup float* tgp_memory) {- // Thread position within simdgroup- const uint thrM = SN != 32 ? simd_lid / SN : 0;- const uint thrN = SN != 32 ? simd_lid % SN : uint(simd_lid);-- // Simdgroup position within threadgroup- const uint sgN = BN != 1 ? (simd_gid % BN) : 0;- const uint simdM = BN != 1 ? SM * (simd_gid / BN) : uint(SM * simd_gid);- const uint simdN = BN != 1 ? SN * (simd_gid % BN) : 0;-- // Thread's work block- int bm = (simdM + thrM) * TM;- int bn = (simdN + thrN) * TN;-- // Output row position- const uint seg_start = expert_offsets[expert_idx];- const uint seg_end = expert_offsets[expert_idx + 1];- uint global_row = seg_start + row_in_expert;-- if (global_row >= seg_end) return;-- // Calculate output h index- int h_row = h_block_idx * BLOCK_M + bm;-- // Adjust tail block to ensure in-bounds reads- if (h_row + TM > d_ff) {- h_row = d_ff > TM ? int(d_ff - TM) : 0;- bm = h_row - int(h_block_idx * BLOCK_M);+ // Thread position within simdgroup+ const uint thrM = SN != 32 ? simd_lid / SN : 0;+ const uint thrN = SN != 32 ? simd_lid % SN : uint(simd_lid);++ // Simdgroup position within threadgroup+ const uint sgN = BN != 1 ? (simd_gid % BN) : 0;+ const uint simdM = BN != 1 ? SM * (simd_gid / BN) : uint(SM * simd_gid);+ const uint simdN = BN != 1 ? SN * (simd_gid % BN) : 0;++ // Thread's work block+ int bm = (simdM + thrM) * TM;+ int bn = (simdN + thrN) * TN;++ // Output row position+ const uint seg_start = expert_offsets[expert_idx];+ const uint seg_end = expert_offsets[expert_idx + 1];+ uint global_row = seg_start + row_in_expert;++ if (global_row >= seg_end)+ return;++ // Calculate output h index+ int h_row = h_block_idx * BLOCK_M + bm;++ // Adjust tail block to ensure in-bounds reads+ if (h_row + TM > d_ff) {+ h_row = d_ff > TM ? int(d_ff - TM) : 0;+ bm = h_row - int(h_block_idx * BLOCK_M);+ }++ if (h_row >= int(d_ff))+ return;++ // Accumulation arrays for up and gate projections+ thread AccumT result_up[TM] = {0};+ thread AccumT result_gate[TM] = {0};+ thread T weights_up[TN];+ thread T weights_gate[TN];+ thread AccumT x_vals[TN];++ // Expert weight base addresses+ const ulong w13_base = (ulong)expert_idx * (ulong)d_model * (ulong)(2 * d_ff);+ const ulong bias_base = (ulong)expert_idx * (ulong)(2 * d_ff);+ const ulong x_row_base = (ulong)global_row * (ulong)d_model;++ // Main accumulation loop over d_model in blocks of BLOCK_N+ const uint n_iter = d_model / BLOCK_N;+ const uint leftover = d_model - (n_iter * BLOCK_N);++ for (uint i = 0; i < n_iter; ++i) {+ // Load x values for this block+ MTL_PRAGMA_UNROLL+ for (uint tn = 0; tn < TN; tn++) {+ uint d_idx = bn + tn;+ x_vals[tn] = AccumT(X_perm[x_row_base + d_idx]);}- if (h_row >= int(d_ff)) return;-- // Accumulation arrays for up and gate projections- thread AccumT result_up[TM] = {0};- thread AccumT result_gate[TM] = {0};- thread T weights_up[TN];- thread T weights_gate[TN];- thread AccumT x_vals[TN];-- // Expert weight base addresses- const ulong w13_base = (ulong)expert_idx * (ulong)d_model * (ulong)(2 * d_ff);- const ulong bias_base = (ulong)expert_idx * (ulong)(2 * d_ff);- const ulong x_row_base = (ulong)global_row * (ulong)d_model;-- // Main accumulation loop over d_model in blocks of BLOCK_N- const uint n_iter = d_model / BLOCK_N;- const uint leftover = d_model - (n_iter * BLOCK_N);-- for (uint i = 0; i < n_iter; ++i) {- // Load x values for this block+ // Accumulate for each output row+ MTL_PRAGMA_UNROLL+ for (uint tm = 0; tm < TM; tm++) {+ uint h_idx = h_row + tm;++ // Load weights for up projection+ MTL_PRAGMA_UNROLL+ for (uint tn = 0; tn < TN; tn++) {+ uint d_idx = bn + tn;+ ulong w_idx = w13_base + (ulong)h_idx * (ulong)d_model + (ulong)d_idx;+ weights_up[tn] = W13_all[w_idx];+ }++ // Accumulate up projection+ MTL_PRAGMA_UNROLL+ for (uint tn = 0; tn < TN; tn++) {+ result_up[tm] += x_vals[tn] * AccumT(weights_up[tn]);+ }++ // Load and accumulate gate projection if needed+ if (GATING_SEL > 1) {MTL_PRAGMA_UNROLLfor (uint tn = 0; tn < TN; tn++) {- uint d_idx = bn + tn;- x_vals[tn] = AccumT(X_perm[x_row_base + d_idx]);+ uint d_idx = bn + tn;+ ulong w_gate_idx =+ w13_base + (ulong)(d_ff + h_idx) * (ulong)d_model + (ulong)d_idx;+ weights_gate[tn] = W13_all[w_gate_idx];}- // Accumulate for each output rowMTL_PRAGMA_UNROLL- for (uint tm = 0; tm < TM; tm++) {- uint h_idx = h_row + tm;-- // Load weights for up projection- MTL_PRAGMA_UNROLL- for (uint tn = 0; tn < TN; tn++) {- uint d_idx = bn + tn;- ulong w_idx = w13_base + (ulong)h_idx * (ulong)d_model + (ulong)d_idx;- weights_up[tn] = W13_all[w_idx];- }-- // Accumulate up projection- MTL_PRAGMA_UNROLL- for (uint tn = 0; tn < TN; tn++) {- result_up[tm] += x_vals[tn] * AccumT(weights_up[tn]);- }-- // Load and accumulate gate projection if needed- if (GATING_SEL > 1) {- MTL_PRAGMA_UNROLL- for (uint tn = 0; tn < TN; tn++) {- uint d_idx = bn + tn;- ulong w_gate_idx = w13_base + (ulong)(d_ff + h_idx) * (ulong)d_model + (ulong)d_idx;- weights_gate[tn] = W13_all[w_gate_idx];- }-- MTL_PRAGMA_UNROLL- for (uint tn = 0; tn < TN; tn++) {- result_gate[tm] += x_vals[tn] * AccumT(weights_gate[tn]);- }- }+ for (uint tn = 0; tn < TN; tn++) {+ result_gate[tm] += x_vals[tn] * AccumT(weights_gate[tn]);}-- bn += BLOCK_N;+ }}- // Handle leftover elements- if (leftover > 0) {- MTL_PRAGMA_UNROLL- for (uint tn = 0; tn < TN; tn++) {- uint d_idx = bn + tn;- x_vals[tn] = (d_idx < d_model) ? AccumT(X_perm[x_row_base + d_idx]) : AccumT(0.0);- }+ bn += BLOCK_N;+ }- MTL_PRAGMA_UNROLL- for (uint tm = 0; tm < TM; tm++) {- uint h_idx = h_row + tm;-- MTL_PRAGMA_UNROLL- for (uint tn = 0; tn < TN; tn++) {- uint d_idx = bn + tn;- if (d_idx < d_model) {- ulong w_idx = w13_base + (ulong)h_idx * (ulong)d_model + (ulong)d_idx;- weights_up[tn] = W13_all[w_idx];- result_up[tm] += x_vals[tn] * AccumT(weights_up[tn]);-- if (GATING_SEL > 1) {- ulong w_gate_idx = w13_base + (ulong)(d_ff + h_idx) * (ulong)d_model + (ulong)d_idx;- weights_gate[tn] = W13_all[w_gate_idx];- result_gate[tm] += x_vals[tn] * AccumT(weights_gate[tn]);- }- }- }- }+ // Handle leftover elements+ if (leftover > 0) {+ MTL_PRAGMA_UNROLL+ for (uint tn = 0; tn < TN; tn++) {+ uint d_idx = bn + tn;+ x_vals[tn] =+ (d_idx < d_model) ? AccumT(X_perm[x_row_base + d_idx]) : AccumT(0.0);}- // Simdgroup reduction across thrN dimensionMTL_PRAGMA_UNROLLfor (uint tm = 0; tm < TM; tm++) {- result_up[tm] = simd_sum(result_up[tm]);- if (GATING_SEL > 1) {- result_gate[tm] = simd_sum(result_gate[tm]);+ uint h_idx = h_row + tm;++ MTL_PRAGMA_UNROLL+ for (uint tn = 0; tn < TN; tn++) {+ uint d_idx = bn + tn;+ if (d_idx < d_model) {+ ulong w_idx = w13_base + (ulong)h_idx * (ulong)d_model + (ulong)d_idx;+ weights_up[tn] = W13_all[w_idx];+ result_up[tm] += x_vals[tn] * AccumT(weights_up[tn]);++ if (GATING_SEL > 1) {+ ulong w_gate_idx = w13_base ++ (ulong)(d_ff + h_idx) * (ulong)d_model ++ (ulong)d_idx;+ weights_gate[tn] = W13_all[w_gate_idx];+ result_gate[tm] += x_vals[tn] * AccumT(weights_gate[tn]);+ }}+ }+ }+ }++ // Simdgroup reduction across thrN dimension+ MTL_PRAGMA_UNROLL+ for (uint tm = 0; tm < TM; tm++) {+ result_up[tm] = simd_sum(result_up[tm]);+ if (GATING_SEL > 1) {+ result_gate[tm] = simd_sum(result_gate[tm]);}+ }- // Threadgroup reduction if BN > 1- if (BN > 1) {- threadgroup float* tgp_up = tgp_memory;- threadgroup float* tgp_gate = tgp_memory + BN * (BLOCK_M + TM);-- if (thrN == 0) {- MTL_PRAGMA_UNROLL- for (uint tm = 0; tm < TM; tm++) {- tgp_up[sgN * (BLOCK_M + TM) + bm + tm] = float(result_up[tm]);- if (GATING_SEL > 1) {- tgp_gate[sgN * (BLOCK_M + TM) + bm + tm] = float(result_gate[tm]);- }- }+ // Threadgroup reduction if BN > 1+ if (BN > 1) {+ threadgroup float* tgp_up = tgp_memory;+ threadgroup float* tgp_gate = tgp_memory + BN * (BLOCK_M + TM);- threadgroup_barrier(mem_flags::mem_threadgroup);-- if (sgN == 0) {- MTL_PRAGMA_UNROLL- for (uint sgn = 1; sgn < BN; sgn++) {- MTL_PRAGMA_UNROLL- for (uint tm = 0; tm < TM; tm++) {- result_up[tm] += AccumT(tgp_up[sgn * (BLOCK_M + TM) + bm + tm]);- if (GATING_SEL > 1) {- result_gate[tm] += AccumT(tgp_gate[sgn * (BLOCK_M + TM) + bm + tm]);- }- }- }- }+ if (thrN == 0) {+ MTL_PRAGMA_UNROLL+ for (uint tm = 0; tm < TM; tm++) {+ tgp_up[sgN * (BLOCK_M + TM) + bm + tm] = float(result_up[tm]);+ if (GATING_SEL > 1) {+ tgp_gate[sgN * (BLOCK_M + TM) + bm + tm] = float(result_gate[tm]);}- }+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);- // Apply activation and write output (only first thread in simdgroup width)- if (simdN == 0 && thrN == 0) {+ if (sgN == 0) {MTL_PRAGMA_UNROLL- for (uint tm = 0; tm < TM; tm++) {- uint h_idx = h_row + tm;- if (h_idx < d_ff) {- // Add bias and clip- AccumT up_val = result_up[tm] + AccumT(up_biases[bias_base + h_idx]);- up_val = AccumT(clamp(float(up_val), up_clip_min, up_clip_max));-- AccumT activated_val;- if (GATING_SEL <= 1) {- activated_val = (GATING_SEL == 0) ? AccumT(gelu_approx(float(up_val))) : AccumT(silu(float(up_val), silu_alpha));- } else {- AccumT gate_val = result_gate[tm] + AccumT(up_biases[bias_base + d_ff + h_idx]);- gate_val = AccumT(clamp(float(gate_val), gate_clip_min, gate_clip_max));- AccumT gate_activated = (GATING_SEL == 2) ? AccumT(silu(float(gate_val), silu_alpha)) : AccumT(gelu_approx(float(gate_val)));- activated_val = gate_activated * up_val;- }-- ulong hidden_idx = (ulong)global_row * (ulong)d_ff + (ulong)h_idx;- hidden_out[hidden_idx] = float(activated_val);+ for (uint sgn = 1; sgn < BN; sgn++) {+ MTL_PRAGMA_UNROLL+ for (uint tm = 0; tm < TM; tm++) {+ result_up[tm] += AccumT(tgp_up[sgn * (BLOCK_M + TM) + bm + tm]);+ if (GATING_SEL > 1) {+ result_gate[tm] +=+ AccumT(tgp_gate[sgn * (BLOCK_M + TM) + bm + tm]);}+ }}+ }}-}+ }-#define MOE_PASS_A_KERNEL(DTYPE, SUFFIX) \-kernel void moe_experts_decode_pass_a_##SUFFIX( \- device const DTYPE* X_perm [[buffer(0)]], \- device const uint* expert_offsets [[buffer(1)]], \- device const DTYPE* W13_all [[buffer(2)]], \- device float* hidden_out [[buffer(3)]], \- device const DTYPE* up_biases [[buffer(4)]], \- constant uint& d_model [[buffer(5)]], \- constant uint& d_ff [[buffer(6)]], \- constant uint& E [[buffer(7)]], \- constant float& gate_clip_min [[buffer(8)]], \- constant float& gate_clip_max [[buffer(9)]], \- constant float& up_clip_min [[buffer(10)]], \- constant float& up_clip_max [[buffer(11)]], \- constant float& silu_alpha [[buffer(12)]], \- uint3 tgpig [[threadgroup_position_in_grid]], \- uint simd_gid [[simdgroup_index_in_threadgroup]], \- uint simd_lid [[thread_index_in_simdgroup]]) \-{ \- constexpr uint tgp_mem_size = (BN > 1) ? 2 * BN * (BLOCK_M + TM) : 1; \- threadgroup float tgp_memory[tgp_mem_size]; \- \- moe_experts_decode_pass_a_impl<DTYPE, float>( \- X_perm, expert_offsets, W13_all, up_biases, hidden_out, \- d_model, d_ff, E, gate_clip_min, gate_clip_max, \- up_clip_min, up_clip_max, silu_alpha, \- tgpig.y, tgpig.z, tgpig.x, \- simd_gid, simd_lid, \- (BN > 1) ? tgp_memory : nullptr); \+ // Apply activation and write output (only first thread in simdgroup width)+ if (simdN == 0 && thrN == 0) {+ MTL_PRAGMA_UNROLL+ for (uint tm = 0; tm < TM; tm++) {+ uint h_idx = h_row + tm;+ if (h_idx < d_ff) {+ // Add bias and clip+ AccumT up_val = result_up[tm] + AccumT(up_biases[bias_base + h_idx]);+ up_val = AccumT(clamp(float(up_val), up_clip_min, up_clip_max));++ AccumT activated_val;+ if (GATING_SEL <= 1) {+ activated_val = (GATING_SEL == 0)+ ? AccumT(gelu_approx(float(up_val)))+ : AccumT(silu(float(up_val), silu_alpha));+ } else {+ AccumT gate_val =+ result_gate[tm] + AccumT(up_biases[bias_base + d_ff + h_idx]);+ gate_val =+ AccumT(clamp(float(gate_val), gate_clip_min, gate_clip_max));+ AccumT gate_activated =+ (GATING_SEL == 2) ? AccumT(silu(float(gate_val), silu_alpha))+ : AccumT(gelu_approx(float(gate_val)));+ activated_val = gate_activated * up_val;+ }++ ulong hidden_idx = (ulong)global_row * (ulong)d_ff + (ulong)h_idx;+ hidden_out[hidden_idx] = float(activated_val);+ }+ }+ }}+#define MOE_PASS_A_KERNEL(DTYPE, SUFFIX) \+ kernel void moe_experts_decode_pass_a_##SUFFIX( \+ device const DTYPE* X_perm [[buffer(0)]], \+ device const uint* expert_offsets [[buffer(1)]], \+ device const DTYPE* W13_all [[buffer(2)]], \+ device float* hidden_out [[buffer(3)]], \+ device const DTYPE* up_biases [[buffer(4)]], \+ constant uint& d_model [[buffer(5)]], \+ constant uint& d_ff [[buffer(6)]], \+ constant uint& E [[buffer(7)]], \+ constant float& gate_clip_min [[buffer(8)]], \+ constant float& gate_clip_max [[buffer(9)]], \+ constant float& up_clip_min [[buffer(10)]], \+ constant float& up_clip_max [[buffer(11)]], \+ constant float& silu_alpha [[buffer(12)]], \+ uint3 tgpig [[threadgroup_position_in_grid]], \+ uint simd_gid [[simdgroup_index_in_threadgroup]], \+ uint simd_lid [[thread_index_in_simdgroup]] \+ ) { \+ constexpr uint tgp_mem_size = (BN > 1) ? 2 * BN * (BLOCK_M + TM) : 1; \+ threadgroup float tgp_memory[tgp_mem_size]; \+ \+ moe_experts_decode_pass_a_impl<DTYPE, float>( \+ X_perm, \+ expert_offsets, \+ W13_all, \+ up_biases, \+ hidden_out, \+ d_model, \+ d_ff, \+ E, \+ gate_clip_min, \+ gate_clip_max, \+ up_clip_min, \+ up_clip_max, \+ silu_alpha, \+ tgpig.y, \+ tgpig.z, \+ tgpig.x, \+ simd_gid, \+ simd_lid, \+ (BN > 1) ? tgp_memory : nullptr \+ ); \+ }+MOE_PASS_A_KERNEL(bfloat, bf16)MOE_PASS_A_KERNEL(half, f16)MOE_PASS_A_KERNEL(float, f32)@@ -290,143 +326,150 @@ MOE_PASS_A_KERNEL(float, f32)// Count tiles per expert: tiles = (num_rows > 0) ? num_rows * h_blocks : 0kernel void moe_pass_a_tile_counts(- device const uint* expert_offsets [[buffer(0)]], // [E+1]- device uint* tile_counts [[buffer(1)]], // [E]+ device const uint* expert_offsets [[buffer(0)]], // [E+1]+ device uint* tile_counts [[buffer(1)]], // [E]constant uint& E [[buffer(2)]],constant uint& h_blocks [[buffer(3)]],uint tid [[thread_position_in_grid]]) {- if (tid >= E) return;- const uint start = expert_offsets[tid];- const uint end = expert_offsets[tid + 1];- const uint num_rows = end - start;- tile_counts[tid] = (num_rows > 0) ? (num_rows * h_blocks) : 0;+ if (tid >= E)+ return;+ const uint start = expert_offsets[tid];+ const uint end = expert_offsets[tid + 1];+ const uint num_rows = end - start;+ tile_counts[tid] = (num_rows > 0) ? (num_rows * h_blocks) : 0;}// Exclusive scan of tile_counts to get tile_offsets and total_tileskernel void moe_pass_a_tile_scan(- device const uint* tile_counts [[buffer(0)]], // [E]- device uint* tile_offsets [[buffer(1)]], // [E+1]- device uint* total_tiles [[buffer(2)]], // [1]+ device const uint* tile_counts [[buffer(0)]], // [E]+ device uint* tile_offsets [[buffer(1)]], // [E+1]+ device uint* total_tiles [[buffer(2)]], // [1]constant uint& E [[buffer(3)]],uint lid [[thread_index_in_threadgroup]],threadgroup uint* scratch [[threadgroup(0)]]) {- // Simple single-threadgroup scan (works for E <= 1024)- const uint idx = lid;-- // Load into threadgroup memory- if (idx < E) {- scratch[idx] = tile_counts[idx];- } else {- scratch[idx] = 0;+ // Simple single-threadgroup scan (works for E <= 1024)+ const uint idx = lid;++ // Load into threadgroup memory+ if (idx < E) {+ scratch[idx] = tile_counts[idx];+ } else {+ scratch[idx] = 0;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);++ // Kogge-Stone scan+ uint val = scratch[idx];+ MTL_PRAGMA_UNROLL+ for (uint offset = 1; offset < 1024; offset *= 2) {+ uint temp = 0;+ if (idx >= offset && idx < E) {+ temp = scratch[idx - offset];}threadgroup_barrier(mem_flags::mem_threadgroup);-- // Kogge-Stone scan- uint val = scratch[idx];- MTL_PRAGMA_UNROLL- for (uint offset = 1; offset < 1024; offset *= 2) {- uint temp = 0;- if (idx >= offset && idx < E) {- temp = scratch[idx - offset];- }- threadgroup_barrier(mem_flags::mem_threadgroup);- if (idx >= offset && idx < E) {- val += temp;- scratch[idx] = val;- }- threadgroup_barrier(mem_flags::mem_threadgroup);+ if (idx >= offset && idx < E) {+ val += temp;+ scratch[idx] = val;}-- // Write exclusive scan (shift right by 1)- if (idx == 0) {- tile_offsets[0] = 0;- }- if (idx < E) {- tile_offsets[idx + 1] = scratch[idx];- if (idx == E - 1) {- total_tiles[0] = scratch[idx];- }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }++ // Write exclusive scan (shift right by 1)+ if (idx == 0) {+ tile_offsets[0] = 0;+ }+ if (idx < E) {+ tile_offsets[idx + 1] = scratch[idx];+ if (idx == E - 1) {+ total_tiles[0] = scratch[idx];}+ }}// Build row→expert map: one thread per routed rowkernel void moe_pass_a_build_row_map(- device const uint* expert_offsets [[buffer(0)]] , // [E+1]- device uint* row_expert_map [[buffer(1)]] , // [total_rows]+ device const uint* expert_offsets [[buffer(0)]], // [E+1]+ device uint* row_expert_map [[buffer(1)]], // [total_rows]constant uint& total_rows [[buffer(2)]],constant uint& E [[buffer(3)]],uint tid [[thread_position_in_grid]]) {- if (tid >= total_rows) return;+ if (tid >= total_rows)+ return;- uint left = 0u;- uint right = E;- const uint row = tid;+ uint left = 0u;+ uint right = E;+ const uint row = tid;- while (left + 1u < right) {- const uint mid = (left + right) >> 1;- if (row < expert_offsets[mid]) {- right = mid;- } else {- left = mid;- }+ while (left + 1u < right) {+ const uint mid = (left + right) >> 1;+ if (row < expert_offsets[mid]) {+ right = mid;+ } else {+ left = mid;}+ }- row_expert_map[row] = left;+ row_expert_map[row] = left;}// Build tile map entries from row→expert mapkernel void moe_pass_a_build_tile_map(- device const uint* expert_offsets [[buffer(0)]], // [E+1]- device const uint* tile_offsets [[buffer(1)]], // [E+1]- device const uint* row_expert_map [[buffer(2)]], // [total_rows]- device uint* tile_map [[buffer(3)]], // [total_tiles * 3]+ device const uint* expert_offsets [[buffer(0)]], // [E+1]+ device const uint* tile_offsets [[buffer(1)]], // [E+1]+ device const uint* row_expert_map [[buffer(2)]], // [total_rows]+ device uint* tile_map [[buffer(3)]], // [total_tiles * 3]constant uint& total_rows [[buffer(4)]],constant uint& h_blocks [[buffer(5)]],uint tid [[thread_position_in_grid]]) {- const uint total_tiles = total_rows * h_blocks;- if (tid >= total_tiles) return;+ const uint total_tiles = total_rows * h_blocks;+ if (tid >= total_tiles)+ return;- const uint row_idx = tid / h_blocks;- const uint h_block = tid % h_blocks;+ const uint row_idx = tid / h_blocks;+ const uint h_block = tid % h_blocks;- if (row_idx >= total_rows) return;+ if (row_idx >= total_rows)+ return;- const uint expert_idx = row_expert_map[row_idx];- const uint row_start = expert_offsets[expert_idx];- const uint row_in_expert = row_idx - row_start;- const uint tile_base = tile_offsets[expert_idx] + row_in_expert * h_blocks + h_block;+ const uint expert_idx = row_expert_map[row_idx];+ const uint row_start = expert_offsets[expert_idx];+ const uint row_in_expert = row_idx - row_start;+ const uint tile_base =+ tile_offsets[expert_idx] + row_in_expert * h_blocks + h_block;- tile_map[tile_base * 3 + 0] = h_block;- tile_map[tile_base * 3 + 1] = expert_idx;- tile_map[tile_base * 3 + 2] = row_in_expert;+ tile_map[tile_base * 3 + 0] = h_block;+ tile_map[tile_base * 3 + 1] = expert_idx;+ tile_map[tile_base * 3 + 2] = row_in_expert;}// Write dispatch args for indirect dispatch (reusable from tiled version)kernel void moe_pass_a_write_dispatch_args(- device const uint* total_tiles [[buffer(0)]], // [1]- device uint* dispatch_args [[buffer(1)]], // [3] - MTLDispatchThreadgroupsIndirectArguments- constant uint& num_tiles_y [[buffer(2)]], // usually 1 for Pass A+ device const uint* total_tiles [[buffer(0)]], // [1]+ device uint* dispatch_args+ [[buffer(1)]], // [3] - MTLDispatchThreadgroupsIndirectArguments+ constant uint& num_tiles_y [[buffer(2)]], // usually 1 for Pass Auint tid [[thread_position_in_grid]]) {- if (tid > 0) return;- dispatch_args[0] = total_tiles[0]; // x dimension = total tiles- dispatch_args[1] = num_tiles_y; // y dimension- dispatch_args[2] = 1; // z dimension+ if (tid > 0)+ return;+ dispatch_args[0] = total_tiles[0]; // x dimension = total tiles+ dispatch_args[1] = num_tiles_y; // y dimension+ dispatch_args[2] = 1; // z dimension}// Modified Pass A that reads from tile map for indirect dispatch-template<typename T>+template <typename T>void moe_experts_decode_pass_a_indirect_impl(device const T* X_perm,device const uint* expert_offsets,device const T* W13_all,device const T* up_biases,device float* hidden_out,- device const uint* tile_map, // [total_tiles * 3]+ device const uint* tile_map, // [total_tiles * 3]uint d_model,uint d_ff,uint E,@@ -435,73 +478,101 @@ void moe_experts_decode_pass_a_indirect_impl(float up_clip_min,float up_clip_max,float silu_alpha,- uint tile_idx, // flat threadgroup index+ uint tile_idx, // flat threadgroup indexuint simd_gid,uint simd_lid,threadgroup float* tgp_memory) {- // Read tile descriptor- const uint h_block_idx = tile_map[tile_idx * 3 + 0];- const uint expert_idx = tile_map[tile_idx * 3 + 1];- const uint row_in_expert = tile_map[tile_idx * 3 + 2];-- // Call original implementation- moe_experts_decode_pass_a_impl<T, float>(- X_perm, expert_offsets, W13_all, up_biases, hidden_out,- d_model, d_ff, E,- gate_clip_min, gate_clip_max,- up_clip_min, up_clip_max, silu_alpha,- expert_idx, row_in_expert, h_block_idx,- simd_gid, simd_lid, tgp_memory- );+ // Read tile descriptor+ const uint h_block_idx = tile_map[tile_idx * 3 + 0];+ const uint expert_idx = tile_map[tile_idx * 3 + 1];+ const uint row_in_expert = tile_map[tile_idx * 3 + 2];++ // Call original implementation+ moe_experts_decode_pass_a_impl<T, float>(+ X_perm,+ expert_offsets,+ W13_all,+ up_biases,+ hidden_out,+ d_model,+ d_ff,+ E,+ gate_clip_min,+ gate_clip_max,+ up_clip_min,+ up_clip_max,+ silu_alpha,+ expert_idx,+ row_in_expert,+ h_block_idx,+ simd_gid,+ simd_lid,+ tgp_memory+ );}-#define MOE_PASS_A_INDIRECT_KERNEL(DTYPE, SUFFIX) \-kernel void moe_experts_decode_pass_a_indirect_##SUFFIX( \- device const DTYPE* X_perm [[buffer(0)]], \- device const uint* expert_offsets [[buffer(1)]], \- device const DTYPE* W13_all [[buffer(2)]], \- device float* hidden_out [[buffer(3)]], \- device const DTYPE* up_biases [[buffer(4)]], \- constant uint& d_model [[buffer(5)]], \- constant uint& d_ff [[buffer(6)]], \- constant uint& E [[buffer(7)]], \- constant float& gate_clip_min [[buffer(8)]], \- constant float& gate_clip_max [[buffer(9)]], \- constant float& up_clip_min [[buffer(10)]], \- constant float& up_clip_max [[buffer(11)]], \- constant float& silu_alpha [[buffer(12)]], \- device const uint* tile_map [[buffer(13)]], \- uint3 tgpig [[threadgroup_position_in_grid]], \- uint simd_gid [[simdgroup_index_in_threadgroup]], \- uint simd_lid [[thread_index_in_simdgroup]]) \-{ \- constexpr uint tgp_mem_size = (BN > 1) ? 2 * BN * (BLOCK_M + TM) : 1; \- threadgroup float tgp_memory[tgp_mem_size]; \- \- moe_experts_decode_pass_a_indirect_impl<DTYPE>( \- X_perm, expert_offsets, W13_all, up_biases, hidden_out, tile_map, \- d_model, d_ff, E, gate_clip_min, gate_clip_max, \- up_clip_min, up_clip_max, silu_alpha, \- tgpig.x, simd_gid, simd_lid, \- (BN > 1) ? tgp_memory : nullptr); \-}+#define MOE_PASS_A_INDIRECT_KERNEL(DTYPE, SUFFIX) \+ kernel void moe_experts_decode_pass_a_indirect_##SUFFIX( \+ device const DTYPE* X_perm [[buffer(0)]], \+ device const uint* expert_offsets [[buffer(1)]], \+ device const DTYPE* W13_all [[buffer(2)]], \+ device float* hidden_out [[buffer(3)]], \+ device const DTYPE* up_biases [[buffer(4)]], \+ constant uint& d_model [[buffer(5)]], \+ constant uint& d_ff [[buffer(6)]], \+ constant uint& E [[buffer(7)]], \+ constant float& gate_clip_min [[buffer(8)]], \+ constant float& gate_clip_max [[buffer(9)]], \+ constant float& up_clip_min [[buffer(10)]], \+ constant float& up_clip_max [[buffer(11)]], \+ constant float& silu_alpha [[buffer(12)]], \+ device const uint* tile_map [[buffer(13)]], \+ uint3 tgpig [[threadgroup_position_in_grid]], \+ uint simd_gid [[simdgroup_index_in_threadgroup]], \+ uint simd_lid [[thread_index_in_simdgroup]] \+ ) { \+ constexpr uint tgp_mem_size = (BN > 1) ? 2 * BN * (BLOCK_M + TM) : 1; \+ threadgroup float tgp_memory[tgp_mem_size]; \+ \+ moe_experts_decode_pass_a_indirect_impl<DTYPE>( \+ X_perm, \+ expert_offsets, \+ W13_all, \+ up_biases, \+ hidden_out, \+ tile_map, \+ d_model, \+ d_ff, \+ E, \+ gate_clip_min, \+ gate_clip_max, \+ up_clip_min, \+ up_clip_max, \+ silu_alpha, \+ tgpig.x, \+ simd_gid, \+ simd_lid, \+ (BN > 1) ? tgp_memory : nullptr \+ ); \+ }MOE_PASS_A_INDIRECT_KERNEL(bfloat, bf16)MOE_PASS_A_INDIRECT_KERNEL(half, f16)MOE_PASS_A_INDIRECT_KERNEL(float, f32)-// === Pass B: Simdgroup cooperation along K for coalescing ===-// W2 layout [E, d_model, d_ff] - 32 threads cooperate on one output, reading consecutive K elements+// W2 layout [E, d_model, d_ff] - 32 threads cooperate on one output, reading+// consecutive K elements-template<typename T, typename AccumT>+template <typename T, typename AccumT>void moe_experts_decode_down_fused_2d_impl(- device const float* hidden, // [total_rows, d_ff] - f32 from Pass A- device const uint* row_expert_map, // [total_rows] - direct row->expert lookup- device const T* w2_all, // [E, d_model, d_ff] - layout- device const T* down_biases, // [E, d_model]- device T* y_out, // [total_rows, d_model]+ device const float* hidden, // [total_rows, d_ff] - f32 from Pass A+ device const uint*+ row_expert_map, // [total_rows] - direct row->expert lookup+ device const T* w2_all, // [E, d_model, d_ff] - layout+ device const T* down_biases, // [E, d_model]+ device T* y_out, // [total_rows, d_model]uint total_rows,uint d_model,uint d_ff,@@ -510,116 +581,143 @@ void moe_experts_decode_down_fused_2d_impl(uint simd_gid,uint simd_lid) {- constexpr uint THREADS_PER_SIMD = 32;- constexpr uint SIMDGROUPS_PER_TG = 8;-- const uint row_idx = tgpig.y;- if (row_idx >= total_rows) return;-- // Each simdgroup computes one output column- const uint my_col = tgpig.x * SIMDGROUPS_PER_TG + simd_gid;- if (my_col >= d_model) return;-- const uint expert_idx = row_expert_map[row_idx];-- // Base addresses for this output column- const ulong hidden_base = (ulong)row_idx * (ulong)d_ff;- const ulong w2_col_base = (ulong)expert_idx * (ulong)d_model * (ulong)d_ff + (ulong)my_col * (ulong)d_ff;-- // Dual accumulators for ILP: breaks FMA dependency chains- AccumT acc0 = AccumT(0.0);- AccumT acc1 = AccumT(0.0);- AccumT acc = AccumT(0.0);-- // Main loop: stride-32 with vec8 loads for ILP- const uint k_iters = d_ff / THREADS_PER_SIMD;- const uint k_vec_iters = k_iters / 8;-- MTL_PRAGMA_UNROLL- for (uint iter = 0; iter < k_vec_iters; ++iter) {- const uint k_base = iter * (8 * THREADS_PER_SIMD) + simd_lid;-- // hidden: stride-32 per lane, already f32 from Pass A- const AccumT h0 = hidden[hidden_base + k_base + 0 * THREADS_PER_SIMD];- const AccumT h1 = hidden[hidden_base + k_base + 1 * THREADS_PER_SIMD];- const AccumT h2 = hidden[hidden_base + k_base + 2 * THREADS_PER_SIMD];- const AccumT h3 = hidden[hidden_base + k_base + 3 * THREADS_PER_SIMD];- const AccumT h4 = hidden[hidden_base + k_base + 4 * THREADS_PER_SIMD];- const AccumT h5 = hidden[hidden_base + k_base + 5 * THREADS_PER_SIMD];- const AccumT h6 = hidden[hidden_base + k_base + 6 * THREADS_PER_SIMD];- const AccumT h7 = hidden[hidden_base + k_base + 7 * THREADS_PER_SIMD];-- // W2: lane-coalesced- const AccumT w0 = AccumT(w2_all[w2_col_base + k_base + 0 * THREADS_PER_SIMD]);- const AccumT w1 = AccumT(w2_all[w2_col_base + k_base + 1 * THREADS_PER_SIMD]);- const AccumT w2 = AccumT(w2_all[w2_col_base + k_base + 2 * THREADS_PER_SIMD]);- const AccumT w3 = AccumT(w2_all[w2_col_base + k_base + 3 * THREADS_PER_SIMD]);- const AccumT w4 = AccumT(w2_all[w2_col_base + k_base + 4 * THREADS_PER_SIMD]);- const AccumT w5 = AccumT(w2_all[w2_col_base + k_base + 5 * THREADS_PER_SIMD]);- const AccumT w6 = AccumT(w2_all[w2_col_base + k_base + 6 * THREADS_PER_SIMD]);- const AccumT w7 = AccumT(w2_all[w2_col_base + k_base + 7 * THREADS_PER_SIMD]);-- // dual trees for ILP- acc0 = fma(h0, w0, acc0);- acc1 = fma(h1, w1, acc1);- acc0 = fma(h2, w2, acc0);- acc1 = fma(h3, w3, acc1);- acc0 = fma(h4, w4, acc0);- acc1 = fma(h5, w5, acc1);- acc0 = fma(h6, w6, acc0);- acc1 = fma(h7, w7, acc1);- }-- // fold the vectorized contribution once- acc += (acc0 + acc1);-- // Handle remaining full iterations- MTL_PRAGMA_UNROLL- for (uint iter = k_vec_iters * 8; iter < k_iters; ++iter) {- const uint k = iter * THREADS_PER_SIMD + simd_lid;- acc = fma(AccumT(hidden[hidden_base + k]), AccumT(w2_all[w2_col_base + k]), acc);- }-- // Handle leftover elements (d_ff % 32)- const uint leftover_start = k_iters * THREADS_PER_SIMD;- if (leftover_start + simd_lid < d_ff) {- const uint k = leftover_start + simd_lid;- acc = fma(AccumT(hidden[hidden_base + k]), AccumT(w2_all[w2_col_base + k]), acc);- }-- // Simdgroup reduction- AccumT result = simd_sum(acc);-- // Lane 0 writes result- if (simd_lid == 0) {- const ulong bias_idx = (ulong)expert_idx * (ulong)d_model + (ulong)my_col;- result += AccumT(down_biases[bias_idx]);-- const ulong out_idx = (ulong)row_idx * (ulong)d_model + (ulong)my_col;- y_out[out_idx] = T(result);- }+ constexpr uint THREADS_PER_SIMD = 32;+ constexpr uint SIMDGROUPS_PER_TG = 8;++ const uint row_idx = tgpig.y;+ if (row_idx >= total_rows)+ return;++ // Each simdgroup computes one output column+ const uint my_col = tgpig.x * SIMDGROUPS_PER_TG + simd_gid;+ if (my_col >= d_model)+ return;++ const uint expert_idx = row_expert_map[row_idx];++ // Base addresses for this output column+ const ulong hidden_base = (ulong)row_idx * (ulong)d_ff;+ const ulong w2_col_base = (ulong)expert_idx * (ulong)d_model * (ulong)d_ff ++ (ulong)my_col * (ulong)d_ff;++ // Dual accumulators for ILP: breaks FMA dependency chains+ AccumT acc0 = AccumT(0.0);+ AccumT acc1 = AccumT(0.0);+ AccumT acc = AccumT(0.0);++ // Main loop: stride-32 with vec8 loads for ILP+ const uint k_iters = d_ff / THREADS_PER_SIMD;+ const uint k_vec_iters = k_iters / 8;++ MTL_PRAGMA_UNROLL+ for (uint iter = 0; iter < k_vec_iters; ++iter) {+ const uint k_base = iter * (8 * THREADS_PER_SIMD) + simd_lid;++ // hidden: stride-32 per lane, already f32 from Pass A+ const AccumT h0 = hidden[hidden_base + k_base + 0 * THREADS_PER_SIMD];+ const AccumT h1 = hidden[hidden_base + k_base + 1 * THREADS_PER_SIMD];+ const AccumT h2 = hidden[hidden_base + k_base + 2 * THREADS_PER_SIMD];+ const AccumT h3 = hidden[hidden_base + k_base + 3 * THREADS_PER_SIMD];+ const AccumT h4 = hidden[hidden_base + k_base + 4 * THREADS_PER_SIMD];+ const AccumT h5 = hidden[hidden_base + k_base + 5 * THREADS_PER_SIMD];+ const AccumT h6 = hidden[hidden_base + k_base + 6 * THREADS_PER_SIMD];+ const AccumT h7 = hidden[hidden_base + k_base + 7 * THREADS_PER_SIMD];++ // W2: lane-coalesced+ const AccumT w0 =+ AccumT(w2_all[w2_col_base + k_base + 0 * THREADS_PER_SIMD]);+ const AccumT w1 =+ AccumT(w2_all[w2_col_base + k_base + 1 * THREADS_PER_SIMD]);+ const AccumT w2 =+ AccumT(w2_all[w2_col_base + k_base + 2 * THREADS_PER_SIMD]);+ const AccumT w3 =+ AccumT(w2_all[w2_col_base + k_base + 3 * THREADS_PER_SIMD]);+ const AccumT w4 =+ AccumT(w2_all[w2_col_base + k_base + 4 * THREADS_PER_SIMD]);+ const AccumT w5 =+ AccumT(w2_all[w2_col_base + k_base + 5 * THREADS_PER_SIMD]);+ const AccumT w6 =+ AccumT(w2_all[w2_col_base + k_base + 6 * THREADS_PER_SIMD]);+ const AccumT w7 =+ AccumT(w2_all[w2_col_base + k_base + 7 * THREADS_PER_SIMD]);++ // dual trees for ILP+ acc0 = fma(h0, w0, acc0);+ acc1 = fma(h1, w1, acc1);+ acc0 = fma(h2, w2, acc0);+ acc1 = fma(h3, w3, acc1);+ acc0 = fma(h4, w4, acc0);+ acc1 = fma(h5, w5, acc1);+ acc0 = fma(h6, w6, acc0);+ acc1 = fma(h7, w7, acc1);+ }++ // fold the vectorized contribution once+ acc += (acc0 + acc1);++ // Handle remaining full iterations+ MTL_PRAGMA_UNROLL+ for (uint iter = k_vec_iters * 8; iter < k_iters; ++iter) {+ const uint k = iter * THREADS_PER_SIMD + simd_lid;+ acc =+ fma(AccumT(hidden[hidden_base + k]),+ AccumT(w2_all[w2_col_base + k]),+ acc);+ }++ // Handle leftover elements (d_ff % 32)+ const uint leftover_start = k_iters * THREADS_PER_SIMD;+ if (leftover_start + simd_lid < d_ff) {+ const uint k = leftover_start + simd_lid;+ acc =+ fma(AccumT(hidden[hidden_base + k]),+ AccumT(w2_all[w2_col_base + k]),+ acc);+ }++ // Simdgroup reduction+ AccumT result = simd_sum(acc);++ // Lane 0 writes result+ if (simd_lid == 0) {+ const ulong bias_idx = (ulong)expert_idx * (ulong)d_model + (ulong)my_col;+ result += AccumT(down_biases[bias_idx]);++ const ulong out_idx = (ulong)row_idx * (ulong)d_model + (ulong)my_col;+ y_out[out_idx] = T(result);+ }}-#define MOE_PASS_B_FUSED_2D_KERNEL(DTYPE, SUFFIX) \-kernel void moe_experts_decode_down_fused_2d_##SUFFIX( \- device const float* hidden [[buffer(0)]], \- device const uint* row_expert_map [[buffer(1)]], \- device const DTYPE* w2_all [[buffer(2)]], \- device const DTYPE* down_biases [[buffer(3)]], \- device DTYPE* y_out [[buffer(4)]], \- constant uint& total_rows [[buffer(5)]], \- constant uint& d_model [[buffer(6)]], \- constant uint& d_ff [[buffer(7)]], \- constant uint& E [[buffer(8)]], \- uint2 tgpig [[threadgroup_position_in_grid]], \- uint simd_gid [[simdgroup_index_in_threadgroup]], \- uint simd_lid [[thread_index_in_simdgroup]]) \-{ \- moe_experts_decode_down_fused_2d_impl<DTYPE, float>( \- hidden, row_expert_map, w2_all, down_biases, y_out, \- total_rows, d_model, d_ff, E, \- tgpig, simd_gid, simd_lid); \-}+#define MOE_PASS_B_FUSED_2D_KERNEL(DTYPE, SUFFIX) \+ kernel void moe_experts_decode_down_fused_2d_##SUFFIX( \+ device const float* hidden [[buffer(0)]], \+ device const uint* row_expert_map [[buffer(1)]], \+ device const DTYPE* w2_all [[buffer(2)]], \+ device const DTYPE* down_biases [[buffer(3)]], \+ device DTYPE* y_out [[buffer(4)]], \+ constant uint& total_rows [[buffer(5)]], \+ constant uint& d_model [[buffer(6)]], \+ constant uint& d_ff [[buffer(7)]], \+ constant uint& E [[buffer(8)]], \+ uint2 tgpig [[threadgroup_position_in_grid]], \+ uint simd_gid [[simdgroup_index_in_threadgroup]], \+ uint simd_lid [[thread_index_in_simdgroup]] \+ ) { \+ moe_experts_decode_down_fused_2d_impl<DTYPE, float>( \+ hidden, \+ row_expert_map, \+ w2_all, \+ down_biases, \+ y_out, \+ total_rows, \+ d_model, \+ d_ff, \+ E, \+ tgpig, \+ simd_gid, \+ simd_lid \+ ); \+ }MOE_PASS_B_FUSED_2D_KERNEL(bfloat, bf16)MOE_PASS_B_FUSED_2D_KERNEL(half, f16)diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metalindex db64af685..03aad6517 100644--- a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal@@ -19,7 +19,7 @@#include <metal_stdlib>#include <metal_simdgroup>#include <metal_simdgroup_matrix>-#include "../quant_matmul/mma.h"+#include "../quant_matmul/mma.h"using namespace metal;@@ -27,50 +27,52 @@ using namespace metal;static inline uint ceil_div(uint a, uint b) { return (a + b - 1u) / b; }static inline uint linear_tid(uint3 tid, uint3 tpg) {- return tid.z * (tpg.x * tpg.y) + tid.y * tpg.x + tid.x;+ return tid.z * (tpg.x * tpg.y) + tid.y * tpg.x + tid.x;}static inline float gelu_approx(float x) {- const float k0 = 0.7978845608f; // sqrt(2/pi)- const float k1 = 0.044715f;- if (x > 10.0f) return x;- if (x < -10.0f) return 0.0f;- float x3 = x * x * x;- float t = clamp(k0 * (x + k1 * x3), -10.0f, 10.0f);- return 0.5f * x * (1.0f + tanh(t));+ const float k0 = 0.7978845608f; // sqrt(2/pi)+ const float k1 = 0.044715f;+ if (x > 10.0f)+ return x;+ if (x < -10.0f)+ return 0.0f;+ float x3 = x * x * x;+ float t = clamp(k0 * (x + k1 * x3), -10.0f, 10.0f);+ return 0.5f * x * (1.0f + tanh(t));}static inline float silu(float x, float alpha) {- // alpha=1 gives standard SiLU; keep alpha param for parity with your API- return x / (1.0f + exp(-alpha * x));+ // alpha=1 gives standard SiLU; keep alpha param for parity with your API+ return x / (1.0f + exp(-alpha * x));}-template<typename T>+template <typename T>inline void store_vec4(device T* dst, ulong base, float4 vals) {- dst[base + 0] = T(vals.x);- dst[base + 1] = T(vals.y);- dst[base + 2] = T(vals.z);- dst[base + 3] = T(vals.w);+ dst[base + 0] = T(vals.x);+ dst[base + 1] = T(vals.y);+ dst[base + 2] = T(vals.z);+ dst[base + 3] = T(vals.w);}-template<>+template <>inline void store_vec4<float>(device float* dst, ulong base, float4 vals) {- *reinterpret_cast<device float4*>(dst + base) = vals;+ *reinterpret_cast<device float4*>(dst + base) = vals;}-template<typename T>+template <typename T>struct StageStorage {- using type = float;+ using type = float;};-template<>+template <>struct StageStorage<half> {- using type = half;+ using type = half;};-template<>+template <>struct StageStorage<bfloat> {- using type = bfloat;+ using type = bfloat;};// 0=GELU(up), 1=SiLU(up), 2=SwiGLU(gate)*up, 3=GEGLU(gate)*up@@ -84,694 +86,917 @@ constant uint PASSA_BK = 64;constant uint PASSA_SG_BM = 8;constant uint PASSA_SG_BN = 16;-template<typename T>+template <typename T>inline void pass_a_impl(- device const T* X_perm, // [total_rows, D]- device const uint* expert_offsets, // [E+1]- device const T* W13_all, // [E, 2*FF, D]- device const T* up_biases, // [E, 2*FF] (up | gate)- device float* hidden_out, // [total_rows, FF]- uint d_model, uint d_ff, uint E,- float gate_clip_min, float gate_clip_max,- float up_clip_min, float up_clip_max,+ device const T* X_perm, // [total_rows, D]+ device const uint* expert_offsets, // [E+1]+ device const T* W13_all, // [E, 2*FF, D]+ device const T* up_biases, // [E, 2*FF] (up | gate)+ device float* hidden_out, // [total_rows, FF]+ uint d_model,+ uint d_ff,+ uint E,+ float gate_clip_min,+ float gate_clip_max,+ float up_clip_min,+ float up_clip_max,float silu_alpha,// work partition- uint tile_seg_start, uint expert_idx, uint tile_m, uint tile_n,+ uint tile_seg_start,+ uint expert_idx,+ uint tile_m,+ uint tile_n,// per-thread info- uint sg_id, uint3 threads_per_tg, uint3 local_tid,+ uint sg_id,+ uint3 threads_per_tg,+ uint3 local_tid,// TG storage- threadgroup typename StageStorage<T>::type* Xs, // [BM,BK]- threadgroup typename StageStorage<T>::type* Wk_up, // [BN,BK]- threadgroup typename StageStorage<T>::type* Wk_gate, // [BN,BK]- threadgroup float* bias_up, // [BN]- threadgroup float* bias_gate // [BN]+ threadgroup typename StageStorage<T>::type* Xs, // [BM,BK]+ threadgroup typename StageStorage<T>::type* Wk_up, // [BN,BK]+ threadgroup typename StageStorage<T>::type* Wk_gate, // [BN,BK]+ threadgroup float* bias_up, // [BN]+ threadgroup float* bias_gate // [BN]) {- using StageT = typename StageStorage<T>::type;- using StageVec4 = metal::vec<StageT, 4>;-- const uint Bm = PASSA_BM, Bn = PASSA_BN, Bk = PASSA_BK;- const uint SgBm = PASSA_SG_BM, SgBn = PASSA_SG_BN;-- if (expert_idx >= E) return;-- const uint seg_start = expert_offsets[expert_idx];- const uint seg_end = expert_offsets[expert_idx + 1];- const uint seg_len = (seg_end > seg_start) ? (seg_end - seg_start) : 0u;- if (seg_len == 0) return;-- const uint row_tg_off = tile_m * Bm;- const uint col_tg_off = tile_n * Bn;- if (row_tg_off >= seg_len || col_tg_off >= d_ff) return;-- const uint m_rows = min(Bm, seg_len - row_tg_off);- const uint n_cols = min(Bn, d_ff - col_tg_off);-- const ulong x_base = ((ulong)tile_seg_start + (ulong)row_tg_off) * (ulong)d_model;- const ulong w13_expert_base = (ulong)expert_idx * (ulong)(2 * d_ff) * (ulong)d_model;- const ulong bias_base = (ulong)expert_idx * (ulong)(2 * d_ff);-- // simdgroup tile mapping- const uint sg_col_count = Bn / SgBn;- const uint row_sg = sg_id / sg_col_count;- const uint col_sg = sg_id % sg_col_count;- const uint row_sg_off = row_sg * SgBm;- const uint col_sg_off = col_sg * SgBn;-- // Guard against misconfigured TG sizes- constexpr uint SG_TILE = 8; // simdgroup fragment dimension (8x8)- constexpr uint SG_EXPECTED = (PASSA_BM / SG_TILE) * (PASSA_BN / SG_TILE);- if (sg_id >= SG_EXPECTED) return;-- // partial accumulators (8x8 tiles per simdgroup)- constexpr uint TEMP = (PASSA_SG_BM / SG_TILE) * (PASSA_SG_BN / SG_TILE);- metal::simdgroup_float8x8 OutUp[TEMP];- metal::simdgroup_float8x8 OutGate[TEMP];- for (uint i = 0; i < TEMP; ++i) {- OutUp[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);- if (GATING_SEL > 1u) {- OutGate[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);- }+ using StageT = typename StageStorage<T>::type;+ using StageVec4 = metal::vec<StageT, 4>;++ const uint Bm = PASSA_BM, Bn = PASSA_BN, Bk = PASSA_BK;+ const uint SgBm = PASSA_SG_BM, SgBn = PASSA_SG_BN;++ if (expert_idx >= E)+ return;++ const uint seg_start = expert_offsets[expert_idx];+ const uint seg_end = expert_offsets[expert_idx + 1];+ const uint seg_len = (seg_end > seg_start) ? (seg_end - seg_start) : 0u;+ if (seg_len == 0)+ return;++ const uint row_tg_off = tile_m * Bm;+ const uint col_tg_off = tile_n * Bn;+ if (row_tg_off >= seg_len || col_tg_off >= d_ff)+ return;++ const uint m_rows = min(Bm, seg_len - row_tg_off);+ const uint n_cols = min(Bn, d_ff - col_tg_off);++ const ulong x_base =+ ((ulong)tile_seg_start + (ulong)row_tg_off) * (ulong)d_model;+ const ulong w13_expert_base =+ (ulong)expert_idx * (ulong)(2 * d_ff) * (ulong)d_model;+ const ulong bias_base = (ulong)expert_idx * (ulong)(2 * d_ff);++ // simdgroup tile mapping+ const uint sg_col_count = Bn / SgBn;+ const uint row_sg = sg_id / sg_col_count;+ const uint col_sg = sg_id % sg_col_count;+ const uint row_sg_off = row_sg * SgBm;+ const uint col_sg_off = col_sg * SgBn;++ // Guard against misconfigured TG sizes+ constexpr uint SG_TILE = 8; // simdgroup fragment dimension (8x8)+ constexpr uint SG_EXPECTED = (PASSA_BM / SG_TILE) * (PASSA_BN / SG_TILE);+ if (sg_id >= SG_EXPECTED)+ return;++ // partial accumulators (8x8 tiles per simdgroup)+ constexpr uint TEMP = (PASSA_SG_BM / SG_TILE) * (PASSA_SG_BN / SG_TILE);+ metal::simdgroup_float8x8 OutUp[TEMP];+ metal::simdgroup_float8x8 OutGate[TEMP];+ for (uint i = 0; i < TEMP; ++i) {+ OutUp[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);+ if (GATING_SEL > 1u) {+ OutGate[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);}+ }- const uint threads_total = threads_per_tg.x * threads_per_tg.y * threads_per_tg.z;- const uint lin = linear_tid(local_tid, threads_per_tg);-- // K-loop (tile along D)- for (uint k_off = 0; k_off < d_model; k_off += Bk) {- const uint valid_k = min(Bk, d_model - k_off);-- threadgroup_barrier(mem_flags::mem_threadgroup);-- // ---- Stage LHS: Xs [Bm,Bk] as float, vectorized 4 ----- {- constexpr uint row_stride = PASSA_BK;- constexpr uint vec_cols = PASSA_BK / 4;- const uint total_vecs = PASSA_BM * vec_cols;- const uint iters = ceil_div(total_vecs, threads_total);--UZU_PRAGMA_UNROLL- for (uint t = 0; t < iters; ++t) {- const uint i = t * threads_total + lin;- if (i < total_vecs) {- const uint r = i / vec_cols;- const uint c4 = i % vec_cols;-- threadgroup StageVec4* dst4 =- reinterpret_cast<threadgroup StageVec4*>(Xs + r * row_stride + (c4 << 2));-- StageVec4 packed = StageVec4(StageT(0.0f));- if (r < m_rows) {- const ulong base = x_base + (ulong)r * (ulong)d_model + (ulong)(k_off + (c4 << 2));- const uint base_vec = (c4 << 2);- float v0 = 0.0f, v1 = 0.0f, v2 = 0.0f, v3 = 0.0f;- if (base_vec + 0 < valid_k) v0 = float(X_perm[base + 0]);- if (base_vec + 1 < valid_k) v1 = float(X_perm[base + 1]);- if (base_vec + 2 < valid_k) v2 = float(X_perm[base + 2]);- if (base_vec + 3 < valid_k) v3 = float(X_perm[base + 3]);- packed = StageVec4(StageT(v0), StageT(v1), StageT(v2), StageT(v3));- }- *dst4 = packed;- }- }- }+ const uint threads_total =+ threads_per_tg.x * threads_per_tg.y * threads_per_tg.z;+ const uint lin = linear_tid(local_tid, threads_per_tg);- // ---- Stage RHS: W_up [Bn,Bk], W_gate [Bn,Bk], vectorized 4 ----- {- constexpr uint row_stride = PASSA_BK;- constexpr uint vec_cols = PASSA_BK / 4;- const uint total_vecs = PASSA_BN * vec_cols;- const uint iters = ceil_div(total_vecs, threads_total);--UZU_PRAGMA_UNROLL- for (uint t = 0; t < iters; ++t) {- const uint i = t * threads_total + lin;- if (i < total_vecs) {- const uint n = i / vec_cols; // column within the BN tile- const uint c4 = i % vec_cols;-- threadgroup StageVec4* up4 =- reinterpret_cast<threadgroup StageVec4*>(Wk_up + n * row_stride + (c4 << 2));- StageVec4 upv = StageVec4(StageT(0.0f));-- threadgroup StageVec4* gt4 =- reinterpret_cast<threadgroup StageVec4*>(Wk_gate + n * row_stride + (c4 << 2));- StageVec4 gtv = StageVec4(StageT(0.0f));-- if (n < n_cols) {- const uint gcol = col_tg_off + n;- const ulong up_base = w13_expert_base + (ulong)gcol * (ulong)d_model + (ulong)(k_off + (c4 << 2));- const ulong gate_base = w13_expert_base + (ulong)(d_ff + gcol) * (ulong)d_model + (ulong)(k_off + (c4 << 2));- const uint base_vec = (c4 << 2);- float up0 = 0.0f, up1 = 0.0f, up2 = 0.0f, up3 = 0.0f;- float gt0 = 0.0f, gt1 = 0.0f, gt2 = 0.0f, gt3 = 0.0f;- if (base_vec + 0 < valid_k) { up0 = float(W13_all[up_base + 0]); gt0 = float(W13_all[gate_base + 0]); }- if (base_vec + 1 < valid_k) { up1 = float(W13_all[up_base + 1]); gt1 = float(W13_all[gate_base + 1]); }- if (base_vec + 2 < valid_k) { up2 = float(W13_all[up_base + 2]); gt2 = float(W13_all[gate_base + 2]); }- if (base_vec + 3 < valid_k) { up3 = float(W13_all[up_base + 3]); gt3 = float(W13_all[gate_base + 3]); }- upv = StageVec4(StageT(up0), StageT(up1), StageT(up2), StageT(up3));- gtv = StageVec4(StageT(gt0), StageT(gt1), StageT(gt2), StageT(gt3));- }-- *up4 = upv;- if (GATING_SEL > 1u) *gt4 = gtv;- }- }- }+ // K-loop (tile along D)+ for (uint k_off = 0; k_off < d_model; k_off += Bk) {+ const uint valid_k = min(Bk, d_model - k_off);- threadgroup_barrier(mem_flags::mem_threadgroup);-- // ---- MMA across the BK tile (in simdgroup-sized chunks) -----UZU_PRAGMA_UNROLL- for (uint kk = 0; kk < PASSA_BK; kk += SG_TILE) {-UZU_PRAGMA_UNROLL- for (uint m_sub = 0; m_sub < PASSA_SG_BM; m_sub += SG_TILE) {- const uint r_idx = m_sub / SG_TILE;- metal::simdgroup_matrix<StageT, 8, 8> lhs_frag;- simdgroup_load(lhs_frag, Xs, PASSA_BK, ulong2(kk, row_sg_off + m_sub));--UZU_PRAGMA_UNROLL- for (uint n_sub = 0; n_sub < PASSA_SG_BN; n_sub += SG_TILE) {- const uint c_idx = n_sub / SG_TILE;- const uint tile = r_idx * (PASSA_SG_BN / SG_TILE) + c_idx;- metal::simdgroup_matrix<StageT, 8, 8> rhs_up;- simdgroup_load(rhs_up, Wk_up, PASSA_BK, ulong2(kk, col_sg_off + n_sub), true);- simdgroup_multiply_accumulate(OutUp[tile], lhs_frag, rhs_up, OutUp[tile]);-- if (GATING_SEL > 1u) {- metal::simdgroup_matrix<StageT, 8, 8> rhs_gate;- simdgroup_load(rhs_gate, Wk_gate, PASSA_BK, ulong2(kk, col_sg_off + n_sub), true);- simdgroup_multiply_accumulate(OutGate[tile], lhs_frag, rhs_gate, OutGate[tile]);- }- }- }- }- } // K loop-- // ---- Bias tile [Bn] ----- // Note: bias layout is [up | gate] in contiguous FF chunks.- for (uint c_local = lin; c_local < Bn; c_local += threads_total) {- const uint c_global = col_tg_off + c_local;- bias_up[c_local] = (c_global < d_ff) ? float(up_biases[bias_base + c_global]) : 0.0f;- if (GATING_SEL > 1u) {- bias_gate[c_local] = (c_global < d_ff) ? float(up_biases[bias_base + d_ff + c_global]) : 0.0f;+ threadgroup_barrier(mem_flags::mem_threadgroup);++ // ---- Stage LHS: Xs [Bm,Bk] as float, vectorized 4 ----+ {+ constexpr uint row_stride = PASSA_BK;+ constexpr uint vec_cols = PASSA_BK / 4;+ const uint total_vecs = PASSA_BM * vec_cols;+ const uint iters = ceil_div(total_vecs, threads_total);++ UZU_PRAGMA_UNROLL+ for (uint t = 0; t < iters; ++t) {+ const uint i = t * threads_total + lin;+ if (i < total_vecs) {+ const uint r = i / vec_cols;+ const uint c4 = i % vec_cols;++ threadgroup StageVec4* dst4 =+ reinterpret_cast<threadgroup StageVec4*>(+ Xs + r * row_stride + (c4 << 2)+ );++ StageVec4 packed = StageVec4(StageT(0.0f));+ if (r < m_rows) {+ const ulong base =+ x_base + (ulong)r * (ulong)d_model + (ulong)(k_off + (c4 << 2));+ const uint base_vec = (c4 << 2);+ float v0 = 0.0f, v1 = 0.0f, v2 = 0.0f, v3 = 0.0f;+ if (base_vec + 0 < valid_k)+ v0 = float(X_perm[base + 0]);+ if (base_vec + 1 < valid_k)+ v1 = float(X_perm[base + 1]);+ if (base_vec + 2 < valid_k)+ v2 = float(X_perm[base + 2]);+ if (base_vec + 3 < valid_k)+ v3 = float(X_perm[base + 3]);+ packed = StageVec4(StageT(v0), StageT(v1), StageT(v2), StageT(v3));+ }+ *dst4 = packed;}+ }}- threadgroup_barrier(mem_flags::mem_threadgroup);-- // ---- Epilogue: activation/gating → hidden_out (f32) ----- const uint sg_lane_start = sg_id * 32;- const uint sg_lane_end = sg_lane_start + 32;- if (lin < sg_lane_start || lin >= sg_lane_end) return;- const uint sg_lin = lin - sg_lane_start;- // Lane→(row,col) mapping for 8x8 simdgroup fragments, matches the fragment layout used above.- const uint lane_qid = sg_lin >> 2;- const uint lane_row = (lane_qid & 4u) + ((sg_lin >> 1) & 3u);- const uint lane_col_base = ((lane_qid & 2u) << 1) + ((sg_lin & 1u) << 1);--UZU_PRAGMA_UNROLL- for (uint n_sub = 0; n_sub < PASSA_SG_BN; n_sub += SG_TILE) {- const uint c_idx = n_sub / SG_TILE;-UZU_PRAGMA_UNROLL- for (uint m_sub = 0; m_sub < PASSA_SG_BM; m_sub += SG_TILE) {- const uint r_idx = m_sub / SG_TILE;- const uint tile = r_idx * (PASSA_SG_BN / SG_TILE) + c_idx;-- const uint tile_row_base = row_sg_off + m_sub;- const uint tile_col_base = col_sg_off + n_sub;- const uint local_row = tile_row_base + lane_row;- if (local_row >= m_rows || tile_col_base >= n_cols) {- continue;+ // ---- Stage RHS: W_up [Bn,Bk], W_gate [Bn,Bk], vectorized 4 ----+ {+ constexpr uint row_stride = PASSA_BK;+ constexpr uint vec_cols = PASSA_BK / 4;+ const uint total_vecs = PASSA_BN * vec_cols;+ const uint iters = ceil_div(total_vecs, threads_total);++ UZU_PRAGMA_UNROLL+ for (uint t = 0; t < iters; ++t) {+ const uint i = t * threads_total + lin;+ if (i < total_vecs) {+ const uint n = i / vec_cols; // column within the BN tile+ const uint c4 = i % vec_cols;++ threadgroup StageVec4* up4 = reinterpret_cast<threadgroup StageVec4*>(+ Wk_up + n * row_stride + (c4 << 2)+ );+ StageVec4 upv = StageVec4(StageT(0.0f));++ threadgroup StageVec4* gt4 = reinterpret_cast<threadgroup StageVec4*>(+ Wk_gate + n * row_stride + (c4 << 2)+ );+ StageVec4 gtv = StageVec4(StageT(0.0f));++ if (n < n_cols) {+ const uint gcol = col_tg_off + n;+ const ulong up_base = w13_expert_base ++ (ulong)gcol * (ulong)d_model ++ (ulong)(k_off + (c4 << 2));+ const ulong gate_base = w13_expert_base ++ (ulong)(d_ff + gcol) * (ulong)d_model ++ (ulong)(k_off + (c4 << 2));+ const uint base_vec = (c4 << 2);+ float up0 = 0.0f, up1 = 0.0f, up2 = 0.0f, up3 = 0.0f;+ float gt0 = 0.0f, gt1 = 0.0f, gt2 = 0.0f, gt3 = 0.0f;+ if (base_vec + 0 < valid_k) {+ up0 = float(W13_all[up_base + 0]);+ gt0 = float(W13_all[gate_base + 0]);}-- const auto up_frag = OutUp[tile].thread_elements();- float gate_frag_0 = 0.0f;- float gate_frag_1 = 0.0f;- if (GATING_SEL > 1u) {- const auto gate_frag = OutGate[tile].thread_elements();- gate_frag_0 = gate_frag[0];- gate_frag_1 = gate_frag[1];+ if (base_vec + 1 < valid_k) {+ up1 = float(W13_all[up_base + 1]);+ gt1 = float(W13_all[gate_base + 1]);}-- const uint col0 = tile_col_base + lane_col_base;- const uint col1 = col0 + 1;- const ulong out_row = seg_start + row_tg_off + local_row;-- if (col0 < n_cols) {- float up_v = clamp(up_frag[0] + bias_up[col0], up_clip_min, up_clip_max);- float out_val;- if (GATING_SEL <= 1u) {- out_val =- (GATING_SEL == 0u) ? gelu_approx(up_v) : silu(up_v, silu_alpha);- } else {- float gate_v =- clamp(gate_frag_0 + bias_gate[col0], gate_clip_min, gate_clip_max);- const float gate_act =- (GATING_SEL == 2u) ? silu(gate_v, silu_alpha) : gelu_approx(gate_v);- out_val = gate_act * up_v;- }- const ulong out_col = col_tg_off + col0;- hidden_out[out_row * (ulong)d_ff + out_col] = out_val;+ if (base_vec + 2 < valid_k) {+ up2 = float(W13_all[up_base + 2]);+ gt2 = float(W13_all[gate_base + 2]);}-- if (col1 < n_cols) {- float up_v = clamp(up_frag[1] + bias_up[col1], up_clip_min, up_clip_max);- float out_val;- if (GATING_SEL <= 1u) {- out_val =- (GATING_SEL == 0u) ? gelu_approx(up_v) : silu(up_v, silu_alpha);- } else {- float gate_v =- clamp(gate_frag_1 + bias_gate[col1], gate_clip_min, gate_clip_max);- const float gate_act =- (GATING_SEL == 2u) ? silu(gate_v, silu_alpha) : gelu_approx(gate_v);- out_val = gate_act * up_v;- }- const ulong out_col = col_tg_off + col1;- hidden_out[out_row * (ulong)d_ff + out_col] = out_val;+ if (base_vec + 3 < valid_k) {+ up3 = float(W13_all[up_base + 3]);+ gt3 = float(W13_all[gate_base + 3]);}+ upv = StageVec4(StageT(up0), StageT(up1), StageT(up2), StageT(up3));+ gtv = StageVec4(StageT(gt0), StageT(gt1), StageT(gt2), StageT(gt3));+ }++ *up4 = upv;+ if (GATING_SEL > 1u)+ *gt4 = gtv;+ }+ }+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ // ---- MMA across the BK tile (in simdgroup-sized chunks) ----+ UZU_PRAGMA_UNROLL+ for (uint kk = 0; kk < PASSA_BK; kk += SG_TILE) {+ UZU_PRAGMA_UNROLL+ for (uint m_sub = 0; m_sub < PASSA_SG_BM; m_sub += SG_TILE) {+ const uint r_idx = m_sub / SG_TILE;+ metal::simdgroup_matrix<StageT, 8, 8> lhs_frag;+ simdgroup_load(lhs_frag, Xs, PASSA_BK, ulong2(kk, row_sg_off + m_sub));++ UZU_PRAGMA_UNROLL+ for (uint n_sub = 0; n_sub < PASSA_SG_BN; n_sub += SG_TILE) {+ const uint c_idx = n_sub / SG_TILE;+ const uint tile = r_idx * (PASSA_SG_BN / SG_TILE) + c_idx;+ metal::simdgroup_matrix<StageT, 8, 8> rhs_up;+ simdgroup_load(+ rhs_up,+ Wk_up,+ PASSA_BK,+ ulong2(kk, col_sg_off + n_sub),+ true+ );+ simdgroup_multiply_accumulate(+ OutUp[tile],+ lhs_frag,+ rhs_up,+ OutUp[tile]+ );++ if (GATING_SEL > 1u) {+ metal::simdgroup_matrix<StageT, 8, 8> rhs_gate;+ simdgroup_load(+ rhs_gate,+ Wk_gate,+ PASSA_BK,+ ulong2(kk, col_sg_off + n_sub),+ true+ );+ simdgroup_multiply_accumulate(+ OutGate[tile],+ lhs_frag,+ rhs_gate,+ OutGate[tile]+ );+ }+ }+ }+ }+ } // K loop++ // ---- Bias tile [Bn] ----+ // Note: bias layout is [up | gate] in contiguous FF chunks.+ for (uint c_local = lin; c_local < Bn; c_local += threads_total) {+ const uint c_global = col_tg_off + c_local;+ bias_up[c_local] =+ (c_global < d_ff) ? float(up_biases[bias_base + c_global]) : 0.0f;+ if (GATING_SEL > 1u) {+ bias_gate[c_local] = (c_global < d_ff)+ ? float(up_biases[bias_base + d_ff + c_global])+ : 0.0f;+ }+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ // ---- Epilogue: activation/gating → hidden_out (f32) ----+ const uint sg_lane_start = sg_id * 32;+ const uint sg_lane_end = sg_lane_start + 32;+ if (lin < sg_lane_start || lin >= sg_lane_end)+ return;+ const uint sg_lin = lin - sg_lane_start;+ // Lane→(row,col) mapping for 8x8 simdgroup fragments, matches the fragment+ // layout used above.+ const uint lane_qid = sg_lin >> 2;+ const uint lane_row = (lane_qid & 4u) + ((sg_lin >> 1) & 3u);+ const uint lane_col_base = ((lane_qid & 2u) << 1) + ((sg_lin & 1u) << 1);++ UZU_PRAGMA_UNROLL+ for (uint n_sub = 0; n_sub < PASSA_SG_BN; n_sub += SG_TILE) {+ const uint c_idx = n_sub / SG_TILE;+ UZU_PRAGMA_UNROLL+ for (uint m_sub = 0; m_sub < PASSA_SG_BM; m_sub += SG_TILE) {+ const uint r_idx = m_sub / SG_TILE;+ const uint tile = r_idx * (PASSA_SG_BN / SG_TILE) + c_idx;++ const uint tile_row_base = row_sg_off + m_sub;+ const uint tile_col_base = col_sg_off + n_sub;+ const uint local_row = tile_row_base + lane_row;+ if (local_row >= m_rows || tile_col_base >= n_cols) {+ continue;+ }++ const auto up_frag = OutUp[tile].thread_elements();+ float gate_frag_0 = 0.0f;+ float gate_frag_1 = 0.0f;+ if (GATING_SEL > 1u) {+ const auto gate_frag = OutGate[tile].thread_elements();+ gate_frag_0 = gate_frag[0];+ gate_frag_1 = gate_frag[1];+ }++ const uint col0 = tile_col_base + lane_col_base;+ const uint col1 = col0 + 1;+ const ulong out_row = seg_start + row_tg_off + local_row;++ if (col0 < n_cols) {+ float up_v =+ clamp(up_frag[0] + bias_up[col0], up_clip_min, up_clip_max);+ float out_val;+ if (GATING_SEL <= 1u) {+ out_val =+ (GATING_SEL == 0u) ? gelu_approx(up_v) : silu(up_v, silu_alpha);+ } else {+ float gate_v = clamp(+ gate_frag_0 + bias_gate[col0],+ gate_clip_min,+ gate_clip_max+ );+ const float gate_act = (GATING_SEL == 2u) ? silu(gate_v, silu_alpha)+ : gelu_approx(gate_v);+ out_val = gate_act * up_v;}+ const ulong out_col = col_tg_off + col0;+ hidden_out[out_row * (ulong)d_ff + out_col] = out_val;+ }++ if (col1 < n_cols) {+ float up_v =+ clamp(up_frag[1] + bias_up[col1], up_clip_min, up_clip_max);+ float out_val;+ if (GATING_SEL <= 1u) {+ out_val =+ (GATING_SEL == 0u) ? gelu_approx(up_v) : silu(up_v, silu_alpha);+ } else {+ float gate_v = clamp(+ gate_frag_1 + bias_gate[col1],+ gate_clip_min,+ gate_clip_max+ );+ const float gate_act = (GATING_SEL == 2u) ? silu(gate_v, silu_alpha)+ : gelu_approx(gate_v);+ out_val = gate_act * up_v;+ }+ const ulong out_col = col_tg_off + col1;+ hidden_out[out_row * (ulong)d_ff + out_col] = out_val;+ }}+ }}// ----- Kernel entry points (Pass A) ------#define MOE_PASS_A_KERNEL(DTYPE, SUFFIX) \-kernel void moe_two_pass_prefill_pass_a_##SUFFIX( \- device const DTYPE* X_perm [[buffer(0)]], \- device const uint* expert_offsets [[buffer(1)]], \- device const DTYPE* W13_all [[buffer(2)]], \- device const DTYPE* up_biases [[buffer(3)]], \- device float* hidden_out [[buffer(4)]], \- constant uint& d_model [[buffer(5)]], \- constant uint& d_ff [[buffer(6)]], \- constant uint& E [[buffer(7)]], \- constant float& gate_clip_min [[buffer(8)]], \- constant float& gate_clip_max [[buffer(9)]], \- constant float& up_clip_min [[buffer(10)]], \- constant float& up_clip_max [[buffer(11)]], \- constant float& silu_alpha [[buffer(12)]], \- uint sg_id [[simdgroup_index_in_threadgroup]], \- uint3 threads_per_tg [[threads_per_threadgroup]], \- uint3 tg_pos [[threadgroup_position_in_grid]], \- uint3 local_tid [[thread_position_in_threadgroup]] ) \-{ \- using StageT = typename StageStorage<DTYPE>::type; \- threadgroup StageT Xs [PASSA_BM * PASSA_BK]; \- threadgroup StageT Wk_up [PASSA_BN * PASSA_BK]; \- threadgroup StageT Wk_gate [PASSA_BN * PASSA_BK]; \- threadgroup float bias_up [PASSA_BN]; \- threadgroup float bias_gate [PASSA_BN]; \- pass_a_impl<DTYPE>( \- X_perm, expert_offsets, W13_all, up_biases, hidden_out, \- d_model, d_ff, E, \- gate_clip_min, gate_clip_max, up_clip_min, up_clip_max, silu_alpha, \- /*tile_seg_start*/ expert_offsets[tg_pos.z], /*expert*/ tg_pos.z, /*tile_m*/ tg_pos.y, /*tile_n*/ tg_pos.x, \- sg_id, threads_per_tg, local_tid, \- Xs, Wk_up, Wk_gate, bias_up, bias_gate); \-}+#define MOE_PASS_A_KERNEL(DTYPE, SUFFIX) \+ kernel void moe_two_pass_prefill_pass_a_##SUFFIX( \+ device const DTYPE* X_perm [[buffer(0)]], \+ device const uint* expert_offsets [[buffer(1)]], \+ device const DTYPE* W13_all [[buffer(2)]], \+ device const DTYPE* up_biases [[buffer(3)]], \+ device float* hidden_out [[buffer(4)]], \+ constant uint& d_model [[buffer(5)]], \+ constant uint& d_ff [[buffer(6)]], \+ constant uint& E [[buffer(7)]], \+ constant float& gate_clip_min [[buffer(8)]], \+ constant float& gate_clip_max [[buffer(9)]], \+ constant float& up_clip_min [[buffer(10)]], \+ constant float& up_clip_max [[buffer(11)]], \+ constant float& silu_alpha [[buffer(12)]], \+ uint sg_id [[simdgroup_index_in_threadgroup]], \+ uint3 threads_per_tg [[threads_per_threadgroup]], \+ uint3 tg_pos [[threadgroup_position_in_grid]], \+ uint3 local_tid [[thread_position_in_threadgroup]] \+ ) { \+ using StageT = typename StageStorage<DTYPE>::type; \+ threadgroup StageT Xs[PASSA_BM * PASSA_BK]; \+ threadgroup StageT Wk_up[PASSA_BN * PASSA_BK]; \+ threadgroup StageT Wk_gate[PASSA_BN * PASSA_BK]; \+ threadgroup float bias_up[PASSA_BN]; \+ threadgroup float bias_gate[PASSA_BN]; \+ pass_a_impl<DTYPE>( \+ X_perm, \+ expert_offsets, \+ W13_all, \+ up_biases, \+ hidden_out, \+ d_model, \+ d_ff, \+ E, \+ gate_clip_min, \+ gate_clip_max, \+ up_clip_min, \+ up_clip_max, \+ silu_alpha, \+ /*tile_seg_start*/ expert_offsets[tg_pos.z], \+ /*expert*/ tg_pos.z, \+ /*tile_m*/ tg_pos.y, \+ /*tile_n*/ tg_pos.x, \+ sg_id, \+ threads_per_tg, \+ local_tid, \+ Xs, \+ Wk_up, \+ Wk_gate, \+ bias_up, \+ bias_gate \+ ); \+ }MOE_PASS_A_KERNEL(bfloat, bf16)-MOE_PASS_A_KERNEL(half, f16)-MOE_PASS_A_KERNEL(float, f32)+MOE_PASS_A_KERNEL(half, f16)+MOE_PASS_A_KERNEL(float, f32)// Indirect variant (consumes [expert, seg_start, tile_row_offset] triples)-template<typename T>+template <typename T>inline void pass_a_indirect_impl(- device const T* X_perm,+ device const T* X_perm,device const uint* expert_offsets,- device const T* W13_all,- device const T* up_biases,- device float* hidden_out,- device const uint* tile_map, // [tiles * 3]: (expert, seg_start, row_offset_elems)- uint d_model, uint d_ff, uint E,- float gate_clip_min, float gate_clip_max,- float up_clip_min, float up_clip_max,+ device const T* W13_all,+ device const T* up_biases,+ device float* hidden_out,+ device const uint*+ tile_map, // [tiles * 3]: (expert, seg_start, row_offset_elems)+ uint d_model,+ uint d_ff,+ uint E,+ float gate_clip_min,+ float gate_clip_max,+ float up_clip_min,+ float up_clip_max,float silu_alpha,- uint row_tile_idx, uint n_tile_idx,- uint sg_id, uint3 threads_per_tg, uint3 local_tid,+ uint row_tile_idx,+ uint n_tile_idx,+ uint sg_id,+ uint3 threads_per_tg,+ uint3 local_tid,threadgroup typename StageStorage<T>::type* Xs,threadgroup typename StageStorage<T>::type* Wk_up,threadgroup typename StageStorage<T>::type* Wk_gate,threadgroup float* bias_up,- threadgroup float* bias_gate)-{- constexpr uint ROW_TILE = PASSA_BM;- const uint base = row_tile_idx * 3u;- const uint expert_idx = tile_map[base + 0u];- if (expert_idx >= E) return;- const uint tile_seg_start = tile_map[base + 1u];- const uint row_off_elems = tile_map[base + 2u];- const uint tile_m = row_off_elems / ROW_TILE;-- pass_a_impl<T>(- X_perm, expert_offsets, W13_all, up_biases, hidden_out,- d_model, d_ff, E,- gate_clip_min, gate_clip_max, up_clip_min, up_clip_max, silu_alpha,- tile_seg_start, expert_idx, tile_m, n_tile_idx,- sg_id, threads_per_tg, local_tid,- Xs, Wk_up, Wk_gate, bias_up, bias_gate);+ threadgroup float* bias_gate+) {+ constexpr uint ROW_TILE = PASSA_BM;+ const uint base = row_tile_idx * 3u;+ const uint expert_idx = tile_map[base + 0u];+ if (expert_idx >= E)+ return;+ const uint tile_seg_start = tile_map[base + 1u];+ const uint row_off_elems = tile_map[base + 2u];+ const uint tile_m = row_off_elems / ROW_TILE;++ pass_a_impl<T>(+ X_perm,+ expert_offsets,+ W13_all,+ up_biases,+ hidden_out,+ d_model,+ d_ff,+ E,+ gate_clip_min,+ gate_clip_max,+ up_clip_min,+ up_clip_max,+ silu_alpha,+ tile_seg_start,+ expert_idx,+ tile_m,+ n_tile_idx,+ sg_id,+ threads_per_tg,+ local_tid,+ Xs,+ Wk_up,+ Wk_gate,+ bias_up,+ bias_gate+ );}-#define MOE_PASS_A_INDIRECT_KERNEL(DTYPE, SUFFIX) \-kernel void moe_two_pass_prefill_pass_a_indirect_##SUFFIX( \- device const DTYPE* X_perm [[buffer(0)]], \- device const uint* expert_offsets [[buffer(1)]], \- device const DTYPE* W13_all [[buffer(2)]], \- device const DTYPE* up_biases [[buffer(3)]], \- device float* hidden_out [[buffer(4)]], \- constant uint& d_model [[buffer(5)]], \- constant uint& d_ff [[buffer(6)]], \- constant uint& E [[buffer(7)]], \- constant float& gate_clip_min [[buffer(8)]], \- constant float& gate_clip_max [[buffer(9)]], \- constant float& up_clip_min [[buffer(10)]], \- constant float& up_clip_max [[buffer(11)]], \- constant float& silu_alpha [[buffer(12)]], \- device const uint* tile_map [[buffer(13)]], \- uint sg_id [[simdgroup_index_in_threadgroup]], \- uint3 threads_per_tg [[threads_per_threadgroup]], \- uint3 tg_pos [[threadgroup_position_in_grid]], \- uint3 local_tid [[thread_position_in_threadgroup]]) \-{ \- using StageT = typename StageStorage<DTYPE>::type; \- threadgroup StageT Xs [PASSA_BM * PASSA_BK]; \- threadgroup StageT Wk_up [PASSA_BN * PASSA_BK]; \- threadgroup StageT Wk_gate [PASSA_BN * PASSA_BK]; \- threadgroup float bias_up [PASSA_BN]; \- threadgroup float bias_gate [PASSA_BN]; \- pass_a_indirect_impl<DTYPE>( \- X_perm, expert_offsets, W13_all, up_biases, hidden_out, tile_map, \- d_model, d_ff, E, \- gate_clip_min, gate_clip_max, up_clip_min, up_clip_max, silu_alpha, \- /*row_tile_idx*/ tg_pos.y, /*n_tile_idx*/ tg_pos.x, \- sg_id, threads_per_tg, local_tid, \- Xs, Wk_up, Wk_gate, bias_up, bias_gate); \-}+#define MOE_PASS_A_INDIRECT_KERNEL(DTYPE, SUFFIX) \+ kernel void moe_two_pass_prefill_pass_a_indirect_##SUFFIX( \+ device const DTYPE* X_perm [[buffer(0)]], \+ device const uint* expert_offsets [[buffer(1)]], \+ device const DTYPE* W13_all [[buffer(2)]], \+ device const DTYPE* up_biases [[buffer(3)]], \+ device float* hidden_out [[buffer(4)]], \+ constant uint& d_model [[buffer(5)]], \+ constant uint& d_ff [[buffer(6)]], \+ constant uint& E [[buffer(7)]], \+ constant float& gate_clip_min [[buffer(8)]], \+ constant float& gate_clip_max [[buffer(9)]], \+ constant float& up_clip_min [[buffer(10)]], \+ constant float& up_clip_max [[buffer(11)]], \+ constant float& silu_alpha [[buffer(12)]], \+ device const uint* tile_map [[buffer(13)]], \+ uint sg_id [[simdgroup_index_in_threadgroup]], \+ uint3 threads_per_tg [[threads_per_threadgroup]], \+ uint3 tg_pos [[threadgroup_position_in_grid]], \+ uint3 local_tid [[thread_position_in_threadgroup]] \+ ) { \+ using StageT = typename StageStorage<DTYPE>::type; \+ threadgroup StageT Xs[PASSA_BM * PASSA_BK]; \+ threadgroup StageT Wk_up[PASSA_BN * PASSA_BK]; \+ threadgroup StageT Wk_gate[PASSA_BN * PASSA_BK]; \+ threadgroup float bias_up[PASSA_BN]; \+ threadgroup float bias_gate[PASSA_BN]; \+ pass_a_indirect_impl<DTYPE>( \+ X_perm, \+ expert_offsets, \+ W13_all, \+ up_biases, \+ hidden_out, \+ tile_map, \+ d_model, \+ d_ff, \+ E, \+ gate_clip_min, \+ gate_clip_max, \+ up_clip_min, \+ up_clip_max, \+ silu_alpha, \+ /*row_tile_idx*/ tg_pos.y, \+ /*n_tile_idx*/ tg_pos.x, \+ sg_id, \+ threads_per_tg, \+ local_tid, \+ Xs, \+ Wk_up, \+ Wk_gate, \+ bias_up, \+ bias_gate \+ ); \+ }MOE_PASS_A_INDIRECT_KERNEL(bfloat, bf16)-MOE_PASS_A_INDIRECT_KERNEL(half, f16)-MOE_PASS_A_INDIRECT_KERNEL(float, f32)+MOE_PASS_A_INDIRECT_KERNEL(half, f16)+MOE_PASS_A_INDIRECT_KERNEL(float, f32)-// ------------------------ Pass B (hidden @ W2 → output) ------------------------+// ------------------------ Pass B (hidden @ W2 → output)+// ------------------------constant uint PASSB_BM = 16;constant uint PASSB_BN = 64;constant uint PASSB_BK = 64;constant uint PASSB_SG_BM = 8;constant uint PASSB_SG_BN = 32;-template<typename T>+template <typename T>inline void pass_b_impl(- device const float* hidden, // [total_rows, FF]- device const uint* expert_offsets, // [E+1]- device const T* W2_all, // [E, D, FF] (FF contiguous)- device const T* down_biases, // [E, D]- device T* output, // [total_rows, D]- uint d_model, uint d_ff, uint E,- uint expert_idx, uint tile_m, uint tile_n,- uint sg_id, uint3 threads_per_tg, uint3 local_tid,- threadgroup float* Hs, // [BM,BK]- threadgroup typename StageStorage<T>::type* Wk, // [BN,BK]- threadgroup float* bias_tile // [BN]+ device const float* hidden, // [total_rows, FF]+ device const uint* expert_offsets, // [E+1]+ device const T* W2_all, // [E, D, FF] (FF contiguous)+ device const T* down_biases, // [E, D]+ device T* output, // [total_rows, D]+ uint d_model,+ uint d_ff,+ uint E,+ uint expert_idx,+ uint tile_m,+ uint tile_n,+ uint sg_id,+ uint3 threads_per_tg,+ uint3 local_tid,+ threadgroup float* Hs, // [BM,BK]+ threadgroup typename StageStorage<T>::type* Wk, // [BN,BK]+ threadgroup float* bias_tile // [BN]) {- using StageT = typename StageStorage<T>::type;- using StageVec4 = metal::vec<StageT, 4>;-- const uint Bm = PASSB_BM, Bn = PASSB_BN, Bk = PASSB_BK;- const uint SgBm = PASSB_SG_BM, SgBn = PASSB_SG_BN;+ using StageT = typename StageStorage<T>::type;+ using StageVec4 = metal::vec<StageT, 4>;++ const uint Bm = PASSB_BM, Bn = PASSB_BN, Bk = PASSB_BK;+ const uint SgBm = PASSB_SG_BM, SgBn = PASSB_SG_BN;++ if (expert_idx >= E)+ return;++ const uint seg_start = expert_offsets[expert_idx];+ const uint seg_end = expert_offsets[expert_idx + 1];+ const uint seg_len = (seg_end > seg_start) ? (seg_end - seg_start) : 0u;+ if (seg_len == 0)+ return;++ const uint row_tg_off = tile_m * Bm;+ const uint col_tg_off = tile_n * Bn;+ if (row_tg_off >= seg_len || col_tg_off >= d_model)+ return;++ const uint m_rows = min(Bm, seg_len - row_tg_off);+ const uint n_cols = min(Bn, d_model - col_tg_off);++ const ulong h_base = (ulong)(seg_start + row_tg_off) * (ulong)d_ff;+ const ulong w2_expert_base = (ulong)expert_idx * (ulong)d_model * (ulong)d_ff;+ const ulong bias_base = (ulong)expert_idx * (ulong)d_model;++ const uint sg_col_count = Bn / SgBn;+ const uint row_sg = sg_id / sg_col_count;+ const uint col_sg = sg_id % sg_col_count;+ const uint row_sg_off = row_sg * SgBm;+ const uint col_sg_off = col_sg * SgBn;++ constexpr uint SG_TILE = 8;+ constexpr uint TEMP = (PASSB_SG_BM / SG_TILE) * (PASSB_SG_BN / SG_TILE);+ metal::simdgroup_float8x8 Out[TEMP];+ for (uint i = 0; i < TEMP; ++i) {+ Out[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);+ }++ const uint threads_total =+ threads_per_tg.x * threads_per_tg.y * threads_per_tg.z;+ const uint lin = linear_tid(local_tid, threads_per_tg);++ // K-loop across FF+ for (uint k_off = 0; k_off < d_ff; k_off += Bk) {+ const uint valid_k = min(Bk, d_ff - k_off);- if (expert_idx >= E) return;-- const uint seg_start = expert_offsets[expert_idx];- const uint seg_end = expert_offsets[expert_idx + 1];- const uint seg_len = (seg_end > seg_start) ? (seg_end - seg_start) : 0u;- if (seg_len == 0) return;-- const uint row_tg_off = tile_m * Bm;- const uint col_tg_off = tile_n * Bn;- if (row_tg_off >= seg_len || col_tg_off >= d_model) return;-- const uint m_rows = min(Bm, seg_len - row_tg_off);- const uint n_cols = min(Bn, d_model - col_tg_off);-- const ulong h_base = (ulong)(seg_start + row_tg_off) * (ulong)d_ff;- const ulong w2_expert_base = (ulong)expert_idx * (ulong)d_model * (ulong)d_ff;- const ulong bias_base = (ulong)expert_idx * (ulong)d_model;-- const uint sg_col_count = Bn / SgBn;- const uint row_sg = sg_id / sg_col_count;- const uint col_sg = sg_id % sg_col_count;- const uint row_sg_off = row_sg * SgBm;- const uint col_sg_off = col_sg * SgBn;-- constexpr uint SG_TILE = 8;- constexpr uint TEMP = (PASSB_SG_BM / SG_TILE) * (PASSB_SG_BN / SG_TILE);- metal::simdgroup_float8x8 Out[TEMP];- for (uint i = 0; i < TEMP; ++i) {- Out[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);- }-- const uint threads_total = threads_per_tg.x * threads_per_tg.y * threads_per_tg.z;- const uint lin = linear_tid(local_tid, threads_per_tg);-- // K-loop across FF- for (uint k_off = 0; k_off < d_ff; k_off += Bk) {- const uint valid_k = min(Bk, d_ff - k_off);-- threadgroup_barrier(mem_flags::mem_threadgroup);-- // ---- Stage hidden [Bm,Bk] as float4 ----- {- constexpr uint row_stride = PASSB_BK;- constexpr uint vec_cols = PASSB_BK / 4;- const uint total_vecs = PASSB_BM * vec_cols;- const uint iters = ceil_div(total_vecs, threads_total);--UZU_PRAGMA_UNROLL- for (uint t = 0; t < iters; ++t) {- const uint i = t * threads_total + lin;- if (i < total_vecs) {- const uint r = i / vec_cols;- const uint c4 = i % vec_cols;-- threadgroup float4* dst4 =- reinterpret_cast<threadgroup float4*>(Hs + r * row_stride + (c4 << 2));-- float4 v = float4(0.0f);- if (r < m_rows) {- const uint base_vec = (c4 << 2);- const ulong base = h_base + (ulong)r * (ulong)d_ff + (ulong)(k_off + base_vec);- if (base_vec + 0u < valid_k) v.x = hidden[base + 0];- if (base_vec + 1u < valid_k) v.y = hidden[base + 1];- if (base_vec + 2u < valid_k) v.z = hidden[base + 2];- if (base_vec + 3u < valid_k) v.w = hidden[base + 3];- }- *dst4 = v;- }- }- }-- // ---- Stage W2 [Bn,Bk] as StageT (column-major for simd transpose) ----- {- constexpr uint row_stride = PASSB_BK;- constexpr uint vec_cols = PASSB_BK / 4;- const uint total_vecs = PASSB_BN * vec_cols;- const uint iters = ceil_div(total_vecs, threads_total);--UZU_PRAGMA_UNROLL- for (uint t = 0; t < iters; ++t) {- const uint i = t * threads_total + lin;- if (i < total_vecs) {- const uint n = i / vec_cols;- const uint c4 = i % vec_cols;-- threadgroup StageVec4* dst4 =- reinterpret_cast<threadgroup StageVec4*>(Wk + n * row_stride + (c4 << 2));-- StageVec4 packed = StageVec4(StageT(0.0f));- if (n < n_cols) {- const uint gcol = col_tg_off + n;- const ulong base = w2_expert_base + (ulong)gcol * (ulong)d_ff + (ulong)(k_off + (c4 << 2));- const uint base_vec = (c4 << 2);- float v0 = 0.0f, v1 = 0.0f, v2 = 0.0f, v3 = 0.0f;- if (base_vec + 0u < valid_k) v0 = float(W2_all[base + 0]);- if (base_vec + 1u < valid_k) v1 = float(W2_all[base + 1]);- if (base_vec + 2u < valid_k) v2 = float(W2_all[base + 2]);- if (base_vec + 3u < valid_k) v3 = float(W2_all[base + 3]);- packed = StageVec4(StageT(v0), StageT(v1), StageT(v2), StageT(v3));- }- *dst4 = packed;- }- }- }+ threadgroup_barrier(mem_flags::mem_threadgroup);- threadgroup_barrier(mem_flags::mem_threadgroup);-- // ---- MMA -----UZU_PRAGMA_UNROLL- for (uint kk = 0; kk < PASSB_BK; kk += SG_TILE) {-UZU_PRAGMA_UNROLL- for (uint m_sub = 0; m_sub < PASSB_SG_BM; m_sub += SG_TILE) {- const uint r_idx = m_sub / SG_TILE;- metal::simdgroup_float8x8 lhs;- simdgroup_load(lhs, Hs, PASSB_BK, ulong2(kk, row_sg_off + m_sub));-UZU_PRAGMA_UNROLL- for (uint n_sub = 0; n_sub < PASSB_SG_BN; n_sub += SG_TILE) {- const uint c_idx = n_sub / SG_TILE;- const uint tile = r_idx * (PASSB_SG_BN / SG_TILE) + c_idx;- metal::simdgroup_matrix<StageT, 8, 8> rhs;- simdgroup_load(rhs, Wk, PASSB_BK, ulong2(kk, col_sg_off + n_sub), true);- simdgroup_multiply_accumulate(Out[tile], lhs, rhs, Out[tile]);- }- }+ // ---- Stage hidden [Bm,Bk] as float4 ----+ {+ constexpr uint row_stride = PASSB_BK;+ constexpr uint vec_cols = PASSB_BK / 4;+ const uint total_vecs = PASSB_BM * vec_cols;+ const uint iters = ceil_div(total_vecs, threads_total);++ UZU_PRAGMA_UNROLL+ for (uint t = 0; t < iters; ++t) {+ const uint i = t * threads_total + lin;+ if (i < total_vecs) {+ const uint r = i / vec_cols;+ const uint c4 = i % vec_cols;++ threadgroup float4* dst4 = reinterpret_cast<threadgroup float4*>(+ Hs + r * row_stride + (c4 << 2)+ );++ float4 v = float4(0.0f);+ if (r < m_rows) {+ const uint base_vec = (c4 << 2);+ const ulong base =+ h_base + (ulong)r * (ulong)d_ff + (ulong)(k_off + base_vec);+ if (base_vec + 0u < valid_k)+ v.x = hidden[base + 0];+ if (base_vec + 1u < valid_k)+ v.y = hidden[base + 1];+ if (base_vec + 2u < valid_k)+ v.z = hidden[base + 2];+ if (base_vec + 3u < valid_k)+ v.w = hidden[base + 3];+ }+ *dst4 = v;}+ }}- // ---- Bias tile ----- for (uint c_local = lin; c_local < Bn; c_local += threads_total) {- const uint c_global = col_tg_off + c_local;- bias_tile[c_local] = (c_global < d_model) ? float(down_biases[bias_base + c_global]) : 0.0f;+ // ---- Stage W2 [Bn,Bk] as StageT (column-major for simd transpose) ----+ {+ constexpr uint row_stride = PASSB_BK;+ constexpr uint vec_cols = PASSB_BK / 4;+ const uint total_vecs = PASSB_BN * vec_cols;+ const uint iters = ceil_div(total_vecs, threads_total);++ UZU_PRAGMA_UNROLL+ for (uint t = 0; t < iters; ++t) {+ const uint i = t * threads_total + lin;+ if (i < total_vecs) {+ const uint n = i / vec_cols;+ const uint c4 = i % vec_cols;++ threadgroup StageVec4* dst4 =+ reinterpret_cast<threadgroup StageVec4*>(+ Wk + n * row_stride + (c4 << 2)+ );++ StageVec4 packed = StageVec4(StageT(0.0f));+ if (n < n_cols) {+ const uint gcol = col_tg_off + n;+ const ulong base = w2_expert_base + (ulong)gcol * (ulong)d_ff ++ (ulong)(k_off + (c4 << 2));+ const uint base_vec = (c4 << 2);+ float v0 = 0.0f, v1 = 0.0f, v2 = 0.0f, v3 = 0.0f;+ if (base_vec + 0u < valid_k)+ v0 = float(W2_all[base + 0]);+ if (base_vec + 1u < valid_k)+ v1 = float(W2_all[base + 1]);+ if (base_vec + 2u < valid_k)+ v2 = float(W2_all[base + 2]);+ if (base_vec + 3u < valid_k)+ v3 = float(W2_all[base + 3]);+ packed = StageVec4(StageT(v0), StageT(v1), StageT(v2), StageT(v3));+ }+ *dst4 = packed;+ }+ }}threadgroup_barrier(mem_flags::mem_threadgroup);- // ---- Writeout ----- const uint sg_lane_start = sg_id * 32;- const uint sg_lane_end = sg_lane_start + 32;- if (lin < sg_lane_start || lin >= sg_lane_end) return;- const uint sg_lin = lin - sg_lane_start;-- // Lane→(row,col) mapping for 8x8 simdgroup fragments, matches PASSB fragment packing.- const uint lane_qid = sg_lin >> 2;- const uint lane_row = (lane_qid & 4u) + ((sg_lin >> 1) & 3u);- const uint lane_col_base = ((lane_qid & 2u) << 1) + ((sg_lin & 1u) << 1);--UZU_PRAGMA_UNROLL- for (uint n_sub = 0; n_sub < PASSB_SG_BN; n_sub += SG_TILE) {- const uint c_idx = n_sub / SG_TILE;-UZU_PRAGMA_UNROLL- for (uint m_sub = 0; m_sub < PASSB_SG_BM; m_sub += SG_TILE) {- const uint r_idx = m_sub / SG_TILE;- const uint tile = r_idx * (PASSB_SG_BN / SG_TILE) + c_idx;-- const uint tile_row_base = row_sg_off + m_sub;- const uint tile_col_base = col_sg_off + n_sub;- const uint local_row = tile_row_base + lane_row;- if (local_row >= m_rows || tile_col_base >= n_cols) {- continue;- }-- const auto accum = Out[tile].thread_elements();-- const uint col0 = tile_col_base + lane_col_base;- const uint col1 = col0 + 1u;- const ulong out_row = seg_start + row_tg_off + local_row;-- if (col0 < n_cols) {- const ulong out_col = col_tg_off + col0;- const float val = accum[0] + bias_tile[col0];- output[out_row * (ulong)d_model + out_col] = T(val);- }-- if (col1 < n_cols) {- const ulong out_col = col_tg_off + col1;- const float val = accum[1] + bias_tile[col1];- output[out_row * (ulong)d_model + out_col] = T(val);- }+ // ---- MMA ----+ UZU_PRAGMA_UNROLL+ for (uint kk = 0; kk < PASSB_BK; kk += SG_TILE) {+ UZU_PRAGMA_UNROLL+ for (uint m_sub = 0; m_sub < PASSB_SG_BM; m_sub += SG_TILE) {+ const uint r_idx = m_sub / SG_TILE;+ metal::simdgroup_float8x8 lhs;+ simdgroup_load(lhs, Hs, PASSB_BK, ulong2(kk, row_sg_off + m_sub));+ UZU_PRAGMA_UNROLL+ for (uint n_sub = 0; n_sub < PASSB_SG_BN; n_sub += SG_TILE) {+ const uint c_idx = n_sub / SG_TILE;+ const uint tile = r_idx * (PASSB_SG_BN / SG_TILE) + c_idx;+ metal::simdgroup_matrix<StageT, 8, 8> rhs;+ simdgroup_load(+ rhs,+ Wk,+ PASSB_BK,+ ulong2(kk, col_sg_off + n_sub),+ true+ );+ simdgroup_multiply_accumulate(Out[tile], lhs, rhs, Out[tile]);}+ }}+ }++ // ---- Bias tile ----+ for (uint c_local = lin; c_local < Bn; c_local += threads_total) {+ const uint c_global = col_tg_off + c_local;+ bias_tile[c_local] =+ (c_global < d_model) ? float(down_biases[bias_base + c_global]) : 0.0f;+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ // ---- Writeout ----+ const uint sg_lane_start = sg_id * 32;+ const uint sg_lane_end = sg_lane_start + 32;+ if (lin < sg_lane_start || lin >= sg_lane_end)+ return;+ const uint sg_lin = lin - sg_lane_start;++ // Lane→(row,col) mapping for 8x8 simdgroup fragments, matches PASSB fragment+ // packing.+ const uint lane_qid = sg_lin >> 2;+ const uint lane_row = (lane_qid & 4u) + ((sg_lin >> 1) & 3u);+ const uint lane_col_base = ((lane_qid & 2u) << 1) + ((sg_lin & 1u) << 1);++ UZU_PRAGMA_UNROLL+ for (uint n_sub = 0; n_sub < PASSB_SG_BN; n_sub += SG_TILE) {+ const uint c_idx = n_sub / SG_TILE;+ UZU_PRAGMA_UNROLL+ for (uint m_sub = 0; m_sub < PASSB_SG_BM; m_sub += SG_TILE) {+ const uint r_idx = m_sub / SG_TILE;+ const uint tile = r_idx * (PASSB_SG_BN / SG_TILE) + c_idx;++ const uint tile_row_base = row_sg_off + m_sub;+ const uint tile_col_base = col_sg_off + n_sub;+ const uint local_row = tile_row_base + lane_row;+ if (local_row >= m_rows || tile_col_base >= n_cols) {+ continue;+ }++ const auto accum = Out[tile].thread_elements();++ const uint col0 = tile_col_base + lane_col_base;+ const uint col1 = col0 + 1u;+ const ulong out_row = seg_start + row_tg_off + local_row;++ if (col0 < n_cols) {+ const ulong out_col = col_tg_off + col0;+ const float val = accum[0] + bias_tile[col0];+ output[out_row * (ulong)d_model + out_col] = T(val);+ }++ if (col1 < n_cols) {+ const ulong out_col = col_tg_off + col1;+ const float val = accum[1] + bias_tile[col1];+ output[out_row * (ulong)d_model + out_col] = T(val);+ }+ }+ }}// ----- Kernel entry points (Pass B) ------#define MOE_PASS_B_KERNEL(DTYPE, SUFFIX) \-kernel void moe_two_pass_prefill_pass_b_##SUFFIX( \- device const float* hidden [[buffer(0)]], \- device const uint* expert_offsets [[buffer(1)]], \- device const DTYPE* W2_all [[buffer(2)]], \- device const DTYPE* down_biases [[buffer(3)]], \- device DTYPE* output [[buffer(4)]], \- constant uint& d_model [[buffer(5)]], \- constant uint& d_ff [[buffer(6)]], \- constant uint& E [[buffer(7)]], \- uint sg_id [[simdgroup_index_in_threadgroup]], \- uint3 threads_per_tg [[threads_per_threadgroup]], \- uint3 tg_pos [[threadgroup_position_in_grid]], \- uint3 local_tid [[thread_position_in_threadgroup]]) \-{ \- using StageT = typename StageStorage<DTYPE>::type; \- threadgroup float Hs [PASSB_BM * PASSB_BK]; \- threadgroup StageT Wk [PASSB_BN * PASSB_BK]; \- threadgroup float bias [PASSB_BN]; \- pass_b_impl<DTYPE>( \- hidden, expert_offsets, W2_all, down_biases, output, \- d_model, d_ff, E, \- /*expert*/ tg_pos.z, /*tile_m*/ tg_pos.y, /*tile_n*/ tg_pos.x, \- sg_id, threads_per_tg, local_tid, \- Hs, Wk, bias); \-}+#define MOE_PASS_B_KERNEL(DTYPE, SUFFIX) \+ kernel void moe_two_pass_prefill_pass_b_##SUFFIX( \+ device const float* hidden [[buffer(0)]], \+ device const uint* expert_offsets [[buffer(1)]], \+ device const DTYPE* W2_all [[buffer(2)]], \+ device const DTYPE* down_biases [[buffer(3)]], \+ device DTYPE* output [[buffer(4)]], \+ constant uint& d_model [[buffer(5)]], \+ constant uint& d_ff [[buffer(6)]], \+ constant uint& E [[buffer(7)]], \+ uint sg_id [[simdgroup_index_in_threadgroup]], \+ uint3 threads_per_tg [[threads_per_threadgroup]], \+ uint3 tg_pos [[threadgroup_position_in_grid]], \+ uint3 local_tid [[thread_position_in_threadgroup]] \+ ) { \+ using StageT = typename StageStorage<DTYPE>::type; \+ threadgroup float Hs[PASSB_BM * PASSB_BK]; \+ threadgroup StageT Wk[PASSB_BN * PASSB_BK]; \+ threadgroup float bias[PASSB_BN]; \+ pass_b_impl<DTYPE>( \+ hidden, \+ expert_offsets, \+ W2_all, \+ down_biases, \+ output, \+ d_model, \+ d_ff, \+ E, \+ /*expert*/ tg_pos.z, \+ /*tile_m*/ tg_pos.y, \+ /*tile_n*/ tg_pos.x, \+ sg_id, \+ threads_per_tg, \+ local_tid, \+ Hs, \+ Wk, \+ bias \+ ); \+ }MOE_PASS_B_KERNEL(bfloat, bf16)-MOE_PASS_B_KERNEL(half, f16)-MOE_PASS_B_KERNEL(float, f32)+MOE_PASS_B_KERNEL(half, f16)+MOE_PASS_B_KERNEL(float, f32)// Indirect variant-template<typename T>+template <typename T>inline void pass_b_indirect_impl(device const float* hidden,- device const uint* expert_offsets,- device const T* W2_all,- device const T* down_biases,- device T* output,- device const uint* tile_map, // [tiles * 3]- uint d_model, uint d_ff, uint E,- uint row_tile_idx, uint n_tile_idx,- uint sg_id, uint3 threads_per_tg, uint3 local_tid,+ device const uint* expert_offsets,+ device const T* W2_all,+ device const T* down_biases,+ device T* output,+ device const uint* tile_map, // [tiles * 3]+ uint d_model,+ uint d_ff,+ uint E,+ uint row_tile_idx,+ uint n_tile_idx,+ uint sg_id,+ uint3 threads_per_tg,+ uint3 local_tid,threadgroup float* Hs,threadgroup typename StageStorage<T>::type* Wk,- threadgroup float* bias_tile)-{- constexpr uint ROW_TILE = PASSB_BM;- const uint base = row_tile_idx * 3u;- const uint expert_idx = tile_map[base + 0u];- if (expert_idx >= E) return;- const uint row_off_elems = tile_map[base + 2u];- const uint tile_m = row_off_elems / ROW_TILE;-- pass_b_impl<T>(- hidden, expert_offsets, W2_all, down_biases, output,- d_model, d_ff, E,- expert_idx, tile_m, n_tile_idx,- sg_id, threads_per_tg, local_tid,- Hs, Wk, bias_tile);+ threadgroup float* bias_tile+) {+ constexpr uint ROW_TILE = PASSB_BM;+ const uint base = row_tile_idx * 3u;+ const uint expert_idx = tile_map[base + 0u];+ if (expert_idx >= E)+ return;+ const uint row_off_elems = tile_map[base + 2u];+ const uint tile_m = row_off_elems / ROW_TILE;++ pass_b_impl<T>(+ hidden,+ expert_offsets,+ W2_all,+ down_biases,+ output,+ d_model,+ d_ff,+ E,+ expert_idx,+ tile_m,+ n_tile_idx,+ sg_id,+ threads_per_tg,+ local_tid,+ Hs,+ Wk,+ bias_tile+ );}-#define MOE_PASS_B_INDIRECT_KERNEL(DTYPE, SUFFIX) \-kernel void moe_two_pass_prefill_pass_b_indirect_##SUFFIX( \- device const float* hidden [[buffer(0)]], \- device const uint* expert_offsets [[buffer(1)]], \- device const DTYPE* W2_all [[buffer(2)]], \- device const DTYPE* down_biases [[buffer(3)]], \- device DTYPE* output [[buffer(4)]], \- constant uint& d_model [[buffer(5)]], \- constant uint& d_ff [[buffer(6)]], \- constant uint& E [[buffer(7)]], \- device const uint* tile_map [[buffer(8)]], \- uint sg_id [[simdgroup_index_in_threadgroup]], \- uint3 threads_per_tg [[threads_per_threadgroup]], \- uint3 tg_pos [[threadgroup_position_in_grid]], \- uint3 local_tid [[thread_position_in_threadgroup]]) \-{ \- using StageT = typename StageStorage<DTYPE>::type; \- threadgroup float Hs [PASSB_BM * PASSB_BK]; \- threadgroup StageT Wk [PASSB_BN * PASSB_BK]; \- threadgroup float bias [PASSB_BN]; \- pass_b_indirect_impl<DTYPE>( \- hidden, expert_offsets, W2_all, down_biases, output, tile_map, \- d_model, d_ff, E, \- /*row_tile_idx*/ tg_pos.y, /*n_tile_idx*/ tg_pos.x, \- sg_id, threads_per_tg, local_tid, \- Hs, Wk, bias); \-}+#define MOE_PASS_B_INDIRECT_KERNEL(DTYPE, SUFFIX) \+ kernel void moe_two_pass_prefill_pass_b_indirect_##SUFFIX( \+ device const float* hidden [[buffer(0)]], \+ device const uint* expert_offsets [[buffer(1)]], \+ device const DTYPE* W2_all [[buffer(2)]], \+ device const DTYPE* down_biases [[buffer(3)]], \+ device DTYPE* output [[buffer(4)]], \+ constant uint& d_model [[buffer(5)]], \+ constant uint& d_ff [[buffer(6)]], \+ constant uint& E [[buffer(7)]], \+ device const uint* tile_map [[buffer(8)]], \+ uint sg_id [[simdgroup_index_in_threadgroup]], \+ uint3 threads_per_tg [[threads_per_threadgroup]], \+ uint3 tg_pos [[threadgroup_position_in_grid]], \+ uint3 local_tid [[thread_position_in_threadgroup]] \+ ) { \+ using StageT = typename StageStorage<DTYPE>::type; \+ threadgroup float Hs[PASSB_BM * PASSB_BK]; \+ threadgroup StageT Wk[PASSB_BN * PASSB_BK]; \+ threadgroup float bias[PASSB_BN]; \+ pass_b_indirect_impl<DTYPE>( \+ hidden, \+ expert_offsets, \+ W2_all, \+ down_biases, \+ output, \+ tile_map, \+ d_model, \+ d_ff, \+ E, \+ /*row_tile_idx*/ tg_pos.y, \+ /*n_tile_idx*/ tg_pos.x, \+ sg_id, \+ threads_per_tg, \+ local_tid, \+ Hs, \+ Wk, \+ bias \+ ); \+ }MOE_PASS_B_INDIRECT_KERNEL(bfloat, bf16)-MOE_PASS_B_INDIRECT_KERNEL(half, f16)-MOE_PASS_B_INDIRECT_KERNEL(float, f32)+MOE_PASS_B_INDIRECT_KERNEL(half, f16)+MOE_PASS_B_INDIRECT_KERNEL(float, f32)diff --git a/crates/uzu/src/backends/metal/kernel/moe/finalize.metal b/crates/uzu/src/backends/metal/kernel/moe/finalize.metalindex 978cec9ba..168394b91 100644--- a/crates/uzu/src/backends/metal/kernel/moe/finalize.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/finalize.metal@@ -5,69 +5,72 @@ using namespace metal;#define BM 32#define BN 64-template<typename T>+template <typename T>inline void moe_finalize_impl(- device const int* tok2row, // [T*K], -1 if dropped- device const T* probs, // [T*K]- device const T* Y_partial, // [sum_k, d_model]- device T* Y, // [T, d_model]+ device const int* tok2row, // [T*K], -1 if dropped+ device const T* probs, // [T*K]+ device const T* Y_partial, // [sum_k, d_model]+ device T* Y, // [T, d_model]constant uint& T_count,constant uint& d_model,constant uint& K,uint lid,- uint3 tgpig)-{- if (T_count == 0u || d_model == 0u) return;- const uint tile_m0 = tgpig.y * BM;- const uint tile_n0 = tgpig.x * BN;- if (tile_m0 >= T_count || tile_n0 >= d_model) return;- const uint m_rows = min((uint)BM, T_count - tile_m0);- const uint n_cols = min((uint)BN, d_model - tile_n0);+ uint3 tgpig+) {+ if (T_count == 0u || d_model == 0u)+ return;+ const uint tile_m0 = tgpig.y * BM;+ const uint tile_n0 = tgpig.x * BN;+ if (tile_m0 >= T_count || tile_n0 >= d_model)+ return;+ const uint m_rows = min((uint)BM, T_count - tile_m0);+ const uint n_cols = min((uint)BN, d_model - tile_n0);- // 128 threads per TG expected- for (uint idx = lid; idx < m_rows * n_cols; idx += 128u) {- const uint mi = idx / n_cols;- const uint nj = idx % n_cols;- const uint t = tile_m0 + mi;- const uint f = tile_n0 + nj;- float acc = 0.0f;- const uint base = t * K;- for (uint k = 0; k < K; ++k) {- const int row = tok2row[base + k];- if (row >= 0) {- const ulong yidx = (ulong)(uint)row * (ulong)d_model + (ulong)f;- float prob = (float)probs[base + k];- if (!isfinite(prob)) {- prob = 0.0f;- }- float val = (float)Y_partial[yidx];- if (!isfinite(val)) {- val = 0.0f;- }- acc = fma(prob, val, acc);- }+ // 128 threads per TG expected+ for (uint idx = lid; idx < m_rows * n_cols; idx += 128u) {+ const uint mi = idx / n_cols;+ const uint nj = idx % n_cols;+ const uint t = tile_m0 + mi;+ const uint f = tile_n0 + nj;+ float acc = 0.0f;+ const uint base = t * K;+ for (uint k = 0; k < K; ++k) {+ const int row = tok2row[base + k];+ if (row >= 0) {+ const ulong yidx = (ulong)(uint)row * (ulong)d_model + (ulong)f;+ float prob = (float)probs[base + k];+ if (!isfinite(prob)) {+ prob = 0.0f;}- if (!isfinite(acc)) {- acc = 0.0f;+ float val = (float)Y_partial[yidx];+ if (!isfinite(val)) {+ val = 0.0f;}- Y[t * d_model + f] = T(acc);+ acc = fma(prob, val, acc);+ }}+ if (!isfinite(acc)) {+ acc = 0.0f;+ }+ Y[t * d_model + f] = T(acc);+ }}-#define DEFINE_MOE_FINALIZE_KERNEL(SUFFIX, T) \-kernel void moe_finalize_##SUFFIX( \- device const int* tok2row [[buffer(0)]], \- device const T* probs [[buffer(1)]], \- device const T* Y_partial [[buffer(2)]], \- device T* Y [[buffer(3)]], \- constant uint& T_count [[buffer(4)]], \- constant uint& d_model [[buffer(5)]], \- constant uint& K [[buffer(6)]], \- uint lid [[thread_index_in_threadgroup]], \- uint3 tgpig [[threadgroup_position_in_grid]]) \-{ \- moe_finalize_impl<T>(tok2row, probs, Y_partial, Y, T_count, d_model, K, lid, tgpig); \-}+#define DEFINE_MOE_FINALIZE_KERNEL(SUFFIX, T) \+ kernel void moe_finalize_##SUFFIX( \+ device const int* tok2row [[buffer(0)]], \+ device const T* probs [[buffer(1)]], \+ device const T* Y_partial [[buffer(2)]], \+ device T* Y [[buffer(3)]], \+ constant uint& T_count [[buffer(4)]], \+ constant uint& d_model [[buffer(5)]], \+ constant uint& K [[buffer(6)]], \+ uint lid [[thread_index_in_threadgroup]], \+ uint3 tgpig [[threadgroup_position_in_grid]] \+ ) { \+ moe_finalize_impl< \+ T>(tok2row, probs, Y_partial, Y, T_count, d_model, K, lid, tgpig); \+ }DEFINE_MOE_FINALIZE_KERNEL(f16, half)DEFINE_MOE_FINALIZE_KERNEL(bf16, bfloat)diff --git a/crates/uzu/src/backends/metal/kernel/moe/prepare.metal b/crates/uzu/src/backends/metal/kernel/moe/prepare.metalindex 2499930b2..fb9589771 100644--- a/crates/uzu/src/backends/metal/kernel/moe/prepare.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/prepare.metal@@ -3,11 +3,11 @@ using namespace metal;// Tiling config for 2D gatherconstant uint ROWS_PER_TG = 8u;-constant uint COL_VECS = 32u; // 256 threads = 8×32-constant uint ELEMS_PER_THREAD = 8u; // each thread copies 8 elements (2× vec4)+constant uint COL_VECS = 32u; // 256 threads = 8×32+constant uint ELEMS_PER_THREAD = 8u; // each thread copies 8 elements (2× vec4)// ========== 2D Tiled Gather Template ==========-template<typename T>+template <typename T>void moe_gather_x_perm_2d_impl(device const T* X,device const int* bucketed_ids,@@ -15,119 +15,133 @@ void moe_gather_x_perm_2d_impl(device const uint* sumk_buf,uint d_model,uint3 tgid,- uint lid)-{- const uint total_rows = sumk_buf[0];- if (total_rows == 0u || d_model == 0u) return;-- // Thread organization: lid = row_in_tg * COL_VECS + col_vec- const uint row_in_tg = lid / COL_VECS;- const uint col_vec = lid % COL_VECS;+ uint lid+) {+ const uint total_rows = sumk_buf[0];+ if (total_rows == 0u || d_model == 0u)+ return;++ // Thread organization: lid = row_in_tg * COL_VECS + col_vec+ const uint row_in_tg = lid / COL_VECS;+ const uint col_vec = lid % COL_VECS;++ // Global row assignment+ const uint base_row = tgid.x * ROWS_PER_TG;+ const uint row = base_row + row_in_tg;++ if (row >= total_rows)+ return;++ int token = bucketed_ids[row];+ if (token < 0)+ return;++ const device T* src = X + (ulong)(uint)token * (ulong)d_model;+ device T* dst = x_perm + (ulong)row * (ulong)d_model;++ // Each thread copies ELEMS_PER_THREAD elements+ // Thread i processes: col_vec*8, col_vec*8+256, col_vec*8+512, ...+ uint col = col_vec * ELEMS_PER_THREAD;+ const uint stride = COL_VECS * ELEMS_PER_THREAD;++ // Vectorized copy: 8 elements = 2× vec4+ using Vec4 = typename metal::vec<T, 4>;+ while (col + 7u < d_model) {+ Vec4 v0 = *(reinterpret_cast<const device Vec4*>(src + col));+ Vec4 v1 = *(reinterpret_cast<const device Vec4*>(src + col + 4u));+ *(reinterpret_cast<device Vec4*>(dst + col)) = v0;+ *(reinterpret_cast<device Vec4*>(dst + col + 4u)) = v1;+ col += stride;+ }++ // Scalar remainder+ if (col < d_model) {+ const uint limit = min(d_model, col + ELEMS_PER_THREAD);+ for (uint tail = col; tail < limit; ++tail) {+ dst[tail] = src[tail];+ }+ }+}- // Global row assignment- const uint base_row = tgid.x * ROWS_PER_TG;- const uint row = base_row + row_in_tg;+// ========== 1D Gather Template ==========+template <typename T>+void moe_gather_x_perm_1d_impl(+ device const T* X,+ device const int* bucketed_ids,+ device T* x_perm,+ device const uint* sumk_buf,+ uint d_model,+ uint lid,+ uint threads_x+) {+ const uint total_rows = sumk_buf[0];+ if (total_rows == 0u || d_model == 0u)+ return;- if (row >= total_rows) return;+ const uint num_vec4 = d_model / 4u;+ // Each thread processes multiple rows sequentially+ for (uint row = lid; row < total_rows; row += threads_x) {int token = bucketed_ids[row];- if (token < 0) return;+ if (token < 0)+ continue;const device T* src = X + (ulong)(uint)token * (ulong)d_model;device T* dst = x_perm + (ulong)row * (ulong)d_model;- // Each thread copies ELEMS_PER_THREAD elements- // Thread i processes: col_vec*8, col_vec*8+256, col_vec*8+512, ...- uint col = col_vec * ELEMS_PER_THREAD;- const uint stride = COL_VECS * ELEMS_PER_THREAD;-- // Vectorized copy: 8 elements = 2× vec4+ // Vectorized copy: 4 elements at a timeusing Vec4 = typename metal::vec<T, 4>;- while (col + 7u < d_model) {- Vec4 v0 = *(reinterpret_cast<const device Vec4*>(src + col));- Vec4 v1 = *(reinterpret_cast<const device Vec4*>(src + col + 4u));- *(reinterpret_cast<device Vec4*>(dst + col)) = v0;- *(reinterpret_cast<device Vec4*>(dst + col + 4u)) = v1;- col += stride;+ for (uint vec_idx = 0u; vec_idx < num_vec4; vec_idx++) {+ uint col = vec_idx * 4u;+ Vec4 v = *(reinterpret_cast<const device Vec4*>(src + col));+ *(reinterpret_cast<device Vec4*>(dst + col)) = v;}// Scalar remainder- if (col < d_model) {- const uint limit = min(d_model, col + ELEMS_PER_THREAD);- for (uint tail = col; tail < limit; ++tail) {- dst[tail] = src[tail];- }- }-}--// ========== 1D Gather Template ==========-template<typename T>-void moe_gather_x_perm_1d_impl(- device const T* X,- device const int* bucketed_ids,- device T* x_perm,- device const uint* sumk_buf,- uint d_model,- uint lid,- uint threads_x)-{- const uint total_rows = sumk_buf[0];- if (total_rows == 0u || d_model == 0u) return;-- const uint num_vec4 = d_model / 4u;-- // Each thread processes multiple rows sequentially- for (uint row = lid; row < total_rows; row += threads_x) {- int token = bucketed_ids[row];- if (token < 0) continue;-- const device T* src = X + (ulong)(uint)token * (ulong)d_model;- device T* dst = x_perm + (ulong)row * (ulong)d_model;-- // Vectorized copy: 4 elements at a time- using Vec4 = typename metal::vec<T, 4>;- for (uint vec_idx = 0u; vec_idx < num_vec4; vec_idx++) {- uint col = vec_idx * 4u;- Vec4 v = *(reinterpret_cast<const device Vec4*>(src + col));- *(reinterpret_cast<device Vec4*>(dst + col)) = v;- }-- // Scalar remainder- for (uint col = num_vec4 * 4u; col < d_model; col++) {- dst[col] = src[col];- }+ for (uint col = num_vec4 * 4u; col < d_model; col++) {+ dst[col] = src[col];}+ }}// ========== Kernel Entry Points ==========// 2D Tiled kernels-#define GATHER_2D_KERNEL(DTYPE, SUFFIX) \-kernel void moe_gather_x_perm_##SUFFIX##_2d( \- device const DTYPE* X [[buffer(0)]], \- device const int* bucketed_ids [[buffer(1)]], \- device DTYPE* x_perm [[buffer(2)]], \- device const uint* sumk_buf [[buffer(3)]], \- constant uint& d_model [[buffer(4)]], \- uint3 tgid [[threadgroup_position_in_grid]], \- uint lid [[thread_index_in_threadgroup]]) \-{ \- moe_gather_x_perm_2d_impl<DTYPE>(X, bucketed_ids, x_perm, sumk_buf, d_model, tgid, lid); \-}+#define GATHER_2D_KERNEL(DTYPE, SUFFIX) \+ kernel void moe_gather_x_perm_##SUFFIX##_2d( \+ device const DTYPE* X [[buffer(0)]], \+ device const int* bucketed_ids [[buffer(1)]], \+ device DTYPE* x_perm [[buffer(2)]], \+ device const uint* sumk_buf [[buffer(3)]], \+ constant uint& d_model [[buffer(4)]], \+ uint3 tgid [[threadgroup_position_in_grid]], \+ uint lid [[thread_index_in_threadgroup]] \+ ) { \+ moe_gather_x_perm_2d_impl< \+ DTYPE>(X, bucketed_ids, x_perm, sumk_buf, d_model, tgid, lid); \+ }// 1D kernels-#define GATHER_1D_KERNEL(DTYPE, SUFFIX) \-kernel void moe_gather_x_perm_##SUFFIX( \- device const DTYPE* X [[buffer(0)]], \- device const int* bucketed_ids [[buffer(1)]], \- device DTYPE* x_perm [[buffer(2)]], \- device const uint* sumk_buf [[buffer(3)]], \- constant uint& d_model [[buffer(4)]], \- uint lid [[thread_index_in_threadgroup]], \- uint3 threads_per_threadgroup [[threads_per_threadgroup]]) \-{ \- moe_gather_x_perm_1d_impl<DTYPE>(X, bucketed_ids, x_perm, sumk_buf, d_model, lid, threads_per_threadgroup.x); \-}+#define GATHER_1D_KERNEL(DTYPE, SUFFIX) \+ kernel void moe_gather_x_perm_##SUFFIX( \+ device const DTYPE* X [[buffer(0)]], \+ device const int* bucketed_ids [[buffer(1)]], \+ device DTYPE* x_perm [[buffer(2)]], \+ device const uint* sumk_buf [[buffer(3)]], \+ constant uint& d_model [[buffer(4)]], \+ uint lid [[thread_index_in_threadgroup]], \+ uint3 threads_per_threadgroup [[threads_per_threadgroup]] \+ ) { \+ moe_gather_x_perm_1d_impl<DTYPE>( \+ X, \+ bucketed_ids, \+ x_perm, \+ sumk_buf, \+ d_model, \+ lid, \+ threads_per_threadgroup.x \+ ); \+ }// Instantiate for all typesGATHER_2D_KERNEL(bfloat, bf16)diff --git a/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal b/crates/uzu/src/backends/metal/kernel/moe/router_topk.metalindex 25b41db03..7b19cf3ce 100644--- a/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal@@ -10,11 +10,11 @@ constant float NEG_INF = -INFINITY;template <typename Vec4T, typename ScalarT, typename ProbT>inline void moe_router_topk_impl(- const device Vec4T* input, // [T, d_model/4]- const device Vec4T* weight, // [E, d_model/4]- const device ScalarT* bias, // [E]- device int* topk_ids, // [T, K]- device ProbT* topk_probs, // [T, K]+ const device Vec4T* input, // [T, d_model/4]+ const device Vec4T* weight, // [E, d_model/4]+ const device ScalarT* bias, // [E]+ device int* topk_ids, // [T, K]+ device ProbT* topk_probs, // [T, K]uint T,uint d_model,uint E,@@ -31,152 +31,168 @@ inline void moe_router_topk_impl(threadgroup float* reduce_tmp,threadgroup uint* reduce_tmp_u,threadgroup uint* shared_best_idx,- threadgroup float* shared_best_val)-{- const uint token_idx = tgpig.y;- if (token_idx >= T || d_model == 0 || E == 0 || K == 0) {- return;- }+ threadgroup float* shared_best_val+) {+ const uint token_idx = tgpig.y;+ if (token_idx >= T || d_model == 0 || E == 0 || K == 0) {+ return;+ }- const uint vecs = d_model / 4u;- const device Vec4T* x_vec =- input + (ulong)token_idx * (ulong)vecs;+ const uint vecs = d_model / 4u;+ const device Vec4T* x_vec = input + (ulong)token_idx * (ulong)vecs;- for (uint c = lid; c < vecs; c += THREADS_PER_TG) {- x_cache[c] = float4(x_vec[c]);- }- threadgroup_barrier(mem_flags::mem_threadgroup);+ for (uint c = lid; c < vecs; c += THREADS_PER_TG) {+ x_cache[c] = float4(x_vec[c]);+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);- for (uint row = simdgroup_idx; row < E; row += simdgroups_per_tg) {- const device Vec4T* w_vec =- weight + (ulong)row * (ulong)vecs;+ for (uint row = simdgroup_idx; row < E; row += simdgroups_per_tg) {+ const device Vec4T* w_vec = weight + (ulong)row * (ulong)vecs;- float4 accum4 = float4(0.0f);- for (uint c = simd_lane; c < vecs; c += 32u) {- const float4 wv = float4(w_vec[c]);- const float4 xv = x_cache[c];- accum4 = fma(wv, xv, accum4);- }- float sum = (accum4.x + accum4.y) + (accum4.z + accum4.w);- sum = simd_sum(sum);- if (simd_is_first()) {- sum += float(bias[row]);- logits_shared[row] = sum;- idx_shared[row] = row;- }+ float4 accum4 = float4(0.0f);+ for (uint c = simd_lane; c < vecs; c += 32u) {+ const float4 wv = float4(w_vec[c]);+ const float4 xv = x_cache[c];+ accum4 = fma(wv, xv, accum4);}- threadgroup_barrier(mem_flags::mem_threadgroup);-- for (uint row = lid + E; row < MAX_EXPERTS; row += THREADS_PER_TG) {- logits_shared[row] = NEG_INF;- idx_shared[row] = row;+ float sum = (accum4.x + accum4.y) + (accum4.z + accum4.w);+ sum = simd_sum(sum);+ if (simd_is_first()) {+ sum += float(bias[row]);+ logits_shared[row] = sum;+ idx_shared[row] = row;}- threadgroup_barrier(mem_flags::mem_threadgroup);+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);- const uint effective_k = min(K, MAX_TOPK);- for (uint sel = 0; sel < effective_k; ++sel) {- float local_best = NEG_INF;- uint local_idx = 0xFFFFFFFFu;- for (uint row = lid; row < E; row += THREADS_PER_TG) {- float candidate = logits_shared[row];- if (candidate > local_best ||- (candidate == local_best && row < local_idx)) {- local_best = candidate;- local_idx = row;- }- }+ for (uint row = lid + E; row < MAX_EXPERTS; row += THREADS_PER_TG) {+ logits_shared[row] = NEG_INF;+ idx_shared[row] = row;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);- float max_val =- threadgroup_cooperative_reduce_max<THREADS_PER_TG>(- local_best,- reduce_tmp,- lid);+ const uint effective_k = min(K, MAX_TOPK);+ for (uint sel = 0; sel < effective_k; ++sel) {+ float local_best = NEG_INF;+ uint local_idx = 0xFFFFFFFFu;+ for (uint row = lid; row < E; row += THREADS_PER_TG) {+ float candidate = logits_shared[row];+ if (candidate > local_best ||+ (candidate == local_best && row < local_idx)) {+ local_best = candidate;+ local_idx = row;+ }+ }- uint candidate_id =- (local_best == max_val) ? local_idx : 0xFFFFFFFFu;- uint best_idx =- threadgroup_cooperative_reduce_min<THREADS_PER_TG>(- candidate_id,- reduce_tmp_u,- lid);+ float max_val = threadgroup_cooperative_reduce_max<THREADS_PER_TG>(+ local_best,+ reduce_tmp,+ lid+ );- if (lid == 0) {- *shared_best_idx = best_idx;- *shared_best_val = max_val;- }- threadgroup_barrier(mem_flags::mem_threadgroup);+ uint candidate_id = (local_best == max_val) ? local_idx : 0xFFFFFFFFu;+ uint best_idx = threadgroup_cooperative_reduce_min<THREADS_PER_TG>(+ candidate_id,+ reduce_tmp_u,+ lid+ );- uint winner_idx = *shared_best_idx;- float winner_val = *shared_best_val;- if (winner_idx < MAX_EXPERTS) {- if (lid == 0) {- logits_shared[winner_idx] = NEG_INF;- idx_shared[winner_idx] = 0xFFFFFFFFu;- device int* out_ids = topk_ids + token_idx * K;- device ProbT* out_probs = topk_probs + token_idx * K;- out_ids[sel] = int(winner_idx);- out_probs[sel] = ProbT(winner_val);- }- }- threadgroup_barrier(mem_flags::mem_threadgroup);+ if (lid == 0) {+ *shared_best_idx = best_idx;+ *shared_best_val = max_val;}+ threadgroup_barrier(mem_flags::mem_threadgroup);- if (lid == 0 && renorm != 0 && effective_k > 0) {+ uint winner_idx = *shared_best_idx;+ float winner_val = *shared_best_val;+ if (winner_idx < MAX_EXPERTS) {+ if (lid == 0) {+ logits_shared[winner_idx] = NEG_INF;+ idx_shared[winner_idx] = 0xFFFFFFFFu;+ device int* out_ids = topk_ids + token_idx * K;device ProbT* out_probs = topk_probs + token_idx * K;- float max_logit = -INFINITY;- for (uint i = 0; i < effective_k; ++i) {- max_logit = fmax(max_logit, float(out_probs[i]));- }- float sum_exp = 0.0f;- for (uint i = 0; i < effective_k; ++i) {- sum_exp += exp(float(out_probs[i]) - max_logit);- }- float default_prob = 1.0f / float(effective_k);- float inv_sum = (sum_exp > 0.0f) ? (1.0f / sum_exp) : default_prob;- for (uint i = 0; i < effective_k; ++i) {- float prob = (sum_exp > 0.0f)- ? exp(float(out_probs[i]) - max_logit) * inv_sum- : default_prob;- out_probs[i] = ProbT(prob);- }- for (uint i = effective_k; i < K; ++i) {- out_probs[i] = ProbT(0.0f);- }+ out_ids[sel] = int(winner_idx);+ out_probs[sel] = ProbT(winner_val);+ }}-}+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }-#define DEFINE_ROUTER_TOPK_KERNEL(SUFFIX, VEC4, SCALAR, PROB) \-kernel void moe_router_topk_##SUFFIX( \- const device VEC4* input [[buffer(0)]], \- const device VEC4* weight [[buffer(1)]], \- const device SCALAR* bias [[buffer(2)]], \- device int* topk_ids [[buffer(3)]], \- device PROB* topk_probs [[buffer(4)]], \- constant uint& T [[buffer(5)]], \- constant uint& d_model [[buffer(6)]], \- constant uint& E [[buffer(7)]], \- constant uint& K [[buffer(8)]], \- constant uint& renorm [[buffer(9)]], \- uint2 tgpig [[threadgroup_position_in_grid]], \- ushort lid [[thread_index_in_threadgroup]], \- ushort simd_lane [[thread_index_in_simdgroup]], \- ushort simdgroup_idx [[simdgroup_index_in_threadgroup]], \- ushort simdgroups_per_tg [[simdgroups_per_threadgroup]]) \-{ \- threadgroup float4 x_cache[1024]; \- threadgroup float logits_shared[MAX_EXPERTS]; \- threadgroup uint idx_shared[MAX_EXPERTS]; \- threadgroup float reduce_tmp[THREADS_PER_TG]; \- threadgroup uint reduce_tmp_u[THREADS_PER_TG]; \- threadgroup uint shared_best_idx_mem; \- threadgroup float shared_best_val_mem; \- moe_router_topk_impl<VEC4, SCALAR, PROB>( \- input, weight, bias, topk_ids, topk_probs, \- T, d_model, E, K, renorm, tgpig, lid, simd_lane, simdgroup_idx, simdgroups_per_tg, \- x_cache, logits_shared, idx_shared, reduce_tmp, reduce_tmp_u, \- &shared_best_idx_mem, &shared_best_val_mem); \+ if (lid == 0 && renorm != 0 && effective_k > 0) {+ device ProbT* out_probs = topk_probs + token_idx * K;+ float max_logit = -INFINITY;+ for (uint i = 0; i < effective_k; ++i) {+ max_logit = fmax(max_logit, float(out_probs[i]));+ }+ float sum_exp = 0.0f;+ for (uint i = 0; i < effective_k; ++i) {+ sum_exp += exp(float(out_probs[i]) - max_logit);+ }+ float default_prob = 1.0f / float(effective_k);+ float inv_sum = (sum_exp > 0.0f) ? (1.0f / sum_exp) : default_prob;+ for (uint i = 0; i < effective_k; ++i) {+ float prob = (sum_exp > 0.0f)+ ? exp(float(out_probs[i]) - max_logit) * inv_sum+ : default_prob;+ out_probs[i] = ProbT(prob);+ }+ for (uint i = effective_k; i < K; ++i) {+ out_probs[i] = ProbT(0.0f);+ }+ }}+#define DEFINE_ROUTER_TOPK_KERNEL(SUFFIX, VEC4, SCALAR, PROB) \+ kernel void moe_router_topk_##SUFFIX( \+ const device VEC4* input [[buffer(0)]], \+ const device VEC4* weight [[buffer(1)]], \+ const device SCALAR* bias [[buffer(2)]], \+ device int* topk_ids [[buffer(3)]], \+ device PROB* topk_probs [[buffer(4)]], \+ constant uint& T [[buffer(5)]], \+ constant uint& d_model [[buffer(6)]], \+ constant uint& E [[buffer(7)]], \+ constant uint& K [[buffer(8)]], \+ constant uint& renorm [[buffer(9)]], \+ uint2 tgpig [[threadgroup_position_in_grid]], \+ ushort lid [[thread_index_in_threadgroup]], \+ ushort simd_lane [[thread_index_in_simdgroup]], \+ ushort simdgroup_idx [[simdgroup_index_in_threadgroup]], \+ ushort simdgroups_per_tg [[simdgroups_per_threadgroup]] \+ ) { \+ threadgroup float4 x_cache[1024]; \+ threadgroup float logits_shared[MAX_EXPERTS]; \+ threadgroup uint idx_shared[MAX_EXPERTS]; \+ threadgroup float reduce_tmp[THREADS_PER_TG]; \+ threadgroup uint reduce_tmp_u[THREADS_PER_TG]; \+ threadgroup uint shared_best_idx_mem; \+ threadgroup float shared_best_val_mem; \+ moe_router_topk_impl<VEC4, SCALAR, PROB>( \+ input, \+ weight, \+ bias, \+ topk_ids, \+ topk_probs, \+ T, \+ d_model, \+ E, \+ K, \+ renorm, \+ tgpig, \+ lid, \+ simd_lane, \+ simdgroup_idx, \+ simdgroups_per_tg, \+ x_cache, \+ logits_shared, \+ idx_shared, \+ reduce_tmp, \+ reduce_tmp_u, \+ &shared_best_idx_mem, \+ &shared_best_val_mem \+ ); \+ }+DEFINE_ROUTER_TOPK_KERNEL(f16, half4, half, half)DEFINE_ROUTER_TOPK_KERNEL(bf16, bfloat4, bfloat, bfloat)DEFINE_ROUTER_TOPK_KERNEL(f32, float4, float, float)diff --git a/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal b/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metalindex d55ca893b..3a2245045 100644--- a/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal@@ -17,34 +17,39 @@ kernel void moe_block_bases_from_partials(constant uint& num_blocks [[buffer(4)]],constant uint& num_tiles [[buffer(5)]],constant uint& capacity_per_expert [[buffer(6)]], // 0 => no clamp- uint gid [[thread_position_in_grid]])-{- const uint entries = num_tiles * TILE_E;- if (gid >= entries) return;- const uint tile_id = gid / TILE_E;- const uint te = gid % TILE_E;- const uint e = tile_id * TILE_E + te;- if (e >= E) return;+ uint gid [[thread_position_in_grid]]+) {+ const uint entries = num_tiles * TILE_E;+ if (gid >= entries)+ return;+ const uint tile_id = gid / TILE_E;+ const uint te = gid % TILE_E;+ const uint e = tile_id * TILE_E + te;+ if (e >= E)+ return;- uint running = 0u;- const bool clamp = (capacity_per_expert > 0u);- for (uint b = 0; b < num_blocks; ++b) {- const uint idx = (b * num_tiles + tile_id) * TILE_E + te;- const uint p = partials[idx];- const uint alloc = clamp ? (running < capacity_per_expert ? min(p, capacity_per_expert - running) : 0u) : p;- block_bases[idx] = running;- block_alloc[idx] = alloc;- running += alloc;- if (clamp && running >= capacity_per_expert) {- // Remainder allocate zero and base stays at capacity- for (uint bb = b + 1; bb < num_blocks; ++bb) {- const uint idx2 = (bb * num_tiles + tile_id) * TILE_E + te;- block_bases[idx2] = running;- block_alloc[idx2] = 0u;- }- break;- }+ uint running = 0u;+ const bool clamp = (capacity_per_expert > 0u);+ for (uint b = 0; b < num_blocks; ++b) {+ const uint idx = (b * num_tiles + tile_id) * TILE_E + te;+ const uint p = partials[idx];+ const uint alloc = clamp ? (running < capacity_per_expert+ ? min(p, capacity_per_expert - running)+ : 0u)+ : p;+ block_bases[idx] = running;+ block_alloc[idx] = alloc;+ running += alloc;+ if (clamp && running >= capacity_per_expert) {+ // Remainder allocate zero and base stays at capacity+ for (uint bb = b + 1; bb < num_blocks; ++bb) {+ const uint idx2 = (bb * num_tiles + tile_id) * TILE_E + te;+ block_bases[idx2] = running;+ block_alloc[idx2] = 0u;+ }+ break;}+ }}// 4B: Scatter into expert-major buckets using atomic-free device writes@@ -58,155 +63,219 @@ inline void moe_scatter_buckets_impl(device int* out_ids,device ProbT* out_probs,device int* tok2row_opt,- uint T, uint E, uint K,- uint num_blocks, uint num_tiles,- uint3 tid3, uint lid, uint3 tgpig,+ uint T,+ uint E,+ uint K,+ uint num_blocks,+ uint num_tiles,+ uint3 tid3,+ uint lid,+ uint3 tgpig,threadgroup _atomic<uint>* sg_counts,- threadgroup uint* sg_base)-{- if (E == 0 || T == 0 || K == 0) return;- const uint block_id = tgpig.x;- const uint t_start = block_id * BLOCK_SIZE;- const uint t_end = min(t_start + BLOCK_SIZE, T);- const uint sg_id = lid / SIMD_WIDTH;- const uint lane = lid % SIMD_WIDTH;+ threadgroup uint* sg_base+) {+ if (E == 0 || T == 0 || K == 0)+ return;+ const uint block_id = tgpig.x;+ const uint t_start = block_id * BLOCK_SIZE;+ const uint t_end = min(t_start + BLOCK_SIZE, T);+ const uint sg_id = lid / SIMD_WIDTH;+ const uint lane = lid % SIMD_WIDTH;- // Per-tile processing- for (uint e0 = 0; e0 < E; e0 += TILE_E) {- const uint tile_e = (e0 + TILE_E <= E) ? TILE_E : (E - e0);+ // Per-tile processing+ for (uint e0 = 0; e0 < E; e0 += TILE_E) {+ const uint tile_e = (e0 + TILE_E <= E) ? TILE_E : (E - e0);- // Zero counts and bases for active tile- for (uint te = lid; te < tile_e; te += BLOCK_SIZE) {- for (uint sg = 0; sg < NUM_SG; ++sg) {- atomic_store_explicit(&sg_counts[sg * TILE_E + te], 0u, memory_order_relaxed);- sg_base[sg * TILE_E + te] = 0u;- }- }- threadgroup_barrier(mem_flags::mem_threadgroup);+ // Zero counts and bases for active tile+ for (uint te = lid; te < tile_e; te += BLOCK_SIZE) {+ for (uint sg = 0; sg < NUM_SG; ++sg) {+ atomic_store_explicit(+ &sg_counts[sg * TILE_E + te],+ 0u,+ memory_order_relaxed+ );+ sg_base[sg * TILE_E + te] = 0u;+ }+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);- // Phase 1: per-sg counting using threadgroup atomics (order-independent)- for (uint t = t_start + lid; t < t_end; t += BLOCK_SIZE) {- const uint base = t * K;- for (uint k = 0; k < K; ++k) {- int eid = topk_ids[base + k];- if (eid >= 0) {- const uint ue = (uint)eid;- if (ue >= e0 && ue < e0 + tile_e) {- const uint te = ue - e0;- atomic_fetch_add_explicit(&sg_counts[sg_id * TILE_E + te], 1u, memory_order_relaxed);- }- }- }+ // Phase 1: per-sg counting using threadgroup atomics (order-independent)+ for (uint t = t_start + lid; t < t_end; t += BLOCK_SIZE) {+ const uint base = t * K;+ for (uint k = 0; k < K; ++k) {+ int eid = topk_ids[base + k];+ if (eid >= 0) {+ const uint ue = (uint)eid;+ if (ue >= e0 && ue < e0 + tile_e) {+ const uint te = ue - e0;+ atomic_fetch_add_explicit(+ &sg_counts[sg_id * TILE_E + te],+ 1u,+ memory_order_relaxed+ );+ }}- threadgroup_barrier(mem_flags::mem_threadgroup);+ }+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);- // Phase 2: compute per-sg bases via prefix across sgs with capacity clamp- for (uint te = lid; te < tile_e; te += BLOCK_SIZE) {- const uint tile_id = e0 / TILE_E;- const uint base_idx_block = (block_id * num_tiles + tile_id) * TILE_E + te;- const uint alloc_total = block_alloc[base_idx_block];- uint prefix = 0u;- for (uint sg = 0; sg < NUM_SG; ++sg) {- const uint idx = sg * TILE_E + te;- const uint c = atomic_load_explicit(&sg_counts[idx], memory_order_relaxed);- const uint room = (prefix < alloc_total) ? (alloc_total - prefix) : 0u;- const uint take = min(c, room);- sg_base[idx] = prefix;- prefix += take;- }- // Reuse sg_counts as write indices: reset to zero- for (uint sg = 0; sg < NUM_SG; ++sg) {- atomic_store_explicit(&sg_counts[sg * TILE_E + te], 0u, memory_order_relaxed);- }- }- threadgroup_barrier(mem_flags::mem_threadgroup);+ // Phase 2: compute per-sg bases via prefix across sgs with capacity clamp+ for (uint te = lid; te < tile_e; te += BLOCK_SIZE) {+ const uint tile_id = e0 / TILE_E;+ const uint base_idx_block =+ (block_id * num_tiles + tile_id) * TILE_E + te;+ const uint alloc_total = block_alloc[base_idx_block];+ uint prefix = 0u;+ for (uint sg = 0; sg < NUM_SG; ++sg) {+ const uint idx = sg * TILE_E + te;+ const uint c =+ atomic_load_explicit(&sg_counts[idx], memory_order_relaxed);+ const uint room = (prefix < alloc_total) ? (alloc_total - prefix) : 0u;+ const uint take = min(c, room);+ sg_base[idx] = prefix;+ prefix += take;+ }+ // Reuse sg_counts as write indices: reset to zero+ for (uint sg = 0; sg < NUM_SG; ++sg) {+ atomic_store_explicit(+ &sg_counts[sg * TILE_E + te],+ 0u,+ memory_order_relaxed+ );+ }+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);- // Phase 3: deterministic write with 32-lane sequencing, no device atomics- for (uint step = 0; step < SIMD_WIDTH; ++step) {- if (lane == step) {- const uint t = t_start + lid;- if (t < t_end) {- const uint base = t * K;- for (uint k = 0; k < K; ++k) {- int eid = topk_ids[base + k];- if (eid >= 0) {- const uint ue = (uint)eid;- if (ue >= e0 && ue < e0 + tile_e) {- const uint te = ue - e0;- const uint tile_id = e0 / TILE_E;- const uint base_idx_block = (block_id * num_tiles + tile_id) * TILE_E + te;- const uint idx_sg_te = sg_id * TILE_E + te;- const uint local = atomic_fetch_add_explicit(&sg_counts[idx_sg_te], 1u, memory_order_relaxed);- const uint base_s = sg_base[idx_sg_te];+ // Phase 3: deterministic write with 32-lane sequencing, no device atomics+ for (uint step = 0; step < SIMD_WIDTH; ++step) {+ if (lane == step) {+ const uint t = t_start + lid;+ if (t < t_end) {+ const uint base = t * K;+ for (uint k = 0; k < K; ++k) {+ int eid = topk_ids[base + k];+ if (eid >= 0) {+ const uint ue = (uint)eid;+ if (ue >= e0 && ue < e0 + tile_e) {+ const uint te = ue - e0;+ const uint tile_id = e0 / TILE_E;+ const uint base_idx_block =+ (block_id * num_tiles + tile_id) * TILE_E + te;+ const uint idx_sg_te = sg_id * TILE_E + te;+ const uint local = atomic_fetch_add_explicit(+ &sg_counts[idx_sg_te],+ 1u,+ memory_order_relaxed+ );+ const uint base_s = sg_base[idx_sg_te];- const uint slot = offsets[ue] + block_bases[base_idx_block] + base_s + local;- out_ids[slot] = (int)t;- out_probs[slot] = topk_probs[base + k];- if (tok2row_opt != nullptr) {- tok2row_opt[base + k] = (int)slot;- }- }- }- }+ const uint slot =+ offsets[ue] + block_bases[base_idx_block] + base_s + local;+ out_ids[slot] = (int)t;+ out_probs[slot] = topk_probs[base + k];+ if (tok2row_opt != nullptr) {+ tok2row_opt[base + k] = (int)slot;}+ }}- threadgroup_barrier(mem_flags::mem_threadgroup);+ }}- threadgroup_barrier(mem_flags::mem_threadgroup);+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);}+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }}-#define DEFINE_MOE_SCATTER_BUCKETS_KERNEL(SUFFIX, ProbT) \-kernel void moe_scatter_buckets_##SUFFIX( \- device const int* topk_ids [[buffer(0)]], \- device const ProbT* topk_probs [[buffer(1)]], \- device const uint* offsets [[buffer(2)]], \- device const uint* block_bases [[buffer(3)]], \- device const uint* block_alloc [[buffer(4)]], \- device int* out_ids [[buffer(5)]], \- device ProbT* out_probs [[buffer(6)]], \- constant uint& T [[buffer(7)]], \- constant uint& E [[buffer(8)]], \- constant uint& K [[buffer(9)]], \- constant uint& num_blocks [[buffer(10)]], \- constant uint& num_tiles [[buffer(11)]], \- uint3 tid3 [[thread_position_in_grid]], \- uint lid [[thread_index_in_threadgroup]], \- uint3 tgpig [[threadgroup_position_in_grid]]) \-{ \- threadgroup _atomic<uint> sg_counts[NUM_SG * TILE_E]; \- threadgroup uint sg_base [NUM_SG * TILE_E]; \- moe_scatter_buckets_impl<ProbT>(topk_ids, topk_probs, offsets, block_bases, block_alloc, out_ids, out_probs, (device int*)nullptr, T, E, K, num_blocks, num_tiles, tid3, lid, tgpig, sg_counts, sg_base); \-}+#define DEFINE_MOE_SCATTER_BUCKETS_KERNEL(SUFFIX, ProbT) \+ kernel void moe_scatter_buckets_##SUFFIX( \+ device const int* topk_ids [[buffer(0)]], \+ device const ProbT* topk_probs [[buffer(1)]], \+ device const uint* offsets [[buffer(2)]], \+ device const uint* block_bases [[buffer(3)]], \+ device const uint* block_alloc [[buffer(4)]], \+ device int* out_ids [[buffer(5)]], \+ device ProbT* out_probs [[buffer(6)]], \+ constant uint& T [[buffer(7)]], \+ constant uint& E [[buffer(8)]], \+ constant uint& K [[buffer(9)]], \+ constant uint& num_blocks [[buffer(10)]], \+ constant uint& num_tiles [[buffer(11)]], \+ uint3 tid3 [[thread_position_in_grid]], \+ uint lid [[thread_index_in_threadgroup]], \+ uint3 tgpig [[threadgroup_position_in_grid]] \+ ) { \+ threadgroup _atomic<uint> sg_counts[NUM_SG * TILE_E]; \+ threadgroup uint sg_base[NUM_SG * TILE_E]; \+ moe_scatter_buckets_impl<ProbT>( \+ topk_ids, \+ topk_probs, \+ offsets, \+ block_bases, \+ block_alloc, \+ out_ids, \+ out_probs, \+ (device int*)nullptr, \+ T, \+ E, \+ K, \+ num_blocks, \+ num_tiles, \+ tid3, \+ lid, \+ tgpig, \+ sg_counts, \+ sg_base \+ ); \+ }DEFINE_MOE_SCATTER_BUCKETS_KERNEL(f16, half)DEFINE_MOE_SCATTER_BUCKETS_KERNEL(f32, float)DEFINE_MOE_SCATTER_BUCKETS_KERNEL(bf16, bfloat)// Map variants-#define DEFINE_MOE_SCATTER_BUCKETS_MAP_KERNEL(SUFFIX, ProbT) \-kernel void moe_scatter_buckets_map_##SUFFIX( \- device const int* topk_ids [[buffer(0)]], \- device const ProbT* topk_probs [[buffer(1)]], \- device const uint* offsets [[buffer(2)]], \- device const uint* block_bases [[buffer(3)]], \- device const uint* block_alloc [[buffer(4)]], \- device int* out_ids [[buffer(5)]], \- device ProbT* out_probs [[buffer(6)]], \- constant uint& T [[buffer(7)]], \- constant uint& E [[buffer(8)]], \- constant uint& K [[buffer(9)]], \- constant uint& num_blocks [[buffer(10)]], \- constant uint& num_tiles [[buffer(11)]], \- device int* tok2row [[buffer(12)]], \- uint3 tid3 [[thread_position_in_grid]], \- uint lid [[thread_index_in_threadgroup]], \- uint3 tgpig [[threadgroup_position_in_grid]]) \-{ \- threadgroup _atomic<uint> sg_counts[NUM_SG * TILE_E]; \- threadgroup uint sg_base [NUM_SG * TILE_E]; \- moe_scatter_buckets_impl<ProbT>(topk_ids, topk_probs, offsets, block_bases, block_alloc, out_ids, out_probs, tok2row, T, E, K, num_blocks, num_tiles, tid3, lid, tgpig, sg_counts, sg_base); \-}+#define DEFINE_MOE_SCATTER_BUCKETS_MAP_KERNEL(SUFFIX, ProbT) \+ kernel void moe_scatter_buckets_map_##SUFFIX( \+ device const int* topk_ids [[buffer(0)]], \+ device const ProbT* topk_probs [[buffer(1)]], \+ device const uint* offsets [[buffer(2)]], \+ device const uint* block_bases [[buffer(3)]], \+ device const uint* block_alloc [[buffer(4)]], \+ device int* out_ids [[buffer(5)]], \+ device ProbT* out_probs [[buffer(6)]], \+ constant uint& T [[buffer(7)]], \+ constant uint& E [[buffer(8)]], \+ constant uint& K [[buffer(9)]], \+ constant uint& num_blocks [[buffer(10)]], \+ constant uint& num_tiles [[buffer(11)]], \+ device int* tok2row [[buffer(12)]], \+ uint3 tid3 [[thread_position_in_grid]], \+ uint lid [[thread_index_in_threadgroup]], \+ uint3 tgpig [[threadgroup_position_in_grid]] \+ ) { \+ threadgroup _atomic<uint> sg_counts[NUM_SG * TILE_E]; \+ threadgroup uint sg_base[NUM_SG * TILE_E]; \+ moe_scatter_buckets_impl<ProbT>( \+ topk_ids, \+ topk_probs, \+ offsets, \+ block_bases, \+ block_alloc, \+ out_ids, \+ out_probs, \+ tok2row, \+ T, \+ E, \+ K, \+ num_blocks, \+ num_tiles, \+ tid3, \+ lid, \+ tgpig, \+ sg_counts, \+ sg_base \+ ); \+ }DEFINE_MOE_SCATTER_BUCKETS_MAP_KERNEL(f16, half)DEFINE_MOE_SCATTER_BUCKETS_MAP_KERNEL(f32, float)diff --git a/crates/uzu/src/backends/metal/kernel/moe/tiles.metal b/crates/uzu/src/backends/metal/kernel/moe/tiles.metalindex f57eff6f8..ccda7f661 100644--- a/crates/uzu/src/backends/metal/kernel/moe/tiles.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/tiles.metal@@ -6,104 +6,109 @@ using namespace metal;// Compute per-expert tile counts: tiles_e = ceil((seg_len)/BM)kernel void moe_tile_counts(- device const uint* offsets [[buffer(0)]], // [E+1]- device uint* tile_counts [[buffer(1)]], // [E]+ device const uint* offsets [[buffer(0)]], // [E+1]+ device uint* tile_counts [[buffer(1)]], // [E]constant uint& E [[buffer(2)]],- uint gid [[thread_position_in_grid]])-{- if (gid >= E) return;- const uint start = offsets[gid];- const uint end = offsets[gid + 1];- const uint seg_len = (end > start) ? (end - start) : 0u;- const uint tiles = (seg_len == 0u) ? 0u : ((seg_len + BM - 1u) / BM);- tile_counts[gid] = tiles;+ uint gid [[thread_position_in_grid]]+) {+ if (gid >= E)+ return;+ const uint start = offsets[gid];+ const uint end = offsets[gid + 1];+ const uint seg_len = (end > start) ? (end - start) : 0u;+ const uint tiles = (seg_len == 0u) ? 0u : ((seg_len + BM - 1u) / BM);+ tile_counts[gid] = tiles;}-// Single-threadgroup exclusive scan over tile_counts -> tile_row_offsets, also writes total_tiles+// Single-threadgroup exclusive scan over tile_counts -> tile_row_offsets, also+// writes total_tileskernel void moe_tile_scan(- device const uint* tile_counts [[buffer(0)]], // [E]- device uint* tile_row_offsets [[buffer(1)]], // [E+1]- device uint* total_tiles_buf [[buffer(2)]], // [>=2]+ device const uint* tile_counts [[buffer(0)]], // [E]+ device uint* tile_row_offsets [[buffer(1)]], // [E+1]+ device uint* total_tiles_buf [[buffer(2)]], // [>=2]constant uint& E [[buffer(3)]],- uint tid [[thread_index_in_threadgroup]])-{- threadgroup uint scratch[1024]; // supports up to E<=1023+ uint tid [[thread_index_in_threadgroup]]+) {+ threadgroup uint scratch[1024]; // supports up to E<=1023+ if (tid < E) {+ scratch[tid] = tile_counts[tid];+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);++ // Inclusive scan in-place over scratch+ // Simple O(log E) scan; E is small+ for (uint offset = 1u; offset < E; offset <<= 1u) {+ uint val = 0u;if (tid < E) {- scratch[tid] = tile_counts[tid];+ val = scratch[tid];+ if (tid >= offset) {+ val += scratch[tid - offset];+ }}threadgroup_barrier(mem_flags::mem_threadgroup);-- // Inclusive scan in-place over scratch- // Simple O(log E) scan; E is small- for (uint offset = 1u; offset < E; offset <<= 1u) {- uint val = 0u;- if (tid < E) {- val = scratch[tid];- if (tid >= offset) {- val += scratch[tid - offset];- }- }- threadgroup_barrier(mem_flags::mem_threadgroup);- if (tid < E) {- scratch[tid] = val;- }- threadgroup_barrier(mem_flags::mem_threadgroup);- }-- if (tid == 0u) {- tile_row_offsets[0] = 0u;- }if (tid < E) {- // exclusive: shift right by one- const uint prefix = (tid == 0u) ? 0u : scratch[tid - 1u];- tile_row_offsets[tid] = prefix;- }- if (tid == 0u) {- tile_row_offsets[E] = (E == 0u) ? 0u : scratch[E - 1u];- total_tiles_buf[0] = tile_row_offsets[E];- total_tiles_buf[1] = 0u;- total_tiles_buf[2] = 0u;- total_tiles_buf[3] = 0u;- total_tiles_buf[4] = 0u;- total_tiles_buf[5] = 0u;- total_tiles_buf[6] = 0u;- total_tiles_buf[7] = 0u;+ scratch[tid] = val;}+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }++ if (tid == 0u) {+ tile_row_offsets[0] = 0u;+ }+ if (tid < E) {+ // exclusive: shift right by one+ const uint prefix = (tid == 0u) ? 0u : scratch[tid - 1u];+ tile_row_offsets[tid] = prefix;+ }+ if (tid == 0u) {+ tile_row_offsets[E] = (E == 0u) ? 0u : scratch[E - 1u];+ total_tiles_buf[0] = tile_row_offsets[E];+ total_tiles_buf[1] = 0u;+ total_tiles_buf[2] = 0u;+ total_tiles_buf[3] = 0u;+ total_tiles_buf[4] = 0u;+ total_tiles_buf[5] = 0u;+ total_tiles_buf[6] = 0u;+ total_tiles_buf[7] = 0u;+ }}// Build flattened tile_map of length total_tiles; each entry is 3 uints:// [3*i+0]=expert_idx, [3*i+1]=seg_start, [3*i+2]=tile_m0kernel void moe_build_tile_map(- device const uint* offsets [[buffer(0)]], // [E+1]- device const uint* tile_row_offsets [[buffer(1)]], // [E+1]- device const uint* tile_counts [[buffer(2)]], // [E]- device uint* tile_map [[buffer(3)]], // [3*total_tiles]+ device const uint* offsets [[buffer(0)]], // [E+1]+ device const uint* tile_row_offsets [[buffer(1)]], // [E+1]+ device const uint* tile_counts [[buffer(2)]], // [E]+ device uint* tile_map [[buffer(3)]], // [3*total_tiles]constant uint& E [[buffer(4)]],- uint gid [[thread_position_in_grid]])-{- if (gid >= E) return;- const uint start = offsets[gid];- const uint tiles = tile_counts[gid];- const uint base = tile_row_offsets[gid];- for (uint t = 0u; t < tiles; ++t) {- const uint idx = base + t;- tile_map[3u * idx + 0u] = gid;- tile_map[3u * idx + 1u] = start;- tile_map[3u * idx + 2u] = t * BM;- }+ uint gid [[thread_position_in_grid]]+) {+ if (gid >= E)+ return;+ const uint start = offsets[gid];+ const uint tiles = tile_counts[gid];+ const uint base = tile_row_offsets[gid];+ for (uint t = 0u; t < tiles; ++t) {+ const uint idx = base + t;+ tile_map[3u * idx + 0u] = gid;+ tile_map[3u * idx + 1u] = start;+ tile_map[3u * idx + 2u] = t * BM;+ }}// Write MTLDispatchThreadgroupsIndirectArguments {x, y, z} where:// x = num_tiles_n (computed on CPU and passed in), y = total_tiles, z = 1kernel void moe_write_dispatch_args(- device const uint* total_tiles_buf [[buffer(0)]], // [>=1], total_tiles_buf[0] = total_rows- device uint* dispatch_args [[buffer(1)]], // [3] u32- constant uint& num_tiles_n [[buffer(2)]], // x dimension- uint gid [[thread_position_in_grid]])-{- if (gid > 0u) return;- const uint total_rows = total_tiles_buf[0];- dispatch_args[0] = num_tiles_n; // x- dispatch_args[1] = total_rows; // y- dispatch_args[2] = 1u; // z+ device const uint* total_tiles_buf+ [[buffer(0)]], // [>=1], total_tiles_buf[0] = total_rows+ device uint* dispatch_args [[buffer(1)]], // [3] u32+ constant uint& num_tiles_n [[buffer(2)]], // x dimension+ uint gid [[thread_position_in_grid]]+) {+ if (gid > 0u)+ return;+ const uint total_rows = total_tiles_buf[0];+ dispatch_args[0] = num_tiles_n; // x+ dispatch_args[1] = total_rows; // y+ dispatch_args[2] = 1u; // z}diff --git a/crates/uzu/src/backends/metal/kernel/moe/topk_select.metal b/crates/uzu/src/backends/metal/kernel/moe/topk_select.metalindex a0f1c85f6..68cf8b73c 100644--- a/crates/uzu/src/backends/metal/kernel/moe/topk_select.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/topk_select.metal@@ -7,7 +7,7 @@ constant uint MAX_EXPERTS = 512;constant uint MAX_TOPK = 128;constant float NEG_INF = -INFINITY;-template<typename LogitT, typename ProbT>+template <typename LogitT, typename ProbT>inline void moe_topk_select_impl(device const LogitT* logits,device int* topk_ids,@@ -23,133 +23,148 @@ inline void moe_topk_select_impl(threadgroup float* reduce_tmp,threadgroup uint* reduce_tmp_u,threadgroup float* selected_vals,- threadgroup uint* selected_ids)-{- if (token >= T || K == 0) {- return;+ threadgroup uint* selected_ids+) {+ if (token >= T || K == 0) {+ return;+ }++ const uint effective_k = min(K, MAX_TOPK);++ for (uint i = lid; i < MAX_EXPERTS; i += THREADS_PER_TG) {+ if (i < E) {+ tg_logits[i] = float(logits[token * E + i]);+ } else {+ tg_logits[i] = NEG_INF;+ }+ tg_indices[i] = i;+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ const uint selections = min(effective_k, E);+ for (uint sel = 0; sel < selections; ++sel) {+ float local_best_val = NEG_INF;+ uint local_best_idx = 0xFFFFFFFFu;++ for (uint i = lid; i < E; i += THREADS_PER_TG) {+ float candidate = tg_logits[i];+ if (candidate > local_best_val ||+ (candidate == local_best_val && i < local_best_idx)) {+ local_best_val = candidate;+ local_best_idx = i;+ }}- const uint effective_k = min(K, MAX_TOPK);+ float max_val = threadgroup_cooperative_reduce_max<THREADS_PER_TG>(+ local_best_val,+ reduce_tmp,+ lid+ );- for (uint i = lid; i < MAX_EXPERTS; i += THREADS_PER_TG) {- if (i < E) {- tg_logits[i] = float(logits[token * E + i]);- } else {- tg_logits[i] = NEG_INF;- }- tg_indices[i] = i;- }+ uint candidate_id =+ (local_best_val == max_val) ? local_best_idx : 0xFFFFFFFFu;+ uint best_idx = threadgroup_cooperative_reduce_min<THREADS_PER_TG>(+ candidate_id,+ reduce_tmp_u,+ lid+ );+ if (lid == 0) {+ selected_vals[sel] = max_val;+ selected_ids[sel] = best_idx;+ tg_logits[best_idx] = NEG_INF;+ }threadgroup_barrier(mem_flags::mem_threadgroup);+ }- const uint selections = min(effective_k, E);- for (uint sel = 0; sel < selections; ++sel) {- float local_best_val = NEG_INF;- uint local_best_idx = 0xFFFFFFFFu;-- for (uint i = lid; i < E; i += THREADS_PER_TG) {- float candidate = tg_logits[i];- if (candidate > local_best_val ||- (candidate == local_best_val && i < local_best_idx)) {- local_best_val = candidate;- local_best_idx = i;- }- }-- float max_val = threadgroup_cooperative_reduce_max<THREADS_PER_TG>(- local_best_val,- reduce_tmp,- lid);-- uint candidate_id =- (local_best_val == max_val) ? local_best_idx : 0xFFFFFFFFu;- uint best_idx = threadgroup_cooperative_reduce_min<THREADS_PER_TG>(- candidate_id,- reduce_tmp_u,- lid);-- if (lid == 0) {- selected_vals[sel] = max_val;- selected_ids[sel] = best_idx;- tg_logits[best_idx] = NEG_INF;- }- threadgroup_barrier(mem_flags::mem_threadgroup);- }+ threadgroup_barrier(mem_flags::mem_threadgroup);- threadgroup_barrier(mem_flags::mem_threadgroup);+ if (lid == 0) {+ device int* out_ids = topk_ids + token * K;+ device ProbT* out_probs = topk_probs + token * K;- if (lid == 0) {- device int* out_ids = topk_ids + token * K;- device ProbT* out_probs = topk_probs + token * K;-- const uint written = selections;- for (uint i = 0; i < written; ++i) {- out_ids[i] = int(selected_ids[i]);- }- for (uint i = written; i < K; ++i) {- out_ids[i] = 0;- }-- if (renorm != 0 && written > 0) {- float max_logit = -INFINITY;- for (uint i = 0; i < written; ++i) {- max_logit = fmax(max_logit, selected_vals[i]);- }-- float exp_values[MAX_TOPK];- float sum_exp = 0.0f;- for (uint i = 0; i < written; ++i) {- float e_val = exp(selected_vals[i] - max_logit);- exp_values[i] = e_val;- sum_exp += e_val;- }-- float default_prob = 1.0f / float(written);- float inv_sum =- (sum_exp > 0.0f) ? (1.0f / sum_exp) : default_prob;-- for (uint i = 0; i < written; ++i) {- float prob = (sum_exp > 0.0f) ? (exp_values[i] * inv_sum)- : default_prob;- out_probs[i] = ProbT(prob);- }- for (uint i = written; i < K; ++i) {- out_probs[i] = ProbT(0.0f);- }- } else {- for (uint i = 0; i < written; ++i) {- out_probs[i] = ProbT(selected_vals[i]);- }- for (uint i = written; i < K; ++i) {- out_probs[i] = ProbT(0.0f);- }- }+ const uint written = selections;+ for (uint i = 0; i < written; ++i) {+ out_ids[i] = int(selected_ids[i]);+ }+ for (uint i = written; i < K; ++i) {+ out_ids[i] = 0;}-}-#define DEFINE_MOE_TOPK_KERNEL(LOGIT_T, PROB_T, SUFFIX) \-kernel void moe_topk_select_##SUFFIX( \- device const LOGIT_T* logits [[buffer(0)]], \- device int* topk_ids [[buffer(1)]], \- device PROB_T* topk_probs [[buffer(2)]], \- constant uint& T [[buffer(3)]], \- constant uint& E [[buffer(4)]], \- constant uint& K [[buffer(5)]], \- constant uint& renorm [[buffer(6)]], \- ushort lid [[thread_index_in_threadgroup]], \- uint3 tgpig [[threadgroup_position_in_grid]]) \-{ \- threadgroup float tg_logits[MAX_EXPERTS]; \- threadgroup uint tg_indices[MAX_EXPERTS]; \- threadgroup float reduce_tmp[THREADS_PER_TG]; \- threadgroup uint reduce_tmp_u[THREADS_PER_TG]; \- threadgroup float selected_vals[MAX_TOPK]; \- threadgroup uint selected_ids[MAX_TOPK]; \- moe_topk_select_impl<LOGIT_T, PROB_T>( \- logits, topk_ids, topk_probs, T, E, K, renorm, lid, tgpig.x, \- tg_logits, tg_indices, reduce_tmp, reduce_tmp_u, selected_vals, selected_ids); \+ if (renorm != 0 && written > 0) {+ float max_logit = -INFINITY;+ for (uint i = 0; i < written; ++i) {+ max_logit = fmax(max_logit, selected_vals[i]);+ }++ float exp_values[MAX_TOPK];+ float sum_exp = 0.0f;+ for (uint i = 0; i < written; ++i) {+ float e_val = exp(selected_vals[i] - max_logit);+ exp_values[i] = e_val;+ sum_exp += e_val;+ }++ float default_prob = 1.0f / float(written);+ float inv_sum = (sum_exp > 0.0f) ? (1.0f / sum_exp) : default_prob;++ for (uint i = 0; i < written; ++i) {+ float prob =+ (sum_exp > 0.0f) ? (exp_values[i] * inv_sum) : default_prob;+ out_probs[i] = ProbT(prob);+ }+ for (uint i = written; i < K; ++i) {+ out_probs[i] = ProbT(0.0f);+ }+ } else {+ for (uint i = 0; i < written; ++i) {+ out_probs[i] = ProbT(selected_vals[i]);+ }+ for (uint i = written; i < K; ++i) {+ out_probs[i] = ProbT(0.0f);+ }+ }+ }}+#define DEFINE_MOE_TOPK_KERNEL(LOGIT_T, PROB_T, SUFFIX) \+ kernel void moe_topk_select_##SUFFIX( \+ device const LOGIT_T* logits [[buffer(0)]], \+ device int* topk_ids [[buffer(1)]], \+ device PROB_T* topk_probs [[buffer(2)]], \+ constant uint& T [[buffer(3)]], \+ constant uint& E [[buffer(4)]], \+ constant uint& K [[buffer(5)]], \+ constant uint& renorm [[buffer(6)]], \+ ushort lid [[thread_index_in_threadgroup]], \+ uint3 tgpig [[threadgroup_position_in_grid]] \+ ) { \+ threadgroup float tg_logits[MAX_EXPERTS]; \+ threadgroup uint tg_indices[MAX_EXPERTS]; \+ threadgroup float reduce_tmp[THREADS_PER_TG]; \+ threadgroup uint reduce_tmp_u[THREADS_PER_TG]; \+ threadgroup float selected_vals[MAX_TOPK]; \+ threadgroup uint selected_ids[MAX_TOPK]; \+ moe_topk_select_impl<LOGIT_T, PROB_T>( \+ logits, \+ topk_ids, \+ topk_probs, \+ T, \+ E, \+ K, \+ renorm, \+ lid, \+ tgpig.x, \+ tg_logits, \+ tg_indices, \+ reduce_tmp, \+ reduce_tmp_u, \+ selected_vals, \+ selected_ids \+ ); \+ }+DEFINE_MOE_TOPK_KERNEL(half, half, f16)DEFINE_MOE_TOPK_KERNEL(float, half, f32)DEFINE_MOE_TOPK_KERNEL(bfloat, bfloat, bf16)diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/mma.h b/crates/uzu/src/backends/metal/kernel/quant_matmul/mma.hindex b75b3241c..56000c057 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/mma.h+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/mma.h@@ -12,37 +12,37 @@ using namespace metal;namespace matmul_utils {-template<typename T>+template <typename T>struct pointer_element_t_impl {- using type = T;+ using type = T;};-template<typename T>+template <typename T>struct pointer_element_t_impl<device T*> {- using type = T;+ using type = T;};-template<typename T>+template <typename T>struct pointer_element_t_impl<threadgroup T*> {- using type = T;+ using type = T;};-template<typename T>+template <typename T>using pointer_element_t = typename pointer_element_t_impl<T>::type;template <typename T, typename U = T>struct TransformNone {- METAL_FUNC static U apply(U val) {- return val;- }+ METAL_FUNC static U apply(U val) { return val; }};template <typename T, int kFragRows_, int kFragCols_>struct BaseMMAFrag {static_assert(kFragRows_ == 8,- "Only 8 x 8 fragment matrices are currently supported");+ "Only 8 x 8 fragment matrices are currently supported"+ );static_assert(kFragCols_ == 8,- "Only 8 x 8 fragment matrices are currently supported");+ "Only 8 x 8 fragment matrices are currently supported"+ );};template <typename T>@@ -55,13 +55,15 @@ struct BaseMMAFrag<T, 8, 8> {static_assert(kElemRows * kElemCols == kElemsPerFrag,- "MMAFrag shape is not consistent with MMAFrag size");+ "MMAFrag shape is not consistent with MMAFrag size"+ );typedef metal::simdgroup_matrix<T, kFragRows, kFragCols> mat_type;typedef metal::vec<T, kElemsPerFrag> frag_type;- METAL_FUNC static constexpr short2 get_coord(ushort simd_lane_id- [[thread_index_in_simdgroup]]) {+ METAL_FUNC static constexpr short2 get_coord(+ ushort simd_lane_id [[thread_index_in_simdgroup]]+ ) {const short qid = simd_lane_id / 4;const short fm = (qid & 4) + ((simd_lane_id / 2) % 4);const short fn = (qid & 2) * 2 + (simd_lane_id % 2) * 2;@@ -69,8 +71,12 @@ struct BaseMMAFrag<T, 8, 8> {}template <typename SrcPtrType, typename StrX, typename StrY>- METAL_FUNC static constexpr void- load(thread frag_type& dst, SrcPtrType src, StrX str_x, StrY str_y) {+ METAL_FUNC static constexpr void load(+ thread frag_type& dst,+ SrcPtrType src,+ StrX str_x,+ StrY str_y+ ) {UZU_PRAGMA_UNROLLfor (short i = 0; i < kElemRows; i++) {UZU_PRAGMA_UNROLL@@ -80,10 +86,13 @@ struct BaseMMAFrag<T, 8, 8> {}}-template <typename DstPtrType, typename StrX, typename StrY>- METAL_FUNC static constexpr void- store(const thread frag_type& src, DstPtrType dst, StrX str_x, StrY str_y) {+ METAL_FUNC static constexpr void store(+ const thread frag_type& src,+ DstPtrType dst,+ StrX str_x,+ StrY str_y+ ) {using U = pointer_element_t<DstPtrType>;UZU_PRAGMA_UNROLL@@ -111,7 +120,8 @@ struct BaseMMAFrag<T, 8, 8> {LimX lim_x,LimY lim_y,OffX off_x = 0,- OffY off_y = 0) {+ OffY off_y = 0+ ) {using U = pointer_element_t<DstPtrType>;UZU_PRAGMA_UNROLL@@ -126,12 +136,12 @@ struct BaseMMAFrag<T, 8, 8> {}}-METAL_FUNC static constexpr void mma(thread frag_type& D,thread frag_type& A,thread frag_type& B,- thread frag_type& C) {+ thread frag_type& C+ ) {mat_type D_mat;mat_type A_mat;mat_type B_mat;@@ -150,7 +160,8 @@ struct BaseMMAFrag<T, 8, 8> {thread mat_type& D,thread mat_type& A,thread mat_type& B,- thread mat_type& C) {+ thread mat_type& C+ ) {simdgroup_multiply_accumulate(D, A, B, C);}};@@ -196,11 +207,11 @@ struct MMATile {METAL_FUNC constexpr const thread frag_type& frag_at(const short i,- const short j) const {+ const short j+ ) const {return val_frags[i * kTileCols + j];}-METAL_FUNC thread elem_type* elems() {return reinterpret_cast<thread elem_type*>(val_frags);}@@ -219,14 +230,15 @@ struct MMATile {frag_at(i, j),&(src[(i * kFragRows) * w_x * str_x +- (j * kFragCols) * w_y * str_y]),+ (j * kFragCols) * w_y * str_y]+ ),str_x,- str_y);+ str_y+ );}}}-template <typename U, int w_x, int w_y>METAL_FUNC void store(device U* dst, const int ld) const {UZU_PRAGMA_UNROLL@@ -237,7 +249,8 @@ struct MMATile {frag_at(i, j),&(dst[(i * kFragRows) * w_x * ld + (j * kFragCols) * w_y]),ld,- 1);+ 1+ );}}}@@ -253,14 +266,18 @@ struct MMATile {frag_at(i, j),&(dst[(i * kFragRows) * w_x * ld + (j * kFragCols) * w_y]),ld,- 1);+ 1+ );}}}template <typename U, int w_x, int w_y>- METAL_FUNC void- store_safe(device U* dst, const int ld, const short2 dst_tile_dims) const {+ METAL_FUNC void store_safe(+ device U* dst,+ const int ld,+ const short2 dst_tile_dims+ ) const {UZU_PRAGMA_UNROLLfor (int i = 0; i < kTileRows; ++i) {UZU_PRAGMA_UNROLL@@ -273,11 +290,11 @@ struct MMATile {dst_tile_dims.y,dst_tile_dims.x,(i * kFragRows) * w_x,- (j * kFragCols) * w_y);+ (j * kFragCols) * w_y+ );}}}-};template <typename T, typename U, int M, int N, int K>@@ -285,7 +302,8 @@ METAL_FUNC void tile_matmad(thread MMATile<T, M, N>& D,thread MMATile<U, M, K>& A,thread MMATile<U, K, N>& B,- thread MMATile<T, M, N>& C) {+ thread MMATile<T, M, N>& C+) {UZU_PRAGMA_UNROLLfor (short m = 0; m < M; ++m) {UZU_PRAGMA_UNROLL@@ -297,7 +315,8 @@ METAL_FUNC void tile_matmad(D.frag_at(m, n_serp),A.frag_at(m, k),B.frag_at(k, n_serp),- C.frag_at(m, n_serp));+ C.frag_at(m, n_serp)+ );}}}@@ -359,7 +378,8 @@ struct BlockMMA {/* Constructor */METAL_FUNC BlockMMA(ushort simd_group_id [[simdgroup_index_in_threadgroup]],- ushort simd_lane_id [[thread_index_in_simdgroup]]) {+ ushort simd_lane_id [[thread_index_in_simdgroup]]+ ) {// Determine thread position in simdgroup matrixshort tm = kFragSize * (simd_group_id / WN);short tn = kFragSize * (simd_group_id % WN);@@ -431,9 +451,11 @@ struct BlockMMA {Ctile.template store_tg<U, WM, WN>(D, ldd);}-- METAL_FUNC void- store_result_safe(device U* D, const int ldd, short2 dst_tile_dims) {+ METAL_FUNC void store_result_safe(+ device U* D,+ const int ldd,+ short2 dst_tile_dims+ ) {// Apply epilogueUZU_PRAGMA_UNROLLfor (short i = 0; i < decltype(Ctile)::kElemsPerTile; i++) {@@ -449,72 +471,69 @@ struct BlockMMA {Ctile.template store_safe<U, WM, WN>(D, ldd, dst_tile_dims);}-};template <typename T,int BM,- int BK,+ int BK,int lda_tgp,int reduction_dim,int tgp_size,int align_M = 1,int align_K = 4>struct BlockLoader {- UZU_MTL_CONST short n_reads = (BM * BK < tgp_size) ? 1 : (BM * BK) / tgp_size;-- const int src_ld;- const short thread_idx;- const short bi;- const short bj;-- threadgroup T* dst;- const device T* src;-- BlockLoader(- const device T* src_,- const int src_ld_,- threadgroup T* dst_,- ushort simd_group_id [[simdgroup_index_in_threadgroup]],- ushort simd_lane_id [[thread_index_in_simdgroup]])- : src_ld(src_ld_),- thread_idx(simd_group_id * 32 + simd_lane_id),- bi(n_reads * thread_idx / BK),- bj((n_reads * thread_idx) % BK),- dst(dst_ + bi * lda_tgp + bj),- src(src_ + bi * src_ld + bj) {}-- void load_unsafe() const {- if (BM * BK < tgp_size && bi >= BM) {- return;- }-- for (int i = 0; i < n_reads; i++) {- dst[i] = src[i];- }+ UZU_MTL_CONST short n_reads = (BM * BK < tgp_size) ? 1 : (BM * BK) / tgp_size;++ const int src_ld;+ const short thread_idx;+ const short bi;+ const short bj;++ threadgroup T* dst;+ const device T* src;++ BlockLoader(+ const device T* src_,+ const int src_ld_,+ threadgroup T* dst_,+ ushort simd_group_id [[simdgroup_index_in_threadgroup]],+ ushort simd_lane_id [[thread_index_in_simdgroup]]+ )+ : src_ld(src_ld_), thread_idx(simd_group_id * 32 + simd_lane_id),+ bi(n_reads * thread_idx / BK), bj((n_reads * thread_idx) % BK),+ dst(dst_ + bi * lda_tgp + bj), src(src_ + bi * src_ld + bj) {}++ void load_unsafe() const {+ if (BM * BK < tgp_size && bi >= BM) {+ return;}-- void load_safe(short2 src_tile_dim) const {- if (BM * BK < tgp_size && bi >= BM) {- return;- }-- if (bi >= src_tile_dim.y || bj >= src_tile_dim.x) {- for (int i = 0; i < n_reads; i++) {- dst[i] = T(0);- }- return;- }-- for (int i = 0; i < n_reads; i++) {- dst[i] = src[i];- }++ for (int i = 0; i < n_reads; i++) {+ dst[i] = src[i];+ }+ }++ void load_safe(short2 src_tile_dim) const {+ if (BM * BK < tgp_size && bi >= BM) {+ return;+ }++ if (bi >= src_tile_dim.y || bj >= src_tile_dim.x) {+ for (int i = 0; i < n_reads; i++) {+ dst[i] = T(0);+ }+ return;}-- void next() {- src += BK; // Advance by tile size++ for (int i = 0; i < n_reads; i++) {+ dst[i] = src[i];}+ }++ void next() {+ src += BK; // Advance by tile size+ }};-}\ No newline at end of file+} // namespace matmul_utils\ No newline at end of filediff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metalindex dc90432d4..a79e2a67a 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal@@ -5,11 +5,11 @@ using namespace metal;#include "mma.h"-// Function constant: true = MLX-style (pre-computed biases), false = AWQ-style (zero-points)+// Function constant: true = MLX-style (pre-computed biases), false = AWQ-style+// (zero-points)constant bool kUseMlxQuant [[function_constant(40)]];constant bool kUseZeroPoints = !kUseMlxQuant;-template <int bits, int wsize = 8>inline constexpr short get_pack_factor() {return (bits == 3 || bits == 5) ? 8 : (bits == 6 ? 4 : wsize / bits);@@ -50,10 +50,10 @@ inline U load_vector_safe(const device T* x, thread U* x_thread, int N) {U sum = 0;if (bits == 4) {const U scale_lut[4] = {- static_cast<U>(1.0f),- static_cast<U>(1.0f / 16.0f),- static_cast<U>(1.0f / 256.0f),- static_cast<U>(1.0f / 4096.0f)+ static_cast<U>(1.0f),+ static_cast<U>(1.0f / 16.0f),+ static_cast<U>(1.0f / 256.0f),+ static_cast<U>(1.0f / 4096.0f)};for (int i = 0; i < values_per_thread; ++i) {@@ -78,8 +78,13 @@ inline U load_vector_safe(const device T* x, thread U* x_thread, int N) {}template <typename U, int values_per_thread, int bits>-inline void-qouter(const thread uint8_t* w, U x, U scale, U bias, thread U* result) {+inline void qouter(+ const thread uint8_t* w,+ U x,+ U scale,+ U bias,+ thread U* result+) {static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");if (bits == 4) {@@ -88,8 +93,7 @@ qouter(const thread uint8_t* w, U x, U scale, U bias, thread U* result) {result[2 * i] += x * (s[0] * (w[i] & 0x0f) + bias);result[2 * i + 1] += x * (s[1] * (w[i] & 0xf0) + bias);}- }- else if (bits == 8) {+ } else if (bits == 8) {for (int i = 0; i < values_per_thread; i++) {result[i] += x * (scale * w[i] + bias);}@@ -102,7 +106,8 @@ inline U qdot(const thread U* x_thread,U scale,U bias,- U sum) {+ U sum+) {static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");U accum = 0;@@ -128,7 +133,8 @@ inline U qdot_zero_point(const device uint8_t* w,const thread U* x_thread,U scale,- U zero_point) {+ U zero_point+) {static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");U accum = 0;@@ -142,13 +148,21 @@ inline U qdot_zero_point(for (int i = 0; i < (values_per_thread / 4); i++) {uint16_t word = ws[i];accum += x_thread[4 * i] *- static_cast<U>(static_cast<int>(word & 0x000f) - static_cast<int>(zp0));+ static_cast<U>(+ static_cast<int>(word & 0x000f) - static_cast<int>(zp0)+ );accum += x_thread[4 * i + 1] *- static_cast<U>(static_cast<int>(word & 0x00f0) - static_cast<int>(zp1));+ static_cast<U>(+ static_cast<int>(word & 0x00f0) - static_cast<int>(zp1)+ );accum += x_thread[4 * i + 2] *- static_cast<U>(static_cast<int>(word & 0x0f00) - static_cast<int>(zp2));+ static_cast<U>(+ static_cast<int>(word & 0x0f00) - static_cast<int>(zp2)+ );accum += x_thread[4 * i + 3] *- static_cast<U>(static_cast<int>(word & 0xf000) - static_cast<int>(zp3));+ static_cast<U>(+ static_cast<int>(word & 0xf000) - static_cast<int>(zp3)+ );}} else if (bits == 8) {for (int i = 0; i < values_per_thread; i++) {@@ -165,7 +179,8 @@ inline U qdot_safe(U scale,U bias,U sum,- int N) {+ int N+) {static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");U accum = 0;@@ -185,9 +200,12 @@ inline U qdot_safe(if (rem > 0) {uint16_t wv = ws[full];int base = 4 * full;- if (rem > 0) accum += x_thread[base] * (wv & 0x000f);- if (rem > 1) accum += x_thread[base + 1] * (wv & 0x00f0);- if (rem > 2) accum += x_thread[base + 2] * (wv & 0x0f00);+ if (rem > 0)+ accum += x_thread[base] * (wv & 0x000f);+ if (rem > 1)+ accum += x_thread[base + 1] * (wv & 0x00f0);+ if (rem > 2)+ accum += x_thread[base + 2] * (wv & 0x0f00);}} else if (bits == 8) {for (int i = 0; i < N; i++) {@@ -199,7 +217,12 @@ inline U qdot_safe(}template <typename U, int N, int bits>-inline void dequantize(const device uint8_t* w, U scale, U bias, threadgroup U* w_local) {+inline void dequantize(+ const device uint8_t* w,+ U scale,+ U bias,+ threadgroup U* w_local+) {static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");if (bits == 4) {@@ -216,30 +239,35 @@ inline void dequantize(const device uint8_t* w, U scale, U bias, threadgroup U*}template <>-inline void dequantize<bfloat, 8, 4>(const device uint8_t* w, bfloat scale, bfloat bias, threadgroup bfloat* w_local) {- const device uint32_t* w_ptr = (const device uint32_t*)w;- uint32_t packed = *w_ptr;-- bfloat4 v0, v1;-- // Low 4 nibbles- v0.x = static_cast<bfloat>(packed & 0xF);- v0.y = static_cast<bfloat>((packed >> 4) & 0xF);- v0.z = static_cast<bfloat>((packed >> 8) & 0xF);- v0.w = static_cast<bfloat>((packed >> 12) & 0xF);-- // High 4 nibbles- v1.x = static_cast<bfloat>((packed >> 16) & 0xF);- v1.y = static_cast<bfloat>((packed >> 20) & 0xF);- v1.z = static_cast<bfloat>((packed >> 24) & 0xF);- v1.w = static_cast<bfloat>((packed >> 28) & 0xF);-- v0 = v0 * scale + bias;- v1 = v1 * scale + bias;-- threadgroup bfloat4* out_ptr = (threadgroup bfloat4*)w_local;- out_ptr[0] = v0;- out_ptr[1] = v1;+inline void dequantize<bfloat, 8, 4>(+ const device uint8_t* w,+ bfloat scale,+ bfloat bias,+ threadgroup bfloat* w_local+) {+ const device uint32_t* w_ptr = (const device uint32_t*)w;+ uint32_t packed = *w_ptr;++ bfloat4 v0, v1;++ // Low 4 nibbles+ v0.x = static_cast<bfloat>(packed & 0xF);+ v0.y = static_cast<bfloat>((packed >> 4) & 0xF);+ v0.z = static_cast<bfloat>((packed >> 8) & 0xF);+ v0.w = static_cast<bfloat>((packed >> 12) & 0xF);++ // High 4 nibbles+ v1.x = static_cast<bfloat>((packed >> 16) & 0xF);+ v1.y = static_cast<bfloat>((packed >> 20) & 0xF);+ v1.z = static_cast<bfloat>((packed >> 24) & 0xF);+ v1.w = static_cast<bfloat>((packed >> 28) & 0xF);++ v0 = v0 * scale + bias;+ v1 = v1 * scale + bias;++ threadgroup bfloat4* out_ptr = (threadgroup bfloat4*)w_local;+ out_ptr[0] = v0;+ out_ptr[1] = v1;}template <@@ -252,14 +280,21 @@ template <short group_size,short bits>struct QuantizedBlockLoaderMlx {- static_assert(BCOLS <= group_size, "Group size should be larger than columns");- static_assert(group_size % BCOLS == 0, "Group size should be divisible by columns");+ static_assert(+ BCOLS <= group_size,+ "Group size should be larger than columns"+ );+ static_assert(+ group_size % BCOLS == 0,+ "Group size should be divisible by columns"+ );static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");UZU_MTL_CONST short pack_factor = get_pack_factor<bits, 8>();UZU_MTL_CONST short bytes_per_pack = get_bytes_per_pack<bits>();UZU_MTL_CONST short BCOLS_PACKED = BCOLS / pack_factor;- UZU_MTL_CONST short n_reads = (BCOLS_PACKED * BROWS < tgp_size) ? 1 : (BCOLS_PACKED * BROWS) / tgp_size;+ UZU_MTL_CONST short n_reads =+ (BCOLS_PACKED * BROWS < tgp_size) ? 1 : (BCOLS_PACKED * BROWS) / tgp_size;UZU_MTL_CONST short group_steps = group_size / BCOLS;const int src_ld;@@ -283,18 +318,20 @@ struct QuantizedBlockLoaderMlx {const int src_ld_,threadgroup T* dst_,ushort simd_group_id [[simdgroup_index_in_threadgroup]],- ushort simd_lane_id [[thread_index_in_simdgroup]])+ ushort simd_lane_id [[thread_index_in_simdgroup]]+ ): src_ld(src_ld_),tile_stride(reduction_dim ? BCOLS_PACKED * bytes_per_pack- : BROWS * src_ld * bytes_per_pack / pack_factor),- group_step_cnt(0),- group_stride(BROWS * src_ld / group_size),+ : BROWS * src_ld * bytes_per_pack / pack_factor+ ),+ group_step_cnt(0), group_stride(BROWS * src_ld / group_size),thread_idx(simd_group_id * 32 + simd_lane_id),bi(n_reads * thread_idx / BCOLS_PACKED),bj((n_reads * thread_idx) % BCOLS_PACKED),dst(dst_ + bi * dst_ld + bj * pack_factor),- src(src_ + bi * src_ld * bytes_per_pack / pack_factor + bj * bytes_per_pack),+ src(src_ + bi * src_ld * bytes_per_pack / pack_factor ++ bj * bytes_per_pack),scales(scales_ + bi * src_ld / group_size),biases(biases_ + bi * src_ld / group_size) {}@@ -307,7 +344,11 @@ struct QuantizedBlockLoaderMlx {T bias = *biases;for (int i = 0; i < n_reads; i++) {dequantize<T, pack_factor, bits>(- src + i * bytes_per_pack, scale, bias, dst + i * pack_factor);+ src + i * bytes_per_pack,+ scale,+ bias,+ dst + i * pack_factor+ );}}@@ -337,7 +378,8 @@ struct QuantizedBlockLoaderMlx {(device uint8_t*)(src + i * bytes_per_pack),scale,bias,- dst + i * pack_factor);+ dst + i * pack_factor+ );}}@@ -373,14 +415,21 @@ template <short bits,bool per_output_layout = false>struct QuantizedBlockLoaderZp {- static_assert(BCOLS <= group_size, "Group size should be larger than columns");- static_assert(group_size % BCOLS == 0, "Group size should be divisible by columns");+ static_assert(+ BCOLS <= group_size,+ "Group size should be larger than columns"+ );+ static_assert(+ group_size % BCOLS == 0,+ "Group size should be divisible by columns"+ );static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");UZU_MTL_CONST short pack_factor = get_pack_factor<bits, 8>();UZU_MTL_CONST short bytes_per_pack = get_bytes_per_pack<bits>();UZU_MTL_CONST short BCOLS_PACKED = BCOLS / pack_factor;- UZU_MTL_CONST short n_reads = (BCOLS_PACKED * BROWS < tgp_size) ? 1 : (BCOLS_PACKED * BROWS) / tgp_size;+ UZU_MTL_CONST short n_reads =+ (BCOLS_PACKED * BROWS < tgp_size) ? 1 : (BCOLS_PACKED * BROWS) / tgp_size;UZU_MTL_CONST short group_steps = group_size / BCOLS;const int src_ld;@@ -414,30 +463,38 @@ struct QuantizedBlockLoaderZp {ushort simd_lane_id [[thread_index_in_simdgroup]],const int out_group_base_ = 0,const int out_groups_total_ = 0,- const int zp_stride_total_ = 0)- : src_ld(src_ld_),- groups_per_row(groups_per_row_),+ const int zp_stride_total_ = 0+ )+ : src_ld(src_ld_), groups_per_row(groups_per_row_),tile_stride(reduction_dim ? BCOLS_PACKED * bytes_per_pack- : BROWS * src_ld * bytes_per_pack / pack_factor),- group_step_cnt(0),- k_base(0),- group_stride(BROWS * groups_per_row_),+ : BROWS * src_ld * bytes_per_pack / pack_factor+ ),+ group_step_cnt(0), k_base(0), group_stride(BROWS * groups_per_row_),thread_idx(simd_group_id * 32 + simd_lane_id),bi(n_reads * thread_idx / BCOLS_PACKED),bj((n_reads * thread_idx) % BCOLS_PACKED),dst(dst_ + bi * dst_ld + bj * pack_factor),- src(src_ + bi * src_ld * bytes_per_pack / pack_factor + bj * bytes_per_pack),+ src(src_ + bi * src_ld * bytes_per_pack / pack_factor ++ bj * bytes_per_pack),scales(reduction_dim == 1 ? (scales_ + bi * groups_per_row_) : scales_),- scales_row_start(reduction_dim == 1 ? (scales_ + bi * groups_per_row_) : scales_),- zps_row_start(reduction_dim == 1- ? (zero_points_row_start_ + bi * (bits == 4 ? ((groups_per_row_ + 1) / 2) : groups_per_row_))- : zero_points_row_start_),+ scales_row_start(+ reduction_dim == 1 ? (scales_ + bi * groups_per_row_) : scales_+ ),+ zps_row_start(+ reduction_dim == 1 ? (zero_points_row_start_ ++ bi * (bits == 4 ? ((groups_per_row_ + 1) / 2)+ : groups_per_row_))+ : zero_points_row_start_+ ),out_group_base(per_output_layout ? out_group_base_ : 0),out_groups_total(per_output_layout ? out_groups_total_ : 0),zp_stride_total(per_output_layout ? zp_stride_total_ : 0) {}- inline void current_scale_bias(thread T& out_scale, thread T& out_bias) const {+ inline void current_scale_bias(+ thread T& out_scale,+ thread T& out_bias+ ) const {uint zp_n;T scale_val;if (per_output_layout) {@@ -445,7 +502,8 @@ struct QuantizedBlockLoaderZp {const int scale_index = row_idx * groups_per_row + out_group_base;scale_val = scales_row_start[scale_index];if (bits == 4) {- const int byte_index = row_idx * zp_stride_total + (out_group_base >> 1);+ const int byte_index =+ row_idx * zp_stride_total + (out_group_base >> 1);uint8_t zp_b = zps_row_start[byte_index];zp_n = (out_group_base & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F);} else {@@ -457,11 +515,11 @@ struct QuantizedBlockLoaderZp {: (int)(scales - scales_row_start);scale_val = reduction_dim == 0 ? scales_row_start[g] : *scales;if (bits == 4) {- const device uint8_t* zp_ptr = zps_row_start + (g >> 1);- uint8_t zp_b = *zp_ptr;- zp_n = (g & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F);+ const device uint8_t* zp_ptr = zps_row_start + (g >> 1);+ uint8_t zp_b = *zp_ptr;+ zp_n = (g & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F);} else {- zp_n = zps_row_start[g];+ zp_n = zps_row_start[g];}}out_scale = scale_val;@@ -478,7 +536,11 @@ struct QuantizedBlockLoaderZp {current_scale_bias(scale, bias);for (int i = 0; i < n_reads; i++) {dequantize<T, pack_factor, bits>(- src + i * bytes_per_pack, scale, bias, dst + i * pack_factor);+ src + i * bytes_per_pack,+ scale,+ bias,+ dst + i * pack_factor+ );}}@@ -506,8 +568,12 @@ struct QuantizedBlockLoaderZp {int pack_idx = bj + i; // global pack index across the BK packsif (pack_idx < valid_packs) {dequantize<T, pack_factor, bits>(- src + i * bytes_per_pack, scale, bias, dst + i * pack_factor);-+ src + i * bytes_per_pack,+ scale,+ bias,+ dst + i * pack_factor+ );+// Mask the last pack if neededif (pack_idx == valid_packs - 1) {int rem = valid_cols - pack_idx * pack_factor;@@ -538,7 +604,11 @@ struct QuantizedBlockLoaderZp {current_scale_bias(scale, bias);for (int i = 0; i < n_reads; i++) {dequantize<T, pack_factor, bits>(- src + i * bytes_per_pack, scale, bias, dst + i * pack_factor);+ src + i * bytes_per_pack,+ scale,+ bias,+ dst + i * pack_factor+ );}}@@ -580,7 +650,8 @@ inline void qmm_core(const constant int& N,threadgroup T* Xs,threadgroup T* Ws,- uint lid [[thread_index_in_threadgroup]]) {+ uint lid [[thread_index_in_threadgroup]]+) {if (num_els < BM) {if ((K % BK) != 0) {const int k_blocks = K / BK;@@ -673,7 +744,8 @@ void qmm_impl(uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]) {+ uint simd_lid [[thread_index_in_simdgroup]]+) {static_assert(BK >= 32, "BK should be larger than SIMD_SIZE");static_assert(BK % 32 == 0, "BK should be divisible by SIMD_SIZE");@@ -687,8 +759,10 @@ void qmm_impl(constexpr int BK_padded = (BK + 16 / sizeof(T));constexpr int BN_padded = (BN + 16 / sizeof(T));- using mma_t = matmul_utils::BlockMMA<T, T, BM, BN, BK, WM, WN, false, false, BK_padded, BN_padded>;- using loader_x_t = matmul_utils::BlockLoader<T, BM, BK, BK_padded, 1, WM * WN * 32, 1, 4>;+ using mma_t = matmul_utils::+ BlockMMA<T, T, BM, BN, BK, WM, WN, false, false, BK_padded, BN_padded>;+ using loader_x_t =+ matmul_utils::BlockLoader<T, BM, BK, BK_padded, 1, WM * WN * 32, 1, 4>;auto wl = (const device uint8_t*)w;@@ -707,19 +781,47 @@ void qmm_impl(mma_t mma_op(simd_gid, simd_lid);if (kUseMlxQuant) {- using loader_w_t = QuantizedBlockLoaderMlx<T, BK, BN, BN_padded, 0, WM * WN * 32, group_size, bits>;+ using loader_w_t = QuantizedBlockLoaderMlx<+ T,+ BK,+ BN,+ BN_padded,+ 0,+ WM * WN * 32,+ group_size,+ bits>;const device T* scales_mlx = scales_base + (y_col / group_size);const device T* biases_mlx = biases_base + (y_col / group_size);loader_w_t loader_w(wl, scales_mlx, biases_mlx, N, Ws, simd_gid, simd_lid);qmm_core<loader_w_t, loader_x_t, mma_t, T, aligned_K, BM, BK, BN>(- loader_x, loader_w, mma_op, num_els, num_outs, K, y, N, Xs, Ws, lid);+ loader_x,+ loader_w,+ mma_op,+ num_els,+ num_outs,+ K,+ y,+ N,+ Xs,+ Ws,+ lid+ );} else {const int out_groups_total = (N + group_size - 1) / group_size;const int groups_per_row = out_groups_total;const int out_group = y_col / group_size;const int zp_stride_out =(bits == 4) ? ((out_groups_total + 1) / 2) : out_groups_total;- using loader_w_t = QuantizedBlockLoaderZp<T, BK, BN, BN_padded, 0, WM * WN * 32, group_size, bits, true>;+ using loader_w_t = QuantizedBlockLoaderZp<+ T,+ BK,+ BN,+ BN_padded,+ 0,+ WM * WN * 32,+ group_size,+ bits,+ true>;loader_w_t loader_w(wl,scales_base,@@ -731,9 +833,21 @@ void qmm_impl(simd_lid,out_group,out_groups_total,- zp_stride_out);+ zp_stride_out+ );qmm_core<loader_w_t, loader_x_t, mma_t, T, aligned_K, BM, BK, BN>(- loader_x, loader_w, mma_op, num_els, num_outs, K, y, N, Xs, Ws, lid);+ loader_x,+ loader_w,+ mma_op,+ num_els,+ num_outs,+ K,+ y,+ N,+ Xs,+ Ws,+ lid+ );}}@@ -757,7 +871,8 @@ inline void qmm_transposed_core(const constant int& N,threadgroup T* Xs,threadgroup T* Ws,- uint lid [[thread_index_in_threadgroup]]) {+ uint lid [[thread_index_in_threadgroup]]+) {if (num_els < BM) {if (!aligned_N && num_outs < BN) {for (int k = 0; k < K; k += BK) {@@ -837,7 +952,8 @@ void qmm_transposed_impl(uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]) {+ uint simd_lid [[thread_index_in_simdgroup]]+) {static_assert(BK >= 32, "BK should be larger than SIMD_SIZE");static_assert(BK % 32 == 0, "BK should be divisible by SIMD_SIZE");@@ -847,8 +963,10 @@ void qmm_transposed_impl(constexpr int bytes_per_pack = get_bytes_per_pack<bits>();constexpr int BK_padded = (BK + 16 / sizeof(T));- using mma_t = matmul_utils::BlockMMA<T, T, BM, BN, BK, WM, WN, false, true, BK_padded, BK_padded>;- using loader_x_t = matmul_utils::BlockLoader<T, BM, BK, BK_padded, 1, WM * WN * 32>;+ using mma_t = matmul_utils::+ BlockMMA<T, T, BM, BN, BK, WM, WN, false, true, BK_padded, BK_padded>;+ using loader_x_t =+ matmul_utils::BlockLoader<T, BM, BK, BK_padded, 1, WM * WN * 32>;const int K_w = K * bytes_per_pack / pack_factor;const int K_g = (K + group_size - 1) / group_size;@@ -869,16 +987,80 @@ void qmm_transposed_impl(mma_t mma_op(simd_gid, simd_lid);if (kUseMlxQuant) {- using loader_w_t = QuantizedBlockLoaderMlx< T, BN, BK, BK_padded, 1, WM * WN * 32, group_size, bits>;+ using loader_w_t = QuantizedBlockLoaderMlx<+ T,+ BN,+ BK,+ BK_padded,+ 1,+ WM * WN * 32,+ group_size,+ bits>;loader_w_t loader_w(w_block, scales, biases, K, Ws, simd_gid, simd_lid);- qmm_transposed_core<loader_w_t, loader_x_t, mma_t, T, aligned_N, BM, BK, BN>(- loader_x, loader_w, mma_op, num_els, num_outs, K, y_block, N, Xs, Ws, lid);+ qmm_transposed_core<+ loader_w_t,+ loader_x_t,+ mma_t,+ T,+ aligned_N,+ BM,+ BK,+ BN>(+ loader_x,+ loader_w,+ mma_op,+ num_els,+ num_outs,+ K,+ y_block,+ N,+ Xs,+ Ws,+ lid+ );} else {- using loader_w_t = QuantizedBlockLoaderZp<T, BN, BK, BK_padded, 1, WM * WN * 32, group_size, bits>;- const device uint8_t* zero_points_row = zero_points + y_col * (bits == 4 ? ((K_g + 1) / 2) : K_g);- loader_w_t loader_w(w_block, scales, zero_points_row, K, K_g, Ws, simd_gid, simd_lid);- qmm_transposed_core<loader_w_t, loader_x_t, mma_t, T, aligned_N, BM, BK, BN>(- loader_x, loader_w, mma_op, num_els, num_outs, K, y_block, N, Xs, Ws, lid);+ using loader_w_t = QuantizedBlockLoaderZp<+ T,+ BN,+ BK,+ BK_padded,+ 1,+ WM * WN * 32,+ group_size,+ bits>;+ const device uint8_t* zero_points_row =+ zero_points + y_col * (bits == 4 ? ((K_g + 1) / 2) : K_g);+ loader_w_t loader_w(+ w_block,+ scales,+ zero_points_row,+ K,+ K_g,+ Ws,+ simd_gid,+ simd_lid+ );+ qmm_transposed_core<+ loader_w_t,+ loader_x_t,+ mma_t,+ T,+ aligned_N,+ BM,+ BK,+ BN>(+ loader_x,+ loader_w,+ mma_op,+ num_els,+ num_outs,+ K,+ y_block,+ N,+ Xs,+ Ws,+ lid+ );}}@@ -894,7 +1076,8 @@ void qmv_impl_dispatch(const constant int& N, // out_vec_sizeuint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]) {+ uint simd_lid [[thread_index_in_simdgroup]]+) {constexpr int num_simdgroups = 2;constexpr int results_per_simdgroup = 4;constexpr int packs_per_thread = 1;@@ -911,10 +1094,12 @@ void qmv_impl_dispatch(thread U result[results_per_simdgroup] = {0};const int in_vec_size_w = K * bytes_per_pack / pack_factor;- const int in_vec_size_g = (K + group_size - 1) / group_size; // ceil(K / group_size)- const device T* scales_base = scales;+ const int in_vec_size_g =+ (K + group_size - 1) / group_size; // ceil(K / group_size)+ const device T* scales_base = scales;const device uint8_t* zero_points_base = zero_points;- const int out_row = tid.y * (num_simdgroups * results_per_simdgroup) + simd_gid * results_per_simdgroup;+ const int out_row = tid.y * (num_simdgroups * results_per_simdgroup) ++ simd_gid * results_per_simdgroup;const int used_out_row = min(N - results_per_simdgroup, out_row);if (out_row >= N) {@@ -922,9 +1107,10 @@ void qmv_impl_dispatch(}if (N < (num_simdgroups * results_per_simdgroup)) {- ws += out_row * in_vec_size_w + simd_lid * packs_per_thread * bytes_per_pack;+ ws +=+ out_row * in_vec_size_w + simd_lid * packs_per_thread * bytes_per_pack;scales += out_row * in_vec_size_g + simd_lid / scale_step_per_thread;-+const int zp_stride = bits == 4 ? ((in_vec_size_g + 1) / 2) : in_vec_size_g;const device uint8_t* zps_row_base = nullptr;const device T* biases_row_base = nullptr;@@ -942,7 +1128,8 @@ void qmv_impl_dispatch(U sum = load_vector<T, U, values_per_thread, bits>(x, x_thread);for (int row = 0; out_row + row < N; row++) {- if (row >= results_per_simdgroup) break;+ if (row >= results_per_simdgroup)+ break;auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);const int row_idx = out_row + row;const device T* sr = scales_base + row_idx * in_vec_size_g;@@ -952,7 +1139,8 @@ void qmv_impl_dispatch(if (UseMlx) {const device T* bl = biases_row_base + row * in_vec_size_g;U b = static_cast<U>(bl[g]);- result[row] += qdot<U, values_per_thread, bits>(wl, x_thread, s, b, sum);+ result[row] +=+ qdot<U, values_per_thread, bits>(wl, x_thread, s, b, sum);} else {const device uint8_t* zl = zps_row_base + row * zp_stride;U zp;@@ -974,50 +1162,57 @@ void qmv_impl_dispatch(const int remaining = clamp(static_cast<int>(K - k - simd_lid * values_per_thread),0,- values_per_thread);+ values_per_thread+ );if (remaining > 0) {- U sum = load_vector_safe<T, U, values_per_thread, bits>(- x, x_thread, remaining);-- for (int row = 0; out_row + row < N; row++) {- if (row >= results_per_simdgroup) break;- auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);- const int row_idx = out_row + row;- const device T* sr = scales_base + row_idx * in_vec_size_g;-- int g = (k + simd_lid * values_per_thread) / group_size;- U s = static_cast<U>(sr[g]);- if (UseMlx) {- const device T* bl = biases_row_base + row * in_vec_size_g;- U b = static_cast<U>(bl[g]);- result[row] += qdot<U, values_per_thread, bits>(wl, x_thread, s, b, sum);- } else {- const device uint8_t* zl = zps_row_base + row * zp_stride;- U zp;- if (bits == 4) {- uint8_t zp_b = zl[g >> 1];- zp = static_cast<U>((g & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F));- } else {- zp = static_cast<U>(zl[g]);- }- result[row] += qdot_zero_point<U, values_per_thread, bits>(- wl, x_thread, s, zp);- }+ U sum = load_vector_safe<T, U, values_per_thread, bits>(+ x,+ x_thread,+ remaining+ );++ for (int row = 0; out_row + row < N; row++) {+ if (row >= results_per_simdgroup)+ break;+ auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);+ const int row_idx = out_row + row;+ const device T* sr = scales_base + row_idx * in_vec_size_g;++ int g = (k + simd_lid * values_per_thread) / group_size;+ U s = static_cast<U>(sr[g]);+ if (UseMlx) {+ const device T* bl = biases_row_base + row * in_vec_size_g;+ U b = static_cast<U>(bl[g]);+ result[row] +=+ qdot<U, values_per_thread, bits>(wl, x_thread, s, b, sum);+ } else {+ const device uint8_t* zl = zps_row_base + row * zp_stride;+ U zp;+ if (bits == 4) {+ uint8_t zp_b = zl[g >> 1];+ zp = static_cast<U>((g & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F));+ } else {+ zp = static_cast<U>(zl[g]);+ }+ result[row] +=+ qdot_zero_point<U, values_per_thread, bits>(wl, x_thread, s, zp);}+ }}for (int row = 0; out_row + row < N; row++) {- if (row >= results_per_simdgroup) break;+ if (row >= results_per_simdgroup)+ break;result[row] = simd_sum(result[row]);if (simd_lid == 0) {y[row] = static_cast<T>(result[row]);}}- }- else {- ws += used_out_row * in_vec_size_w + simd_lid * packs_per_thread * bytes_per_pack;+ } else {+ ws += used_out_row * in_vec_size_w ++ simd_lid * packs_per_thread * bytes_per_pack;scales += used_out_row * in_vec_size_g + simd_lid / scale_step_per_thread;-+const int zp_stride = bits == 4 ? ((in_vec_size_g + 1) / 2) : in_vec_size_g;const device uint8_t* zps_row_base = nullptr;const device T* biases_row_base = nullptr;@@ -1044,7 +1239,8 @@ void qmv_impl_dispatch(if (UseMlx) {const device T* bl = biases_row_base + row * in_vec_size_g;U b = static_cast<U>(bl[g]);- result[row] += qdot<U, values_per_thread, bits>(wl, x_thread, s, b, sum);+ result[row] +=+ qdot<U, values_per_thread, bits>(wl, x_thread, s, b, sum);} else {const device uint8_t* zl = zps_row_base + row * zp_stride;U zp;@@ -1066,37 +1262,47 @@ void qmv_impl_dispatch(const int remaining = clamp(static_cast<int>(K - k - simd_lid * values_per_thread),0,- values_per_thread);+ values_per_thread+ );if (remaining > 0) {- U sum = load_vector_safe<T, U, values_per_thread, bits>(- x, x_thread, remaining);-- for (int row = 0; row < results_per_simdgroup; row++) {- auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);- const int row_idx = used_out_row + row;- const device T* sr = scales_base + row_idx * in_vec_size_g;-- int g = (k + simd_lid * values_per_thread) / group_size;- U s = static_cast<U>(sr[g]);- if (UseMlx) {- const device T* bl = biases_row_base + row * in_vec_size_g;- U b = static_cast<U>(bl[g]);- result[row] += qdot_safe<U, values_per_thread, bits>(- wl, x_thread, s, b, sum, remaining);- } else {- const device uint8_t* zl = zps_row_base + row * zp_stride;- U zp;- if (bits == 4) {- uint8_t zp_b = zl[g >> 1];- zp = static_cast<U>((g & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F));- } else {- zp = static_cast<U>(zl[g]);- }- result[row] += qdot_zero_point<U, values_per_thread, bits>(- wl, x_thread, s, zp);- }+ U sum = load_vector_safe<T, U, values_per_thread, bits>(+ x,+ x_thread,+ remaining+ );++ for (int row = 0; row < results_per_simdgroup; row++) {+ auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);+ const int row_idx = used_out_row + row;+ const device T* sr = scales_base + row_idx * in_vec_size_g;++ int g = (k + simd_lid * values_per_thread) / group_size;+ U s = static_cast<U>(sr[g]);+ if (UseMlx) {+ const device T* bl = biases_row_base + row * in_vec_size_g;+ U b = static_cast<U>(bl[g]);+ result[row] += qdot_safe<U, values_per_thread, bits>(+ wl,+ x_thread,+ s,+ b,+ sum,+ remaining+ );+ } else {+ const device uint8_t* zl = zps_row_base + row * zp_stride;+ U zp;+ if (bits == 4) {+ uint8_t zp_b = zl[g >> 1];+ zp = static_cast<U>((g & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F));+ } else {+ zp = static_cast<U>(zl[g]);+ }+ result[row] +=+ qdot_zero_point<U, values_per_thread, bits>(wl, x_thread, s, zp);}+ }}for (int row = 0; row < results_per_simdgroup; row++) {@@ -1120,13 +1326,36 @@ void qmv_impl(const constant int& N,uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]) {+ uint simd_lid [[thread_index_in_simdgroup]]+) {if (kUseMlxQuant) {qmv_impl_dispatch<T, group_size, bits, true>(- w, scales, zero_points, biases, x, y, K, N, tid, simd_gid, simd_lid);+ w,+ scales,+ zero_points,+ biases,+ x,+ y,+ K,+ N,+ tid,+ simd_gid,+ simd_lid+ );} else {qmv_impl_dispatch<T, group_size, bits, false>(- w, scales, zero_points, biases, x, y, K, N, tid, simd_gid, simd_lid);+ w,+ scales,+ zero_points,+ biases,+ x,+ y,+ K,+ N,+ tid,+ simd_gid,+ simd_lid+ );}}@@ -1142,7 +1371,8 @@ void qmv_fast_impl(const constant int& out_vec_size,uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]) {+ uint simd_lid [[thread_index_in_simdgroup]]+) {constexpr int packs_per_thread = bits == 2 ? 1 : 2;constexpr int num_simdgroups = 2;constexpr int results_per_simdgroup = 4;@@ -1159,10 +1389,10 @@ void qmv_fast_impl(const int in_vec_size_w = in_vec_size * bytes_per_pack / pack_factor;const int in_vec_size_g = in_vec_size / group_size;const int out_row = tid.y * (num_simdgroups * results_per_simdgroup) +- simd_gid * results_per_simdgroup;+ simd_gid * results_per_simdgroup;ws += out_row * in_vec_size_w + simd_lid * packs_per_thread * bytes_per_pack;scales += out_row * in_vec_size_g + simd_lid / scale_step_per_thread;-+int zp_stride = 0;const device uint8_t* zps = nullptr;bool high_nibble = false;@@ -1171,15 +1401,15 @@ void qmv_fast_impl(biases += out_row * in_vec_size_g + simd_lid / scale_step_per_thread;} else {if (bits == 4) {- zp_stride = (in_vec_size_g + 1) / 2;- zps = zero_points + out_row * zp_stride;- int g_offset = simd_lid / scale_step_per_thread;- zps += g_offset / 2;- high_nibble = (g_offset & 1);+ zp_stride = (in_vec_size_g + 1) / 2;+ zps = zero_points + out_row * zp_stride;+ int g_offset = simd_lid / scale_step_per_thread;+ zps += g_offset / 2;+ high_nibble = (g_offset & 1);} else {- zp_stride = in_vec_size_g;- zps = zero_points + out_row * zp_stride;- zps += simd_lid / scale_step_per_thread;+ zp_stride = in_vec_size_g;+ zps = zero_points + out_row * zp_stride;+ zps += simd_lid / scale_step_per_thread;}}@@ -1205,29 +1435,45 @@ void qmv_fast_impl(U b1 = static_cast<U>(biases[in_vec_size_g]);U b2 = static_cast<U>(biases[2 * in_vec_size_g]);U b3 = static_cast<U>(biases[3 * in_vec_size_g]);- result[0] += qdot<U, values_per_thread, bits>(wl0, x_thread, s0, b0, sum);- result[1] += qdot<U, values_per_thread, bits>(wl1, x_thread, s1, b1, sum);- result[2] += qdot<U, values_per_thread, bits>(wl2, x_thread, s2, b2, sum);- result[3] += qdot<U, values_per_thread, bits>(wl3, x_thread, s3, b3, sum);+ result[0] +=+ qdot<U, values_per_thread, bits>(wl0, x_thread, s0, b0, sum);+ result[1] +=+ qdot<U, values_per_thread, bits>(wl1, x_thread, s1, b1, sum);+ result[2] +=+ qdot<U, values_per_thread, bits>(wl2, x_thread, s2, b2, sum);+ result[3] +=+ qdot<U, values_per_thread, bits>(wl3, x_thread, s3, b3, sum);} else {uint8_t zp_byte0 = zps[0];uint8_t zp_byte1 = zps[zp_stride];uint8_t zp_byte2 = zps[2 * zp_stride];uint8_t zp_byte3 = zps[3 * zp_stride];- U zp0 = static_cast<U>((bits == 4 && high_nibble) ? (zp_byte0 >> 4) : (zp_byte0 & 0x0F));- U zp1 = static_cast<U>((bits == 4 && high_nibble) ? (zp_byte1 >> 4) : (zp_byte1 & 0x0F));- U zp2 = static_cast<U>((bits == 4 && high_nibble) ? (zp_byte2 >> 4) : (zp_byte2 & 0x0F));- U zp3 = static_cast<U>((bits == 4 && high_nibble) ? (zp_byte3 >> 4) : (zp_byte3 & 0x0F));+ U zp0 = static_cast<U>(+ (bits == 4 && high_nibble) ? (zp_byte0 >> 4) : (zp_byte0 & 0x0F)+ );+ U zp1 = static_cast<U>(+ (bits == 4 && high_nibble) ? (zp_byte1 >> 4) : (zp_byte1 & 0x0F)+ );+ U zp2 = static_cast<U>(+ (bits == 4 && high_nibble) ? (zp_byte2 >> 4) : (zp_byte2 & 0x0F)+ );+ U zp3 = static_cast<U>(+ (bits == 4 && high_nibble) ? (zp_byte3 >> 4) : (zp_byte3 & 0x0F)+ );if (bits == 8) {zp0 = static_cast<U>(zp_byte0);zp1 = static_cast<U>(zp_byte1);zp2 = static_cast<U>(zp_byte2);zp3 = static_cast<U>(zp_byte3);}- result[0] += qdot_zero_point<U, values_per_thread, bits>(wl0, x_thread, s0, zp0);- result[1] += qdot_zero_point<U, values_per_thread, bits>(wl1, x_thread, s1, zp1);- result[2] += qdot_zero_point<U, values_per_thread, bits>(wl2, x_thread, s2, zp2);- result[3] += qdot_zero_point<U, values_per_thread, bits>(wl3, x_thread, s3, zp3);+ result[0] +=+ qdot_zero_point<U, values_per_thread, bits>(wl0, x_thread, s0, zp0);+ result[1] +=+ qdot_zero_point<U, values_per_thread, bits>(wl1, x_thread, s1, zp1);+ result[2] +=+ qdot_zero_point<U, values_per_thread, bits>(wl2, x_thread, s2, zp2);+ result[3] +=+ qdot_zero_point<U, values_per_thread, bits>(wl3, x_thread, s3, zp3);}}@@ -1237,9 +1483,9 @@ void qmv_fast_impl(biases += block_size / group_size;} else {if (bits == 4) {- zps += (block_size / group_size) / 2;+ zps += (block_size / group_size) / 2;} else {- zps += block_size / group_size;+ zps += block_size / group_size;}}x += block_size;@@ -1265,7 +1511,8 @@ void qvm_impl_core(const int out_vec_size,uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]) {+ uint simd_lid [[thread_index_in_simdgroup]]+) {constexpr int num_simdgroups = 2;constexpr int pack_factor = get_pack_factor<bits, 32>();constexpr int bytes_per_pack = get_bytes_per_pack<bits, 32>();@@ -1283,18 +1530,20 @@ void qvm_impl_core(const int out_vec_size_w = out_vec_size * bytes_per_pack / pack_factor;const int out_vec_size_g = out_vec_size / group_size;const int out_col = pack_factor * tn * (tid.y * num_simdgroups + simd_gid);-+if (out_col >= out_vec_size) {return;}const int out_group = out_col / group_size;- const int zp_row_stride = use_zero_points ? ((bits == 4) ? ((out_vec_size_g + 1) / 2) : out_vec_size_g) : 0;-- const device uint8_t* ws_ptr =- (const device uint8_t*)ws +- out_col * bytes_per_pack / pack_factor +- simd_lid * out_vec_size_w;+ const int zp_row_stride =+ use_zero_points+ ? ((bits == 4) ? ((out_vec_size_g + 1) / 2) : out_vec_size_g)+ : 0;++ const device uint8_t* ws_ptr = (const device uint8_t*)ws ++ out_col * bytes_per_pack / pack_factor ++ simd_lid * out_vec_size_w;const device T* scales_base = scales;const device T* biases_base = biases;x += tid.x * in_vec_size + simd_lid;@@ -1303,11 +1552,14 @@ void qvm_impl_core(for (int k_base = 0; k_base < in_vec_size; k_base += block_size) {const int k_index = k_base + simd_lid;const bool active = k_index < in_vec_size;-+U x_local = active ? static_cast<U>(*x) : U(0);- U scale = active ? static_cast<U>(scales_base[k_index * out_vec_size_g + out_group]) : U(0);+ U scale =+ active+ ? static_cast<U>(scales_base[k_index * out_vec_size_g + out_group])+ : U(0);U bias;-+if (use_zero_points && active) {U zp;if (bits == 4) {@@ -1315,7 +1567,9 @@ void qvm_impl_core(const device uint8_t* zp_ptr = row_ptr + (out_group >> 1);uint8_t zp_byte = *zp_ptr;bool high_nibble = (out_group & 1) != 0;- zp = static_cast<U>(high_nibble ? ((zp_byte >> 4) & 0x0F) : (zp_byte & 0x0F));+ zp = static_cast<U>(+ high_nibble ? ((zp_byte >> 4) & 0x0F) : (zp_byte & 0x0F)+ );} else {const device uint8_t* row_ptr = zero_points + k_index * zp_row_stride;const device uint8_t* zp_ptr = row_ptr + out_group;@@ -1324,19 +1578,24 @@ void qvm_impl_core(bias = -scale * zp;} else if (active) {bias = static_cast<U>(biases_base[k_index * out_vec_size_g + out_group]);- } else {+ } else {bias = U(0);}-+if (active) {w_local = *((device vec_w*)ws_ptr);}- qouter<U, tn * pack_factor, bits>(- (thread uint8_t*)&w_local, x_local, scale, bias, result);+ qouter<U, tn * pack_factor, bits>(+ (thread uint8_t*)&w_local,+ x_local,+ scale,+ bias,+ result+ );- x += block_size;- scales += block_size * out_vec_size_g;+ x += block_size;+ scales += block_size * out_vec_size_g;if (!use_zero_points) {biases += block_size * out_vec_size_g;}@@ -1367,9 +1626,21 @@ void qvm_impl_mlx(const int out_vec_size,uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]) {+ uint simd_lid [[thread_index_in_simdgroup]]+) {qvm_impl_core<T, group_size, bits, false>(- ws, scales, biases, nullptr, x, y, in_vec_size, out_vec_size, tid, simd_gid, simd_lid);+ ws,+ scales,+ biases,+ nullptr,+ x,+ y,+ in_vec_size,+ out_vec_size,+ tid,+ simd_gid,+ simd_lid+ );}template <typename T, int group_size, int bits>@@ -1383,9 +1654,21 @@ void qvm_impl_zeropoint(const int out_vec_size,uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]) {+ uint simd_lid [[thread_index_in_simdgroup]]+) {qvm_impl_core<T, group_size, bits, true>(- w, scales, nullptr, zero_points, x, y, in_vec_size, out_vec_size, tid, simd_gid, simd_lid);+ w,+ scales,+ nullptr,+ zero_points,+ x,+ y,+ in_vec_size,+ out_vec_size,+ tid,+ simd_gid,+ simd_lid+ );}template <typename T, int group_size, int bits>@@ -1400,17 +1683,37 @@ void qvm_impl(const constant int& N, // out_vec_sizeuint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]) {+ uint simd_lid [[thread_index_in_simdgroup]]+) {if (kUseMlxQuant) {qvm_impl_mlx<T, group_size, bits>(- w, scales, biases, x, y, K, N, tid, simd_gid, simd_lid);+ w,+ scales,+ biases,+ x,+ y,+ K,+ N,+ tid,+ simd_gid,+ simd_lid+ );} else {qvm_impl_zeropoint<T, group_size, bits>(- w, scales, zero_points, x, y, K, N, tid, simd_gid, simd_lid);+ w,+ scales,+ zero_points,+ x,+ y,+ K,+ N,+ tid,+ simd_gid,+ simd_lid+ );}}-template <typename T,const int group_size,@@ -1422,7 +1725,8 @@ template <[[kernel]] void qmm(const device uint32_t* w [[buffer(0)]],const device T* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],+ const device uint8_t* zero_points+ [[buffer(2), function_constant(kUseZeroPoints)]],const device T* biases [[buffer(2), function_constant(kUseMlxQuant)]],const device T* x [[buffer(3)]],device T* y [[buffer(4)]],@@ -1432,7 +1736,8 @@ template <uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]) {+ uint simd_lid [[thread_index_in_simdgroup]]+) {(void)lid;@@ -1443,7 +1748,22 @@ template <threadgroup T Ws[BK * BN_padded];qmm_impl<T, group_size, bits, aligned_K, BM, BK, BN>(- w, scales, zero_points, biases, x, y, Xs, Ws, K, N, M, tid, lid, simd_gid, simd_lid);+ w,+ scales,+ zero_points,+ biases,+ x,+ y,+ Xs,+ Ws,+ K,+ N,+ M,+ tid,+ lid,+ simd_gid,+ simd_lid+ );}template <@@ -1457,7 +1777,8 @@ template <[[kernel]] void qmm_transposed(const device uint32_t* w [[buffer(0)]],const device T* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],+ const device uint8_t* zero_points+ [[buffer(2), function_constant(kUseZeroPoints)]],const device T* biases [[buffer(2), function_constant(kUseMlxQuant)]],const device T* x [[buffer(3)]],device T* y [[buffer(4)]],@@ -1467,7 +1788,8 @@ template <uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]) {+ uint simd_lid [[thread_index_in_simdgroup]]+) {(void)lid;@@ -1477,14 +1799,30 @@ template <threadgroup T Ws[BN * BK_padded];qmm_transposed_impl<T, group_size, bits, aligned_N, BM, BK, BN>(- w, scales, zero_points, biases, x, y, Xs, Ws, K, N, M, tid, lid, simd_gid, simd_lid);+ w,+ scales,+ zero_points,+ biases,+ x,+ y,+ Xs,+ Ws,+ K,+ N,+ M,+ tid,+ lid,+ simd_gid,+ simd_lid+ );}template <typename T, int group_size, int bits>[[kernel]] void qmv(const device uint32_t* w [[buffer(0)]],const device T* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],+ const device uint8_t* zero_points+ [[buffer(2), function_constant(kUseZeroPoints)]],const device T* biases [[buffer(2), function_constant(kUseMlxQuant)]],const device T* x [[buffer(3)]],device T* y [[buffer(4)]],@@ -1492,16 +1830,30 @@ template <typename T, int group_size, int bits>const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]) {-- qmv_impl<T, group_size, bits>(w, scales, zero_points, biases, x, y, K, N, tid, simd_gid, simd_lid);+ uint simd_lid [[thread_index_in_simdgroup]]+) {++ qmv_impl<T, group_size, bits>(+ w,+ scales,+ zero_points,+ biases,+ x,+ y,+ K,+ N,+ tid,+ simd_gid,+ simd_lid+ );}template <typename T, int group_size, int bits>[[kernel]] void qvm(const device uint32_t* w [[buffer(0)]],const device T* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],+ const device uint8_t* zero_points+ [[buffer(2), function_constant(kUseZeroPoints)]],const device T* biases [[buffer(2), function_constant(kUseMlxQuant)]],const device T* x [[buffer(3)]],device T* y [[buffer(4)]],@@ -1509,17 +1861,30 @@ template <typename T, int group_size, int bits>const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]) {+ uint simd_lid [[thread_index_in_simdgroup]]+) {qvm_impl<T, group_size, bits>(- w, scales, zero_points, biases, x, y, K, N, tid, simd_gid, simd_lid);+ w,+ scales,+ zero_points,+ biases,+ x,+ y,+ K,+ N,+ tid,+ simd_gid,+ simd_lid+ );}template <typename T, int group_size, int bits>[[kernel]] void qmv_fast(const device uint32_t* w [[buffer(0)]],const device T* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],+ const device uint8_t* zero_points+ [[buffer(2), function_constant(kUseZeroPoints)]],const device T* biases [[buffer(2), function_constant(kUseMlxQuant)]],const device T* x [[buffer(3)]],device T* y [[buffer(4)]],@@ -1527,12 +1892,27 @@ template <typename T, int group_size, int bits>const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]) {- qmv_fast_impl<T, group_size, bits>(w, scales, zero_points, biases, x, y, K, N, tid, simd_gid, simd_lid);+ uint simd_lid [[thread_index_in_simdgroup]]+) {+ qmv_fast_impl<T, group_size, bits>(+ w,+ scales,+ zero_points,+ biases,+ x,+ y,+ K,+ N,+ tid,+ simd_gid,+ simd_lid+ );}-template [[host_name("qmv_f16_g32_b4_fast")]]-[[kernel]] void qmv_fast<half, 32, 4>(+template [[host_name("qmv_f16_g32_b4_fast")]] [[kernel]] void qmv_fast<+ half,+ 32,+ 4>(const device uint32_t* w [[buffer(0)]],const device half* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -1543,10 +1923,13 @@ template [[host_name("qmv_f16_g32_b4_fast")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmv_f16_g64_b4_fast")]]-[[kernel]] void qmv_fast<half, 64, 4>(+template [[host_name("qmv_f16_g64_b4_fast")]] [[kernel]] void qmv_fast<+ half,+ 64,+ 4>(const device uint32_t* w [[buffer(0)]],const device half* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -1557,10 +1940,13 @@ template [[host_name("qmv_f16_g64_b4_fast")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmv_f16_g128_b4_fast")]]-[[kernel]] void qmv_fast<half, 128, 4>(+template [[host_name("qmv_f16_g128_b4_fast")]] [[kernel]] void qmv_fast<+ half,+ 128,+ 4>(const device uint32_t* w [[buffer(0)]],const device half* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -1571,10 +1957,13 @@ template [[host_name("qmv_f16_g128_b4_fast")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmv_bf16_g32_b4_fast")]]-[[kernel]] void qmv_fast<bfloat, 32, 4>(+template [[host_name("qmv_bf16_g32_b4_fast")]] [[kernel]] void qmv_fast<+ bfloat,+ 32,+ 4>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -1585,10 +1974,13 @@ template [[host_name("qmv_bf16_g32_b4_fast")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmv_bf16_g64_b4_fast")]]-[[kernel]] void qmv_fast<bfloat, 64, 4>(+template [[host_name("qmv_bf16_g64_b4_fast")]] [[kernel]] void qmv_fast<+ bfloat,+ 64,+ 4>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -1599,10 +1991,13 @@ template [[host_name("qmv_bf16_g64_b4_fast")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmv_bf16_g128_b4_fast")]]-[[kernel]] void qmv_fast<bfloat, 128, 4>(+template [[host_name("qmv_bf16_g128_b4_fast")]] [[kernel]] void qmv_fast<+ bfloat,+ 128,+ 4>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -1613,10 +2008,13 @@ template [[host_name("qmv_bf16_g128_b4_fast")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmv_f32_g32_b4_fast")]]-[[kernel]] void qmv_fast<float, 32, 4>(+template [[host_name("qmv_f32_g32_b4_fast")]] [[kernel]] void qmv_fast<+ float,+ 32,+ 4>(const device uint32_t* w [[buffer(0)]],const device float* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -1627,10 +2025,13 @@ template [[host_name("qmv_f32_g32_b4_fast")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmv_f32_g64_b4_fast")]]-[[kernel]] void qmv_fast<float, 64, 4>(+template [[host_name("qmv_f32_g64_b4_fast")]] [[kernel]] void qmv_fast<+ float,+ 64,+ 4>(const device uint32_t* w [[buffer(0)]],const device float* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -1641,10 +2042,13 @@ template [[host_name("qmv_f32_g64_b4_fast")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmv_f32_g128_b4_fast")]]-[[kernel]] void qmv_fast<float, 128, 4>(+template [[host_name("qmv_f32_g128_b4_fast")]] [[kernel]] void qmv_fast<+ float,+ 128,+ 4>(const device uint32_t* w [[buffer(0)]],const device float* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -1655,10 +2059,13 @@ template [[host_name("qmv_f32_g128_b4_fast")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmv_f16_g32_b8_fast")]]-[[kernel]] void qmv_fast<half, 32, 8>(+template [[host_name("qmv_f16_g32_b8_fast")]] [[kernel]] void qmv_fast<+ half,+ 32,+ 8>(const device uint32_t* w [[buffer(0)]],const device half* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -1669,10 +2076,13 @@ template [[host_name("qmv_f16_g32_b8_fast")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmv_f16_g64_b8_fast")]]-[[kernel]] void qmv_fast<half, 64, 8>(+template [[host_name("qmv_f16_g64_b8_fast")]] [[kernel]] void qmv_fast<+ half,+ 64,+ 8>(const device uint32_t* w [[buffer(0)]],const device half* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -1683,10 +2093,13 @@ template [[host_name("qmv_f16_g64_b8_fast")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmv_f16_g128_b8_fast")]]-[[kernel]] void qmv_fast<half, 128, 8>(+template [[host_name("qmv_f16_g128_b8_fast")]] [[kernel]] void qmv_fast<+ half,+ 128,+ 8>(const device uint32_t* w [[buffer(0)]],const device half* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -1697,10 +2110,13 @@ template [[host_name("qmv_f16_g128_b8_fast")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmv_bf16_g32_b8_fast")]]-[[kernel]] void qmv_fast<bfloat, 32, 8>(+template [[host_name("qmv_bf16_g32_b8_fast")]] [[kernel]] void qmv_fast<+ bfloat,+ 32,+ 8>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -1711,10 +2127,13 @@ template [[host_name("qmv_bf16_g32_b8_fast")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmv_bf16_g64_b8_fast")]]-[[kernel]] void qmv_fast<bfloat, 64, 8>(+template [[host_name("qmv_bf16_g64_b8_fast")]] [[kernel]] void qmv_fast<+ bfloat,+ 64,+ 8>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -1725,10 +2144,13 @@ template [[host_name("qmv_bf16_g64_b8_fast")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmv_bf16_g128_b8_fast")]]-[[kernel]] void qmv_fast<bfloat, 128, 8>(+template [[host_name("qmv_bf16_g128_b8_fast")]] [[kernel]] void qmv_fast<+ bfloat,+ 128,+ 8>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -1739,10 +2161,13 @@ template [[host_name("qmv_bf16_g128_b8_fast")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmv_f32_g32_b8_fast")]]-[[kernel]] void qmv_fast<float, 32, 8>(+template [[host_name("qmv_f32_g32_b8_fast")]] [[kernel]] void qmv_fast<+ float,+ 32,+ 8>(const device uint32_t* w [[buffer(0)]],const device float* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -1753,10 +2178,13 @@ template [[host_name("qmv_f32_g32_b8_fast")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmv_f32_g64_b8_fast")]]-[[kernel]] void qmv_fast<float, 64, 8>(+template [[host_name("qmv_f32_g64_b8_fast")]] [[kernel]] void qmv_fast<+ float,+ 64,+ 8>(const device uint32_t* w [[buffer(0)]],const device float* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -1767,10 +2195,13 @@ template [[host_name("qmv_f32_g64_b8_fast")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmv_f32_g128_b8_fast")]]-[[kernel]] void qmv_fast<float, 128, 8>(+template [[host_name("qmv_f32_g128_b8_fast")]] [[kernel]] void qmv_fast<+ float,+ 128,+ 8>(const device uint32_t* w [[buffer(0)]],const device float* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -1781,11 +2212,15 @@ template [[host_name("qmv_f32_g128_b8_fast")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);// Group size 32 (F16)-template [[host_name("qmm_f16_g32_b4")]]-[[kernel]] void qmm<half, 32, 4, false>(+template [[host_name("qmm_f16_g32_b4")]] [[kernel]] void qmm<+ half,+ 32,+ 4,+ false>(const device uint32_t* w [[buffer(0)]],const device half* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -1798,10 +2233,11 @@ template [[host_name("qmm_f16_g32_b4")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_transposed_f16_g32_b4")]]-[[kernel]] void qmm_transposed<half, 32, 4, true>(+template [[host_name("qmm_transposed_f16_g32_b4")]] [[kernel]] void+qmm_transposed<half, 32, 4, true>(const device uint32_t* w [[buffer(0)]],const device half* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -1814,10 +2250,11 @@ template [[host_name("qmm_transposed_f16_g32_b4")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_transposed_f16_g32_b4_unaligned")]]-[[kernel]] void qmm_transposed<half, 32, 4, false>(+template [[host_name("qmm_transposed_f16_g32_b4_unaligned")]] [[kernel]] void+qmm_transposed<half, 32, 4, false>(const device uint32_t* w [[buffer(0)]],const device half* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -1830,10 +2267,14 @@ template [[host_name("qmm_transposed_f16_g32_b4_unaligned")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_f16_g32_b4_alignedk")]]-[[kernel]] void qmm<half, 32, 4, true>(+template [[host_name("qmm_f16_g32_b4_alignedk")]] [[kernel]] void qmm<+ half,+ 32,+ 4,+ true>(const device uint32_t* w [[buffer(0)]],const device half* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -1846,10 +2287,10 @@ template [[host_name("qmm_f16_g32_b4_alignedk")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmv_f16_g32_b4")]]-[[kernel]] void qmv<half, 32, 4>(+template [[host_name("qmv_f16_g32_b4")]] [[kernel]] void qmv<half, 32, 4>(const device uint32_t* w [[buffer(0)]],const device half* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -1860,13 +2301,14 @@ template [[host_name("qmv_f16_g32_b4")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qvm_f16_g32_b4")]]-[[kernel]] void qvm<half, 32, 4>(+template [[host_name("qvm_f16_g32_b4")]] [[kernel]] void qvm<half, 32, 4>(const device uint32_t* w [[buffer(0)]],const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],+ const device uint8_t* zero_points+ [[buffer(2), function_constant(kUseZeroPoints)]],const device half* biases [[buffer(2), function_constant(kUseMlxQuant)]],const device half* x [[buffer(3)]],device half* y [[buffer(4)]],@@ -1874,11 +2316,15 @@ template [[host_name("qvm_f16_g32_b4")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);// Group size 32 (F16) - 8 bit-template [[host_name("qmm_f16_g32_b8")]]-[[kernel]] void qmm<half, 32, 8, false>(+template [[host_name("qmm_f16_g32_b8")]] [[kernel]] void qmm<+ half,+ 32,+ 8,+ false>(const device uint32_t* w [[buffer(0)]],const device half* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -1891,10 +2337,11 @@ template [[host_name("qmm_f16_g32_b8")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_transposed_f16_g32_b8")]]-[[kernel]] void qmm_transposed<half, 32, 8, true>(+template [[host_name("qmm_transposed_f16_g32_b8")]] [[kernel]] void+qmm_transposed<half, 32, 8, true>(const device uint32_t* w [[buffer(0)]],const device half* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -1907,10 +2354,11 @@ template [[host_name("qmm_transposed_f16_g32_b8")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_transposed_f16_g32_b8_unaligned")]]-[[kernel]] void qmm_transposed<half, 32, 8, false>(+template [[host_name("qmm_transposed_f16_g32_b8_unaligned")]] [[kernel]] void+qmm_transposed<half, 32, 8, false>(const device uint32_t* w [[buffer(0)]],const device half* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -1923,10 +2371,14 @@ template [[host_name("qmm_transposed_f16_g32_b8_unaligned")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_f16_g32_b8_alignedk")]]-[[kernel]] void qmm<half, 32, 8, true>(+template [[host_name("qmm_f16_g32_b8_alignedk")]] [[kernel]] void qmm<+ half,+ 32,+ 8,+ true>(const device uint32_t* w [[buffer(0)]],const device half* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -1939,10 +2391,10 @@ template [[host_name("qmm_f16_g32_b8_alignedk")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmv_f16_g32_b8")]]-[[kernel]] void qmv<half, 32, 8>(+template [[host_name("qmv_f16_g32_b8")]] [[kernel]] void qmv<half, 32, 8>(const device uint32_t* w [[buffer(0)]],const device half* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -1953,13 +2405,14 @@ template [[host_name("qmv_f16_g32_b8")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qvm_f16_g32_b8")]]-[[kernel]] void qvm<half, 32, 8>(+template [[host_name("qvm_f16_g32_b8")]] [[kernel]] void qvm<half, 32, 8>(const device uint32_t* w [[buffer(0)]],const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],+ const device uint8_t* zero_points+ [[buffer(2), function_constant(kUseZeroPoints)]],const device half* biases [[buffer(2), function_constant(kUseMlxQuant)]],const device half* x [[buffer(3)]],device half* y [[buffer(4)]],@@ -1967,11 +2420,15 @@ template [[host_name("qvm_f16_g32_b8")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);// Group size 64 (F16)-template [[host_name("qmm_f16_g64_b4")]]-[[kernel]] void qmm<half, 64, 4, false>(+template [[host_name("qmm_f16_g64_b4")]] [[kernel]] void qmm<+ half,+ 64,+ 4,+ false>(const device uint32_t* w [[buffer(0)]],const device half* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -1984,10 +2441,14 @@ template [[host_name("qmm_f16_g64_b4")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_f16_g64_b4_alignedk")]]-[[kernel]] void qmm<half, 64, 4, true>(+template [[host_name("qmm_f16_g64_b4_alignedk")]] [[kernel]] void qmm<+ half,+ 64,+ 4,+ true>(const device uint32_t* w [[buffer(0)]],const device half* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2000,10 +2461,11 @@ template [[host_name("qmm_f16_g64_b4_alignedk")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_transposed_f16_g64_b4")]]-[[kernel]] void qmm_transposed<half, 64, 4, true>(+template [[host_name("qmm_transposed_f16_g64_b4")]] [[kernel]] void+qmm_transposed<half, 64, 4, true>(const device uint32_t* w [[buffer(0)]],const device half* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2016,10 +2478,11 @@ template [[host_name("qmm_transposed_f16_g64_b4")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_transposed_f16_g64_b4_unaligned")]]-[[kernel]] void qmm_transposed<half, 64, 4, false>(+template [[host_name("qmm_transposed_f16_g64_b4_unaligned")]] [[kernel]] void+qmm_transposed<half, 64, 4, false>(const device uint32_t* w [[buffer(0)]],const device half* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2032,10 +2495,10 @@ template [[host_name("qmm_transposed_f16_g64_b4_unaligned")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmv_f16_g64_b4")]]-[[kernel]] void qmv<half, 64, 4>(+template [[host_name("qmv_f16_g64_b4")]] [[kernel]] void qmv<half, 64, 4>(const device uint32_t* w [[buffer(0)]],const device half* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2046,13 +2509,14 @@ template [[host_name("qmv_f16_g64_b4")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qvm_f16_g64_b4")]]-[[kernel]] void qvm<half, 64, 4>(+template [[host_name("qvm_f16_g64_b4")]] [[kernel]] void qvm<half, 64, 4>(const device uint32_t* w [[buffer(0)]],const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],+ const device uint8_t* zero_points+ [[buffer(2), function_constant(kUseZeroPoints)]],const device half* biases [[buffer(2), function_constant(kUseMlxQuant)]],const device half* x [[buffer(3)]],device half* y [[buffer(4)]],@@ -2060,11 +2524,15 @@ template [[host_name("qvm_f16_g64_b4")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);// Group size 64 (F16) - 8 bit-template [[host_name("qmm_f16_g64_b8")]]-[[kernel]] void qmm<half, 64, 8, false>(+template [[host_name("qmm_f16_g64_b8")]] [[kernel]] void qmm<+ half,+ 64,+ 8,+ false>(const device uint32_t* w [[buffer(0)]],const device half* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2077,10 +2545,14 @@ template [[host_name("qmm_f16_g64_b8")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_f16_g64_b8_alignedk")]]-[[kernel]] void qmm<half, 64, 8, true>(+template [[host_name("qmm_f16_g64_b8_alignedk")]] [[kernel]] void qmm<+ half,+ 64,+ 8,+ true>(const device uint32_t* w [[buffer(0)]],const device half* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2093,10 +2565,11 @@ template [[host_name("qmm_f16_g64_b8_alignedk")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_transposed_f16_g64_b8")]]-[[kernel]] void qmm_transposed<half, 64, 8, true>(+template [[host_name("qmm_transposed_f16_g64_b8")]] [[kernel]] void+qmm_transposed<half, 64, 8, true>(const device uint32_t* w [[buffer(0)]],const device half* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2109,10 +2582,11 @@ template [[host_name("qmm_transposed_f16_g64_b8")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_transposed_f16_g64_b8_unaligned")]]-[[kernel]] void qmm_transposed<half, 64, 8, false>(+template [[host_name("qmm_transposed_f16_g64_b8_unaligned")]] [[kernel]] void+qmm_transposed<half, 64, 8, false>(const device uint32_t* w [[buffer(0)]],const device half* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2125,10 +2599,10 @@ template [[host_name("qmm_transposed_f16_g64_b8_unaligned")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmv_f16_g64_b8")]]-[[kernel]] void qmv<half, 64, 8>(+template [[host_name("qmv_f16_g64_b8")]] [[kernel]] void qmv<half, 64, 8>(const device uint32_t* w [[buffer(0)]],const device half* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2139,13 +2613,14 @@ template [[host_name("qmv_f16_g64_b8")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qvm_f16_g64_b8")]]-[[kernel]] void qvm<half, 64, 8>(+template [[host_name("qvm_f16_g64_b8")]] [[kernel]] void qvm<half, 64, 8>(const device uint32_t* w [[buffer(0)]],const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],+ const device uint8_t* zero_points+ [[buffer(2), function_constant(kUseZeroPoints)]],const device half* biases [[buffer(2), function_constant(kUseMlxQuant)]],const device half* x [[buffer(3)]],device half* y [[buffer(4)]],@@ -2153,11 +2628,15 @@ template [[host_name("qvm_f16_g64_b8")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);// Group size 128 (F16)-template [[host_name("qmm_f16_g128_b4")]]-[[kernel]] void qmm<half, 128, 4, false>(+template [[host_name("qmm_f16_g128_b4")]] [[kernel]] void qmm<+ half,+ 128,+ 4,+ false>(const device uint32_t* w [[buffer(0)]],const device half* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2170,10 +2649,14 @@ template [[host_name("qmm_f16_g128_b4")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_f16_g128_b4_alignedk")]]-[[kernel]] void qmm<half, 128, 4, true>(+template [[host_name("qmm_f16_g128_b4_alignedk")]] [[kernel]] void qmm<+ half,+ 128,+ 4,+ true>(const device uint32_t* w [[buffer(0)]],const device half* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2186,10 +2669,11 @@ template [[host_name("qmm_f16_g128_b4_alignedk")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_transposed_f16_g128_b4")]]-[[kernel]] void qmm_transposed<half, 128, 4, true>(+template [[host_name("qmm_transposed_f16_g128_b4")]] [[kernel]] void+qmm_transposed<half, 128, 4, true>(const device uint32_t* w [[buffer(0)]],const device half* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2202,10 +2686,11 @@ template [[host_name("qmm_transposed_f16_g128_b4")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_transposed_f16_g128_b4_unaligned")]]-[[kernel]] void qmm_transposed<half, 128, 4, false>(+template [[host_name("qmm_transposed_f16_g128_b4_unaligned")]] [[kernel]] void+qmm_transposed<half, 128, 4, false>(const device uint32_t* w [[buffer(0)]],const device half* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2218,10 +2703,10 @@ template [[host_name("qmm_transposed_f16_g128_b4_unaligned")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmv_f16_g128_b4")]]-[[kernel]] void qmv<half, 128, 4>(+template [[host_name("qmv_f16_g128_b4")]] [[kernel]] void qmv<half, 128, 4>(const device uint32_t* w [[buffer(0)]],const device half* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2232,13 +2717,14 @@ template [[host_name("qmv_f16_g128_b4")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qvm_f16_g128_b4")]]-[[kernel]] void qvm<half, 128, 4>(+template [[host_name("qvm_f16_g128_b4")]] [[kernel]] void qvm<half, 128, 4>(const device uint32_t* w [[buffer(0)]],const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],+ const device uint8_t* zero_points+ [[buffer(2), function_constant(kUseZeroPoints)]],const device half* biases [[buffer(2), function_constant(kUseMlxQuant)]],const device half* x [[buffer(3)]],device half* y [[buffer(4)]],@@ -2246,11 +2732,15 @@ template [[host_name("qvm_f16_g128_b4")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);// Group size 128 (F16) - 8 bit-template [[host_name("qmm_f16_g128_b8")]]-[[kernel]] void qmm<half, 128, 8, false>(+template [[host_name("qmm_f16_g128_b8")]] [[kernel]] void qmm<+ half,+ 128,+ 8,+ false>(const device uint32_t* w [[buffer(0)]],const device half* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2263,10 +2753,14 @@ template [[host_name("qmm_f16_g128_b8")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_f16_g128_b8_alignedk")]]-[[kernel]] void qmm<half, 128, 8, true>(+template [[host_name("qmm_f16_g128_b8_alignedk")]] [[kernel]] void qmm<+ half,+ 128,+ 8,+ true>(const device uint32_t* w [[buffer(0)]],const device half* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2279,10 +2773,11 @@ template [[host_name("qmm_f16_g128_b8_alignedk")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_transposed_f16_g128_b8")]]-[[kernel]] void qmm_transposed<half, 128, 8, true>(+template [[host_name("qmm_transposed_f16_g128_b8")]] [[kernel]] void+qmm_transposed<half, 128, 8, true>(const device uint32_t* w [[buffer(0)]],const device half* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2295,10 +2790,11 @@ template [[host_name("qmm_transposed_f16_g128_b8")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_transposed_f16_g128_b8_unaligned")]]-[[kernel]] void qmm_transposed<half, 128, 8, false>(+template [[host_name("qmm_transposed_f16_g128_b8_unaligned")]] [[kernel]] void+qmm_transposed<half, 128, 8, false>(const device uint32_t* w [[buffer(0)]],const device half* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2311,10 +2807,10 @@ template [[host_name("qmm_transposed_f16_g128_b8_unaligned")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmv_f16_g128_b8")]]-[[kernel]] void qmv<half, 128, 8>(+template [[host_name("qmv_f16_g128_b8")]] [[kernel]] void qmv<half, 128, 8>(const device uint32_t* w [[buffer(0)]],const device half* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2325,13 +2821,14 @@ template [[host_name("qmv_f16_g128_b8")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qvm_f16_g128_b8")]]-[[kernel]] void qvm<half, 128, 8>(+template [[host_name("qvm_f16_g128_b8")]] [[kernel]] void qvm<half, 128, 8>(const device uint32_t* w [[buffer(0)]],const device half* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],+ const device uint8_t* zero_points+ [[buffer(2), function_constant(kUseZeroPoints)]],const device half* biases [[buffer(2), function_constant(kUseMlxQuant)]],const device half* x [[buffer(3)]],device half* y [[buffer(4)]],@@ -2339,11 +2836,15 @@ template [[host_name("qvm_f16_g128_b8")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);// Group size 32 (F32)-template [[host_name("qmm_f32_g32_b4")]]-[[kernel]] void qmm<float, 32, 4, false>(+template [[host_name("qmm_f32_g32_b4")]] [[kernel]] void qmm<+ float,+ 32,+ 4,+ false>(const device uint32_t* w [[buffer(0)]],const device float* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2356,10 +2857,11 @@ template [[host_name("qmm_f32_g32_b4")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_transposed_f32_g32_b4")]]-[[kernel]] void qmm_transposed<float, 32, 4, true>(+template [[host_name("qmm_transposed_f32_g32_b4")]] [[kernel]] void+qmm_transposed<float, 32, 4, true>(const device uint32_t* w [[buffer(0)]],const device float* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2372,10 +2874,11 @@ template [[host_name("qmm_transposed_f32_g32_b4")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_transposed_f32_g32_b4_unaligned")]]-[[kernel]] void qmm_transposed<float, 32, 4, false>(+template [[host_name("qmm_transposed_f32_g32_b4_unaligned")]] [[kernel]] void+qmm_transposed<float, 32, 4, false>(const device uint32_t* w [[buffer(0)]],const device float* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2388,10 +2891,14 @@ template [[host_name("qmm_transposed_f32_g32_b4_unaligned")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_f32_g32_b4_alignedk")]]-[[kernel]] void qmm<float, 32, 4, true>(+template [[host_name("qmm_f32_g32_b4_alignedk")]] [[kernel]] void qmm<+ float,+ 32,+ 4,+ true>(const device uint32_t* w [[buffer(0)]],const device float* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2404,10 +2911,10 @@ template [[host_name("qmm_f32_g32_b4_alignedk")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmv_f32_g32_b4")]]-[[kernel]] void qmv<float, 32, 4>(+template [[host_name("qmv_f32_g32_b4")]] [[kernel]] void qmv<float, 32, 4>(const device uint32_t* w [[buffer(0)]],const device float* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2418,13 +2925,14 @@ template [[host_name("qmv_f32_g32_b4")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qvm_f32_g32_b4")]]-[[kernel]] void qvm<float, 32, 4>(+template [[host_name("qvm_f32_g32_b4")]] [[kernel]] void qvm<float, 32, 4>(const device uint32_t* w [[buffer(0)]],const device float* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],+ const device uint8_t* zero_points+ [[buffer(2), function_constant(kUseZeroPoints)]],const device float* biases [[buffer(2), function_constant(kUseMlxQuant)]],const device float* x [[buffer(3)]],device float* y [[buffer(4)]],@@ -2432,11 +2940,15 @@ template [[host_name("qvm_f32_g32_b4")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);// Group size 32 (F32) - 8 bit-template [[host_name("qmm_f32_g32_b8")]]-[[kernel]] void qmm<float, 32, 8, false>(+template [[host_name("qmm_f32_g32_b8")]] [[kernel]] void qmm<+ float,+ 32,+ 8,+ false>(const device uint32_t* w [[buffer(0)]],const device float* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2449,10 +2961,11 @@ template [[host_name("qmm_f32_g32_b8")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_transposed_f32_g32_b8")]]-[[kernel]] void qmm_transposed<float, 32, 8, true>(+template [[host_name("qmm_transposed_f32_g32_b8")]] [[kernel]] void+qmm_transposed<float, 32, 8, true>(const device uint32_t* w [[buffer(0)]],const device float* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2465,10 +2978,11 @@ template [[host_name("qmm_transposed_f32_g32_b8")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_transposed_f32_g32_b8_unaligned")]]-[[kernel]] void qmm_transposed<float, 32, 8, false>(+template [[host_name("qmm_transposed_f32_g32_b8_unaligned")]] [[kernel]] void+qmm_transposed<float, 32, 8, false>(const device uint32_t* w [[buffer(0)]],const device float* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2481,10 +2995,14 @@ template [[host_name("qmm_transposed_f32_g32_b8_unaligned")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_f32_g32_b8_alignedk")]]-[[kernel]] void qmm<float, 32, 8, true>(+template [[host_name("qmm_f32_g32_b8_alignedk")]] [[kernel]] void qmm<+ float,+ 32,+ 8,+ true>(const device uint32_t* w [[buffer(0)]],const device float* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2497,10 +3015,10 @@ template [[host_name("qmm_f32_g32_b8_alignedk")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmv_f32_g32_b8")]]-[[kernel]] void qmv<float, 32, 8>(+template [[host_name("qmv_f32_g32_b8")]] [[kernel]] void qmv<float, 32, 8>(const device uint32_t* w [[buffer(0)]],const device float* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2511,13 +3029,14 @@ template [[host_name("qmv_f32_g32_b8")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qvm_f32_g32_b8")]]-[[kernel]] void qvm<float, 32, 8>(+template [[host_name("qvm_f32_g32_b8")]] [[kernel]] void qvm<float, 32, 8>(const device uint32_t* w [[buffer(0)]],const device float* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],+ const device uint8_t* zero_points+ [[buffer(2), function_constant(kUseZeroPoints)]],const device float* biases [[buffer(2), function_constant(kUseMlxQuant)]],const device float* x [[buffer(3)]],device float* y [[buffer(4)]],@@ -2525,11 +3044,15 @@ template [[host_name("qvm_f32_g32_b8")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);// Group size 64 (F32)-template [[host_name("qmm_f32_g64_b4")]]-[[kernel]] void qmm<float, 64, 4, false>(+template [[host_name("qmm_f32_g64_b4")]] [[kernel]] void qmm<+ float,+ 64,+ 4,+ false>(const device uint32_t* w [[buffer(0)]],const device float* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2542,10 +3065,14 @@ template [[host_name("qmm_f32_g64_b4")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_f32_g64_b4_alignedk")]]-[[kernel]] void qmm<float, 64, 4, true>(+template [[host_name("qmm_f32_g64_b4_alignedk")]] [[kernel]] void qmm<+ float,+ 64,+ 4,+ true>(const device uint32_t* w [[buffer(0)]],const device float* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2558,10 +3085,11 @@ template [[host_name("qmm_f32_g64_b4_alignedk")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_transposed_f32_g64_b4")]]-[[kernel]] void qmm_transposed<float, 64, 4, true>(+template [[host_name("qmm_transposed_f32_g64_b4")]] [[kernel]] void+qmm_transposed<float, 64, 4, true>(const device uint32_t* w [[buffer(0)]],const device float* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2574,10 +3102,11 @@ template [[host_name("qmm_transposed_f32_g64_b4")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_transposed_f32_g64_b4_unaligned")]]-[[kernel]] void qmm_transposed<float, 64, 4, false>(+template [[host_name("qmm_transposed_f32_g64_b4_unaligned")]] [[kernel]] void+qmm_transposed<float, 64, 4, false>(const device uint32_t* w [[buffer(0)]],const device float* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2590,10 +3119,10 @@ template [[host_name("qmm_transposed_f32_g64_b4_unaligned")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmv_f32_g64_b4")]]-[[kernel]] void qmv<float, 64, 4>(+template [[host_name("qmv_f32_g64_b4")]] [[kernel]] void qmv<float, 64, 4>(const device uint32_t* w [[buffer(0)]],const device float* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2604,13 +3133,14 @@ template [[host_name("qmv_f32_g64_b4")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qvm_f32_g64_b4")]]-[[kernel]] void qvm<float, 64, 4>(+template [[host_name("qvm_f32_g64_b4")]] [[kernel]] void qvm<float, 64, 4>(const device uint32_t* w [[buffer(0)]],const device float* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],+ const device uint8_t* zero_points+ [[buffer(2), function_constant(kUseZeroPoints)]],const device float* biases [[buffer(2), function_constant(kUseMlxQuant)]],const device float* x [[buffer(3)]],device float* y [[buffer(4)]],@@ -2618,11 +3148,15 @@ template [[host_name("qvm_f32_g64_b4")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);// Group size 64 (F32) - 8 bit-template [[host_name("qmm_f32_g64_b8")]]-[[kernel]] void qmm<float, 64, 8, false>(+template [[host_name("qmm_f32_g64_b8")]] [[kernel]] void qmm<+ float,+ 64,+ 8,+ false>(const device uint32_t* w [[buffer(0)]],const device float* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2635,10 +3169,14 @@ template [[host_name("qmm_f32_g64_b8")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_f32_g64_b8_alignedk")]]-[[kernel]] void qmm<float, 64, 8, true>(+template [[host_name("qmm_f32_g64_b8_alignedk")]] [[kernel]] void qmm<+ float,+ 64,+ 8,+ true>(const device uint32_t* w [[buffer(0)]],const device float* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2651,10 +3189,11 @@ template [[host_name("qmm_f32_g64_b8_alignedk")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_transposed_f32_g64_b8")]]-[[kernel]] void qmm_transposed<float, 64, 8, true>(+template [[host_name("qmm_transposed_f32_g64_b8")]] [[kernel]] void+qmm_transposed<float, 64, 8, true>(const device uint32_t* w [[buffer(0)]],const device float* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2667,10 +3206,11 @@ template [[host_name("qmm_transposed_f32_g64_b8")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_transposed_f32_g64_b8_unaligned")]]-[[kernel]] void qmm_transposed<float, 64, 8, false>(+template [[host_name("qmm_transposed_f32_g64_b8_unaligned")]] [[kernel]] void+qmm_transposed<float, 64, 8, false>(const device uint32_t* w [[buffer(0)]],const device float* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2683,10 +3223,10 @@ template [[host_name("qmm_transposed_f32_g64_b8_unaligned")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmv_f32_g64_b8")]]-[[kernel]] void qmv<float, 64, 8>(+template [[host_name("qmv_f32_g64_b8")]] [[kernel]] void qmv<float, 64, 8>(const device uint32_t* w [[buffer(0)]],const device float* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2697,13 +3237,14 @@ template [[host_name("qmv_f32_g64_b8")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qvm_f32_g64_b8")]]-[[kernel]] void qvm<float, 64, 8>(+template [[host_name("qvm_f32_g64_b8")]] [[kernel]] void qvm<float, 64, 8>(const device uint32_t* w [[buffer(0)]],const device float* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],+ const device uint8_t* zero_points+ [[buffer(2), function_constant(kUseZeroPoints)]],const device float* biases [[buffer(2), function_constant(kUseMlxQuant)]],const device float* x [[buffer(3)]],device float* y [[buffer(4)]],@@ -2711,11 +3252,15 @@ template [[host_name("qvm_f32_g64_b8")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);// Group size 128 (F32)-template [[host_name("qmm_f32_g128_b4")]]-[[kernel]] void qmm<float, 128, 4, false>(+template [[host_name("qmm_f32_g128_b4")]] [[kernel]] void qmm<+ float,+ 128,+ 4,+ false>(const device uint32_t* w [[buffer(0)]],const device float* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2728,10 +3273,14 @@ template [[host_name("qmm_f32_g128_b4")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_f32_g128_b4_alignedk")]]-[[kernel]] void qmm<float, 128, 4, true>(+template [[host_name("qmm_f32_g128_b4_alignedk")]] [[kernel]] void qmm<+ float,+ 128,+ 4,+ true>(const device uint32_t* w [[buffer(0)]],const device float* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2744,10 +3293,11 @@ template [[host_name("qmm_f32_g128_b4_alignedk")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_transposed_f32_g128_b4")]]-[[kernel]] void qmm_transposed<float, 128, 4, true>(+template [[host_name("qmm_transposed_f32_g128_b4")]] [[kernel]] void+qmm_transposed<float, 128, 4, true>(const device uint32_t* w [[buffer(0)]],const device float* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2760,10 +3310,11 @@ template [[host_name("qmm_transposed_f32_g128_b4")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_transposed_f32_g128_b4_unaligned")]]-[[kernel]] void qmm_transposed<float, 128, 4, false>(+template [[host_name("qmm_transposed_f32_g128_b4_unaligned")]] [[kernel]] void+qmm_transposed<float, 128, 4, false>(const device uint32_t* w [[buffer(0)]],const device float* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2776,10 +3327,10 @@ template [[host_name("qmm_transposed_f32_g128_b4_unaligned")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmv_f32_g128_b4")]]-[[kernel]] void qmv<float, 128, 4>(+template [[host_name("qmv_f32_g128_b4")]] [[kernel]] void qmv<float, 128, 4>(const device uint32_t* w [[buffer(0)]],const device float* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2790,13 +3341,14 @@ template [[host_name("qmv_f32_g128_b4")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qvm_f32_g128_b4")]]-[[kernel]] void qvm<float, 128, 4>(+template [[host_name("qvm_f32_g128_b4")]] [[kernel]] void qvm<float, 128, 4>(const device uint32_t* w [[buffer(0)]],const device float* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],+ const device uint8_t* zero_points+ [[buffer(2), function_constant(kUseZeroPoints)]],const device float* biases [[buffer(2), function_constant(kUseMlxQuant)]],const device float* x [[buffer(3)]],device float* y [[buffer(4)]],@@ -2804,11 +3356,15 @@ template [[host_name("qvm_f32_g128_b4")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);// Group size 128 (F32) - 8 bit-template [[host_name("qmm_f32_g128_b8")]]-[[kernel]] void qmm<float, 128, 8, false>(+template [[host_name("qmm_f32_g128_b8")]] [[kernel]] void qmm<+ float,+ 128,+ 8,+ false>(const device uint32_t* w [[buffer(0)]],const device float* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2821,10 +3377,14 @@ template [[host_name("qmm_f32_g128_b8")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_f32_g128_b8_alignedk")]]-[[kernel]] void qmm<float, 128, 8, true>(+template [[host_name("qmm_f32_g128_b8_alignedk")]] [[kernel]] void qmm<+ float,+ 128,+ 8,+ true>(const device uint32_t* w [[buffer(0)]],const device float* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2837,10 +3397,11 @@ template [[host_name("qmm_f32_g128_b8_alignedk")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_transposed_f32_g128_b8")]]-[[kernel]] void qmm_transposed<float, 128, 8, true>(+template [[host_name("qmm_transposed_f32_g128_b8")]] [[kernel]] void+qmm_transposed<float, 128, 8, true>(const device uint32_t* w [[buffer(0)]],const device float* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2853,10 +3414,11 @@ template [[host_name("qmm_transposed_f32_g128_b8")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_transposed_f32_g128_b8_unaligned")]]-[[kernel]] void qmm_transposed<float, 128, 8, false>(+template [[host_name("qmm_transposed_f32_g128_b8_unaligned")]] [[kernel]] void+qmm_transposed<float, 128, 8, false>(const device uint32_t* w [[buffer(0)]],const device float* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2869,10 +3431,10 @@ template [[host_name("qmm_transposed_f32_g128_b8_unaligned")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmv_f32_g128_b8")]]-[[kernel]] void qmv<float, 128, 8>(+template [[host_name("qmv_f32_g128_b8")]] [[kernel]] void qmv<float, 128, 8>(const device uint32_t* w [[buffer(0)]],const device float* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2883,13 +3445,14 @@ template [[host_name("qmv_f32_g128_b8")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qvm_f32_g128_b8")]]-[[kernel]] void qvm<float, 128, 8>(+template [[host_name("qvm_f32_g128_b8")]] [[kernel]] void qvm<float, 128, 8>(const device uint32_t* w [[buffer(0)]],const device float* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],+ const device uint8_t* zero_points+ [[buffer(2), function_constant(kUseZeroPoints)]],const device float* biases [[buffer(2), function_constant(kUseMlxQuant)]],const device float* x [[buffer(3)]],device float* y [[buffer(4)]],@@ -2897,11 +3460,15 @@ template [[host_name("qvm_f32_g128_b8")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);// Group size 32 (BF16)-template [[host_name("qmm_bf16_g32_b4")]]-[[kernel]] void qmm<bfloat, 32, 4, false>(+template [[host_name("qmm_bf16_g32_b4")]] [[kernel]] void qmm<+ bfloat,+ 32,+ 4,+ false>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2914,10 +3481,14 @@ template [[host_name("qmm_bf16_g32_b4")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_bf16_g32_b4_alignedk")]]-[[kernel]] void qmm<bfloat, 32, 4, true>(+template [[host_name("qmm_bf16_g32_b4_alignedk")]] [[kernel]] void qmm<+ bfloat,+ 32,+ 4,+ true>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2930,10 +3501,11 @@ template [[host_name("qmm_bf16_g32_b4_alignedk")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_transposed_bf16_g32_b4")]]-[[kernel]] void qmm_transposed<bfloat, 32, 4, true>(+template [[host_name("qmm_transposed_bf16_g32_b4")]] [[kernel]] void+qmm_transposed<bfloat, 32, 4, true>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2946,10 +3518,11 @@ template [[host_name("qmm_transposed_bf16_g32_b4")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_transposed_bf16_g32_b4_unaligned")]]-[[kernel]] void qmm_transposed<bfloat, 32, 4, false>(+template [[host_name("qmm_transposed_bf16_g32_b4_unaligned")]] [[kernel]] void+qmm_transposed<bfloat, 32, 4, false>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2962,10 +3535,10 @@ template [[host_name("qmm_transposed_bf16_g32_b4_unaligned")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmv_bf16_g32_b4")]]-[[kernel]] void qmv<bfloat, 32, 4>(+template [[host_name("qmv_bf16_g32_b4")]] [[kernel]] void qmv<bfloat, 32, 4>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -2976,13 +3549,14 @@ template [[host_name("qmv_bf16_g32_b4")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qvm_bf16_g32_b4")]]-[[kernel]] void qvm<bfloat, 32, 4>(+template [[host_name("qvm_bf16_g32_b4")]] [[kernel]] void qvm<bfloat, 32, 4>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],+ const device uint8_t* zero_points+ [[buffer(2), function_constant(kUseZeroPoints)]],const device bfloat* biases [[buffer(2), function_constant(kUseMlxQuant)]],const device bfloat* x [[buffer(3)]],device bfloat* y [[buffer(4)]],@@ -2990,11 +3564,15 @@ template [[host_name("qvm_bf16_g32_b4")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);// Group size 32 (BF16) - 8 bit-template [[host_name("qmm_bf16_g32_b8")]]-[[kernel]] void qmm<bfloat, 32, 8, false>(+template [[host_name("qmm_bf16_g32_b8")]] [[kernel]] void qmm<+ bfloat,+ 32,+ 8,+ false>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -3007,10 +3585,14 @@ template [[host_name("qmm_bf16_g32_b8")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_bf16_g32_b8_alignedk")]]-[[kernel]] void qmm<bfloat, 32, 8, true>(+template [[host_name("qmm_bf16_g32_b8_alignedk")]] [[kernel]] void qmm<+ bfloat,+ 32,+ 8,+ true>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -3023,10 +3605,11 @@ template [[host_name("qmm_bf16_g32_b8_alignedk")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_transposed_bf16_g32_b8")]]-[[kernel]] void qmm_transposed<bfloat, 32, 8, true>(+template [[host_name("qmm_transposed_bf16_g32_b8")]] [[kernel]] void+qmm_transposed<bfloat, 32, 8, true>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -3039,10 +3622,11 @@ template [[host_name("qmm_transposed_bf16_g32_b8")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_transposed_bf16_g32_b8_unaligned")]]-[[kernel]] void qmm_transposed<bfloat, 32, 8, false>(+template [[host_name("qmm_transposed_bf16_g32_b8_unaligned")]] [[kernel]] void+qmm_transposed<bfloat, 32, 8, false>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -3055,10 +3639,10 @@ template [[host_name("qmm_transposed_bf16_g32_b8_unaligned")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmv_bf16_g32_b8")]]-[[kernel]] void qmv<bfloat, 32, 8>(+template [[host_name("qmv_bf16_g32_b8")]] [[kernel]] void qmv<bfloat, 32, 8>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -3069,13 +3653,14 @@ template [[host_name("qmv_bf16_g32_b8")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qvm_bf16_g32_b8")]]-[[kernel]] void qvm<bfloat, 32, 8>(+template [[host_name("qvm_bf16_g32_b8")]] [[kernel]] void qvm<bfloat, 32, 8>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],+ const device uint8_t* zero_points+ [[buffer(2), function_constant(kUseZeroPoints)]],const device bfloat* biases [[buffer(2), function_constant(kUseMlxQuant)]],const device bfloat* x [[buffer(3)]],device bfloat* y [[buffer(4)]],@@ -3083,11 +3668,15 @@ template [[host_name("qvm_bf16_g32_b8")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);// Group size 64 (BF16)-template [[host_name("qmm_bf16_g64_b4")]]-[[kernel]] void qmm<bfloat, 64, 4, false>(+template [[host_name("qmm_bf16_g64_b4")]] [[kernel]] void qmm<+ bfloat,+ 64,+ 4,+ false>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -3100,10 +3689,14 @@ template [[host_name("qmm_bf16_g64_b4")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_bf16_g64_b4_alignedk")]]-[[kernel]] void qmm<bfloat, 64, 4, true>(+template [[host_name("qmm_bf16_g64_b4_alignedk")]] [[kernel]] void qmm<+ bfloat,+ 64,+ 4,+ true>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -3116,10 +3709,11 @@ template [[host_name("qmm_bf16_g64_b4_alignedk")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_transposed_bf16_g64_b4")]]-[[kernel]] void qmm_transposed<bfloat, 64, 4, true>(+template [[host_name("qmm_transposed_bf16_g64_b4")]] [[kernel]] void+qmm_transposed<bfloat, 64, 4, true>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -3132,10 +3726,11 @@ template [[host_name("qmm_transposed_bf16_g64_b4")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_transposed_bf16_g64_b4_unaligned")]]-[[kernel]] void qmm_transposed<bfloat, 64, 4, false>(+template [[host_name("qmm_transposed_bf16_g64_b4_unaligned")]] [[kernel]] void+qmm_transposed<bfloat, 64, 4, false>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -3148,10 +3743,10 @@ template [[host_name("qmm_transposed_bf16_g64_b4_unaligned")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmv_bf16_g64_b4")]]-[[kernel]] void qmv<bfloat, 64, 4>(+template [[host_name("qmv_bf16_g64_b4")]] [[kernel]] void qmv<bfloat, 64, 4>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -3162,13 +3757,14 @@ template [[host_name("qmv_bf16_g64_b4")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qvm_bf16_g64_b4")]]-[[kernel]] void qvm<bfloat, 64, 4>(+template [[host_name("qvm_bf16_g64_b4")]] [[kernel]] void qvm<bfloat, 64, 4>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],+ const device uint8_t* zero_points+ [[buffer(2), function_constant(kUseZeroPoints)]],const device bfloat* biases [[buffer(2), function_constant(kUseMlxQuant)]],const device bfloat* x [[buffer(3)]],device bfloat* y [[buffer(4)]],@@ -3176,11 +3772,15 @@ template [[host_name("qvm_bf16_g64_b4")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);// Group size 64 (BF16) - 8 bit-template [[host_name("qmm_bf16_g64_b8")]]-[[kernel]] void qmm<bfloat, 64, 8, false>(+template [[host_name("qmm_bf16_g64_b8")]] [[kernel]] void qmm<+ bfloat,+ 64,+ 8,+ false>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -3193,10 +3793,14 @@ template [[host_name("qmm_bf16_g64_b8")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_bf16_g64_b8_alignedk")]]-[[kernel]] void qmm<bfloat, 64, 8, true>(+template [[host_name("qmm_bf16_g64_b8_alignedk")]] [[kernel]] void qmm<+ bfloat,+ 64,+ 8,+ true>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -3209,10 +3813,11 @@ template [[host_name("qmm_bf16_g64_b8_alignedk")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_transposed_bf16_g64_b8")]]-[[kernel]] void qmm_transposed<bfloat, 64, 8, true>(+template [[host_name("qmm_transposed_bf16_g64_b8")]] [[kernel]] void+qmm_transposed<bfloat, 64, 8, true>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -3225,10 +3830,11 @@ template [[host_name("qmm_transposed_bf16_g64_b8")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_transposed_bf16_g64_b8_unaligned")]]-[[kernel]] void qmm_transposed<bfloat, 64, 8, false>(+template [[host_name("qmm_transposed_bf16_g64_b8_unaligned")]] [[kernel]] void+qmm_transposed<bfloat, 64, 8, false>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -3241,10 +3847,10 @@ template [[host_name("qmm_transposed_bf16_g64_b8_unaligned")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmv_bf16_g64_b8")]]-[[kernel]] void qmv<bfloat, 64, 8>(+template [[host_name("qmv_bf16_g64_b8")]] [[kernel]] void qmv<bfloat, 64, 8>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -3255,13 +3861,14 @@ template [[host_name("qmv_bf16_g64_b8")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qvm_bf16_g64_b8")]]-[[kernel]] void qvm<bfloat, 64, 8>(+template [[host_name("qvm_bf16_g64_b8")]] [[kernel]] void qvm<bfloat, 64, 8>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],+ const device uint8_t* zero_points+ [[buffer(2), function_constant(kUseZeroPoints)]],const device bfloat* biases [[buffer(2), function_constant(kUseMlxQuant)]],const device bfloat* x [[buffer(3)]],device bfloat* y [[buffer(4)]],@@ -3269,11 +3876,15 @@ template [[host_name("qvm_bf16_g64_b8")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);// Group size 128 (BF16)-template [[host_name("qmm_bf16_g128_b4")]]-[[kernel]] void qmm<bfloat, 128, 4, false>(+template [[host_name("qmm_bf16_g128_b4")]] [[kernel]] void qmm<+ bfloat,+ 128,+ 4,+ false>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -3286,10 +3897,14 @@ template [[host_name("qmm_bf16_g128_b4")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_bf16_g128_b4_alignedk")]]-[[kernel]] void qmm<bfloat, 128, 4, true>(+template [[host_name("qmm_bf16_g128_b4_alignedk")]] [[kernel]] void qmm<+ bfloat,+ 128,+ 4,+ true>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -3302,10 +3917,11 @@ template [[host_name("qmm_bf16_g128_b4_alignedk")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_transposed_bf16_g128_b4")]]-[[kernel]] void qmm_transposed<bfloat, 128, 4, true>(+template [[host_name("qmm_transposed_bf16_g128_b4")]] [[kernel]] void+qmm_transposed<bfloat, 128, 4, true>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -3318,10 +3934,11 @@ template [[host_name("qmm_transposed_bf16_g128_b4")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_transposed_bf16_g128_b4_unaligned")]]-[[kernel]] void qmm_transposed<bfloat, 128, 4, false>(+template [[host_name("qmm_transposed_bf16_g128_b4_unaligned")]] [[kernel]] void+qmm_transposed<bfloat, 128, 4, false>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -3334,10 +3951,10 @@ template [[host_name("qmm_transposed_bf16_g128_b4_unaligned")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmv_bf16_g128_b4")]]-[[kernel]] void qmv<bfloat, 128, 4>(+template [[host_name("qmv_bf16_g128_b4")]] [[kernel]] void qmv<bfloat, 128, 4>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -3348,13 +3965,14 @@ template [[host_name("qmv_bf16_g128_b4")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qvm_bf16_g128_b4")]]-[[kernel]] void qvm<bfloat, 128, 4>(+template [[host_name("qvm_bf16_g128_b4")]] [[kernel]] void qvm<bfloat, 128, 4>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],+ const device uint8_t* zero_points+ [[buffer(2), function_constant(kUseZeroPoints)]],const device bfloat* biases [[buffer(2), function_constant(kUseMlxQuant)]],const device bfloat* x [[buffer(3)]],device bfloat* y [[buffer(4)]],@@ -3362,11 +3980,15 @@ template [[host_name("qvm_bf16_g128_b4")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);// Group size 128 (BF16) - 8 bit-template [[host_name("qmm_bf16_g128_b8")]]-[[kernel]] void qmm<bfloat, 128, 8, false>(+template [[host_name("qmm_bf16_g128_b8")]] [[kernel]] void qmm<+ bfloat,+ 128,+ 8,+ false>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -3379,10 +4001,14 @@ template [[host_name("qmm_bf16_g128_b8")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_bf16_g128_b8_alignedk")]]-[[kernel]] void qmm<bfloat, 128, 8, true>(+template [[host_name("qmm_bf16_g128_b8_alignedk")]] [[kernel]] void qmm<+ bfloat,+ 128,+ 8,+ true>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -3395,10 +4021,11 @@ template [[host_name("qmm_bf16_g128_b8_alignedk")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_transposed_bf16_g128_b8")]]-[[kernel]] void qmm_transposed<bfloat, 128, 8, true>(+template [[host_name("qmm_transposed_bf16_g128_b8")]] [[kernel]] void+qmm_transposed<bfloat, 128, 8, true>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -3411,10 +4038,11 @@ template [[host_name("qmm_transposed_bf16_g128_b8")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_transposed_bf16_g128_b8_unaligned")]]-[[kernel]] void qmm_transposed<bfloat, 128, 8, false>(+template [[host_name("qmm_transposed_bf16_g128_b8_unaligned")]] [[kernel]] void+qmm_transposed<bfloat, 128, 8, false>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -3427,10 +4055,10 @@ template [[host_name("qmm_transposed_bf16_g128_b8_unaligned")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmv_bf16_g128_b8")]]-[[kernel]] void qmv<bfloat, 128, 8>(+template [[host_name("qmv_bf16_g128_b8")]] [[kernel]] void qmv<bfloat, 128, 8>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -3441,13 +4069,14 @@ template [[host_name("qmv_bf16_g128_b8")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qvm_bf16_g128_b8")]]-[[kernel]] void qvm<bfloat, 128, 8>(+template [[host_name("qvm_bf16_g128_b8")]] [[kernel]] void qvm<bfloat, 128, 8>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],- const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],+ const device uint8_t* zero_points+ [[buffer(2), function_constant(kUseZeroPoints)]],const device bfloat* biases [[buffer(2), function_constant(kUseMlxQuant)]],const device bfloat* x [[buffer(3)]],device bfloat* y [[buffer(4)]],@@ -3455,10 +4084,11 @@ template [[host_name("qvm_bf16_g128_b8")]]const constant int& N [[buffer(6)]],uint3 tid [[threadgroup_position_in_grid]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_transposed_bf16_g128_b4_64x64")]]-[[kernel]] void qmm_transposed<bfloat, 128, 4, true, 64, 32, 64>(+template [[host_name("qmm_transposed_bf16_g128_b4_64x64")]] [[kernel]] void+qmm_transposed<bfloat, 128, 4, true, 64, 32, 64>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -3471,10 +4101,11 @@ template [[host_name("qmm_transposed_bf16_g128_b4_64x64")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_transposed_bf16_g128_b4_64x128")]]-[[kernel]] void qmm_transposed<bfloat, 128, 4, true, 64, 32, 128>(+template [[host_name("qmm_transposed_bf16_g128_b4_64x128")]] [[kernel]] void+qmm_transposed<bfloat, 128, 4, true, 64, 32, 128>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -3487,11 +4118,11 @@ template [[host_name("qmm_transposed_bf16_g128_b4_64x128")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);-+ uint simd_lid [[thread_index_in_simdgroup]]+);-template [[host_name("qmm_transposed_bf16_g128_b4_128x64")]]-[[kernel]] void qmm_transposed<bfloat, 128, 4, true, 128, 32, 64>(+template [[host_name("qmm_transposed_bf16_g128_b4_128x64")]] [[kernel]] void+qmm_transposed<bfloat, 128, 4, true, 128, 32, 64>(const device uint32_t* w [[buffer(0)]],const device bfloat* scales [[buffer(1)]],const device uint8_t* zero_points [[buffer(2)]],@@ -3504,4 +4135,5 @@ template [[host_name("qmm_transposed_bf16_g128_b4_128x64")]]uint3 tid [[threadgroup_position_in_grid]],uint lid [[thread_index_in_threadgroup]],uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]);+ uint simd_lid [[thread_index_in_simdgroup]]+);diff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metalindex 30e0b7573..060fb09d8 100644--- a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal+++ b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal@@ -8,7 +8,7 @@ using namespace metal;#define GRAIN_SIZE 4// Main template kernel - single implementation for all types-template<typename InputT, typename ScaleT, typename OutputT, typename AccumT>+template <typename InputT, typename ScaleT, typename OutputT, typename AccumT>void rms_norm_core(const device InputT* input_data,const device ScaleT* scales_data,@@ -20,279 +20,299 @@ void rms_norm_core(uint thread_in_row,bool full_layer) {- AccumT partial_sum = static_cast<AccumT>(0.0f);+ AccumT partial_sum = static_cast<AccumT>(0.0f);- // Compute thread local partial sum- for (uint base_i = thread_in_row * GRAIN_SIZE; base_i < element_count; base_i += BLOCK_SIZE * GRAIN_SIZE) {- AccumT vals[GRAIN_SIZE];-- for (uint j = 0; j < GRAIN_SIZE; ++j) {- uint i = base_i + j;- vals[j] = (i < element_count) ? static_cast<AccumT>(input_data[i]) : 0.0f;- }-- for (uint j = 0; j < GRAIN_SIZE; ++j) {- partial_sum += vals[j] * vals[j];- }+ // Compute thread local partial sum+ for (uint base_i = thread_in_row * GRAIN_SIZE; base_i < element_count;+ base_i += BLOCK_SIZE * GRAIN_SIZE) {+ AccumT vals[GRAIN_SIZE];++ for (uint j = 0; j < GRAIN_SIZE; ++j) {+ uint i = base_i + j;+ vals[j] = (i < element_count) ? static_cast<AccumT>(input_data[i]) : 0.0f;+ }++ for (uint j = 0; j < GRAIN_SIZE; ++j) {+ partial_sum += vals[j] * vals[j];}+ }++ // Compute total sum across threadgroup+ AccumT total_sum = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(+ partial_sum,+ shared_sum,+ thread_in_row+ );++ // Compute RMS norm factor+ AccumT mean_square =+ static_cast<AccumT>(total_sum) / static_cast<AccumT>(element_count);+ AccumT rms_norm = rsqrt(mean_square + static_cast<AccumT>(epsilon));- // Compute total sum across threadgroup- AccumT total_sum = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(partial_sum, shared_sum, thread_in_row);+ // Apply normalization and scaling using the same vectorized pattern+ for (uint base_i = thread_in_row * GRAIN_SIZE; base_i < element_count;+ base_i += BLOCK_SIZE * GRAIN_SIZE) {+ AccumT vals[GRAIN_SIZE];+ AccumT scaled_vals[GRAIN_SIZE];- // Compute RMS norm factor- AccumT mean_square = static_cast<AccumT>(total_sum) / static_cast<AccumT>(element_count);- AccumT rms_norm = rsqrt(mean_square + static_cast<AccumT>(epsilon));+ // Load GRAIN_SIZE input elements+ for (uint j = 0; j < GRAIN_SIZE; ++j) {+ uint i = base_i + j;+ vals[j] = (i < element_count) ? static_cast<AccumT>(input_data[i]) : 0.0f;+ }++ // Process GRAIN_SIZE elements: normalize and scale+ for (uint j = 0; j < GRAIN_SIZE; ++j) {+ uint i = base_i + j;+ if (i >= element_count) {+ continue;+ }- // Apply normalization and scaling using the same vectorized pattern- for (uint base_i = thread_in_row * GRAIN_SIZE; base_i < element_count; base_i += BLOCK_SIZE * GRAIN_SIZE) {- AccumT vals[GRAIN_SIZE];- AccumT scaled_vals[GRAIN_SIZE];-- // Load GRAIN_SIZE input elements- for (uint j = 0; j < GRAIN_SIZE; ++j) {- uint i = base_i + j;- vals[j] = (i < element_count) ? static_cast<AccumT>(input_data[i]) : 0.0f;- }-- // Process GRAIN_SIZE elements: normalize and scale- for (uint j = 0; j < GRAIN_SIZE; ++j) {- uint i = base_i + j;- if (i >= element_count) { continue; }+ AccumT normalized_high = vals[j] * rms_norm;- AccumT normalized_high = vals[j] * rms_norm;+ if (full_layer) {+ // Full-layer: keep everything in accumulation precision+ AccumT scale_value_high = static_cast<AccumT>(scales_data[i]) ++ static_cast<AccumT>(scale_offset);+ scaled_vals[j] = normalized_high * scale_value_high;+ } else {+ // Only-normalization: cast down for the scale multiply+ OutputT normalized_low = static_cast<OutputT>(normalized_high);+ OutputT scale_value_low = static_cast<OutputT>(+ static_cast<AccumT>(scales_data[i]) ++ static_cast<AccumT>(scale_offset)+ );+ OutputT product_low = normalized_low * scale_value_low;+ scaled_vals[j] = static_cast<AccumT>(product_low);+ }+ }- if (full_layer) {- // Full-layer: keep everything in accumulation precision- AccumT scale_value_high = static_cast<AccumT>(scales_data[i]) + static_cast<AccumT>(scale_offset);- scaled_vals[j] = normalized_high * scale_value_high;- } else {- // Only-normalization: cast down for the scale multiply- OutputT normalized_low = static_cast<OutputT>(normalized_high);- OutputT scale_value_low = static_cast<OutputT>(static_cast<AccumT>(scales_data[i]) + static_cast<AccumT>(scale_offset));- OutputT product_low = normalized_low * scale_value_low;- scaled_vals[j] = static_cast<AccumT>(product_low);- }- }-- // Store GRAIN_SIZE output elements- for (uint j = 0; j < GRAIN_SIZE; ++j) {- uint i = base_i + j;- if (i < element_count) {- output_data[i] = static_cast<OutputT>(scaled_vals[j]);- }- }+ // Store GRAIN_SIZE output elements+ for (uint j = 0; j < GRAIN_SIZE; ++j) {+ uint i = base_i + j;+ if (i < element_count) {+ output_data[i] = static_cast<OutputT>(scaled_vals[j]);+ }}+ }}// X-macro table defining all supported type combinations-// Format: (InputType, ScaleType, OutputType, AccumType, full_layer_mode, suffix)-#define FOREACH_RMS_COMBO(_) \- /* float combinations */ \- _(float, float, float, float, false, f32_f32_f32_f32_norm) \- _(float, float, float, float, true, f32_f32_f32_f32_full) \- _(float, float, half, float, false, f32_f32_f16_f32_norm) \- _(float, float, half, float, true, f32_f32_f16_f32_full) \- _(float, float, bfloat, float, false, f32_f32_bf16_f32_norm) \- _(float, float, bfloat, float, true, f32_f32_bf16_f32_full) \- _(float, half, float, float, false, f32_f16_f32_f32_norm) \- _(float, half, float, float, true, f32_f16_f32_f32_full) \- _(float, half, half, float, false, f32_f16_f16_f32_norm) \- _(float, half, half, float, true, f32_f16_f16_f32_full) \- _(float, half, bfloat, float, false, f32_f16_bf16_f32_norm) \- _(float, half, bfloat, float, true, f32_f16_bf16_f32_full) \- _(float, bfloat, float, float, false, f32_bf16_f32_f32_norm) \- _(float, bfloat, float, float, true, f32_bf16_f32_f32_full) \- _(float, bfloat, half, float, false, f32_bf16_f16_f32_norm) \- _(float, bfloat, half, float, true, f32_bf16_f16_f32_full) \- _(float, bfloat, bfloat, float, false, f32_bf16_bf16_f32_norm) \- _(float, bfloat, bfloat, float, true, f32_bf16_bf16_f32_full) \- \- /* half combinations */ \- _(half, float, float, float, false, f16_f32_f32_f32_norm) \- _(half, float, float, float, true, f16_f32_f32_f32_full) \- _(half, float, half, float, false, f16_f32_f16_f32_norm) \- _(half, float, half, float, true, f16_f32_f16_f32_full) \- _(half, float, bfloat, float, false, f16_f32_bf16_f32_norm) \- _(half, float, bfloat, float, true, f16_f32_bf16_f32_full) \- _(half, half, float, float, false, f16_f16_f32_f32_norm) \- _(half, half, float, float, true, f16_f16_f32_f32_full) \- _(half, half, half, float, false, f16_f16_f16_f32_norm) \- _(half, half, half, float, true, f16_f16_f16_f32_full) \- _(half, half, bfloat, float, false, f16_f16_bf16_f32_norm) \- _(half, half, bfloat, float, true, f16_f16_bf16_f32_full) \- _(half, bfloat, float, float, false, f16_bf16_f32_f32_norm) \- _(half, bfloat, float, float, true, f16_bf16_f32_f32_full) \- _(half, bfloat, half, float, false, f16_bf16_f16_f32_norm) \- _(half, bfloat, half, float, true, f16_bf16_f16_f32_full) \- _(half, bfloat, bfloat, float, false, f16_bf16_bf16_f32_norm) \- _(half, bfloat, bfloat, float, true, f16_bf16_bf16_f32_full) \- \- /* bfloat combinations */ \- _(bfloat, float, float, float, false, bf16_f32_f32_f32_norm) \- _(bfloat, float, float, float, true, bf16_f32_f32_f32_full) \- _(bfloat, float, half, float, false, bf16_f32_f16_f32_norm) \- _(bfloat, float, half, float, true, bf16_f32_f16_f32_full) \- _(bfloat, float, bfloat, float, false, bf16_f32_bf16_f32_norm) \- _(bfloat, float, bfloat, float, true, bf16_f32_bf16_f32_full) \- _(bfloat, half, float, float, false, bf16_f16_f32_f32_norm) \- _(bfloat, half, float, float, true, bf16_f16_f32_f32_full) \- _(bfloat, half, half, float, false, bf16_f16_f16_f32_norm) \- _(bfloat, half, half, float, true, bf16_f16_f16_f32_full) \- _(bfloat, half, bfloat, float, false, bf16_f16_bf16_f32_norm) \- _(bfloat, half, bfloat, float, true, bf16_f16_bf16_f32_full) \- _(bfloat, bfloat, float, float, false, bf16_bf16_f32_f32_norm) \- _(bfloat, bfloat, float, float, true, bf16_bf16_f32_f32_full) \- _(bfloat, bfloat, half, float, false, bf16_bf16_f16_f32_norm) \- _(bfloat, bfloat, half, float, true, bf16_bf16_f16_f32_full) \- _(bfloat, bfloat, bfloat, float, false, bf16_bf16_bf16_f32_norm) \- _(bfloat, bfloat, bfloat, float, true, bf16_bf16_bf16_f32_full) \- \- /* All combinations with half accumulation */ \- _(float, float, float, half, false, f32_f32_f32_f16_norm) \- _(float, float, float, half, true, f32_f32_f32_f16_full) \- _(float, float, half, half, false, f32_f32_f16_f16_norm) \- _(float, float, half, half, true, f32_f32_f16_f16_full) \- _(float, float, bfloat, half, false, f32_f32_bf16_f16_norm) \- _(float, float, bfloat, half, true, f32_f32_bf16_f16_full) \- _(float, half, float, half, false, f32_f16_f32_f16_norm) \- _(float, half, float, half, true, f32_f16_f32_f16_full) \- _(float, half, half, half, false, f32_f16_f16_f16_norm) \- _(float, half, half, half, true, f32_f16_f16_f16_full) \- _(float, half, bfloat, half, false, f32_f16_bf16_f16_norm) \- _(float, half, bfloat, half, true, f32_f16_bf16_f16_full) \- _(float, bfloat, float, half, false, f32_bf16_f32_f16_norm) \- _(float, bfloat, float, half, true, f32_bf16_f32_f16_full) \- _(float, bfloat, half, half, false, f32_bf16_f16_f16_norm) \- _(float, bfloat, half, half, true, f32_bf16_f16_f16_full) \- _(float, bfloat, bfloat, half, false, f32_bf16_bf16_f16_norm) \- _(float, bfloat, bfloat, half, true, f32_bf16_bf16_f16_full) \- \- _(half, float, float, half, false, f16_f32_f32_f16_norm) \- _(half, float, float, half, true, f16_f32_f32_f16_full) \- _(half, float, half, half, false, f16_f32_f16_f16_norm) \- _(half, float, half, half, true, f16_f32_f16_f16_full) \- _(half, float, bfloat, half, false, f16_f32_bf16_f16_norm) \- _(half, float, bfloat, half, true, f16_f32_bf16_f16_full) \- _(half, half, float, half, false, f16_f16_f32_f16_norm) \- _(half, half, float, half, true, f16_f16_f32_f16_full) \- _(half, half, half, half, false, f16_f16_f16_f16_norm) \- _(half, half, half, half, true, f16_f16_f16_f16_full) \- _(half, half, bfloat, half, false, f16_f16_bf16_f16_norm) \- _(half, half, bfloat, half, true, f16_f16_bf16_f16_full) \- _(half, bfloat, float, half, false, f16_bf16_f32_f16_norm) \- _(half, bfloat, float, half, true, f16_bf16_f32_f16_full) \- _(half, bfloat, half, half, false, f16_bf16_f16_f16_norm) \- _(half, bfloat, half, half, true, f16_bf16_f16_f16_full) \- _(half, bfloat, bfloat, half, false, f16_bf16_bf16_f16_norm) \- _(half, bfloat, bfloat, half, true, f16_bf16_bf16_f16_full) \- \- _(bfloat, float, float, half, false, bf16_f32_f32_f16_norm) \- _(bfloat, float, float, half, true, bf16_f32_f32_f16_full) \- _(bfloat, float, half, half, false, bf16_f32_f16_f16_norm) \- _(bfloat, float, half, half, true, bf16_f32_f16_f16_full) \- _(bfloat, float, bfloat, half, false, bf16_f32_bf16_f16_norm) \- _(bfloat, float, bfloat, half, true, bf16_f32_bf16_f16_full) \- _(bfloat, half, float, half, false, bf16_f16_f32_f16_norm) \- _(bfloat, half, float, half, true, bf16_f16_f32_f16_full) \- _(bfloat, half, half, half, false, bf16_f16_f16_f16_norm) \- _(bfloat, half, half, half, true, bf16_f16_f16_f16_full) \- _(bfloat, half, bfloat, half, false, bf16_f16_bf16_f16_norm) \- _(bfloat, half, bfloat, half, true, bf16_f16_bf16_f16_full) \- _(bfloat, bfloat, float, half, false, bf16_bf16_f32_f16_norm) \- _(bfloat, bfloat, float, half, true, bf16_bf16_f32_f16_full) \- _(bfloat, bfloat, half, half, false, bf16_bf16_f16_f16_norm) \- _(bfloat, bfloat, half, half, true, bf16_bf16_f16_f16_full) \- _(bfloat, bfloat, bfloat, half, false, bf16_bf16_bf16_f16_norm) \- _(bfloat, bfloat, bfloat, half, true, bf16_bf16_bf16_f16_full)+// Format: (InputType, ScaleType, OutputType, AccumType, full_layer_mode,+// suffix)+#define FOREACH_RMS_COMBO(_) \+ /* float combinations */ \+ _(float, float, float, float, false, f32_f32_f32_f32_norm) \+ _(float, float, float, float, true, f32_f32_f32_f32_full) \+ _(float, float, half, float, false, f32_f32_f16_f32_norm) \+ _(float, float, half, float, true, f32_f32_f16_f32_full) \+ _(float, float, bfloat, float, false, f32_f32_bf16_f32_norm) \+ _(float, float, bfloat, float, true, f32_f32_bf16_f32_full) \+ _(float, half, float, float, false, f32_f16_f32_f32_norm) \+ _(float, half, float, float, true, f32_f16_f32_f32_full) \+ _(float, half, half, float, false, f32_f16_f16_f32_norm) \+ _(float, half, half, float, true, f32_f16_f16_f32_full) \+ _(float, half, bfloat, float, false, f32_f16_bf16_f32_norm) \+ _(float, half, bfloat, float, true, f32_f16_bf16_f32_full) \+ _(float, bfloat, float, float, false, f32_bf16_f32_f32_norm) \+ _(float, bfloat, float, float, true, f32_bf16_f32_f32_full) \+ _(float, bfloat, half, float, false, f32_bf16_f16_f32_norm) \+ _(float, bfloat, half, float, true, f32_bf16_f16_f32_full) \+ _(float, bfloat, bfloat, float, false, f32_bf16_bf16_f32_norm) \+ _(float, bfloat, bfloat, float, true, f32_bf16_bf16_f32_full) \+ \+ /* half combinations */ \+ _(half, float, float, float, false, f16_f32_f32_f32_norm) \+ _(half, float, float, float, true, f16_f32_f32_f32_full) \+ _(half, float, half, float, false, f16_f32_f16_f32_norm) \+ _(half, float, half, float, true, f16_f32_f16_f32_full) \+ _(half, float, bfloat, float, false, f16_f32_bf16_f32_norm) \+ _(half, float, bfloat, float, true, f16_f32_bf16_f32_full) \+ _(half, half, float, float, false, f16_f16_f32_f32_norm) \+ _(half, half, float, float, true, f16_f16_f32_f32_full) \+ _(half, half, half, float, false, f16_f16_f16_f32_norm) \+ _(half, half, half, float, true, f16_f16_f16_f32_full) \+ _(half, half, bfloat, float, false, f16_f16_bf16_f32_norm) \+ _(half, half, bfloat, float, true, f16_f16_bf16_f32_full) \+ _(half, bfloat, float, float, false, f16_bf16_f32_f32_norm) \+ _(half, bfloat, float, float, true, f16_bf16_f32_f32_full) \+ _(half, bfloat, half, float, false, f16_bf16_f16_f32_norm) \+ _(half, bfloat, half, float, true, f16_bf16_f16_f32_full) \+ _(half, bfloat, bfloat, float, false, f16_bf16_bf16_f32_norm) \+ _(half, bfloat, bfloat, float, true, f16_bf16_bf16_f32_full) \+ \+ /* bfloat combinations */ \+ _(bfloat, float, float, float, false, bf16_f32_f32_f32_norm) \+ _(bfloat, float, float, float, true, bf16_f32_f32_f32_full) \+ _(bfloat, float, half, float, false, bf16_f32_f16_f32_norm) \+ _(bfloat, float, half, float, true, bf16_f32_f16_f32_full) \+ _(bfloat, float, bfloat, float, false, bf16_f32_bf16_f32_norm) \+ _(bfloat, float, bfloat, float, true, bf16_f32_bf16_f32_full) \+ _(bfloat, half, float, float, false, bf16_f16_f32_f32_norm) \+ _(bfloat, half, float, float, true, bf16_f16_f32_f32_full) \+ _(bfloat, half, half, float, false, bf16_f16_f16_f32_norm) \+ _(bfloat, half, half, float, true, bf16_f16_f16_f32_full) \+ _(bfloat, half, bfloat, float, false, bf16_f16_bf16_f32_norm) \+ _(bfloat, half, bfloat, float, true, bf16_f16_bf16_f32_full) \+ _(bfloat, bfloat, float, float, false, bf16_bf16_f32_f32_norm) \+ _(bfloat, bfloat, float, float, true, bf16_bf16_f32_f32_full) \+ _(bfloat, bfloat, half, float, false, bf16_bf16_f16_f32_norm) \+ _(bfloat, bfloat, half, float, true, bf16_bf16_f16_f32_full) \+ _(bfloat, bfloat, bfloat, float, false, bf16_bf16_bf16_f32_norm) \+ _(bfloat, bfloat, bfloat, float, true, bf16_bf16_bf16_f32_full) \+ \+ /* All combinations with half accumulation */ \+ _(float, float, float, half, false, f32_f32_f32_f16_norm) \+ _(float, float, float, half, true, f32_f32_f32_f16_full) \+ _(float, float, half, half, false, f32_f32_f16_f16_norm) \+ _(float, float, half, half, true, f32_f32_f16_f16_full) \+ _(float, float, bfloat, half, false, f32_f32_bf16_f16_norm) \+ _(float, float, bfloat, half, true, f32_f32_bf16_f16_full) \+ _(float, half, float, half, false, f32_f16_f32_f16_norm) \+ _(float, half, float, half, true, f32_f16_f32_f16_full) \+ _(float, half, half, half, false, f32_f16_f16_f16_norm) \+ _(float, half, half, half, true, f32_f16_f16_f16_full) \+ _(float, half, bfloat, half, false, f32_f16_bf16_f16_norm) \+ _(float, half, bfloat, half, true, f32_f16_bf16_f16_full) \+ _(float, bfloat, float, half, false, f32_bf16_f32_f16_norm) \+ _(float, bfloat, float, half, true, f32_bf16_f32_f16_full) \+ _(float, bfloat, half, half, false, f32_bf16_f16_f16_norm) \+ _(float, bfloat, half, half, true, f32_bf16_f16_f16_full) \+ _(float, bfloat, bfloat, half, false, f32_bf16_bf16_f16_norm) \+ _(float, bfloat, bfloat, half, true, f32_bf16_bf16_f16_full) \+ \+ _(half, float, float, half, false, f16_f32_f32_f16_norm) \+ _(half, float, float, half, true, f16_f32_f32_f16_full) \+ _(half, float, half, half, false, f16_f32_f16_f16_norm) \+ _(half, float, half, half, true, f16_f32_f16_f16_full) \+ _(half, float, bfloat, half, false, f16_f32_bf16_f16_norm) \+ _(half, float, bfloat, half, true, f16_f32_bf16_f16_full) \+ _(half, half, float, half, false, f16_f16_f32_f16_norm) \+ _(half, half, float, half, true, f16_f16_f32_f16_full) \+ _(half, half, half, half, false, f16_f16_f16_f16_norm) \+ _(half, half, half, half, true, f16_f16_f16_f16_full) \+ _(half, half, bfloat, half, false, f16_f16_bf16_f16_norm) \+ _(half, half, bfloat, half, true, f16_f16_bf16_f16_full) \+ _(half, bfloat, float, half, false, f16_bf16_f32_f16_norm) \+ _(half, bfloat, float, half, true, f16_bf16_f32_f16_full) \+ _(half, bfloat, half, half, false, f16_bf16_f16_f16_norm) \+ _(half, bfloat, half, half, true, f16_bf16_f16_f16_full) \+ _(half, bfloat, bfloat, half, false, f16_bf16_bf16_f16_norm) \+ _(half, bfloat, bfloat, half, true, f16_bf16_bf16_f16_full) \+ \+ _(bfloat, float, float, half, false, bf16_f32_f32_f16_norm) \+ _(bfloat, float, float, half, true, bf16_f32_f32_f16_full) \+ _(bfloat, float, half, half, false, bf16_f32_f16_f16_norm) \+ _(bfloat, float, half, half, true, bf16_f32_f16_f16_full) \+ _(bfloat, float, bfloat, half, false, bf16_f32_bf16_f16_norm) \+ _(bfloat, float, bfloat, half, true, bf16_f32_bf16_f16_full) \+ _(bfloat, half, float, half, false, bf16_f16_f32_f16_norm) \+ _(bfloat, half, float, half, true, bf16_f16_f32_f16_full) \+ _(bfloat, half, half, half, false, bf16_f16_f16_f16_norm) \+ _(bfloat, half, half, half, true, bf16_f16_f16_f16_full) \+ _(bfloat, half, bfloat, half, false, bf16_f16_bf16_f16_norm) \+ _(bfloat, half, bfloat, half, true, bf16_f16_bf16_f16_full) \+ _(bfloat, bfloat, float, half, false, bf16_bf16_f32_f16_norm) \+ _(bfloat, bfloat, float, half, true, bf16_bf16_f32_f16_full) \+ _(bfloat, bfloat, half, half, false, bf16_bf16_f16_f16_norm) \+ _(bfloat, bfloat, half, half, true, bf16_bf16_f16_f16_full) \+ _(bfloat, bfloat, bfloat, half, false, bf16_bf16_bf16_f16_norm) \+ _(bfloat, bfloat, bfloat, half, true, bf16_bf16_bf16_f16_full)// Generate RMS norm kernels-#define DEFINE_RMS_KERNEL(IN, SC, OUT, ACC, FULL_LAYER, SUF) \-[[max_total_threads_per_threadgroup(1024)]] kernel void rms_norm_##SUF( \- const device IN* input [[buffer(0)]], \- const device SC* scales [[buffer(1)]], \- device OUT* output [[buffer(2)]], \- constant uint& batch_size [[buffer(3)]], \- constant uint& model_dim [[buffer(4)]], \- constant float& epsilon [[buffer(5)]], \- constant float& scale_offset [[buffer(6)]], \- uint batch_idx [[threadgroup_position_in_grid]], \- uint thread_in_row [[thread_position_in_threadgroup]] \-) { \- if (batch_idx >= batch_size) return; \- \- threadgroup ACC shared_sum[SIMD_SIZE]; \- const uint input_offset = batch_idx * model_dim; \- \- rms_norm_core<IN, SC, OUT, ACC>( \- input + input_offset, \- scales, \- output + input_offset, \- model_dim, \- epsilon, \- scale_offset, \- shared_sum, \- thread_in_row, \- FULL_LAYER \- ); \-}+#define DEFINE_RMS_KERNEL(IN, SC, OUT, ACC, FULL_LAYER, SUF) \+ [[max_total_threads_per_threadgroup(1024)]] kernel void rms_norm_##SUF( \+ const device IN* input [[buffer(0)]], \+ const device SC* scales [[buffer(1)]], \+ device OUT* output [[buffer(2)]], \+ constant uint& batch_size [[buffer(3)]], \+ constant uint& model_dim [[buffer(4)]], \+ constant float& epsilon [[buffer(5)]], \+ constant float& scale_offset [[buffer(6)]], \+ uint batch_idx [[threadgroup_position_in_grid]], \+ uint thread_in_row [[thread_position_in_threadgroup]] \+ ) { \+ if (batch_idx >= batch_size) \+ return; \+ \+ threadgroup ACC shared_sum[SIMD_SIZE]; \+ const uint input_offset = batch_idx * model_dim; \+ \+ rms_norm_core<IN, SC, OUT, ACC>( \+ input + input_offset, \+ scales, \+ output + input_offset, \+ model_dim, \+ epsilon, \+ scale_offset, \+ shared_sum, \+ thread_in_row, \+ FULL_LAYER \+ ); \+ }FOREACH_RMS_COMBO(DEFINE_RMS_KERNEL)#undef DEFINE_RMS_KERNEL// Generate QK norm kernels-#define DEFINE_QK_KERNEL(IN, SC, OUT, ACC, FULL_LAYER, SUF) \-[[max_total_threads_per_threadgroup(1024)]] kernel void qk_norm_##SUF( \- const device IN* qkv_input [[buffer(0)]], \- const device SC* scales [[buffer(1)]], \- device OUT* qkv_output [[buffer(2)]], \- constant uint& batch_size [[buffer(3)]], \- constant uint& num_q_heads [[buffer(4)]], \- constant uint& num_kv_heads [[buffer(5)]], \- constant uint& head_dim [[buffer(6)]], \- constant float& epsilon [[buffer(7)]], \- constant float& scale_offset [[buffer(8)]], \- constant uint& active_type_mask [[buffer(9)]], \- uint3 threadgroup_position_in_grid [[threadgroup_position_in_grid]], \- uint3 thread_position_in_threadgroup [[thread_position_in_threadgroup]] \-) { \- uint batch_idx = threadgroup_position_in_grid.x; \- uint sequential_head_idx = threadgroup_position_in_grid.y; \- uint thread_in_threadgroup = thread_position_in_threadgroup.x; \- \- if (batch_idx >= batch_size) return; \- \- uint qk_type, actual_head_idx, logical_head_idx; \- if (sequential_head_idx < num_q_heads) { \- qk_type = 0; \- actual_head_idx = sequential_head_idx; \- logical_head_idx = actual_head_idx; \- } else { \- qk_type = 1; \- actual_head_idx = sequential_head_idx - num_q_heads; \- logical_head_idx = num_q_heads + actual_head_idx; \- } \- \- if ((active_type_mask & (1 << qk_type)) == 0) return; \- if (qk_type == 0 && actual_head_idx >= num_q_heads) return; \- if (qk_type == 1 && actual_head_idx >= num_kv_heads) return; \- \- const uint total_heads_in_buffer = num_q_heads + 2 * num_kv_heads; \- const uint slice_offset = batch_idx * total_heads_in_buffer * head_dim + logical_head_idx * head_dim; \- \- threadgroup ACC shared_sum[SIMD_SIZE]; \- \- rms_norm_core<IN, SC, OUT, ACC>( \- qkv_input + slice_offset, \- scales, \- qkv_output + slice_offset, \- head_dim, \- epsilon, \- scale_offset, \- shared_sum, \- thread_in_threadgroup, \- FULL_LAYER \- ); \-}+#define DEFINE_QK_KERNEL(IN, SC, OUT, ACC, FULL_LAYER, SUF) \+ [[max_total_threads_per_threadgroup(1024)]] kernel void qk_norm_##SUF( \+ const device IN* qkv_input [[buffer(0)]], \+ const device SC* scales [[buffer(1)]], \+ device OUT* qkv_output [[buffer(2)]], \+ constant uint& batch_size [[buffer(3)]], \+ constant uint& num_q_heads [[buffer(4)]], \+ constant uint& num_kv_heads [[buffer(5)]], \+ constant uint& head_dim [[buffer(6)]], \+ constant float& epsilon [[buffer(7)]], \+ constant float& scale_offset [[buffer(8)]], \+ constant uint& active_type_mask [[buffer(9)]], \+ uint3 threadgroup_position_in_grid [[threadgroup_position_in_grid]], \+ uint3 thread_position_in_threadgroup [[thread_position_in_threadgroup]] \+ ) { \+ uint batch_idx = threadgroup_position_in_grid.x; \+ uint sequential_head_idx = threadgroup_position_in_grid.y; \+ uint thread_in_threadgroup = thread_position_in_threadgroup.x; \+ \+ if (batch_idx >= batch_size) \+ return; \+ \+ uint qk_type, actual_head_idx, logical_head_idx; \+ if (sequential_head_idx < num_q_heads) { \+ qk_type = 0; \+ actual_head_idx = sequential_head_idx; \+ logical_head_idx = actual_head_idx; \+ } else { \+ qk_type = 1; \+ actual_head_idx = sequential_head_idx - num_q_heads; \+ logical_head_idx = num_q_heads + actual_head_idx; \+ } \+ \+ if ((active_type_mask & (1 << qk_type)) == 0) \+ return; \+ if (qk_type == 0 && actual_head_idx >= num_q_heads) \+ return; \+ if (qk_type == 1 && actual_head_idx >= num_kv_heads) \+ return; \+ \+ const uint total_heads_in_buffer = num_q_heads + 2 * num_kv_heads; \+ const uint slice_offset = batch_idx * total_heads_in_buffer * head_dim + \+ logical_head_idx * head_dim; \+ \+ threadgroup ACC shared_sum[SIMD_SIZE]; \+ \+ rms_norm_core<IN, SC, OUT, ACC>( \+ qkv_input + slice_offset, \+ scales, \+ qkv_output + slice_offset, \+ head_dim, \+ epsilon, \+ scale_offset, \+ shared_sum, \+ thread_in_threadgroup, \+ FULL_LAYER \+ ); \+ }FOREACH_RMS_COMBO(DEFINE_QK_KERNEL)-#undef DEFINE_QK_KERNEL+#undef DEFINE_QK_KERNELdiff --git a/crates/uzu/src/backends/metal/kernel/rng.metal b/crates/uzu/src/backends/metal/kernel/rng.metalindex 565a901df..97979277f 100644--- a/crates/uzu/src/backends/metal/kernel/rng.metal+++ b/crates/uzu/src/backends/metal/kernel/rng.metal@@ -6,105 +6,125 @@ using namespace metal;/* ───────────── Philox4x32-10 RNG ───────────── */-#define PHILOX_W32_0 0x9E3779B9u-#define PHILOX_W32_1 0xBB67AE85u+#define PHILOX_W32_0 0x9E3779B9u+#define PHILOX_W32_1 0xBB67AE85u#define PHILOX_M4x32_0 0xD2511F53u#define PHILOX_M4x32_1 0xCD9E8D57ustruct PhiloxState {- uint key[2];- uint ctr[4];- int state_idx;- uint output[4];+ uint key[2];+ uint ctr[4];+ int state_idx;+ uint output[4];};inline uint mulhilo32(uint a, uint b, thread uint* hip) {- uint64_t product = (uint64_t(a)) * (uint64_t(b));- *hip = uint(product >> 32);- return uint(product);+ uint64_t product = (uint64_t(a)) * (uint64_t(b));+ *hip = uint(product >> 32);+ return uint(product);}inline void philox4x32round(thread uint* ctr, thread uint* key) {- uint hi0, hi1;- uint lo0 = mulhilo32(PHILOX_M4x32_0, ctr[0], &hi0);- uint lo1 = mulhilo32(PHILOX_M4x32_1, ctr[2], &hi1);-- uint new_ctr[4];- new_ctr[0] = hi1 ^ ctr[1] ^ key[0];- new_ctr[1] = lo1;- new_ctr[2] = hi0 ^ ctr[3] ^ key[1];- new_ctr[3] = lo0;-- for(int i = 0; i < 4; i++) {- ctr[i] = new_ctr[i];- }+ uint hi0, hi1;+ uint lo0 = mulhilo32(PHILOX_M4x32_0, ctr[0], &hi0);+ uint lo1 = mulhilo32(PHILOX_M4x32_1, ctr[2], &hi1);++ uint new_ctr[4];+ new_ctr[0] = hi1 ^ ctr[1] ^ key[0];+ new_ctr[1] = lo1;+ new_ctr[2] = hi0 ^ ctr[3] ^ key[1];+ new_ctr[3] = lo0;++ for (int i = 0; i < 4; i++) {+ ctr[i] = new_ctr[i];+ }}inline void philox4x32bumpkey(thread uint* key) {- key[0] += PHILOX_W32_0;- key[1] += PHILOX_W32_1;+ key[0] += PHILOX_W32_0;+ key[1] += PHILOX_W32_1;}-inline void curand_philox4x32_10(thread uint* ctr, thread uint* key, thread uint* out) {- uint local_ctr[4] = {ctr[0], ctr[1], ctr[2], ctr[3]};- uint local_key[2] = {key[0], key[1]};-- // 10 rounds- philox4x32round(local_ctr, local_key); // 1- philox4x32bumpkey(local_key); philox4x32round(local_ctr, local_key); // 2- philox4x32bumpkey(local_key); philox4x32round(local_ctr, local_key); // 3- philox4x32bumpkey(local_key); philox4x32round(local_ctr, local_key); // 4- philox4x32bumpkey(local_key); philox4x32round(local_ctr, local_key); // 5- philox4x32bumpkey(local_key); philox4x32round(local_ctr, local_key); // 6- philox4x32bumpkey(local_key); philox4x32round(local_ctr, local_key); // 7- philox4x32bumpkey(local_key); philox4x32round(local_ctr, local_key); // 8- philox4x32bumpkey(local_key); philox4x32round(local_ctr, local_key); // 9- philox4x32bumpkey(local_key); philox4x32round(local_ctr, local_key); // 10-- for(int i = 0; i < 4; i++) {- out[i] = local_ctr[i];- }+inline void curand_philox4x32_10(+ thread uint* ctr,+ thread uint* key,+ thread uint* out+) {+ uint local_ctr[4] = {ctr[0], ctr[1], ctr[2], ctr[3]};+ uint local_key[2] = {key[0], key[1]};++ // 10 rounds+ philox4x32round(local_ctr, local_key); // 1+ philox4x32bumpkey(local_key);+ philox4x32round(local_ctr, local_key); // 2+ philox4x32bumpkey(local_key);+ philox4x32round(local_ctr, local_key); // 3+ philox4x32bumpkey(local_key);+ philox4x32round(local_ctr, local_key); // 4+ philox4x32bumpkey(local_key);+ philox4x32round(local_ctr, local_key); // 5+ philox4x32bumpkey(local_key);+ philox4x32round(local_ctr, local_key); // 6+ philox4x32bumpkey(local_key);+ philox4x32round(local_ctr, local_key); // 7+ philox4x32bumpkey(local_key);+ philox4x32round(local_ctr, local_key); // 8+ philox4x32bumpkey(local_key);+ philox4x32round(local_ctr, local_key); // 9+ philox4x32bumpkey(local_key);+ philox4x32round(local_ctr, local_key); // 10++ for (int i = 0; i < 4; i++) {+ out[i] = local_ctr[i];+ }}inline void philox_state_incr(thread PhiloxState* s) {- ++s->ctr[0];- if(s->ctr[0] != 0) return;- ++s->ctr[1];- if(s->ctr[1] != 0) return;- ++s->ctr[2];- if(s->ctr[2] != 0) return;- ++s->ctr[3];+ ++s->ctr[0];+ if (s->ctr[0] != 0)+ return;+ ++s->ctr[1];+ if (s->ctr[1] != 0)+ return;+ ++s->ctr[2];+ if (s->ctr[2] != 0)+ return;+ ++s->ctr[3];}-inline void philox_init(thread PhiloxState* state, uint64_t seed, uint64_t offset) {- // Initialize key from seed- state->key[0] = uint(seed);- state->key[1] = uint(seed >> 32);-- // Initialize counter from offset- state->ctr[0] = uint(offset);- state->ctr[1] = uint(offset >> 32);- state->ctr[2] = 0;- state->ctr[3] = 0;-- state->state_idx = 0;-- // Generate first batch of random numbers- curand_philox4x32_10(state->ctr, state->key, state->output);- philox_state_incr(state);+inline void philox_init(+ thread PhiloxState* state,+ uint64_t seed,+ uint64_t offset+) {+ // Initialize key from seed+ state->key[0] = uint(seed);+ state->key[1] = uint(seed >> 32);++ // Initialize counter from offset+ state->ctr[0] = uint(offset);+ state->ctr[1] = uint(offset >> 32);+ state->ctr[2] = 0;+ state->ctr[3] = 0;++ state->state_idx = 0;++ // Generate first batch of random numbers+ curand_philox4x32_10(state->ctr, state->key, state->output);+ philox_state_incr(state);}inline uint philox_next(thread PhiloxState* state) {- if(state->state_idx >= 4) {- curand_philox4x32_10(state->ctr, state->key, state->output);- philox_state_incr(state);- state->state_idx = 0;- }- return state->output[state->state_idx++];+ if (state->state_idx >= 4) {+ curand_philox4x32_10(state->ctr, state->key, state->output);+ philox_state_incr(state);+ state->state_idx = 0;+ }+ return state->output[state->state_idx++];}inline float uniform_float(thread PhiloxState* state) {- return float(philox_next(state)) * (1.0f/4294967296.0f); /* (0,1) */+ return float(philox_next(state)) * (1.0f / 4294967296.0f); /* (0,1) */}#endif /* rng_metal */diff --git a/crates/uzu/src/backends/metal/kernel/rope/rope.metal b/crates/uzu/src/backends/metal/kernel/rope/rope.metalindex 5d03ebb00..1f61c9b92 100644--- a/crates/uzu/src/backends/metal/kernel/rope/rope.metal+++ b/crates/uzu/src/backends/metal/kernel/rope/rope.metal@@ -11,89 +11,127 @@ inline T applyRopeTransform(T cosVal,T sinVal) {- T inputVal = qkvTensorView(tokenIdx, headIdx, dimIdx);- T pairedVal = (dimIdx < halfDim) ?- -qkvTensorView(tokenIdx, headIdx, dimIdx + halfDim) :- qkvTensorView(tokenIdx, headIdx, dimIdx - halfDim);- return inputVal * cosVal + pairedVal * sinVal;+ T inputVal = qkvTensorView(tokenIdx, headIdx, dimIdx);+ T pairedVal = (dimIdx < halfDim)+ ? -qkvTensorView(tokenIdx, headIdx, dimIdx + halfDim)+ : qkvTensorView(tokenIdx, headIdx, dimIdx - halfDim);+ return inputVal * cosVal + pairedVal * sinVal;}template <typename T>void applyRope(- device const T* qkv, // [suffix_len, (num_heads + 2*num_groups) * head_dim]- device const T* cosines, // [max_seq_len, head_dim]- device const T* sines, // [max_seq_len, head_dim]- device const int* tokenPositions, // [suffix_len] - actual token positions- device T* rotatedQueries, // [num_heads, suffix_len, head_dim]- device T* rotatedKeys, // [num_groups, suffix_len, head_dim]+ device const T* qkv, // [suffix_len, (num_heads + 2*num_groups) * head_dim]+ device const T* cosines, // [max_seq_len, head_dim]+ device const T* sines, // [max_seq_len, head_dim]+ device const int* tokenPositions, // [suffix_len] - actual token positions+ device T* rotatedQueries, // [num_heads, suffix_len, head_dim]+ device T* rotatedKeys, // [num_groups, suffix_len, head_dim]constant int& headDim,constant int& numHeads,constant int& numGroups,constant int& suffixLength,constant int& maxSequenceLength,- const uint3 position // x: headIdx, y: tokenIdx, z: dimIdx+ const uint3 position // x: headIdx, y: tokenIdx, z: dimIdx) {- const uint headIndex = position.x;- const uint tokenIndex = position.y;- const uint dimensionIndex = position.z;+ const uint headIndex = position.x;+ const uint tokenIndex = position.y;+ const uint dimensionIndex = position.z;- if (headIndex >= numHeads || tokenIndex >= suffixLength || dimensionIndex >= headDim) return;- if (headDim & 1) return; // head_dim must be even- if (numHeads % numGroups != 0) return;+ if (headIndex >= numHeads || tokenIndex >= suffixLength ||+ dimensionIndex >= headDim)+ return;+ if (headDim & 1)+ return; // head_dim must be even+ if (numHeads % numGroups != 0)+ return;- const uint groupIndex = headIndex / (numHeads / numGroups); // which KV group this head belongs to- const uint totalHeads = numHeads + 2 * numGroups;+ const uint groupIndex =+ headIndex / (numHeads / numGroups); // which KV group this head belongs to+ const uint totalHeads = numHeads + 2 * numGroups;- // Use actual token position from buffer- const uint rawPosition = tokenPositions[tokenIndex];- const uint absolutePosition = rawPosition > maxSequenceLength ? 0 : rawPosition;+ // Use actual token position from buffer+ const uint rawPosition = tokenPositions[tokenIndex];+ const uint absolutePosition =+ rawPosition > maxSequenceLength ? 0 : rawPosition;- const uint halfDimension = headDim / 2;+ const uint halfDimension = headDim / 2;- TensorView3D<const T> qkvTensorView = TensorView3D<const T>(qkv).shaped(suffixLength, totalHeads, headDim);- TensorView2D<const T> cosinesTensorView = TensorView2D<const T>(cosines).shaped(maxSequenceLength, headDim);- TensorView2D<const T> sinesTensorView = TensorView2D<const T>(sines).shaped(maxSequenceLength, headDim);- TensorView3D<T> rotatedQueriesTensorView = TensorView3D<T>(rotatedQueries).shaped(numHeads, suffixLength, headDim);- TensorView3D<T> rotatedKeysTensorView = TensorView3D<T>(rotatedKeys).shaped(numGroups, suffixLength, headDim);-- const T cosVal = cosinesTensorView(absolutePosition, dimensionIndex);- const T sinVal = sinesTensorView(absolutePosition, dimensionIndex);+ TensorView3D<const T> qkvTensorView =+ TensorView3D<const T>(qkv).shaped(suffixLength, totalHeads, headDim);+ TensorView2D<const T> cosinesTensorView =+ TensorView2D<const T>(cosines).shaped(maxSequenceLength, headDim);+ TensorView2D<const T> sinesTensorView =+ TensorView2D<const T>(sines).shaped(maxSequenceLength, headDim);+ TensorView3D<T> rotatedQueriesTensorView =+ TensorView3D<T>(rotatedQueries).shaped(numHeads, suffixLength, headDim);+ TensorView3D<T> rotatedKeysTensorView =+ TensorView3D<T>(rotatedKeys).shaped(numGroups, suffixLength, headDim);- /* ---------- QUERIES ---------- */- T queryResult = applyRopeTransform(qkvTensorView, tokenIndex, headIndex, dimensionIndex, halfDimension, cosVal, sinVal);- rotatedQueriesTensorView(headIndex, tokenIndex, dimensionIndex) = queryResult;+ const T cosVal = cosinesTensorView(absolutePosition, dimensionIndex);+ const T sinVal = sinesTensorView(absolutePosition, dimensionIndex);- /* ---------- KEYS & VALUES (only first head of each group processes) ---------- */- uint firstHeadInGroup = groupIndex * (numHeads / numGroups);- if (headIndex == firstHeadInGroup) {- /* ---------- keys ---------- */- uint keyHeadIndex = numHeads + groupIndex; // Keys start after all query heads- T keyResult = applyRopeTransform(qkvTensorView, tokenIndex, keyHeadIndex, dimensionIndex, halfDimension, cosVal, sinVal);- rotatedKeysTensorView(groupIndex, tokenIndex, dimensionIndex) = keyResult;- }+ /* ---------- QUERIES ---------- */+ T queryResult = applyRopeTransform(+ qkvTensorView,+ tokenIndex,+ headIndex,+ dimensionIndex,+ halfDimension,+ cosVal,+ sinVal+ );+ rotatedQueriesTensorView(headIndex, tokenIndex, dimensionIndex) = queryResult;++ /* ---------- KEYS & VALUES (only first head of each group processes)+ * ---------- */+ uint firstHeadInGroup = groupIndex * (numHeads / numGroups);+ if (headIndex == firstHeadInGroup) {+ /* ---------- keys ---------- */+ uint keyHeadIndex =+ numHeads + groupIndex; // Keys start after all query heads+ T keyResult = applyRopeTransform(+ qkvTensorView,+ tokenIndex,+ keyHeadIndex,+ dimensionIndex,+ halfDimension,+ cosVal,+ sinVal+ );+ rotatedKeysTensorView(groupIndex, tokenIndex, dimensionIndex) = keyResult;+ }}/* ---------- boiler-plate kernel wrappers ---------- */-#define outerArguments(T) \-(device const T* qkv [[ buffer(0) ]], \- device const T* cosines [[ buffer(1) ]], \- device const T* sines [[ buffer(2) ]], \- device const int* tokenPositions [[ buffer(3) ]], \- device T* rotatedQueries [[ buffer(4) ]], \- device T* rotatedKeys [[ buffer(5) ]], \- constant int& headDim [[ buffer(6) ]], \- constant int& numHeads [[ buffer(7) ]], \- constant int& numGroups [[ buffer(8) ]], \- constant int& suffixLength [[ buffer(9) ]], \- constant int& maxSequenceLength [[ buffer(10) ]], \- const uint3 position [[ thread_position_in_grid ]]) \+#define outerArguments(T) \+ (device const T* qkv [[buffer(0)]], \+ device const T* cosines [[buffer(1)]], \+ device const T* sines [[buffer(2)]], \+ device const int* tokenPositions [[buffer(3)]], \+ device T* rotatedQueries [[buffer(4)]], \+ device T* rotatedKeys [[buffer(5)]], \+ constant int& headDim [[buffer(6)]], \+ constant int& numHeads [[buffer(7)]], \+ constant int& numGroups [[buffer(8)]], \+ constant int& suffixLength [[buffer(9)]], \+ constant int& maxSequenceLength [[buffer(10)]], \+ const uint3 position [[thread_position_in_grid]])-#define innerArguments \-(qkv, cosines, sines, tokenPositions, rotatedQueries, rotatedKeys, \- headDim, numHeads, numGroups, suffixLength, maxSequenceLength, position) \+#define innerArguments \+ (qkv, \+ cosines, \+ sines, \+ tokenPositions, \+ rotatedQueries, \+ rotatedKeys, \+ headDim, \+ numHeads, \+ numGroups, \+ suffixLength, \+ maxSequenceLength, \+ position)generateKernels(applyRope)#undef outerArguments#undef innerArguments-diff --git a/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal b/crates/uzu/src/backends/metal/kernel/sampling/argmax.metalindex c3966c0c3..7cb44b155 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal+++ b/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal@@ -6,302 +6,379 @@#define GRAIN_SIZE 4struct ArgmaxPair {- float value;- uint index;+ float value;+ uint index;};constant ArgmaxPair ARGMAX_INIT = {-INFINITY, UINT_MAX};bool argmax_is_better(ArgmaxPair a, ArgmaxPair b) {- return a.value > b.value || (a.value == b.value && a.index < b.index);+ return a.value > b.value || (a.value == b.value && a.index < b.index);}ArgmaxPair argmax_combine(ArgmaxPair a, ArgmaxPair b) {- return argmax_is_better(a, b) ? a : b;+ return argmax_is_better(a, b) ? a : b;}//------------------------------------------------------------------------------------------------//// Cooperative threadgroup argmax reduction (2-level hierarchical)-template<ushort BLOCK_SIZE_PARAM>-static ArgmaxPair threadgroup_cooperative_argmax(ArgmaxPair value, threadgroup ArgmaxPair* shared, const ushort lid) {- const ushort simd_group_id = lid / 32;- const ushort simd_lane_id = lid % 32;-- // Reduce within simdgroup using manual shuffle operations- ArgmaxPair local_result = value;- for (ushort offset = 16; offset > 0; offset /= 2) {- ArgmaxPair other = {- simd_shuffle_down(local_result.value, offset),- simd_shuffle_down(local_result.index, offset)- };- local_result = argmax_combine(local_result, other);- }-- // First thread in each simdgroup writes to shared memory- if (simd_lane_id == 0) {- shared[simd_group_id] = local_result;- }-- // Synchronize across the threadgroup- threadgroup_barrier(mem_flags::mem_threadgroup);-- // Reduce across simdgroups- const ushort num_simd_groups = (BLOCK_SIZE_PARAM + 31) / 32;- ArgmaxPair total_result = ARGMAX_INIT;- if (lid < num_simd_groups) {- total_result = shared[lid];- }-- // Final simdgroup reduction- for (ushort offset = 16; offset > 0; offset /= 2) {- ArgmaxPair other = {- simd_shuffle_down(total_result.value, offset),- simd_shuffle_down(total_result.index, offset)- };- total_result = argmax_combine(total_result, other);- }-- // Broadcast the result to all threads- if (lid == 0) {- shared[0] = total_result;- }- threadgroup_barrier(mem_flags::mem_threadgroup);-- return shared[0];+template <ushort BLOCK_SIZE_PARAM>+static ArgmaxPair threadgroup_cooperative_argmax(+ ArgmaxPair value,+ threadgroup ArgmaxPair* shared,+ const ushort lid+) {+ const ushort simd_group_id = lid / 32;+ const ushort simd_lane_id = lid % 32;++ // Reduce within simdgroup using manual shuffle operations+ ArgmaxPair local_result = value;+ for (ushort offset = 16; offset > 0; offset /= 2) {+ ArgmaxPair other = {+ simd_shuffle_down(local_result.value, offset),+ simd_shuffle_down(local_result.index, offset)+ };+ local_result = argmax_combine(local_result, other);+ }++ // First thread in each simdgroup writes to shared memory+ if (simd_lane_id == 0) {+ shared[simd_group_id] = local_result;+ }++ // Synchronize across the threadgroup+ threadgroup_barrier(mem_flags::mem_threadgroup);++ // Reduce across simdgroups+ const ushort num_simd_groups = (BLOCK_SIZE_PARAM + 31) / 32;+ ArgmaxPair total_result = ARGMAX_INIT;+ if (lid < num_simd_groups) {+ total_result = shared[lid];+ }++ // Final simdgroup reduction+ for (ushort offset = 16; offset > 0; offset /= 2) {+ ArgmaxPair other = {+ simd_shuffle_down(total_result.value, offset),+ simd_shuffle_down(total_result.index, offset)+ };+ total_result = argmax_combine(total_result, other);+ }++ // Broadcast the result to all threads+ if (lid == 0) {+ shared[0] = total_result;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);++ return shared[0];}-template<ushort GRAIN_SIZE_PARAM>-static inline ArgmaxPair thread_argmax_reduce(thread ArgmaxPair (&values)[GRAIN_SIZE_PARAM]) {- ArgmaxPair result = values[0];- for (ushort i = 1; i < GRAIN_SIZE_PARAM; i++) {- result = argmax_combine(result, values[i]);- }- return result;+template <ushort GRAIN_SIZE_PARAM>+static inline ArgmaxPair thread_argmax_reduce(+ thread ArgmaxPair (&values)[GRAIN_SIZE_PARAM]+) {+ ArgmaxPair result = values[0];+ for (ushort i = 1; i < GRAIN_SIZE_PARAM; i++) {+ result = argmax_combine(result, values[i]);+ }+ return result;}-template<ushort GRAIN_SIZE_PARAM, typename T>-static void load_blocked_argmax_batched(thread ArgmaxPair (&values)[GRAIN_SIZE_PARAM],- const device T* logits_data,- const ushort local_id,- const uint batch_idx,- const uint vocab_offset,- const uint vocab_size) {- for (ushort i = 0; i < GRAIN_SIZE_PARAM; i++) {- uint vocab_idx = vocab_offset + local_id * GRAIN_SIZE_PARAM + i;- if (vocab_idx < vocab_size) {- uint global_idx = batch_idx * vocab_size + vocab_idx;- float value;- value = static_cast<float>(logits_data[global_idx]);- values[i] = {value, vocab_idx};- } else {- values[i] = ARGMAX_INIT;- }+template <ushort GRAIN_SIZE_PARAM, typename T>+static void load_blocked_argmax_batched(+ thread ArgmaxPair (&values)[GRAIN_SIZE_PARAM],+ const device T* logits_data,+ const ushort local_id,+ const uint batch_idx,+ const uint vocab_offset,+ const uint vocab_size+) {+ for (ushort i = 0; i < GRAIN_SIZE_PARAM; i++) {+ uint vocab_idx = vocab_offset + local_id * GRAIN_SIZE_PARAM + i;+ if (vocab_idx < vocab_size) {+ uint global_idx = batch_idx * vocab_size + vocab_idx;+ float value;+ value = static_cast<float>(logits_data[global_idx]);+ values[i] = {value, vocab_idx};+ } else {+ values[i] = ARGMAX_INIT;}+ }}-template<ushort BLOCK_SIZE_PARAM>-static ArgmaxPair threadgroup_raking_argmax(ArgmaxPair value, threadgroup ArgmaxPair* shared, const ushort lid) {- shared[lid] = value;- threadgroup_barrier(mem_flags::mem_threadgroup);-- if (lid < 32) {- const ushort values_per_thread = BLOCK_SIZE_PARAM / 32;- const ushort first_index = lid * values_per_thread;-- ArgmaxPair thread_result = shared[first_index];- for (ushort i = first_index + 1; i < first_index + values_per_thread; i++) {- thread_result = argmax_combine(thread_result, shared[i]);- }-- ArgmaxPair simd_result = thread_result;-- for (ushort offset = 16; offset > 0; offset /= 2) {- ArgmaxPair other = {- simd_shuffle_down(simd_result.value, offset),- simd_shuffle_down(simd_result.index, offset)- };- simd_result = argmax_combine(simd_result, other);- }-- ArgmaxPair final_result = {- simd_broadcast(simd_result.value, 0),- simd_broadcast(simd_result.index, 0)- };-- for (ushort i = first_index; i < first_index + values_per_thread; i++) {- shared[i] = final_result;- }+template <ushort BLOCK_SIZE_PARAM>+static ArgmaxPair threadgroup_raking_argmax(+ ArgmaxPair value,+ threadgroup ArgmaxPair* shared,+ const ushort lid+) {+ shared[lid] = value;+ threadgroup_barrier(mem_flags::mem_threadgroup);++ if (lid < 32) {+ const ushort values_per_thread = BLOCK_SIZE_PARAM / 32;+ const ushort first_index = lid * values_per_thread;++ ArgmaxPair thread_result = shared[first_index];+ for (ushort i = first_index + 1; i < first_index + values_per_thread; i++) {+ thread_result = argmax_combine(thread_result, shared[i]);}- threadgroup_barrier(mem_flags::mem_threadgroup);- return shared[lid];-}--template<typename T>-void batched_argmax_main_impl(const device T* logits_data,- device ArgmaxPair* partial_results,- constant uint& batch_size,- constant uint& vocab_size,- threadgroup ArgmaxPair* shared,- uint batch_idx,- uint vocab_group_idx,- ushort local_id) {-- if (batch_idx >= batch_size) return;-- const uint elements_per_group = BLOCK_SIZE * GRAIN_SIZE;- const uint vocab_offset = vocab_group_idx * elements_per_group;+ ArgmaxPair simd_result = thread_result;- ArgmaxPair values[GRAIN_SIZE];- load_blocked_argmax_batched<GRAIN_SIZE>(values, logits_data, local_id, batch_idx, vocab_offset, vocab_size);-- ArgmaxPair thread_result = thread_argmax_reduce<GRAIN_SIZE>(values);- ArgmaxPair group_result = threadgroup_cooperative_argmax<BLOCK_SIZE>(thread_result, shared, local_id);-- if (local_id == 0) {- uint vocab_groups_per_batch = (vocab_size + elements_per_group - 1) / elements_per_group;- uint partial_idx = batch_idx * vocab_groups_per_batch + vocab_group_idx;- partial_results[partial_idx] = group_result;+ for (ushort offset = 16; offset > 0; offset /= 2) {+ ArgmaxPair other = {+ simd_shuffle_down(simd_result.value, offset),+ simd_shuffle_down(simd_result.index, offset)+ };+ simd_result = argmax_combine(simd_result, other);}-}--void batched_argmax_final_impl(const device ArgmaxPair* partial_results,- device uint* final_tokens,- constant uint& batch_size,- constant uint& vocab_size,- threadgroup ArgmaxPair* shared,- uint batch_idx,- ushort local_id) {-- if (batch_idx >= batch_size) return;- const uint elements_per_group = BLOCK_SIZE * GRAIN_SIZE;- const uint vocab_groups_per_batch = (vocab_size + elements_per_group - 1) / elements_per_group;- const uint partial_offset = batch_idx * vocab_groups_per_batch;+ ArgmaxPair final_result = {+ simd_broadcast(simd_result.value, 0),+ simd_broadcast(simd_result.index, 0)+ };- ArgmaxPair thread_result = ARGMAX_INIT;-- for (uint i = local_id; i < vocab_groups_per_batch; i += BLOCK_SIZE) {- thread_result = argmax_combine(thread_result, partial_results[partial_offset + i]);+ for (ushort i = first_index; i < first_index + values_per_thread; i++) {+ shared[i] = final_result;}+ }- ArgmaxPair result = threadgroup_cooperative_argmax<BLOCK_SIZE>(thread_result, shared, local_id);+ threadgroup_barrier(mem_flags::mem_threadgroup);+ return shared[lid];+}- if (local_id == 0) {- final_tokens[batch_idx] = result.index;- }+template <typename T>+void batched_argmax_main_impl(+ const device T* logits_data,+ device ArgmaxPair* partial_results,+ constant uint& batch_size,+ constant uint& vocab_size,+ threadgroup ArgmaxPair* shared,+ uint batch_idx,+ uint vocab_group_idx,+ ushort local_id+) {++ if (batch_idx >= batch_size)+ return;++ const uint elements_per_group = BLOCK_SIZE * GRAIN_SIZE;+ const uint vocab_offset = vocab_group_idx * elements_per_group;++ ArgmaxPair values[GRAIN_SIZE];+ load_blocked_argmax_batched<GRAIN_SIZE>(+ values,+ logits_data,+ local_id,+ batch_idx,+ vocab_offset,+ vocab_size+ );++ ArgmaxPair thread_result = thread_argmax_reduce<GRAIN_SIZE>(values);+ ArgmaxPair group_result = threadgroup_cooperative_argmax<BLOCK_SIZE>(+ thread_result,+ shared,+ local_id+ );++ if (local_id == 0) {+ uint vocab_groups_per_batch =+ (vocab_size + elements_per_group - 1) / elements_per_group;+ uint partial_idx = batch_idx * vocab_groups_per_batch + vocab_group_idx;+ partial_results[partial_idx] = group_result;+ }}-// Generate argmax main kernels using macro approach-#define outerArguments(T) \-(device const T* logits_data [[ buffer(0) ]], \-device ArgmaxPair* partial_results [[ buffer(1) ]], \-constant uint& batch_size [[ buffer(2) ]], \-constant uint& vocab_size [[ buffer(3) ]], \-uint2 group_position [[ threadgroup_position_in_grid ]], \-uint2 local_position [[ thread_position_in_threadgroup ]])--#define innerArguments \-(logits_data, partial_results, batch_size, vocab_size, shared, group_position.x, group_position.y, (ushort)local_position.x)--#define generateArgmaxMainKernel(functionName, T, outerArgs, innerArgs) \-[[max_total_threads_per_threadgroup(1024)]] kernel void functionName##_##T outerArgs { \- threadgroup ArgmaxPair shared[BLOCK_SIZE]; \- batched_argmax_main_impl<T> innerArgs; \+void batched_argmax_final_impl(+ const device ArgmaxPair* partial_results,+ device uint* final_tokens,+ constant uint& batch_size,+ constant uint& vocab_size,+ threadgroup ArgmaxPair* shared,+ uint batch_idx,+ ushort local_id+) {++ if (batch_idx >= batch_size)+ return;++ const uint elements_per_group = BLOCK_SIZE * GRAIN_SIZE;+ const uint vocab_groups_per_batch =+ (vocab_size + elements_per_group - 1) / elements_per_group;+ const uint partial_offset = batch_idx * vocab_groups_per_batch;++ ArgmaxPair thread_result = ARGMAX_INIT;++ for (uint i = local_id; i < vocab_groups_per_batch; i += BLOCK_SIZE) {+ thread_result =+ argmax_combine(thread_result, partial_results[partial_offset + i]);+ }++ ArgmaxPair result = threadgroup_cooperative_argmax<BLOCK_SIZE>(+ thread_result,+ shared,+ local_id+ );++ if (local_id == 0) {+ final_tokens[batch_idx] = result.index;+ }}-generateArgmaxMainKernel(batched_argmax_main, float, outerArguments(float), innerArguments)-generateArgmaxMainKernel(batched_argmax_main, half, outerArguments(half), innerArguments)-generateArgmaxMainKernel(batched_argmax_main, bfloat, outerArguments(bfloat), innerArguments)+// Generate argmax main kernels using macro approach+#define outerArguments(T) \+ (device const T* logits_data [[buffer(0)]], \+ device ArgmaxPair* partial_results [[buffer(1)]], \+ constant uint& batch_size [[buffer(2)]], \+ constant uint& vocab_size [[buffer(3)]], \+ uint2 group_position [[threadgroup_position_in_grid]], \+ uint2 local_position [[thread_position_in_threadgroup]])++#define innerArguments \+ (logits_data, \+ partial_results, \+ batch_size, \+ vocab_size, \+ shared, \+ group_position.x, \+ group_position.y, \+ (ushort)local_position.x)++#define generateArgmaxMainKernel(functionName, T, outerArgs, innerArgs) \+ [[max_total_threads_per_threadgroup( \+ 1024 \+ )]] kernel void functionName##_##T outerArgs { \+ threadgroup ArgmaxPair shared[BLOCK_SIZE]; \+ batched_argmax_main_impl<T> innerArgs; \+ }++generateArgmaxMainKernel(batched_argmax_main, float, outerArguments(float), innerArguments) generateArgmaxMainKernel(+ batched_argmax_main,+ half,+ outerArguments(half),+ innerArguments+) generateArgmaxMainKernel(batched_argmax_main, bfloat, outerArguments(bfloat), innerArguments)#undef outerArguments#undef innerArguments#undef generateArgmaxMainKernel-// Generate argmax final kernels using macro approach - note: no T parameter needed for final stage-#define outerArguments(T) \-(device const ArgmaxPair* partial_results [[ buffer(0) ]], \-device uint* final_tokens [[ buffer(1) ]], \-constant uint& batch_size [[ buffer(2) ]], \-constant uint& vocab_size [[ buffer(3) ]], \-uint batch_idx [[ threadgroup_position_in_grid ]], \-uint local_id [[ thread_position_in_threadgroup ]])--#define innerArguments \-(partial_results, final_tokens, batch_size, vocab_size, shared, batch_idx, (ushort)local_id)--#define generateArgmaxFinalKernel(functionName, T, outerArgs, innerArgs) \-[[max_total_threads_per_threadgroup(1024)]] kernel void functionName##_##T outerArgs { \- threadgroup ArgmaxPair shared[BLOCK_SIZE]; \- batched_argmax_final_impl innerArgs; \-}--generateArgmaxFinalKernel(batched_argmax_final, float, outerArguments(float), innerArguments)-generateArgmaxFinalKernel(batched_argmax_final, half, outerArguments(half), innerArguments)-generateArgmaxFinalKernel(batched_argmax_final, bfloat, outerArguments(bfloat), innerArguments)+// Generate argmax final kernels using macro approach - note: no T parameter+// needed for final stage+#define outerArguments(T) \+ (device const ArgmaxPair* partial_results [[buffer(0)]], \+ device uint* final_tokens [[buffer(1)]], \+ constant uint& batch_size [[buffer(2)]], \+ constant uint& vocab_size [[buffer(3)]], \+ uint batch_idx [[threadgroup_position_in_grid]], \+ uint local_id [[thread_position_in_threadgroup]])++#define innerArguments \+ (partial_results, \+ final_tokens, \+ batch_size, \+ vocab_size, \+ shared, \+ batch_idx, \+ (ushort)local_id)++#define generateArgmaxFinalKernel(functionName, T, outerArgs, innerArgs) \+ [[max_total_threads_per_threadgroup( \+ 1024 \+ )]] kernel void functionName##_##T outerArgs { \+ threadgroup ArgmaxPair shared[BLOCK_SIZE]; \+ batched_argmax_final_impl innerArgs; \+ }++ generateArgmaxFinalKernel(batched_argmax_final, float, outerArguments(float), innerArguments) generateArgmaxFinalKernel(+ batched_argmax_final,+ half,+ outerArguments(half),+ innerArguments+ ) generateArgmaxFinalKernel(batched_argmax_final, bfloat, outerArguments(bfloat), innerArguments)#undef outerArguments#undef innerArguments#undef generateArgmaxFinalKernel-template<typename T>-void batched_argmax_single_impl(const device T* logits_data,- device uint* final_tokens,- constant uint& batch_size,- constant uint& vocab_size,- threadgroup ArgmaxPair* shared,- uint batch_idx,- ushort local_id)-{- if (batch_idx >= batch_size) return;-- const uint elements_per_tile = BLOCK_SIZE * GRAIN_SIZE;- ArgmaxPair best = ARGMAX_INIT;-- for (uint vocab_offset = 0; vocab_offset < vocab_size; vocab_offset += elements_per_tile) {- ArgmaxPair buf[GRAIN_SIZE];- load_blocked_argmax_batched<GRAIN_SIZE>(buf,- logits_data,- local_id,- batch_idx,- vocab_offset,- vocab_size);- ArgmaxPair tile_best = thread_argmax_reduce<GRAIN_SIZE>(buf);- best = argmax_combine(best, tile_best);- }-- ArgmaxPair group_best = threadgroup_raking_argmax<BLOCK_SIZE>(best, shared, local_id);-- if (local_id == 0) {- final_tokens[batch_idx] = group_best.index;- }+ template <typename T>+ void batched_argmax_single_impl(+ const device T* logits_data,+ device uint* final_tokens,+ constant uint& batch_size,+ constant uint& vocab_size,+ threadgroup ArgmaxPair* shared,+ uint batch_idx,+ ushort local_id+ ) {+ if (batch_idx >= batch_size)+ return;++ const uint elements_per_tile = BLOCK_SIZE * GRAIN_SIZE;+ ArgmaxPair best = ARGMAX_INIT;++ for (uint vocab_offset = 0; vocab_offset < vocab_size;+ vocab_offset += elements_per_tile) {+ ArgmaxPair buf[GRAIN_SIZE];+ load_blocked_argmax_batched<GRAIN_SIZE>(+ buf,+ logits_data,+ local_id,+ batch_idx,+ vocab_offset,+ vocab_size+ );+ ArgmaxPair tile_best = thread_argmax_reduce<GRAIN_SIZE>(buf);+ best = argmax_combine(best, tile_best);+ }++ ArgmaxPair group_best =+ threadgroup_raking_argmax<BLOCK_SIZE>(best, shared, local_id);++ if (local_id == 0) {+ final_tokens[batch_idx] = group_best.index;+ }}// Generate argmax single pass kernels using macro approach-#define outerArguments(T) \-(device const T* logits_data [[ buffer(0) ]], \-device uint* final_tokens [[ buffer(1) ]], \-constant uint& batch_size [[ buffer(2) ]], \-constant uint& vocab_size [[ buffer(3) ]], \-uint batch_idx [[ threadgroup_position_in_grid ]], \-uint local_id [[ thread_position_in_threadgroup ]])--#define innerArguments \-(logits_data, final_tokens, batch_size, vocab_size, shared, batch_idx, (ushort)local_id)--#define generateArgmaxSingleKernel(functionName, T, outerArgs, innerArgs) \-[[max_total_threads_per_threadgroup(1024)]] kernel void functionName##_##T outerArgs { \- threadgroup ArgmaxPair shared[BLOCK_SIZE]; \- batched_argmax_single_impl<T> innerArgs; \-}--generateArgmaxSingleKernel(batched_argmax_single, float, outerArguments(float), innerArguments)-generateArgmaxSingleKernel(batched_argmax_single, half, outerArguments(half), innerArguments)-generateArgmaxSingleKernel(batched_argmax_single, bfloat, outerArguments(bfloat), innerArguments)+#define outerArguments(T) \+ (device const T* logits_data [[buffer(0)]], \+ device uint* final_tokens [[buffer(1)]], \+ constant uint& batch_size [[buffer(2)]], \+ constant uint& vocab_size [[buffer(3)]], \+ uint batch_idx [[threadgroup_position_in_grid]], \+ uint local_id [[thread_position_in_threadgroup]])++#define innerArguments \+ (logits_data, \+ final_tokens, \+ batch_size, \+ vocab_size, \+ shared, \+ batch_idx, \+ (ushort)local_id)++#define generateArgmaxSingleKernel(functionName, T, outerArgs, innerArgs) \+ [[max_total_threads_per_threadgroup( \+ 1024 \+ )]] kernel void functionName##_##T outerArgs { \+ threadgroup ArgmaxPair shared[BLOCK_SIZE]; \+ batched_argmax_single_impl<T> innerArgs; \+ }++generateArgmaxSingleKernel(batched_argmax_single, float, outerArguments(float), innerArguments) generateArgmaxSingleKernel(+ batched_argmax_single,+ half,+ outerArguments(half),+ innerArguments+) generateArgmaxSingleKernel(batched_argmax_single, bfloat, outerArguments(bfloat), innerArguments)#undef outerArguments#undef innerArguments#undef generateArgmaxSingleKernel#undef BLOCK_SIZE-#undef GRAIN_SIZE+#undef GRAIN_SIZEdiff --git a/crates/uzu/src/backends/metal/kernel/sampling/bitmask.metal b/crates/uzu/src/backends/metal/kernel/sampling/bitmask.metalindex 999f7b150..27e7c0780 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/bitmask.metal+++ b/crates/uzu/src/backends/metal/kernel/sampling/bitmask.metal@@ -16,38 +16,66 @@ void batched_bitmask(uint batch_idx,uint thread_idx) {- uint base_idx = batch_idx * vocab_size + group_idx * BLOCK_SIZE * GRAIN_SIZE + thread_idx;- uint batch_end = batch_idx * vocab_size + vocab_size;-- #pragma unroll(4)- for (uint i = 0; i < GRAIN_SIZE; i++) {- uint global_idx = base_idx + i * BLOCK_SIZE;- if (global_idx < batch_end) {- bool mask = (bitmask_data[global_idx / BITS_IN_U32] >> (global_idx % BITS_IN_U32)) & 0b1;- processed_logits[global_idx] = select(T(-INFINITY), logits_data[global_idx], mask);- }+ uint base_idx =+ batch_idx * vocab_size + group_idx * BLOCK_SIZE * GRAIN_SIZE + thread_idx;+ uint batch_end = batch_idx * vocab_size + vocab_size;++#pragma unroll(4)+ for (uint i = 0; i < GRAIN_SIZE; i++) {+ uint global_idx = base_idx + i * BLOCK_SIZE;+ if (global_idx < batch_end) {+ bool mask = (bitmask_data[global_idx / BITS_IN_U32] >>+ (global_idx % BITS_IN_U32)) &+ 0b1;+ processed_logits[global_idx] =+ select(T(-INFINITY), logits_data[global_idx], mask);}+ }}-#define outerArguments(T) \-(device const T* logits_data [[ buffer(0) ]], \-device const uint32_t* bitmask_data [[ buffer(1) ]], \-device T* processed_logits [[ buffer(2) ]], \-constant uint& vocab_size [[ buffer(3) ]], \-uint3 threadgroup_idx [[ threadgroup_position_in_grid ]], \-uint3 thread_idx [[ thread_position_in_threadgroup ]])+#define outerArguments(T) \+ (device const T* logits_data [[buffer(0)]], \+ device const uint32_t* bitmask_data [[buffer(1)]], \+ device T* processed_logits [[buffer(2)]], \+ constant uint& vocab_size [[buffer(3)]], \+ uint3 threadgroup_idx [[threadgroup_position_in_grid]], \+ uint3 thread_idx [[thread_position_in_threadgroup]])-#define innerArguments (logits_data, bitmask_data, processed_logits, vocab_size, threadgroup_idx.x, threadgroup_idx.y, thread_idx.x)+#define innerArguments \+ (logits_data, \+ bitmask_data, \+ processed_logits, \+ vocab_size, \+ threadgroup_idx.x, \+ threadgroup_idx.y, \+ thread_idx.x)-#define generateBitmaskKernel(functionName, scalarType, outerArgs, innerArgs) \-[[max_total_threads_per_threadgroup(1024)]] kernel void functionName##_##scalarType outerArgs { \- functionName innerArgs; \-}+#define generateBitmaskKernel(functionName, scalarType, outerArgs, innerArgs) \+ [[max_total_threads_per_threadgroup( \+ 1024 \+ )]] kernel void functionName##_##scalarType outerArgs { \+ functionName innerArgs; \+ }-#define generateBitmaskKernels(functionName) \-generateBitmaskKernel(functionName, float, outerArguments(float), innerArguments); \-generateBitmaskKernel(functionName, bfloat, outerArguments(bfloat), innerArguments); \-generateBitmaskKernel(functionName, half, outerArguments(half), innerArguments);+#define generateBitmaskKernels(functionName) \+ generateBitmaskKernel( \+ functionName, \+ float, \+ outerArguments(float), \+ innerArguments \+ ); \+ generateBitmaskKernel( \+ functionName, \+ bfloat, \+ outerArguments(bfloat), \+ innerArguments \+ ); \+ generateBitmaskKernel( \+ functionName, \+ half, \+ outerArguments(half), \+ innerArguments \+ );generateBitmaskKernels(batched_bitmask)@@ -55,4 +83,3 @@ generateBitmaskKernels(batched_bitmask)#undef innerArguments#undef generatebitmaskKernel#undef generatebitmaskKernels-diff --git a/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metal b/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metalindex 47b804e33..b51b9bb94 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metal+++ b/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metal@@ -17,46 +17,73 @@ void batched_gumbel(uint batch_idx,uint thread_idx) {- uint batch_start = vocab_size * batch_idx;- uint batch_end = batch_start + vocab_size;+ uint batch_start = vocab_size * batch_idx;+ uint batch_end = batch_start + vocab_size;- uint grain_offset_in_batch = group_idx * BLOCK_SIZE * GRAIN_SIZE + thread_idx;+ uint grain_offset_in_batch = group_idx * BLOCK_SIZE * GRAIN_SIZE + thread_idx;- uint grain_offset = batch_start + grain_offset_in_batch;+ uint grain_offset = batch_start + grain_offset_in_batch;- uint64_t rng_seed = batch_seeds[batch_idx];- uint64_t rng_offset = (group_idx * BLOCK_SIZE + thread_idx) * (GRAIN_SIZE + WORDS_PER_OFFSET - 1) / WORDS_PER_OFFSET;- PhiloxState rng;- philox_init(&rng, rng_seed, rng_offset);+ uint64_t rng_seed = batch_seeds[batch_idx];+ uint64_t rng_offset = (group_idx * BLOCK_SIZE + thread_idx) *+ (GRAIN_SIZE + WORDS_PER_OFFSET - 1) / WORDS_PER_OFFSET;+ PhiloxState rng;+ philox_init(&rng, rng_seed, rng_offset);- #pragma unroll(4)- for (uint i = 0; i < GRAIN_SIZE; i++) {- uint global_idx = grain_offset + i * BLOCK_SIZE;- if (global_idx < batch_end) {- processed_logits[global_idx] = logits_data[global_idx] + T(-fast::log(-fast::log(uniform_float(&rng))));- }+#pragma unroll(4)+ for (uint i = 0; i < GRAIN_SIZE; i++) {+ uint global_idx = grain_offset + i * BLOCK_SIZE;+ if (global_idx < batch_end) {+ processed_logits[global_idx] =+ logits_data[global_idx] ++ T(-fast::log(-fast::log(uniform_float(&rng))));}+ }}-#define outerArguments(T) \-(device const T* logits_data [[ buffer(0) ]], \-device const uint64_t* batch_seeds [[ buffer(1) ]], \-device T* processed_logits [[ buffer(2) ]], \-constant uint& vocab_size [[ buffer(3) ]], \-uint3 threadgroup_idx [[ threadgroup_position_in_grid ]], \-uint3 thread_idx [[ thread_position_in_threadgroup ]])+#define outerArguments(T) \+ (device const T* logits_data [[buffer(0)]], \+ device const uint64_t* batch_seeds [[buffer(1)]], \+ device T* processed_logits [[buffer(2)]], \+ constant uint& vocab_size [[buffer(3)]], \+ uint3 threadgroup_idx [[threadgroup_position_in_grid]], \+ uint3 thread_idx [[thread_position_in_threadgroup]])-#define innerArguments (logits_data, batch_seeds, processed_logits, vocab_size, threadgroup_idx.x, threadgroup_idx.y, thread_idx.x)+#define innerArguments \+ (logits_data, \+ batch_seeds, \+ processed_logits, \+ vocab_size, \+ threadgroup_idx.x, \+ threadgroup_idx.y, \+ thread_idx.x)-#define generateGumbelKernel(functionName, scalarType, outerArgs, innerArgs) \-[[max_total_threads_per_threadgroup(1024)]] kernel void functionName##_##scalarType outerArgs { \- functionName innerArgs; \-}+#define generateGumbelKernel(functionName, scalarType, outerArgs, innerArgs) \+ [[max_total_threads_per_threadgroup( \+ 1024 \+ )]] kernel void functionName##_##scalarType outerArgs { \+ functionName innerArgs; \+ }-#define generateGumbelKernels(functionName) \-generateGumbelKernel(functionName, float, outerArguments(float), innerArguments); \-generateGumbelKernel(functionName, bfloat, outerArguments(bfloat), innerArguments); \-generateGumbelKernel(functionName, half, outerArguments(half), innerArguments);+#define generateGumbelKernels(functionName) \+ generateGumbelKernel( \+ functionName, \+ float, \+ outerArguments(float), \+ innerArguments \+ ); \+ generateGumbelKernel( \+ functionName, \+ bfloat, \+ outerArguments(bfloat), \+ innerArguments \+ ); \+ generateGumbelKernel( \+ functionName, \+ half, \+ outerArguments(half), \+ innerArguments \+ );generateGumbelKernels(batched_gumbel)diff --git a/crates/uzu/src/backends/metal/kernel/sampling/temperature.metal b/crates/uzu/src/backends/metal/kernel/sampling/temperature.metalindex 50b2e83e7..027813031 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/temperature.metal+++ b/crates/uzu/src/backends/metal/kernel/sampling/temperature.metal@@ -14,37 +14,68 @@ void batched_temperature(uint batch_idx,uint thread_idx) {- uint base_idx = batch_idx * vocab_size + group_idx * BLOCK_SIZE * GRAIN_SIZE + thread_idx;- uint batch_end = batch_idx * vocab_size + vocab_size;-- #pragma unroll(4)- for (uint i = 0; i < GRAIN_SIZE; i++) {- uint global_idx = base_idx + i * BLOCK_SIZE;- if (global_idx < batch_end) {- processed_logits[global_idx] = T(float(logits_data[global_idx]) / temperature);- }+ uint base_idx =+ batch_idx * vocab_size + group_idx * BLOCK_SIZE * GRAIN_SIZE + thread_idx;+ uint batch_end = batch_idx * vocab_size + vocab_size;++#pragma unroll(4)+ for (uint i = 0; i < GRAIN_SIZE; i++) {+ uint global_idx = base_idx + i * BLOCK_SIZE;+ if (global_idx < batch_end) {+ processed_logits[global_idx] =+ T(float(logits_data[global_idx]) / temperature);}+ }}-#define outerArguments(T) \-(device const T* logits_data [[ buffer(0) ]], \-device T* processed_logits [[ buffer(1) ]], \-constant uint& vocab_size [[ buffer(2) ]], \-constant float& temperature [[ buffer(3) ]], \-uint3 threadgroup_idx [[ threadgroup_position_in_grid ]], \-uint3 thread_idx [[ thread_position_in_threadgroup ]])+#define outerArguments(T) \+ (device const T* logits_data [[buffer(0)]], \+ device T* processed_logits [[buffer(1)]], \+ constant uint& vocab_size [[buffer(2)]], \+ constant float& temperature [[buffer(3)]], \+ uint3 threadgroup_idx [[threadgroup_position_in_grid]], \+ uint3 thread_idx [[thread_position_in_threadgroup]])-#define innerArguments (logits_data, processed_logits, vocab_size, temperature, threadgroup_idx.x, threadgroup_idx.y, thread_idx.x)+#define innerArguments \+ (logits_data, \+ processed_logits, \+ vocab_size, \+ temperature, \+ threadgroup_idx.x, \+ threadgroup_idx.y, \+ thread_idx.x)-#define generateTemperatureKernel(functionName, scalarType, outerArgs, innerArgs) \-[[max_total_threads_per_threadgroup(1024)]] kernel void functionName##_##scalarType outerArgs { \- functionName innerArgs; \-}+#define generateTemperatureKernel( \+ functionName, \+ scalarType, \+ outerArgs, \+ innerArgs \+) \+ [[max_total_threads_per_threadgroup( \+ 1024 \+ )]] kernel void functionName##_##scalarType outerArgs { \+ functionName innerArgs; \+ }-#define generateTemperatureKernels(functionName) \-generateTemperatureKernel(functionName, float, outerArguments(float), innerArguments); \-generateTemperatureKernel(functionName, bfloat, outerArguments(bfloat), innerArguments); \-generateTemperatureKernel(functionName, half, outerArguments(half), innerArguments);+#define generateTemperatureKernels(functionName) \+ generateTemperatureKernel( \+ functionName, \+ float, \+ outerArguments(float), \+ innerArguments \+ ); \+ generateTemperatureKernel( \+ functionName, \+ bfloat, \+ outerArguments(bfloat), \+ innerArguments \+ ); \+ generateTemperatureKernel( \+ functionName, \+ half, \+ outerArguments(half), \+ innerArguments \+ );generateTemperatureKernels(batched_temperature)diff --git a/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal b/crates/uzu/src/backends/metal/kernel/sampling/top_k.metalindex 69f7f70c1..51d7ef616 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal+++ b/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal@@ -14,77 +14,112 @@ void batched_topk(uint batch_idx,uint thread_idx) {- uint batch_start = batch_idx * vocab_size;+ uint batch_start = batch_idx * vocab_size;- // Find min and max logit for binary search- float local_max = -INFINITY;- float local_min = INFINITY;- #pragma unroll(4)- for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {- float logit_value = float(logits_data[batch_start + i]);- local_max = fmax(local_max, logit_value);- local_min = select(local_min, fmin(local_min, logit_value), logit_value > -INFINITY);- }- float max_logit = threadgroup_cooperative_reduce_max<BLOCK_SIZE>(local_max, shared_reduce_buffer, thread_idx);- float min_logit = threadgroup_cooperative_reduce_min<BLOCK_SIZE>(local_min, shared_reduce_buffer, thread_idx);+ // Find min and max logit for binary search+ float local_max = -INFINITY;+ float local_min = INFINITY;+#pragma unroll(4)+ for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {+ float logit_value = float(logits_data[batch_start + i]);+ local_max = fmax(local_max, logit_value);+ local_min = select(+ local_min,+ fmin(local_min, logit_value),+ logit_value > -INFINITY+ );+ }+ float max_logit = threadgroup_cooperative_reduce_max<BLOCK_SIZE>(+ local_max,+ shared_reduce_buffer,+ thread_idx+ );+ float min_logit = threadgroup_cooperative_reduce_min<BLOCK_SIZE>(+ local_min,+ shared_reduce_buffer,+ thread_idx+ );+ // Do the binary search on the threshold+ float low = min_logit;+ float high = max_logit;+ float threshold = (min_logit + max_logit) / 2.0;- // Do the binary search on the threshold- float low = min_logit;- float high = max_logit;- float threshold = (min_logit + max_logit) / 2.0;-- for (uint iter = 0; iter < MAX_ITERS; iter++) {- // Find the mass above threshold- uint local_num_above_threshold = 0;- #pragma unroll(4)- for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {- float logit_value = float(logits_data[batch_start + i]);- local_num_above_threshold += select(0, 1, logit_value >= threshold);- }- uint num_above_threshold = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(local_num_above_threshold, (threadgroup uint*)shared_reduce_buffer, thread_idx);+ for (uint iter = 0; iter < MAX_ITERS; iter++) {+ // Find the mass above threshold+ uint local_num_above_threshold = 0;+#pragma unroll(4)+ for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {+ float logit_value = float(logits_data[batch_start + i]);+ local_num_above_threshold += select(0, 1, logit_value >= threshold);+ }+ uint num_above_threshold = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(+ local_num_above_threshold,+ (threadgroup uint*)shared_reduce_buffer,+ thread_idx+ );- // Update binary search- if (num_above_threshold == top_k) {- break;- } else if (num_above_threshold > top_k) {- low = threshold;- } else {- high = threshold;- }- threshold = (low + high) / 2.0;+ // Update binary search+ if (num_above_threshold == top_k) {+ break;+ } else if (num_above_threshold > top_k) {+ low = threshold;+ } else {+ high = threshold;}+ threshold = (low + high) / 2.0;+ }- T t_threshold = T(threshold);+ T t_threshold = T(threshold);- // We know the threshold, just mask everything below it- #pragma unroll(4)- for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {- T logit_value = logits_data[batch_start + i];- processed_logits[batch_start + i] = select(T(-INFINITY), logit_value, logit_value >= t_threshold);- }+// We know the threshold, just mask everything below it+#pragma unroll(4)+ for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {+ T logit_value = logits_data[batch_start + i];+ processed_logits[batch_start + i] =+ select(T(-INFINITY), logit_value, logit_value >= t_threshold);+ }}-#define outerArguments(T) \-(device const T* logits_data [[ buffer(0) ]], \-device T* processed_logits [[ buffer(1) ]], \-constant uint& vocab_size [[ buffer(2) ]], \-constant uint& top_k [[ buffer(3) ]], \-uint3 threadgroup_idx [[ threadgroup_position_in_grid ]], \-uint3 thread_idx [[ thread_position_in_threadgroup ]])+#define outerArguments(T) \+ (device const T* logits_data [[buffer(0)]], \+ device T* processed_logits [[buffer(1)]], \+ constant uint& vocab_size [[buffer(2)]], \+ constant uint& top_k [[buffer(3)]], \+ uint3 threadgroup_idx [[threadgroup_position_in_grid]], \+ uint3 thread_idx [[thread_position_in_threadgroup]])-#define innerArguments (logits_data, processed_logits, shared_reduce_buffer, vocab_size, top_k, threadgroup_idx.x, thread_idx.x)+#define innerArguments \+ (logits_data, \+ processed_logits, \+ shared_reduce_buffer, \+ vocab_size, \+ top_k, \+ threadgroup_idx.x, \+ thread_idx.x)-#define generateTopkKernel(functionName, scalarType, outerArgs, innerArgs) \-[[max_total_threads_per_threadgroup(1024)]] kernel void functionName##_##scalarType outerArgs { \- threadgroup float shared_reduce_buffer[BLOCK_SIZE]; \- functionName innerArgs; \-}+#define generateTopkKernel(functionName, scalarType, outerArgs, innerArgs) \+ [[max_total_threads_per_threadgroup( \+ 1024 \+ )]] kernel void functionName##_##scalarType outerArgs { \+ threadgroup float shared_reduce_buffer[BLOCK_SIZE]; \+ functionName innerArgs; \+ }-#define generateTopkKernels(functionName) \-generateTopkKernel(functionName, float, outerArguments(float), innerArguments); \-generateTopkKernel(functionName, bfloat, outerArguments(bfloat), innerArguments); \-generateTopkKernel(functionName, half, outerArguments(half), innerArguments);+#define generateTopkKernels(functionName) \+ generateTopkKernel( \+ functionName, \+ float, \+ outerArguments(float), \+ innerArguments \+ ); \+ generateTopkKernel( \+ functionName, \+ bfloat, \+ outerArguments(bfloat), \+ innerArguments \+ ); \+ generateTopkKernel(functionName, half, outerArguments(half), innerArguments);generateTopkKernels(batched_topk)diff --git a/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal b/crates/uzu/src/backends/metal/kernel/sampling/top_p.metalindex 4bbcc8478..f16125107 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal+++ b/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal@@ -16,94 +16,146 @@ void batched_topp(uint batch_idx,uint thread_idx) {- uint batch_start = batch_idx * vocab_size;-- // Find min (for binary search) and max (for binary search and softmax)- float local_max = -INFINITY;- float local_min = INFINITY;- #pragma unroll(4)+ uint batch_start = batch_idx * vocab_size;++ // Find min (for binary search) and max (for binary search and softmax)+ float local_max = -INFINITY;+ float local_min = INFINITY;+#pragma unroll(4)+ for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {+ float logit_value = float(logits_data[batch_start + i]);+ local_max = fmax(local_max, logit_value);+ local_min = select(+ local_min,+ fmin(local_min, logit_value),+ logit_value > -INFINITY+ );+ }+ float max_logit = threadgroup_cooperative_reduce_max<BLOCK_SIZE>(+ local_max,+ shared_reduce_buffer,+ thread_idx+ );+ float min_logit = threadgroup_cooperative_reduce_min<BLOCK_SIZE>(+ local_min,+ shared_reduce_buffer,+ thread_idx+ );++ // Find denominator for softmax+ float local_sum = 0.0f;+#pragma unroll(4)+ for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {+ float logit_value = float(logits_data[batch_start + i]);+ local_sum += select(+ 0.0,+ fast::exp(logit_value - max_logit),+ logit_value > -INFINITY+ );+ }+ float total_sum = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(+ local_sum,+ shared_reduce_buffer,+ thread_idx+ );++ // Do the binary search on the threshold+ float target_mass = top_p * total_sum;++ float low = min_logit;+ float high = max_logit;+ float threshold = (min_logit + max_logit) / 2.0;++ for (uint iter = 0; iter < MAX_ITERS; iter++) {+ // Find the mass above threshold+ float local_sum_above_threshold = 0.0f;+ float local_min_above_threshold = INFINITY;+#pragma unroll(4)for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {- float logit_value = float(logits_data[batch_start + i]);- local_max = fmax(local_max, logit_value);- local_min = select(local_min, fmin(local_min, logit_value), logit_value > -INFINITY);+ float logit_value = float(logits_data[batch_start + i]);+ float logit_mass = fast::exp(logit_value - max_logit);+ local_sum_above_threshold +=+ select(0.0, logit_mass, logit_value >= threshold);+ local_min_above_threshold = fmin(+ local_min_above_threshold,+ select(INFINITY, logit_mass, logit_value >= threshold)+ );}- float max_logit = threadgroup_cooperative_reduce_max<BLOCK_SIZE>(local_max, shared_reduce_buffer, thread_idx);- float min_logit = threadgroup_cooperative_reduce_min<BLOCK_SIZE>(local_min, shared_reduce_buffer, thread_idx);-- // Find denominator for softmax- float local_sum = 0.0f;- #pragma unroll(4)- for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {- float logit_value = float(logits_data[batch_start + i]);- local_sum += select(0.0, fast::exp(logit_value - max_logit), logit_value > -INFINITY);+ float sum_above_threshold = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(+ local_sum_above_threshold,+ shared_reduce_buffer,+ thread_idx+ );+ float min_above_threshold = threadgroup_cooperative_reduce_min<BLOCK_SIZE>(+ local_min_above_threshold,+ shared_reduce_buffer,+ thread_idx+ );++ // Early exit+ if (sum_above_threshold >= target_mass &&+ sum_above_threshold - min_above_threshold < target_mass) {+ break;}- float total_sum = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(local_sum, shared_reduce_buffer, thread_idx);--- // Do the binary search on the threshold- float target_mass = top_p * total_sum;-- float low = min_logit;- float high = max_logit;- float threshold = (min_logit + max_logit) / 2.0;-- for (uint iter = 0; iter < MAX_ITERS; iter++) {- // Find the mass above threshold- float local_sum_above_threshold = 0.0f;- float local_min_above_threshold = INFINITY;- #pragma unroll(4)- for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {- float logit_value = float(logits_data[batch_start + i]);- float logit_mass = fast::exp(logit_value - max_logit);- local_sum_above_threshold += select(0.0, logit_mass, logit_value >= threshold);- local_min_above_threshold = fmin(local_min_above_threshold, select(INFINITY, logit_mass, logit_value >= threshold));- }- float sum_above_threshold = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(local_sum_above_threshold, shared_reduce_buffer, thread_idx);- float min_above_threshold = threadgroup_cooperative_reduce_min<BLOCK_SIZE>(local_min_above_threshold, shared_reduce_buffer, thread_idx);-- // Early exit- if (sum_above_threshold >= target_mass && sum_above_threshold - min_above_threshold < target_mass) {- break;- }- // Update binary search- if (sum_above_threshold >= target_mass) {- low = threshold;- } else {- high = threshold;- }- threshold = (low + high) / 2.0;+ // Update binary search+ if (sum_above_threshold >= target_mass) {+ low = threshold;+ } else {+ high = threshold;}-- T t_threshold = T(threshold);-- // We know the threshold, just mask everything below it- #pragma unroll(4)- for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {- T logit_value = logits_data[batch_start + i];- processed_logits[batch_start + i] = select(T(-INFINITY), logit_value, logit_value >= t_threshold);- }-}--#define outerArguments(T) \-(device const T* logits_data [[ buffer(0) ]], \-device T* processed_logits [[ buffer(1) ]], \-constant uint& vocab_size [[ buffer(2) ]], \-constant float& top_p [[ buffer(3) ]], \-uint3 threadgroup_idx [[ threadgroup_position_in_grid ]], \-uint3 thread_idx [[ thread_position_in_threadgroup ]])--#define innerArguments (logits_data, processed_logits, shared_reduce_buffer, vocab_size, top_p, threadgroup_idx.x, thread_idx.x)--#define generateToppKernel(functionName, scalarType, outerArgs, innerArgs) \-[[max_total_threads_per_threadgroup(1024)]] kernel void functionName##_##scalarType outerArgs { \- threadgroup float shared_reduce_buffer[BLOCK_SIZE_IN_SIMDS]; \- functionName innerArgs; \+ threshold = (low + high) / 2.0;+ }++ T t_threshold = T(threshold);++// We know the threshold, just mask everything below it+#pragma unroll(4)+ for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {+ T logit_value = logits_data[batch_start + i];+ processed_logits[batch_start + i] =+ select(T(-INFINITY), logit_value, logit_value >= t_threshold);+ }}-#define generateToppKernels(functionName) \-generateToppKernel(functionName, float, outerArguments(float), innerArguments); \-generateToppKernel(functionName, bfloat, outerArguments(bfloat), innerArguments); \-generateToppKernel(functionName, half, outerArguments(half), innerArguments);+#define outerArguments(T) \+ (device const T* logits_data [[buffer(0)]], \+ device T* processed_logits [[buffer(1)]], \+ constant uint& vocab_size [[buffer(2)]], \+ constant float& top_p [[buffer(3)]], \+ uint3 threadgroup_idx [[threadgroup_position_in_grid]], \+ uint3 thread_idx [[thread_position_in_threadgroup]])++#define innerArguments \+ (logits_data, \+ processed_logits, \+ shared_reduce_buffer, \+ vocab_size, \+ top_p, \+ threadgroup_idx.x, \+ thread_idx.x)++#define generateToppKernel(functionName, scalarType, outerArgs, innerArgs) \+ [[max_total_threads_per_threadgroup( \+ 1024 \+ )]] kernel void functionName##_##scalarType outerArgs { \+ threadgroup float shared_reduce_buffer[BLOCK_SIZE_IN_SIMDS]; \+ functionName innerArgs; \+ }++#define generateToppKernels(functionName) \+ generateToppKernel( \+ functionName, \+ float, \+ outerArguments(float), \+ innerArguments \+ ); \+ generateToppKernel( \+ functionName, \+ bfloat, \+ outerArguments(bfloat), \+ innerArguments \+ ); \+ generateToppKernel(functionName, half, outerArguments(half), innerArguments);generateToppKernels(batched_topp)diff --git a/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.metal b/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.metalindex dc477a170..369dc79ea 100644--- a/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.metal+++ b/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.metal@@ -13,240 +13,241 @@ constant uint CONV_SCAN_THREADS = 32u;template <typename T>inline T apply_silu(T x) {- float xf = float(x);- float y = 1.0f / (1.0f + fast::exp(-fabs(xf)));- float out = (xf < 0.0f) ? (1.0f - y) * xf : y * xf;- return static_cast<T>(out);+ float xf = float(x);+ float y = 1.0f / (1.0f + fast::exp(-fabs(xf)));+ float out = (xf < 0.0f) ? (1.0f - y) * xf : y * xf;+ return static_cast<T>(out);}template <typename T>inline T apply_gelu(T x) {- float xf = float(x);- return static_cast<T>(0.5f * xf * (1.0f + fast::tanh(0.797885f * (xf + 0.044715f * xf * xf * xf))));+ float xf = float(x);+ return static_cast<T>(+ 0.5f * xf *+ (1.0f + fast::tanh(0.797885f * (xf + 0.044715f * xf * xf * xf)))+ );}template <typename T>inline T apply_activation_fn(T x, int activation_type) {- if (activation_type == ACTIVATION_SILU) {- return apply_silu(x);- } else if (activation_type == ACTIVATION_GELU) {- return apply_gelu(x);- } else {- return x; // Identity- }+ if (activation_type == ACTIVATION_SILU) {+ return apply_silu(x);+ } else if (activation_type == ACTIVATION_GELU) {+ return apply_gelu(x);+ } else {+ return x; // Identity+ }}template <typename T>kernel void conv1d_pack_prefix_kernel(- device const T* state_in [[ buffer(0) ]],- device const T* x [[ buffer(1) ]],- device T* padded [[ buffer(2) ]],- constant const size_t& state_stride [[ buffer(3) ]],- constant const size_t& row_stride [[ buffer(4) ]],- constant const size_t& suffix_len [[ buffer(5) ]],- constant const uint& num_channels [[ buffer(6) ]],- uint2 grid_idx [[ thread_position_in_grid ]]+ device const T* state_in [[buffer(0)]],+ device const T* x [[buffer(1)]],+ device T* padded [[buffer(2)]],+ constant const size_t& state_stride [[buffer(3)]],+ constant const size_t& row_stride [[buffer(4)]],+ constant const size_t& suffix_len [[buffer(5)]],+ constant const uint& num_channels [[buffer(6)]],+ uint2 grid_idx [[thread_position_in_grid]]) {- const uint channel_idx = grid_idx.x;- const uint row_idx = grid_idx.y;- const size_t padded_rows = state_stride + suffix_len;- if (channel_idx >= num_channels || row_idx >= padded_rows) {- return;- }- const size_t padded_index =- size_t(row_idx) * row_stride + size_t(channel_idx);- if (row_idx < state_stride) {- const size_t state_index =- size_t(channel_idx) * state_stride + size_t(row_idx);- padded[padded_index] = state_in[state_index];- } else {- const size_t token = row_idx - state_stride;- const size_t x_index =- size_t(token) * row_stride + size_t(channel_idx);- padded[padded_index] = x[x_index];- }+ const uint channel_idx = grid_idx.x;+ const uint row_idx = grid_idx.y;+ const size_t padded_rows = state_stride + suffix_len;+ if (channel_idx >= num_channels || row_idx >= padded_rows) {+ return;+ }+ const size_t padded_index =+ size_t(row_idx) * row_stride + size_t(channel_idx);+ if (row_idx < state_stride) {+ const size_t state_index =+ size_t(channel_idx) * state_stride + size_t(row_idx);+ padded[padded_index] = state_in[state_index];+ } else {+ const size_t token = row_idx - state_stride;+ const size_t x_index = size_t(token) * row_stride + size_t(channel_idx);+ padded[padded_index] = x[x_index];+ }}template <typename T>kernel void conv1d_scan_kernel(- device const T* padded [[ buffer(0) ]], // (prefix+suffix, channels)- device const T* w [[ buffer(1) ]], // (channels, kernel)- device const T* b [[ buffer(2) ]], // optional (channels)- device T* x_out [[ buffer(3) ]], // (suffix, inner_dim)- device T* b_out [[ buffer(4) ]], // (suffix, proj_dim)- device T* c_out [[ buffer(5) ]], // (suffix, proj_dim)- device T* state_out [[ buffer(6) ]], // (channels, kernel-1)- constant const size_t& suffix_len [[ buffer(7) ]],- constant const int& kernel_size [[ buffer(8) ]],- constant const size_t& row_stride [[ buffer(9) ]],- constant const size_t& state_stride [[ buffer(10) ]],- constant const uint& num_channels [[ buffer(11) ]],- constant const uint& inner_dim [[ buffer(12) ]],- constant const uint& proj_dim [[ buffer(13) ]],- uint3 grid_idx [[ thread_position_in_grid ]]+ device const T* padded [[buffer(0)]], // (prefix+suffix, channels)+ device const T* w [[buffer(1)]], // (channels, kernel)+ device const T* b [[buffer(2)]], // optional (channels)+ device T* x_out [[buffer(3)]], // (suffix, inner_dim)+ device T* b_out [[buffer(4)]], // (suffix, proj_dim)+ device T* c_out [[buffer(5)]], // (suffix, proj_dim)+ device T* state_out [[buffer(6)]], // (channels, kernel-1)+ constant const size_t& suffix_len [[buffer(7)]],+ constant const int& kernel_size [[buffer(8)]],+ constant const size_t& row_stride [[buffer(9)]],+ constant const size_t& state_stride [[buffer(10)]],+ constant const uint& num_channels [[buffer(11)]],+ constant const uint& inner_dim [[buffer(12)]],+ constant const uint& proj_dim [[buffer(13)]],+ uint3 grid_idx [[thread_position_in_grid]]) {- const int kernel_value = kernel_size;- if (kernel_value <= 0) {- return;- }+ const int kernel_value = kernel_size;+ if (kernel_value <= 0) {+ return;+ }- const int tap_count = max(kernel_value - 1, 0);- const size_t work_len = suffix_len + size_t(tap_count);+ const int tap_count = max(kernel_value - 1, 0);+ const size_t work_len = suffix_len + size_t(tap_count);- const uint token_idx = grid_idx.x;- const uint channel_idx = grid_idx.y;- if (channel_idx >= num_channels || token_idx >= work_len) {- return;- }+ const uint token_idx = grid_idx.x;+ const uint channel_idx = grid_idx.y;+ if (channel_idx >= num_channels || token_idx >= work_len) {+ return;+ }- const size_t state_offset = size_t(channel_idx) * state_stride;- const size_t weight_offset = size_t(channel_idx) * size_t(kernel_value);- const device T* w_row = w + weight_offset;- const bool has_bias = b != nullptr;+ const size_t state_offset = size_t(channel_idx) * state_stride;+ const size_t weight_offset = size_t(channel_idx) * size_t(kernel_value);+ const device T* w_row = w + weight_offset;+ const bool has_bias = b != nullptr;- if (token_idx < suffix_len) {- float acc = has_bias ? float(b[channel_idx]) : 0.0f;- for (int tap = 0; tap < kernel_value; ++tap) {- const size_t padded_idx = size_t(token_idx) + size_t(tap);- float sample = 0.0f;- const size_t padded_index =- size_t(padded_idx) * row_stride + channel_idx;- sample = float(padded[padded_index]);- acc += float(w_row[tap]) * sample;- }+ if (token_idx < suffix_len) {+ float acc = has_bias ? float(b[channel_idx]) : 0.0f;+ for (int tap = 0; tap < kernel_value; ++tap) {+ const size_t padded_idx = size_t(token_idx) + size_t(tap);+ float sample = 0.0f;+ const size_t padded_index = size_t(padded_idx) * row_stride + channel_idx;+ sample = float(padded[padded_index]);+ acc += float(w_row[tap]) * sample;+ }- const T activated = apply_activation_fn(static_cast<T>(acc), activation_type);- if (channel_idx < inner_dim) {- const size_t dst = size_t(token_idx) * inner_dim + channel_idx;- x_out[dst] = activated;- } else if (channel_idx < inner_dim + proj_dim) {- const size_t dst = size_t(token_idx) * proj_dim + (channel_idx - inner_dim);- b_out[dst] = activated;- } else if (channel_idx < inner_dim + 2 * proj_dim) {- const size_t dst = size_t(token_idx) * proj_dim + (channel_idx - inner_dim - proj_dim);- c_out[dst] = activated;- }- } else if (tap_count > 0) {- const size_t tap = size_t(token_idx - suffix_len);- if (tap >= size_t(tap_count)) {- return;- }- const size_t padded_index =- size_t(token_idx) * row_stride + channel_idx;- const float sample = float(padded[padded_index]);- state_out[state_offset + tap] = static_cast<T>(sample);+ const T activated =+ apply_activation_fn(static_cast<T>(acc), activation_type);+ if (channel_idx < inner_dim) {+ const size_t dst = size_t(token_idx) * inner_dim + channel_idx;+ x_out[dst] = activated;+ } else if (channel_idx < inner_dim + proj_dim) {+ const size_t dst =+ size_t(token_idx) * proj_dim + (channel_idx - inner_dim);+ b_out[dst] = activated;+ } else if (channel_idx < inner_dim + 2 * proj_dim) {+ const size_t dst =+ size_t(token_idx) * proj_dim + (channel_idx - inner_dim - proj_dim);+ c_out[dst] = activated;+ }+ } else if (tap_count > 0) {+ const size_t tap = size_t(token_idx - suffix_len);+ if (tap >= size_t(tap_count)) {+ return;}+ const size_t padded_index = size_t(token_idx) * row_stride + channel_idx;+ const float sample = float(padded[padded_index]);+ state_out[state_offset + tap] = static_cast<T>(sample);+ }}template <typename T>kernel void conv1d_decode_kernel(- device const T* x [[ buffer(0) ]],- device const T* w [[ buffer(1) ]],- device const T* b [[ buffer(2) ]],- device const T* state [[ buffer(3) ]],- device T* x_out [[ buffer(4) ]],- device T* b_out [[ buffer(5) ]],- device T* c_out [[ buffer(6) ]],- device T* next_state [[ buffer(7) ]],- constant const int& kernel_size [[ buffer(8) ]],- constant const size_t& row_stride [[ buffer(9) ]],- constant const size_t& state_stride [[ buffer(10) ]],- constant const uint& num_channels [[ buffer(11) ]],- constant const size_t& suffix_len [[ buffer(12) ]],- constant const uint& inner_dim [[ buffer(13) ]],- constant const uint& proj_dim [[ buffer(14) ]],- uint2 grid_idx [[ thread_position_in_grid ]]+ device const T* x [[buffer(0)]],+ device const T* w [[buffer(1)]],+ device const T* b [[buffer(2)]],+ device const T* state [[buffer(3)]],+ device T* x_out [[buffer(4)]],+ device T* b_out [[buffer(5)]],+ device T* c_out [[buffer(6)]],+ device T* next_state [[buffer(7)]],+ constant const int& kernel_size [[buffer(8)]],+ constant const size_t& row_stride [[buffer(9)]],+ constant const size_t& state_stride [[buffer(10)]],+ constant const uint& num_channels [[buffer(11)]],+ constant const size_t& suffix_len [[buffer(12)]],+ constant const uint& inner_dim [[buffer(13)]],+ constant const uint& proj_dim [[buffer(14)]],+ uint2 grid_idx [[thread_position_in_grid]]) {- const int kernel_value = kernel_size;- if (kernel_value <= 0) {- return;- }+ const int kernel_value = kernel_size;+ if (kernel_value <= 0) {+ return;+ }- const uint token_idx = grid_idx.x;- const uint channel_idx = grid_idx.y;- if (channel_idx >= num_channels || token_idx >= suffix_len) {- return;- }+ const uint token_idx = grid_idx.x;+ const uint channel_idx = grid_idx.y;+ if (channel_idx >= num_channels || token_idx >= suffix_len) {+ return;+ }- const size_t x_idx =- size_t(token_idx) * row_stride + size_t(channel_idx);- const size_t state_offset =- size_t(channel_idx) * state_stride;- const device T* w_row =- w + size_t(channel_idx) * size_t(kernel_value);- const bool has_bias = b != nullptr;+ const size_t x_idx = size_t(token_idx) * row_stride + size_t(channel_idx);+ const size_t state_offset = size_t(channel_idx) * state_stride;+ const device T* w_row = w + size_t(channel_idx) * size_t(kernel_value);+ const bool has_bias = b != nullptr;- float acc = has_bias ? float(b[channel_idx]) : 0.0f;- const int state_taps = max(kernel_value - 1, 0);- for (int tap = 0; tap < state_taps; ++tap) {- const size_t state_index = state_offset + size_t(tap);- const float sample = float(state[state_index]);- acc += float(w_row[tap]) * sample;- }+ float acc = has_bias ? float(b[channel_idx]) : 0.0f;+ const int state_taps = max(kernel_value - 1, 0);+ for (int tap = 0; tap < state_taps; ++tap) {+ const size_t state_index = state_offset + size_t(tap);+ const float sample = float(state[state_index]);+ acc += float(w_row[tap]) * sample;+ }- const float current = float(x[x_idx]);- const int current_tap_index = state_taps;- if (current_tap_index < kernel_value) {- acc += float(w_row[current_tap_index]) * current;- }+ const float current = float(x[x_idx]);+ const int current_tap_index = state_taps;+ if (current_tap_index < kernel_value) {+ acc += float(w_row[current_tap_index]) * current;+ }- const T activated = apply_activation_fn(static_cast<T>(acc), activation_type);- if (channel_idx < inner_dim) {- const size_t dst = size_t(token_idx) * inner_dim + channel_idx;- x_out[dst] = activated;- } else if (channel_idx < inner_dim + proj_dim) {- const size_t dst = size_t(token_idx) * proj_dim + (channel_idx - inner_dim);- b_out[dst] = activated;- } else if (channel_idx < inner_dim + 2 * proj_dim) {- const size_t dst = size_t(token_idx) * proj_dim + (channel_idx - inner_dim - proj_dim);- c_out[dst] = activated;- }+ const T activated = apply_activation_fn(static_cast<T>(acc), activation_type);+ if (channel_idx < inner_dim) {+ const size_t dst = size_t(token_idx) * inner_dim + channel_idx;+ x_out[dst] = activated;+ } else if (channel_idx < inner_dim + proj_dim) {+ const size_t dst = size_t(token_idx) * proj_dim + (channel_idx - inner_dim);+ b_out[dst] = activated;+ } else if (channel_idx < inner_dim + 2 * proj_dim) {+ const size_t dst =+ size_t(token_idx) * proj_dim + (channel_idx - inner_dim - proj_dim);+ c_out[dst] = activated;+ }- if (state_taps == 0) {- return;- }+ if (state_taps == 0) {+ return;+ }- for (int tap = 0; tap < state_taps - 1; ++tap) {- const size_t dst_index = state_offset + size_t(tap);- const size_t src_index = state_offset + size_t(tap + 1);- next_state[dst_index] = state[src_index];- }+ for (int tap = 0; tap < state_taps - 1; ++tap) {+ const size_t dst_index = state_offset + size_t(tap);+ const size_t src_index = state_offset + size_t(tap + 1);+ next_state[dst_index] = state[src_index];+ }- const size_t tail_index = state_offset + size_t(state_taps - 1);- next_state[tail_index] = static_cast<T>(current);+ const size_t tail_index = state_offset + size_t(state_taps - 1);+ next_state[tail_index] = static_cast<T>(current);}-#define instantiate_conv1d_pack_prefix_kernel(type_name, type) \- template [[host_name("conv1d_pack_prefix_kernel_" #type_name)]] \- kernel void conv1d_pack_prefix_kernel<type>( \- device const type* state_in [[ buffer(0) ]], \- device const type* x [[ buffer(1) ]], \- device type* padded [[ buffer(2) ]], \- constant const size_t& state_stride [[ buffer(3) ]], \- constant const size_t& row_stride [[ buffer(4) ]], \- constant const size_t& suffix_len [[ buffer(5) ]], \- constant const uint& num_channels [[ buffer(6) ]], \- uint2 grid_idx [[ thread_position_in_grid ]] \+#define instantiate_conv1d_pack_prefix_kernel(type_name, type) \+ template [[host_name("conv1d_pack_prefix_kernel_" #type_name)]] \+ kernel void conv1d_pack_prefix_kernel<type>( \+ device const type* state_in [[buffer(0)]], \+ device const type* x [[buffer(1)]], \+ device type* padded [[buffer(2)]], \+ constant const size_t& state_stride [[buffer(3)]], \+ constant const size_t& row_stride [[buffer(4)]], \+ constant const size_t& suffix_len [[buffer(5)]], \+ constant const uint& num_channels [[buffer(6)]], \+ uint2 grid_idx [[thread_position_in_grid]] \);-#define instantiate_conv1d_scan_kernel(type_name, type) \- template [[host_name("conv1d_scan_kernel_" #type_name)]] \- kernel void conv1d_scan_kernel<type>( \- device const type* padded [[ buffer(0) ]], \- device const type* w [[ buffer(1) ]], \- device const type* b [[ buffer(2) ]], \- device type* x_out [[ buffer(3) ]], \- device type* b_out [[ buffer(4) ]], \- device type* c_out [[ buffer(5) ]], \- device type* state_out [[ buffer(6) ]], \- constant const size_t& suffix_len [[ buffer(7) ]], \- constant const int& kernel_size [[ buffer(8) ]], \- constant const size_t& row_stride [[ buffer(9) ]], \- constant const size_t& state_stride [[ buffer(10) ]], \- constant const uint& num_channels [[ buffer(11) ]], \- constant const uint& inner_dim [[ buffer(12) ]], \- constant const uint& proj_dim [[ buffer(13) ]], \- uint3 grid_idx [[ thread_position_in_grid ]] \+#define instantiate_conv1d_scan_kernel(type_name, type) \+ template [[host_name("conv1d_scan_kernel_" #type_name)]] \+ kernel void conv1d_scan_kernel<type>( \+ device const type* padded [[buffer(0)]], \+ device const type* w [[buffer(1)]], \+ device const type* b [[buffer(2)]], \+ device type* x_out [[buffer(3)]], \+ device type* b_out [[buffer(4)]], \+ device type* c_out [[buffer(5)]], \+ device type* state_out [[buffer(6)]], \+ constant const size_t& suffix_len [[buffer(7)]], \+ constant const int& kernel_size [[buffer(8)]], \+ constant const size_t& row_stride [[buffer(9)]], \+ constant const size_t& state_stride [[buffer(10)]], \+ constant const uint& num_channels [[buffer(11)]], \+ constant const uint& inner_dim [[buffer(12)]], \+ constant const uint& proj_dim [[buffer(13)]], \+ uint3 grid_idx [[thread_position_in_grid]] \);instantiate_conv1d_pack_prefix_kernel(float, float);@@ -257,25 +258,25 @@ instantiate_conv1d_scan_kernel(float, float);instantiate_conv1d_scan_kernel(bfloat, bfloat);instantiate_conv1d_scan_kernel(half, half);-#define instantiate_conv1d_decode_kernel(type_name, type) \- template [[host_name("conv1d_decode_kernel_" #type_name)]] \- kernel void conv1d_decode_kernel<type>( \- device const type* x [[ buffer(0) ]], \- device const type* w [[ buffer(1) ]], \- device const type* b [[ buffer(2) ]], \- device const type* state [[ buffer(3) ]], \- device type* x_out [[ buffer(4) ]], \- device type* b_out [[ buffer(5) ]], \- device type* c_out [[ buffer(6) ]], \- device type* next_state [[ buffer(7) ]], \- constant const int& kernel_size [[ buffer(8) ]], \- constant const size_t& row_stride [[ buffer(9) ]], \- constant const size_t& state_stride [[ buffer(10) ]], \- constant const uint& num_channels [[ buffer(11) ]], \- constant const size_t& suffix_len [[ buffer(12) ]], \- constant const uint& inner_dim [[ buffer(13) ]], \- constant const uint& proj_dim [[ buffer(14) ]], \- uint2 grid_idx [[ thread_position_in_grid ]] \+#define instantiate_conv1d_decode_kernel(type_name, type) \+ template [[host_name("conv1d_decode_kernel_" #type_name)]] \+ kernel void conv1d_decode_kernel<type>( \+ device const type* x [[buffer(0)]], \+ device const type* w [[buffer(1)]], \+ device const type* b [[buffer(2)]], \+ device const type* state [[buffer(3)]], \+ device type* x_out [[buffer(4)]], \+ device type* b_out [[buffer(5)]], \+ device type* c_out [[buffer(6)]], \+ device type* next_state [[buffer(7)]], \+ constant const int& kernel_size [[buffer(8)]], \+ constant const size_t& row_stride [[buffer(9)]], \+ constant const size_t& state_stride [[buffer(10)]], \+ constant const uint& num_channels [[buffer(11)]], \+ constant const size_t& suffix_len [[buffer(12)]], \+ constant const uint& inner_dim [[buffer(13)]], \+ constant const uint& proj_dim [[buffer(14)]], \+ uint2 grid_idx [[thread_position_in_grid]] \);#undef instantiate_conv1d_pack_prefix_kerneldiff --git a/crates/uzu/src/backends/metal/kernel/ssm/split_inproj.metal b/crates/uzu/src/backends/metal/kernel/ssm/split_inproj.metalindex 9f6565fd2..ae0066ead 100644--- a/crates/uzu/src/backends/metal/kernel/ssm/split_inproj.metal+++ b/crates/uzu/src/backends/metal/kernel/ssm/split_inproj.metal@@ -5,53 +5,53 @@ using namespace metal;template <typename T>kernel void ssm_split_inproj_kernel(- device const T* input [[ buffer(0) ]],- device T* conv_out [[ buffer(1) ]],- device T* z_out [[ buffer(2) ]],- device T* dt_out [[ buffer(3) ]],- device const T* z_bias [[ buffer(4) ]],- constant const int& total_dim [[ buffer(5) ]],- constant const int& conv_dim [[ buffer(6) ]],- constant const int& inner_dim [[ buffer(7) ]],- constant const int& num_heads [[ buffer(8) ]],- uint2 grid_idx [[ thread_position_in_grid ]],- uint2 grid_size [[ threads_per_grid ]]+ device const T* input [[buffer(0)]],+ device T* conv_out [[buffer(1)]],+ device T* z_out [[buffer(2)]],+ device T* dt_out [[buffer(3)]],+ device const T* z_bias [[buffer(4)]],+ constant const int& total_dim [[buffer(5)]],+ constant const int& conv_dim [[buffer(6)]],+ constant const int& inner_dim [[buffer(7)]],+ constant const int& num_heads [[buffer(8)]],+ uint2 grid_idx [[thread_position_in_grid]],+ uint2 grid_size [[threads_per_grid]]) {- const int row = static_cast<int>(grid_idx.x);- const int col = static_cast<int>(grid_idx.y);+ const int row = static_cast<int>(grid_idx.x);+ const int col = static_cast<int>(grid_idx.y);- if (row >= grid_size.x || col >= grid_size.y || col >= total_dim) {- return;- }+ if (row >= grid_size.x || col >= grid_size.y || col >= total_dim) {+ return;+ }- const int input_idx = row * total_dim + col;- if (col < conv_dim) {- const int dst = row * conv_dim + col;- conv_out[dst] = input[input_idx];- } else if (col < conv_dim + inner_dim) {- const int dst = row * inner_dim + (col - conv_dim);- const int bias_idx = col - conv_dim;- z_out[dst] = input[input_idx] + z_bias[bias_idx];- } else if (col < conv_dim + inner_dim + num_heads) {- const int dst = row * num_heads + (col - conv_dim - inner_dim);- dt_out[dst] = input[input_idx];- }+ const int input_idx = row * total_dim + col;+ if (col < conv_dim) {+ const int dst = row * conv_dim + col;+ conv_out[dst] = input[input_idx];+ } else if (col < conv_dim + inner_dim) {+ const int dst = row * inner_dim + (col - conv_dim);+ const int bias_idx = col - conv_dim;+ z_out[dst] = input[input_idx] + z_bias[bias_idx];+ } else if (col < conv_dim + inner_dim + num_heads) {+ const int dst = row * num_heads + (col - conv_dim - inner_dim);+ dt_out[dst] = input[input_idx];+ }}-#define instantiate_ssm_split_inproj_kernel(type_name, type) \- template [[host_name("ssm_split_inproj_kernel_" #type_name)]] \- kernel void ssm_split_inproj_kernel<type>( \- device const type* input [[ buffer(0) ]], \- device type* conv_out [[ buffer(1) ]], \- device type* z_out [[ buffer(2) ]], \- device type* dt_out [[ buffer(3) ]], \- device const type* z_bias [[ buffer(4) ]], \- constant const int& total_dim [[ buffer(5) ]], \- constant const int& conv_dim [[ buffer(6) ]], \- constant const int& inner_dim [[ buffer(7) ]], \- constant const int& num_heads [[ buffer(8) ]], \- uint2 grid_idx [[ thread_position_in_grid ]], \- uint2 grid_size [[ threads_per_grid ]] \+#define instantiate_ssm_split_inproj_kernel(type_name, type) \+ template [[host_name("ssm_split_inproj_kernel_" #type_name)]] \+ kernel void ssm_split_inproj_kernel<type>( \+ device const type* input [[buffer(0)]], \+ device type* conv_out [[buffer(1)]], \+ device type* z_out [[buffer(2)]], \+ device type* dt_out [[buffer(3)]], \+ device const type* z_bias [[buffer(4)]], \+ constant const int& total_dim [[buffer(5)]], \+ constant const int& conv_dim [[buffer(6)]], \+ constant const int& inner_dim [[buffer(7)]], \+ constant const int& num_heads [[buffer(8)]], \+ uint2 grid_idx [[thread_position_in_grid]], \+ uint2 grid_size [[threads_per_grid]] \);instantiate_ssm_split_inproj_kernel(float, float);diff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal b/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metalindex c39c50ec8..72c0bebbe 100644--- a/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal+++ b/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal@@ -18,141 +18,143 @@ struct SILU {template <typename T>inline T softplus(T x) {- float xf = float(x);- if (xf > 20.0f) {- return x;- }- return static_cast<T>(log(1.0f + fast::exp(xf)));+ float xf = float(x);+ if (xf > 20.0f) {+ return x;+ }+ return static_cast<T>(log(1.0f + fast::exp(xf)));}template <typename T>kernel void ssd_prefill_kernel_64(- device const T* x [[ buffer(0) ]], // (suffix, h, dh)- device const T* dt_raw [[ buffer(1) ]], // (suffix, h) - raw dt values- device const T* B [[ buffer(2) ]], // (suffix, g, n)- device const T* C [[ buffer(3) ]], // (suffix, g, n)- device const T* D [[ buffer(4) ]], // (h)- device const T* z [[ buffer(5) ]], // (suffix, h, dh)- device T* state [[ buffer(6) ]], // (h, dh, n)- device T* y [[ buffer(7) ]], // (suffix, h, dh)- constant const size_t& suffix_len [[ buffer(8) ]],- constant const int& group_size [[ buffer(9) ]],- constant const int& state_size [[ buffer(10) ]],- constant const size_t* x_strides [[ buffer(11) ]],- constant const size_t* dt_strides [[ buffer(12) ]],- constant const size_t* cb_strides [[ buffer(13) ]],- constant const size_t* state_strides [[ buffer(14) ]],- constant const uint& num_heads [[ buffer(15) ]],- constant const uint& head_dim [[ buffer(16) ]],- uint3 tg_pos [[ threadgroup_position_in_grid ]],- ushort lane [[ thread_index_in_threadgroup ]]+ device const T* x [[buffer(0)]], // (suffix, h, dh)+ device const T* dt_raw [[buffer(1)]], // (suffix, h) - raw dt values+ device const T* B [[buffer(2)]], // (suffix, g, n)+ device const T* C [[buffer(3)]], // (suffix, g, n)+ device const T* D [[buffer(4)]], // (h)+ device const T* z [[buffer(5)]], // (suffix, h, dh)+ device T* state [[buffer(6)]], // (h, dh, n)+ device T* y [[buffer(7)]], // (suffix, h, dh)+ constant const size_t& suffix_len [[buffer(8)]],+ constant const int& group_size [[buffer(9)]],+ constant const int& state_size [[buffer(10)]],+ constant const size_t* x_strides [[buffer(11)]],+ constant const size_t* dt_strides [[buffer(12)]],+ constant const size_t* cb_strides [[buffer(13)]],+ constant const size_t* state_strides [[buffer(14)]],+ constant const uint& num_heads [[buffer(15)]],+ constant const uint& head_dim [[buffer(16)]],+ uint3 tg_pos [[threadgroup_position_in_grid]],+ ushort lane [[thread_index_in_threadgroup]]) {- const uint simd_width = 32;- const int state_dim = state_size;+ const uint simd_width = 32;+ const int state_dim = state_size;- const uint total_pairs = num_heads * head_dim;- const uint pair_idx = tg_pos.x;- if (pair_idx >= total_pairs) {- return;- }+ const uint total_pairs = num_heads * head_dim;+ const uint pair_idx = tg_pos.x;+ if (pair_idx >= total_pairs) {+ return;+ }- const uint lane_idx = uint(lane);- const uint h_idx = pair_idx / head_dim;- const uint dh_idx = pair_idx % head_dim;- const uint safe_group = uint(max(group_size, 1));- const uint group_idx = h_idx / safe_group;-- const size_t x_token_stride = x_strides[0];- const size_t x_head_stride = x_strides[1];- const size_t x_dim_stride = x_strides[2];- const size_t dt_token_stride = dt_strides[0];- const size_t dt_head_stride = dt_strides[1];- const size_t cb_token_stride = cb_strides[0];- const size_t cb_group_stride = cb_strides[1];- const size_t cb_state_stride = cb_strides[2];- const size_t state_head_stride = state_strides[0];- const size_t state_dim_stride = state_strides[1];- const size_t state_inner_stride = state_strides[2];-- const size_t x_base = size_t(h_idx) * x_head_stride- + size_t(dh_idx) * x_dim_stride;- const size_t dt_base = size_t(h_idx) * dt_head_stride;- const size_t state_base =- size_t(h_idx) * state_head_stride + size_t(dh_idx) * state_dim_stride;- const size_t cb_group_base = size_t(group_idx) * cb_group_stride;- const float d_scalar = float(D[h_idx]);-- const uint idx0 = lane_idx;- const uint idx1 = lane_idx + simd_width;-- float state0 = 0.0f;- float state1 = 0.0f;- size_t state_idx0 = 0;- size_t state_idx1 = 0;- state_idx0 = state_base + size_t(idx0) * state_inner_stride;- state0 = float(state[state_idx0]);-- state_idx1 = state_base + size_t(idx1) * state_inner_stride;- state1 = float(state[state_idx1]);-- size_t cb_idx0 = 0;- size_t cb_idx1 = 0;- cb_idx0 = cb_group_base + size_t(idx0) * cb_state_stride;- cb_idx1 = cb_group_base + size_t(idx1) * cb_state_stride;-- for (size_t token = 0; token < suffix_len; ++token) {- const size_t x_idx = token * x_token_stride + x_base;- const size_t dt_idx = token * dt_token_stride + dt_base;-- const float x_val = float(x[x_idx]);- const float decay_val = fast::exp(-float(softplus(float(dt_raw[dt_idx]))));- const float gate = float(SILU{}(z[x_idx]));- const float skip = d_scalar * x_val;- const float dt_scaled_input = x_val;-- float contrib = 0.0f;- const float new_state0 = decay_val * state0 + dt_scaled_input * float(B[cb_idx0]);- state0 = new_state0;- contrib += new_state0 * float(C[cb_idx0]);- cb_idx0 += cb_token_stride;-- const float new_state1 = decay_val * state1 + dt_scaled_input * float(B[cb_idx1]);- state1 = new_state1;- contrib += new_state1 * float(C[cb_idx1]);- cb_idx1 += cb_token_stride;-- float dot = simd_sum(contrib);- if (lane_idx == 0) {- y[x_idx] = static_cast<T>((skip + dot) * gate);- }+ const uint lane_idx = uint(lane);+ const uint h_idx = pair_idx / head_dim;+ const uint dh_idx = pair_idx % head_dim;+ const uint safe_group = uint(max(group_size, 1));+ const uint group_idx = h_idx / safe_group;++ const size_t x_token_stride = x_strides[0];+ const size_t x_head_stride = x_strides[1];+ const size_t x_dim_stride = x_strides[2];+ const size_t dt_token_stride = dt_strides[0];+ const size_t dt_head_stride = dt_strides[1];+ const size_t cb_token_stride = cb_strides[0];+ const size_t cb_group_stride = cb_strides[1];+ const size_t cb_state_stride = cb_strides[2];+ const size_t state_head_stride = state_strides[0];+ const size_t state_dim_stride = state_strides[1];+ const size_t state_inner_stride = state_strides[2];++ const size_t x_base =+ size_t(h_idx) * x_head_stride + size_t(dh_idx) * x_dim_stride;+ const size_t dt_base = size_t(h_idx) * dt_head_stride;+ const size_t state_base =+ size_t(h_idx) * state_head_stride + size_t(dh_idx) * state_dim_stride;+ const size_t cb_group_base = size_t(group_idx) * cb_group_stride;+ const float d_scalar = float(D[h_idx]);++ const uint idx0 = lane_idx;+ const uint idx1 = lane_idx + simd_width;++ float state0 = 0.0f;+ float state1 = 0.0f;+ size_t state_idx0 = 0;+ size_t state_idx1 = 0;+ state_idx0 = state_base + size_t(idx0) * state_inner_stride;+ state0 = float(state[state_idx0]);++ state_idx1 = state_base + size_t(idx1) * state_inner_stride;+ state1 = float(state[state_idx1]);++ size_t cb_idx0 = 0;+ size_t cb_idx1 = 0;+ cb_idx0 = cb_group_base + size_t(idx0) * cb_state_stride;+ cb_idx1 = cb_group_base + size_t(idx1) * cb_state_stride;++ for (size_t token = 0; token < suffix_len; ++token) {+ const size_t x_idx = token * x_token_stride + x_base;+ const size_t dt_idx = token * dt_token_stride + dt_base;++ const float x_val = float(x[x_idx]);+ const float decay_val = fast::exp(-float(softplus(float(dt_raw[dt_idx]))));+ const float gate = float(SILU{}(z[x_idx]));+ const float skip = d_scalar * x_val;+ const float dt_scaled_input = x_val;++ float contrib = 0.0f;+ const float new_state0 =+ decay_val * state0 + dt_scaled_input * float(B[cb_idx0]);+ state0 = new_state0;+ contrib += new_state0 * float(C[cb_idx0]);+ cb_idx0 += cb_token_stride;++ const float new_state1 =+ decay_val * state1 + dt_scaled_input * float(B[cb_idx1]);+ state1 = new_state1;+ contrib += new_state1 * float(C[cb_idx1]);+ cb_idx1 += cb_token_stride;++ float dot = simd_sum(contrib);+ if (lane_idx == 0) {+ y[x_idx] = static_cast<T>((skip + dot) * gate);}+ }- state[state_idx0] = static_cast<T>(state0);- state[state_idx1] = static_cast<T>(state1);+ state[state_idx0] = static_cast<T>(state0);+ state[state_idx1] = static_cast<T>(state1);}-#define instantiate_ssd_prefill_kernel_64(type_name, type) \- template [[host_name("ssd_prefill_kernel_64_" #type_name)]] \- kernel void ssd_prefill_kernel_64<type>( \- device const type* x [[ buffer(0) ]], \- device const type* dt_raw [[ buffer(1) ]], \- device const type* B [[ buffer(2) ]], \- device const type* C [[ buffer(3) ]], \- device const type* D [[ buffer(4) ]], \- device const type* z [[ buffer(5) ]], \- device type* state [[ buffer(6) ]], \- device type* y [[ buffer(7) ]], \- constant const size_t& suffix_len [[ buffer(8) ]], \- constant const int& group_size [[ buffer(9) ]], \- constant const int& state_size [[ buffer(10) ]], \- constant const size_t* x_strides [[ buffer(11) ]], \- constant const size_t* dt_strides [[ buffer(12) ]], \- constant const size_t* cb_strides [[ buffer(13) ]], \- constant const size_t* state_strides [[ buffer(14) ]], \- constant const uint& num_heads [[ buffer(15) ]], \- constant const uint& head_dim [[ buffer(16) ]], \- uint3 tg_pos [[ threadgroup_position_in_grid ]], \- ushort lane [[ thread_index_in_threadgroup ]] \+#define instantiate_ssd_prefill_kernel_64(type_name, type) \+ template [[host_name("ssd_prefill_kernel_64_" #type_name)]] \+ kernel void ssd_prefill_kernel_64<type>( \+ device const type* x [[buffer(0)]], \+ device const type* dt_raw [[buffer(1)]], \+ device const type* B [[buffer(2)]], \+ device const type* C [[buffer(3)]], \+ device const type* D [[buffer(4)]], \+ device const type* z [[buffer(5)]], \+ device type* state [[buffer(6)]], \+ device type* y [[buffer(7)]], \+ constant const size_t& suffix_len [[buffer(8)]], \+ constant const int& group_size [[buffer(9)]], \+ constant const int& state_size [[buffer(10)]], \+ constant const size_t* x_strides [[buffer(11)]], \+ constant const size_t* dt_strides [[buffer(12)]], \+ constant const size_t* cb_strides [[buffer(13)]], \+ constant const size_t* state_strides [[buffer(14)]], \+ constant const uint& num_heads [[buffer(15)]], \+ constant const uint& head_dim [[buffer(16)]], \+ uint3 tg_pos [[threadgroup_position_in_grid]], \+ ushort lane [[thread_index_in_threadgroup]] \);instantiate_ssd_prefill_kernel_64(float, float);@@ -163,133 +165,135 @@ instantiate_ssd_prefill_kernel_64(half, half);template <typename T>kernel void ssd_prefill_kernel(- device const T* x [[ buffer(0) ]], // (suffix, h, dh)- device const T* dt_raw [[ buffer(1) ]], // (suffix, h) - raw dt values- device const T* B [[ buffer(2) ]], // (suffix, g, n)- device const T* C [[ buffer(3) ]], // (suffix, g, n)- device const T* D [[ buffer(4) ]], // (h)- device const T* z [[ buffer(5) ]], // (suffix, h, dh)- device T* state [[ buffer(6) ]], // (h, dh, n)- device T* y [[ buffer(7) ]], // (suffix, h, dh)- constant const size_t& suffix_len [[ buffer(8) ]],- constant const int& group_size [[ buffer(9) ]],- constant const int& state_size [[ buffer(10) ]],- constant const size_t* x_strides [[ buffer(11) ]],- constant const size_t* dt_strides [[ buffer(12) ]],- constant const size_t* cb_strides [[ buffer(13) ]],- constant const size_t* state_strides [[ buffer(14) ]],- constant const uint& num_heads [[ buffer(15) ]],- constant const uint& head_dim [[ buffer(16) ]],- uint3 tg_pos [[ threadgroup_position_in_grid ]],- ushort lane [[ thread_index_in_threadgroup ]]+ device const T* x [[buffer(0)]], // (suffix, h, dh)+ device const T* dt_raw [[buffer(1)]], // (suffix, h) - raw dt values+ device const T* B [[buffer(2)]], // (suffix, g, n)+ device const T* C [[buffer(3)]], // (suffix, g, n)+ device const T* D [[buffer(4)]], // (h)+ device const T* z [[buffer(5)]], // (suffix, h, dh)+ device T* state [[buffer(6)]], // (h, dh, n)+ device T* y [[buffer(7)]], // (suffix, h, dh)+ constant const size_t& suffix_len [[buffer(8)]],+ constant const int& group_size [[buffer(9)]],+ constant const int& state_size [[buffer(10)]],+ constant const size_t* x_strides [[buffer(11)]],+ constant const size_t* dt_strides [[buffer(12)]],+ constant const size_t* cb_strides [[buffer(13)]],+ constant const size_t* state_strides [[buffer(14)]],+ constant const uint& num_heads [[buffer(15)]],+ constant const uint& head_dim [[buffer(16)]],+ uint3 tg_pos [[threadgroup_position_in_grid]],+ ushort lane [[thread_index_in_threadgroup]]) {- const uint simd_width = 32;- const int state_dim = state_size;+ const uint simd_width = 32;+ const int state_dim = state_size;- const uint total_pairs = num_heads * head_dim;- const uint pair_idx = tg_pos.x;- if (pair_idx >= total_pairs) {- return;- }+ const uint total_pairs = num_heads * head_dim;+ const uint pair_idx = tg_pos.x;+ if (pair_idx >= total_pairs) {+ return;+ }- const uint lane_idx = uint(lane);- const uint h_idx = pair_idx / head_dim;- const uint dh_idx = pair_idx % head_dim;- const uint safe_group = uint(max(group_size, 1));- const uint group_idx = h_idx / safe_group;-- const size_t x_token_stride = x_strides[0];- const size_t x_head_stride = x_strides[1];- const size_t x_dim_stride = x_strides[2];- const size_t dt_token_stride = dt_strides[0];- const size_t dt_head_stride = dt_strides[1];- const size_t cb_token_stride = cb_strides[0];- const size_t cb_group_stride = cb_strides[1];- const size_t cb_state_stride = cb_strides[2];- const size_t state_head_stride = state_strides[0];- const size_t state_dim_stride = state_strides[1];- const size_t state_inner_stride = state_strides[2];-- const size_t x_base = size_t(h_idx) * x_head_stride + size_t(dh_idx) * x_dim_stride;- const size_t dt_base = size_t(h_idx) * dt_head_stride;- const size_t state_base = size_t(h_idx) * state_head_stride + size_t(dh_idx) * state_dim_stride;- const size_t cb_group_base = size_t(group_idx) * cb_group_stride;- const float d_scalar = float(D[h_idx]);-- const int max_chunks = SSM_PREFILL_MAX_STATE / simd_width;- const int chunk_count =- (state_dim + int(simd_width) - 1) / int(simd_width);- if (state_dim <= 0 || chunk_count > max_chunks) {- return;- }+ const uint lane_idx = uint(lane);+ const uint h_idx = pair_idx / head_dim;+ const uint dh_idx = pair_idx % head_dim;+ const uint safe_group = uint(max(group_size, 1));+ const uint group_idx = h_idx / safe_group;++ const size_t x_token_stride = x_strides[0];+ const size_t x_head_stride = x_strides[1];+ const size_t x_dim_stride = x_strides[2];+ const size_t dt_token_stride = dt_strides[0];+ const size_t dt_head_stride = dt_strides[1];+ const size_t cb_token_stride = cb_strides[0];+ const size_t cb_group_stride = cb_strides[1];+ const size_t cb_state_stride = cb_strides[2];+ const size_t state_head_stride = state_strides[0];+ const size_t state_dim_stride = state_strides[1];+ const size_t state_inner_stride = state_strides[2];++ const size_t x_base =+ size_t(h_idx) * x_head_stride + size_t(dh_idx) * x_dim_stride;+ const size_t dt_base = size_t(h_idx) * dt_head_stride;+ const size_t state_base =+ size_t(h_idx) * state_head_stride + size_t(dh_idx) * state_dim_stride;+ const size_t cb_group_base = size_t(group_idx) * cb_group_stride;+ const float d_scalar = float(D[h_idx]);++ const int max_chunks = SSM_PREFILL_MAX_STATE / simd_width;+ const int chunk_count = (state_dim + int(simd_width) - 1) / int(simd_width);+ if (state_dim <= 0 || chunk_count > max_chunks) {+ return;+ }++ thread float lane_states[SSM_PREFILL_MAX_STATE / 32];- thread float lane_states[SSM_PREFILL_MAX_STATE / 32];+#pragma unroll+ for (int chunk = 0; chunk < chunk_count; ++chunk) {+ const int idx = chunk * int(simd_width) + int(lane_idx);+ const size_t state_idx = state_base + size_t(idx) * state_inner_stride;+ lane_states[chunk] = float(state[state_idx]);+ }- #pragma unroll+ for (size_t token = 0; token < suffix_len; ++token) {+ const size_t x_idx = token * x_token_stride + x_base;+ const size_t dt_idx = token * dt_token_stride + dt_base;++ const float x_val = float(x[x_idx]);+ const float decay_val = fast::exp(-float(softplus(float(dt_raw[dt_idx]))));+ const float gate = float(SILU{}(z[x_idx]));+ const float skip = d_scalar * x_val;+ const float dt_scaled_input = x_val;++ float contrib_sum = 0.0f;+#pragma unrollfor (int chunk = 0; chunk < chunk_count; ++chunk) {- const int idx = chunk * int(simd_width) + int(lane_idx);- const size_t state_idx =- state_base + size_t(idx) * state_inner_stride;- lane_states[chunk] = float(state[state_idx]);+ const int idx = chunk * int(simd_width) + int(lane_idx);+ const size_t cb_idx = cb_group_base + size_t(idx) * cb_state_stride ++ token * cb_token_stride;++ const float new_state =+ decay_val * lane_states[chunk] + dt_scaled_input * float(B[cb_idx]);+ lane_states[chunk] = new_state;+ contrib_sum += new_state * float(C[cb_idx]);}- for (size_t token = 0; token < suffix_len; ++token) {- const size_t x_idx = token * x_token_stride + x_base;- const size_t dt_idx = token * dt_token_stride + dt_base;-- const float x_val = float(x[x_idx]);- const float decay_val = fast::exp(-float(softplus(float(dt_raw[dt_idx]))));- const float gate = float(SILU{}(z[x_idx]));- const float skip = d_scalar * x_val;- const float dt_scaled_input = x_val;-- float contrib_sum = 0.0f;- #pragma unroll- for (int chunk = 0; chunk < chunk_count; ++chunk) {- const int idx = chunk * int(simd_width) + int(lane_idx);- const size_t cb_idx = cb_group_base + size_t(idx) * cb_state_stride + token * cb_token_stride;-- const float new_state = decay_val * lane_states[chunk] + dt_scaled_input * float(B[cb_idx]);- lane_states[chunk] = new_state;- contrib_sum += new_state * float(C[cb_idx]);- }-- float dot = simd_sum(contrib_sum);- if (lane_idx == 0) {- y[x_idx] = static_cast<T>((skip + dot) * gate);- }+ float dot = simd_sum(contrib_sum);+ if (lane_idx == 0) {+ y[x_idx] = static_cast<T>((skip + dot) * gate);}+ }- #pragma unroll- for (int chunk = 0; chunk < chunk_count; ++chunk) {- const int idx = chunk * int(simd_width) + int(lane_idx);- const size_t state_idx = state_base + size_t(idx) * state_inner_stride;- state[state_idx] = static_cast<T>(lane_states[chunk]);- }+#pragma unroll+ for (int chunk = 0; chunk < chunk_count; ++chunk) {+ const int idx = chunk * int(simd_width) + int(lane_idx);+ const size_t state_idx = state_base + size_t(idx) * state_inner_stride;+ state[state_idx] = static_cast<T>(lane_states[chunk]);+ }}-#define instantiate_ssd_prefill_kernel(type_name, type) \- template [[host_name("ssd_prefill_kernel_" #type_name)]] \- kernel void ssd_prefill_kernel<type>( \- device const type* x [[ buffer(0) ]], \- device const type* dt_raw [[ buffer(1) ]], \- device const type* B [[ buffer(2) ]], \- device const type* C [[ buffer(3) ]], \- device const type* D [[ buffer(4) ]], \- device const type* z [[ buffer(5) ]], \- device type* state [[ buffer(6) ]], \- device type* y [[ buffer(7) ]], \- constant const size_t& suffix_len [[ buffer(8) ]], \- constant const int& group_size [[ buffer(9) ]], \- constant const int& state_size [[ buffer(10) ]], \- constant const size_t* x_strides [[ buffer(11) ]], \- constant const size_t* dt_strides [[ buffer(12) ]], \- constant const size_t* cb_strides [[ buffer(13) ]], \- constant const size_t* state_strides [[ buffer(14) ]], \- constant const uint& num_heads [[ buffer(15) ]], \- constant const uint& head_dim [[ buffer(16) ]], \- uint3 tg_pos [[ threadgroup_position_in_grid ]], \- ushort lane [[ thread_index_in_threadgroup ]] \+#define instantiate_ssd_prefill_kernel(type_name, type) \+ template [[host_name("ssd_prefill_kernel_" #type_name)]] \+ kernel void ssd_prefill_kernel<type>( \+ device const type* x [[buffer(0)]], \+ device const type* dt_raw [[buffer(1)]], \+ device const type* B [[buffer(2)]], \+ device const type* C [[buffer(3)]], \+ device const type* D [[buffer(4)]], \+ device const type* z [[buffer(5)]], \+ device type* state [[buffer(6)]], \+ device type* y [[buffer(7)]], \+ constant const size_t& suffix_len [[buffer(8)]], \+ constant const int& group_size [[buffer(9)]], \+ constant const int& state_size [[buffer(10)]], \+ constant const size_t* x_strides [[buffer(11)]], \+ constant const size_t* dt_strides [[buffer(12)]], \+ constant const size_t* cb_strides [[buffer(13)]], \+ constant const size_t* state_strides [[buffer(14)]], \+ constant const uint& num_heads [[buffer(15)]], \+ constant const uint& head_dim [[buffer(16)]], \+ uint3 tg_pos [[threadgroup_position_in_grid]], \+ ushort lane [[thread_index_in_threadgroup]] \);instantiate_ssd_prefill_kernel(float, float);@@ -300,101 +304,99 @@ instantiate_ssd_prefill_kernel(half, half);template <typename T>kernel void ssd_prefill_kernel_sequential(- device const T* x [[ buffer(0) ]], // (suffix, h, dh)- device const T* dt_raw [[ buffer(1) ]], // (suffix, h) - raw dt values- device const T* B [[ buffer(2) ]], // (suffix, g, n)- device const T* C [[ buffer(3) ]], // (suffix, g, n)- device const T* D [[ buffer(4) ]], // (h)- device const T* z [[ buffer(5) ]], // (suffix, h, dh)- device T* state [[ buffer(6) ]], // (h, dh, n)- device T* y [[ buffer(7) ]], // (suffix, h, dh)- constant const size_t& suffix_len [[ buffer(8) ]],- constant const int& group_size [[ buffer(9) ]],- constant const int& state_size [[ buffer(10) ]],- constant const size_t* x_strides [[ buffer(11) ]],- constant const size_t* dt_strides [[ buffer(12) ]],- constant const size_t* cb_strides [[ buffer(13) ]],- constant const size_t* state_strides [[ buffer(14) ]],- uint3 tid [[ thread_position_in_grid ]],- uint3 grid_dim [[ threads_per_grid ]]+ device const T* x [[buffer(0)]], // (suffix, h, dh)+ device const T* dt_raw [[buffer(1)]], // (suffix, h) - raw dt values+ device const T* B [[buffer(2)]], // (suffix, g, n)+ device const T* C [[buffer(3)]], // (suffix, g, n)+ device const T* D [[buffer(4)]], // (h)+ device const T* z [[buffer(5)]], // (suffix, h, dh)+ device T* state [[buffer(6)]], // (h, dh, n)+ device T* y [[buffer(7)]], // (suffix, h, dh)+ constant const size_t& suffix_len [[buffer(8)]],+ constant const int& group_size [[buffer(9)]],+ constant const int& state_size [[buffer(10)]],+ constant const size_t* x_strides [[buffer(11)]],+ constant const size_t* dt_strides [[buffer(12)]],+ constant const size_t* cb_strides [[buffer(13)]],+ constant const size_t* state_strides [[buffer(14)]],+ uint3 tid [[thread_position_in_grid]],+ uint3 grid_dim [[threads_per_grid]]) {- const uint h_idx = tid.x;- const uint dh_idx = tid.y;- if (h_idx >= grid_dim.x || dh_idx >= grid_dim.y) {- return;- }+ const uint h_idx = tid.x;+ const uint dh_idx = tid.y;+ if (h_idx >= grid_dim.x || dh_idx >= grid_dim.y) {+ return;+ }- const uint safe_group = uint(max(group_size, 1));- const uint group_idx = h_idx / safe_group;- device T* state_row =- state + size_t(h_idx) * state_strides[0] + size_t(dh_idx) * state_strides[1];-- for (size_t token = 0; token < suffix_len; ++token) {- const size_t x_idx = token * x_strides[0]- + size_t(h_idx) * x_strides[1]- + size_t(dh_idx) * x_strides[2];- const size_t dt_idx =- token * dt_strides[0] + size_t(h_idx) * dt_strides[1];- const size_t cb_base =- token * cb_strides[0] + group_idx * cb_strides[1];-- const T this_x = x[x_idx];- const T dt_raw_val = dt_raw[dt_idx];- const T this_dt = softplus(dt_raw_val);- const T this_decay = static_cast<T>(fast::exp(-float(this_dt)));- const T this_D = D[h_idx];- const T this_z = SILU{}(z[x_idx]);- const T dt_scaled_input = this_x;-- T acc = T(0);- int s = 0;- const int vec_bound = (state_size / 4) * 4;- for (; s < vec_bound; s += 4) {- const size_t state_idx = s * state_strides[2];- const size_t cb_idx = cb_base + s * cb_strides[2];- auto prev_state = *reinterpret_cast<device vec<T, 4>*>(state_row + state_idx);- auto b_vec = *reinterpret_cast<device const vec<T, 4>*>(B + cb_idx);- auto c_vec = *reinterpret_cast<device const vec<T, 4>*>(C + cb_idx);- vec<T, 4> new_state = prev_state * this_decay + b_vec * dt_scaled_input;- *reinterpret_cast<device vec<T, 4>*>(state_row + state_idx) = new_state;- vec<T, 4> prod = new_state * c_vec;- acc += prod.x + prod.y + prod.z + prod.w;- }- for (; s < state_size; ++s) {- const size_t state_idx = s * state_strides[2];- const T prev_state = state_row[state_idx];- const size_t cb_idx = cb_base + s * cb_strides[2];- const T new_state = prev_state * this_decay + B[cb_idx] * dt_scaled_input;- state_row[state_idx] = new_state;- acc += new_state * C[cb_idx];- }-- acc += this_D * this_x;- acc *= this_z;- y[x_idx] = acc;+ const uint safe_group = uint(max(group_size, 1));+ const uint group_idx = h_idx / safe_group;+ device T* state_row = state + size_t(h_idx) * state_strides[0] ++ size_t(dh_idx) * state_strides[1];++ for (size_t token = 0; token < suffix_len; ++token) {+ const size_t x_idx = token * x_strides[0] + size_t(h_idx) * x_strides[1] ++ size_t(dh_idx) * x_strides[2];+ const size_t dt_idx = token * dt_strides[0] + size_t(h_idx) * dt_strides[1];+ const size_t cb_base = token * cb_strides[0] + group_idx * cb_strides[1];++ const T this_x = x[x_idx];+ const T dt_raw_val = dt_raw[dt_idx];+ const T this_dt = softplus(dt_raw_val);+ const T this_decay = static_cast<T>(fast::exp(-float(this_dt)));+ const T this_D = D[h_idx];+ const T this_z = SILU{}(z[x_idx]);+ const T dt_scaled_input = this_x;++ T acc = T(0);+ int s = 0;+ const int vec_bound = (state_size / 4) * 4;+ for (; s < vec_bound; s += 4) {+ const size_t state_idx = s * state_strides[2];+ const size_t cb_idx = cb_base + s * cb_strides[2];+ auto prev_state =+ *reinterpret_cast<device vec<T, 4>*>(state_row + state_idx);+ auto b_vec = *reinterpret_cast<device const vec<T, 4>*>(B + cb_idx);+ auto c_vec = *reinterpret_cast<device const vec<T, 4>*>(C + cb_idx);+ vec<T, 4> new_state = prev_state * this_decay + b_vec * dt_scaled_input;+ *reinterpret_cast<device vec<T, 4>*>(state_row + state_idx) = new_state;+ vec<T, 4> prod = new_state * c_vec;+ acc += prod.x + prod.y + prod.z + prod.w;+ }+ for (; s < state_size; ++s) {+ const size_t state_idx = s * state_strides[2];+ const T prev_state = state_row[state_idx];+ const size_t cb_idx = cb_base + s * cb_strides[2];+ const T new_state = prev_state * this_decay + B[cb_idx] * dt_scaled_input;+ state_row[state_idx] = new_state;+ acc += new_state * C[cb_idx];}++ acc += this_D * this_x;+ acc *= this_z;+ y[x_idx] = acc;+ }}-#define instantiate_ssd_prefill_kernel_sequential(type_name, type) \- template [[host_name("ssd_prefill_kernel_sequential_" #type_name)]] \- kernel void ssd_prefill_kernel_sequential<type>( \- device const type* x [[ buffer(0) ]], \- device const type* dt_raw [[ buffer(1) ]], \- device const type* B [[ buffer(2) ]], \- device const type* C [[ buffer(3) ]], \- device const type* D [[ buffer(4) ]], \- device const type* z [[ buffer(5) ]], \- device type* state [[ buffer(6) ]], \- device type* y [[ buffer(7) ]], \- constant const size_t& suffix_len [[ buffer(8) ]], \- constant const int& group_size [[ buffer(9) ]], \- constant const int& state_size [[ buffer(10) ]], \- constant const size_t* x_strides [[ buffer(11) ]], \- constant const size_t* dt_strides [[ buffer(12) ]], \- constant const size_t* cb_strides [[ buffer(13) ]], \- constant const size_t* state_strides [[ buffer(14) ]], \- uint3 tid [[ thread_position_in_grid ]], \- uint3 grid_dim [[ threads_per_grid ]] \+#define instantiate_ssd_prefill_kernel_sequential(type_name, type) \+ template [[host_name("ssd_prefill_kernel_sequential_" #type_name)]] \+ kernel void ssd_prefill_kernel_sequential<type>( \+ device const type* x [[buffer(0)]], \+ device const type* dt_raw [[buffer(1)]], \+ device const type* B [[buffer(2)]], \+ device const type* C [[buffer(3)]], \+ device const type* D [[buffer(4)]], \+ device const type* z [[buffer(5)]], \+ device type* state [[buffer(6)]], \+ device type* y [[buffer(7)]], \+ constant const size_t& suffix_len [[buffer(8)]], \+ constant const int& group_size [[buffer(9)]], \+ constant const int& state_size [[buffer(10)]], \+ constant const size_t* x_strides [[buffer(11)]], \+ constant const size_t* dt_strides [[buffer(12)]], \+ constant const size_t* cb_strides [[buffer(13)]], \+ constant const size_t* state_strides [[buffer(14)]], \+ uint3 tid [[thread_position_in_grid]], \+ uint3 grid_dim [[threads_per_grid]] \);instantiate_ssd_prefill_kernel_sequential(float, float);diff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal b/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metalindex 048023aca..085fc63eb 100644--- a/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal+++ b/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal@@ -15,90 +15,100 @@ struct SILU {template <typename T>inline T softplus(T x) {- float xf = float(x);- if (xf > 20.0f) {- return x;- }- return static_cast<T>(log(1.0f + fast::exp(xf)));+ float xf = float(x);+ if (xf > 20.0f) {+ return x;+ }+ return static_cast<T>(log(1.0f + fast::exp(xf)));}template <typename T>kernel void ssd_update_kernel(- // Input- device const T* x [[ buffer(0) ]], // (b, h, dh)- device const T* dt_raw [[ buffer(1) ]], // (b, h)- device const T* B [[ buffer(2) ]], // (b, g, n)- device const T* C [[ buffer(3) ]], // (b, g, n)- device const T* D [[ buffer(4) ]], // (h)- device const T* z [[ buffer(5) ]], // (b, d)- device const T* state [[ buffer(6) ]], // (b, h, dh, n)- device T* y [[ buffer(7) ]],- device T* next_state [[ buffer(8) ]],+ // Input+ device const T* x [[buffer(0)]], // (b, h, dh)+ device const T* dt_raw [[buffer(1)]], // (b, h)+ device const T* B [[buffer(2)]], // (b, g, n)+ device const T* C [[buffer(3)]], // (b, g, n)+ device const T* D [[buffer(4)]], // (h)+ device const T* z [[buffer(5)]], // (b, d)+ device const T* state [[buffer(6)]], // (b, h, dh, n)+ device T* y [[buffer(7)]],+ device T* next_state [[buffer(8)]],// Parameters- constant const int& group_size [[ buffer(9) ]], // h = group_size * g- constant const int& state_size [[ buffer(10) ]],+ constant const int& group_size [[buffer(9)]], // h = group_size * g+ constant const int& state_size [[buffer(10)]],// Strides- constant const size_t* x_strides [[ buffer(11) ]],- constant const size_t* dt_strides [[ buffer(12) ]],- constant const size_t* CB_strides [[ buffer(13) ]],- constant const size_t* state_strides [[ buffer(14) ]],+ constant const size_t* x_strides [[buffer(11)]],+ constant const size_t* dt_strides [[buffer(12)]],+ constant const size_t* CB_strides [[buffer(13)]],+ constant const size_t* state_strides [[buffer(14)]],// Grid- uint3 grid_idx [[ thread_position_in_grid ]],- uint3 grid_size [[ threads_per_grid ]]+ uint3 grid_idx [[thread_position_in_grid]],+ uint3 grid_size [[threads_per_grid]]) {- const int b_idx = grid_idx.x;- const int h_idx = grid_idx.y;- const int dh_idx = grid_idx.z;+ const int b_idx = grid_idx.x;+ const int h_idx = grid_idx.y;+ const int dh_idx = grid_idx.z;- const int CB_start_idx = static_cast<int>(b_idx * CB_strides[0] + (h_idx / group_size) * CB_strides[1]);- const int x_idx = static_cast<int>(b_idx * x_strides[0] + h_idx * x_strides[1] + dh_idx * x_strides[2]);- const int dt_idx = static_cast<int>(b_idx * dt_strides[0] + h_idx * dt_strides[1]);- const int state_start_idx = static_cast<int>(b_idx * state_strides[0] + h_idx * state_strides[1] + dh_idx * state_strides[2]);+ const int CB_start_idx = static_cast<int>(+ b_idx * CB_strides[0] + (h_idx / group_size) * CB_strides[1]+ );+ const int x_idx = static_cast<int>(+ b_idx * x_strides[0] + h_idx * x_strides[1] + dh_idx * x_strides[2]+ );+ const int dt_idx =+ static_cast<int>(b_idx * dt_strides[0] + h_idx * dt_strides[1]);+ const int state_start_idx = static_cast<int>(+ b_idx * state_strides[0] + h_idx * state_strides[1] ++ dh_idx * state_strides[2]+ );- // load data- T this_x = x[x_idx];- T dt_raw_val = dt_raw[dt_idx];- T this_dt = softplus(dt_raw_val);- T this_decay = static_cast<T>(fast::exp(-float(this_dt)));- T this_D = D[h_idx];- T this_z = z[x_idx];- this_z = SILU{}(this_z);- T dt_scaled_input = this_x;+ // load data+ T this_x = x[x_idx];+ T dt_raw_val = dt_raw[dt_idx];+ T this_dt = softplus(dt_raw_val);+ T this_decay = static_cast<T>(fast::exp(-float(this_dt)));+ T this_D = D[h_idx];+ T this_z = z[x_idx];+ this_z = SILU{}(this_z);+ T dt_scaled_input = this_x;- T temp = T(0);- #pragma unroll- for (int i = 0; i < state_size; ++i) {- int CB_idx = CB_start_idx + i;- int state_idx = state_start_idx + i;- T this_new_state = state[state_idx] * this_decay + B[CB_idx] * dt_scaled_input;- next_state[state_idx] = this_new_state;- temp = temp + this_new_state * C[CB_idx];- }- temp = temp + this_D * this_x;- temp = temp * this_z;- y[x_idx] = temp;+ T temp = T(0);+#pragma unroll+ for (int i = 0; i < state_size; ++i) {+ int CB_idx = CB_start_idx + i;+ int state_idx = state_start_idx + i;+ T this_new_state =+ state[state_idx] * this_decay + B[CB_idx] * dt_scaled_input;+ next_state[state_idx] = this_new_state;+ temp = temp + this_new_state * C[CB_idx];+ }+ temp = temp + this_D * this_x;+ temp = temp * this_z;+ y[x_idx] = temp;}-#define instantiate_ssd_update_kernel(type_name, type) \- template [[host_name("ssd_update_kernel_" #type_name)]] \- kernel void ssd_update_kernel<type>( \- device const type* x [[buffer(0)]], \- device const type* dt_raw [[buffer(1)]], \- device const type* B [[buffer(2)]], \- device const type* C [[buffer(3)]], \- device const type* D [[buffer(4)]], \- device const type* z [[buffer(5)]], \- device const type* state [[buffer(6)]], \- device type* y [[buffer(7)]], \- device type* next_state [[buffer(8)]], \- constant const int& group_size [[buffer(9)]], \- constant const int& state_size [[buffer(10)]], \- constant const size_t* x_strides [[buffer(11)]], \- constant const size_t* dt_strides [[buffer(12)]], \- constant const size_t* CB_strides [[buffer(13)]], \- constant const size_t* state_strides [[buffer(14)]], \- uint3 grid_idx [[thread_position_in_grid]], \- uint3 grid_size [[threads_per_grid]]);+#define instantiate_ssd_update_kernel(type_name, type) \+ template [[host_name("ssd_update_kernel_" #type_name)]] \+ kernel void ssd_update_kernel<type>( \+ device const type* x [[buffer(0)]], \+ device const type* dt_raw [[buffer(1)]], \+ device const type* B [[buffer(2)]], \+ device const type* C [[buffer(3)]], \+ device const type* D [[buffer(4)]], \+ device const type* z [[buffer(5)]], \+ device const type* state [[buffer(6)]], \+ device type* y [[buffer(7)]], \+ device type* next_state [[buffer(8)]], \+ constant const int& group_size [[buffer(9)]], \+ constant const int& state_size [[buffer(10)]], \+ constant const size_t* x_strides [[buffer(11)]], \+ constant const size_t* dt_strides [[buffer(12)]], \+ constant const size_t* CB_strides [[buffer(13)]], \+ constant const size_t* state_strides [[buffer(14)]], \+ uint3 grid_idx [[thread_position_in_grid]], \+ uint3 grid_size [[threads_per_grid]] \+ );instantiate_ssd_update_kernel(float, float);instantiate_ssd_update_kernel(bfloat, bfloat);diff --git a/crates/uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal b/crates/uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metalindex 45b57d9ed..ca865bfb4 100644--- a/crates/uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal+++ b/crates/uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal@@ -10,31 +10,23 @@ void tensorAddBias(constant int& length,const uint position) {- if (position < length) {- int col = position % numCols;- output[position] = input[position] + bias[col];- }+ if (position < length) {+ int col = position % numCols;+ output[position] = input[position] + bias[col];+ }}-#define outerArguments(T) \-(const device T* input [[ buffer(0) ]], \-const device T* bias [[ buffer(1) ]], \-device T* output [[ buffer(2) ]], \-constant int& numCols [[ buffer(3) ]], \-constant int& length [[ buffer(4) ]], \-const uint position [[ thread_position_in_grid ]])+#define outerArguments(T) \+ (const device T* input [[buffer(0)]], \+ const device T* bias [[buffer(1)]], \+ device T* output [[buffer(2)]], \+ constant int& numCols [[buffer(3)]], \+ constant int& length [[buffer(4)]], \+ const uint position [[thread_position_in_grid]])-#define innerArguments \-(input, \-bias, \-output, \-numCols, \-length, \-position)+#define innerArguments (input, bias, output, numCols, length, position)generateKernels(tensorAddBias)#undef outerArguments#undef innerArguments--diff --git a/crates/uzu/src/backends/metal/kernel/tensor_add_swap/tensor_add_swap.metal b/crates/uzu/src/backends/metal/kernel/tensor_add_swap/tensor_add_swap.metalindex 42a5e235b..2b7d32ab5 100644--- a/crates/uzu/src/backends/metal/kernel/tensor_add_swap/tensor_add_swap.metal+++ b/crates/uzu/src/backends/metal/kernel/tensor_add_swap/tensor_add_swap.metal@@ -8,27 +8,23 @@ void tensorAddSwap(constant int& length,const uint position) {- if (position < length) {- T skipValue = skipBuffer[position];- T mainValue = mainBuffer[position];- T result = mainValue + skipValue;-- mainBuffer[position] = result;- skipBuffer[position] = result;- }+ if (position < length) {+ T skipValue = skipBuffer[position];+ T mainValue = mainBuffer[position];+ T result = mainValue + skipValue;++ mainBuffer[position] = result;+ skipBuffer[position] = result;+ }}-#define outerArguments(T) \-(device T* skipBuffer [[ buffer(0) ]], \-device T* mainBuffer [[ buffer(1) ]], \-constant int& length [[ buffer(2) ]], \-const uint position [[ thread_position_in_grid ]])\+#define outerArguments(T) \+ (device T * skipBuffer [[buffer(0)]], \+ device T * mainBuffer [[buffer(1)]], \+ constant int& length [[buffer(2)]], \+ const uint position [[thread_position_in_grid]])-#define innerArguments \-(skipBuffer, \-mainBuffer, \-length, \-position) \+#define innerArguments (skipBuffer, mainBuffer, length, position)generateKernels(tensorAddSwap)diff --git a/crates/uzu/src/backends/metal/kernel/tensor_copy/tensor_copy.metal b/crates/uzu/src/backends/metal/kernel/tensor_copy/tensor_copy.metalindex 21733a303..6ba08dfd8 100644--- a/crates/uzu/src/backends/metal/kernel/tensor_copy/tensor_copy.metal+++ b/crates/uzu/src/backends/metal/kernel/tensor_copy/tensor_copy.metal@@ -8,24 +8,20 @@ void tensorCopy(constant int& length,const uint position) {- if (position < length) {- destinationBuffer[position] = sourceBuffer[position];- }+ if (position < length) {+ destinationBuffer[position] = sourceBuffer[position];+ }}-#define outerArguments(T) \-(const device T* sourceBuffer [[ buffer(0) ]], \-device T* destinationBuffer [[ buffer(1) ]], \-constant int& length [[ buffer(2) ]], \-const uint position [[ thread_position_in_grid ]])\+#define outerArguments(T) \+ (const device T* sourceBuffer [[buffer(0)]], \+ device T* destinationBuffer [[buffer(1)]], \+ constant int& length [[buffer(2)]], \+ const uint position [[thread_position_in_grid]])-#define innerArguments \-(sourceBuffer, \-destinationBuffer, \-length, \-position) \+#define innerArguments (sourceBuffer, destinationBuffer, length, position)generateKernels(tensorCopy)#undef outerArguments-#undef innerArguments\ No newline at end of file+#undef innerArguments\ No newline at end of filediff --git a/crates/uzu/src/backends/metal/kernel/token_copy/token_copy.metal b/crates/uzu/src/backends/metal/kernel/token_copy/token_copy.metalindex b9f458c16..cba725880 100644--- a/crates/uzu/src/backends/metal/kernel/token_copy/token_copy.metal+++ b/crates/uzu/src/backends/metal/kernel/token_copy/token_copy.metal@@ -8,9 +8,9 @@ kernel void copy_sampled_token(device uint64_t* dst [[buffer(1)]],uint idx [[thread_position_in_grid]]) {- if (idx == 0) {- dst[0] = static_cast<uint64_t>(src[0]);- }+ if (idx == 0) {+ dst[0] = static_cast<uint64_t>(src[0]);+ }}/// Copies a single sampled token (u32) to an indexed results buffer (u32).@@ -20,7 +20,7 @@ kernel void copy_token_to_results(device uint32_t* dst [[buffer(1)]],uint idx [[thread_position_in_grid]]) {- if (idx == 0) {- dst[0] = src[0];- }+ if (idx == 0) {+ dst[0] = src[0];+ }}diff --git a/scripts/clang-format.sh b/scripts/clang-format.shnew file mode 100755index 000000000..1295a4fcb--- /dev/null+++ b/scripts/clang-format.sh@@ -0,0 +1,63 @@+#!/usr/bin/env bash+set -euo pipefail++# Format C/C++/Metal headers and sources according to .clang-format at repo root.+# Usage:+# ./scripts/clang-format.sh+# ./scripts/clang-format.sh --check # only check (no changes), non-zero exit on diffs+#+# Excludes:+# - target/ (Rust build)+# - external/ (third-party deps)+# - .git/++ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)"+cd "$ROOT_DIR"++# Ensure clang-format is available+if ! command -v clang-format >/dev/null 2>&1; then+ echo "Error: clang-format not found. Install it with: brew install clang-format" >&2+ exit 127+fi++MODE="fix"+if [[ "${1:-}" == "--check" ]]; then+ MODE="check"+fi++# Files to consider (zsh/bash compatible)+FILES=()+while IFS= read -r -d '' file; do+ FILES+=("$file")+done < <(find . \+ -path './target' -prune -o \+ -path './external' -prune -o \+ -path './.git' -prune -o \+ -type f \( \+ -name '*.h' -o -name '*.hpp' -o -name '*.hh' -o \+ -name '*.c' -o -name '*.cc' -o -name '*.cpp' -o \+ -name '*.metal' \+ \) -print0)++if [[ ${#FILES[@]} -eq 0 ]]; then+ echo "No files to format."+ exit 0+fi++if [[ "$MODE" == "check" ]]; then+ # Show diffs without writing+ FAILED=0+ for f in "${FILES[@]}"; do+ if ! diff -u "$f" <(clang-format "$f"); then+ FAILED=1+ fi+ done+ if [[ $FAILED -ne 0 ]]; then+ echo "\nFormatting check failed. Run ./scripts/clang-format.sh to fix." >&2+ exit 1+ fi+else+ # In-place rewrite+ printf '%s\0' "${FILES[@]}" | xargs -0 -n 50 clang-format -i+ echo "Formatted ${#FILES[@]} file(s)."+fi