Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

format

eugeneaac41e1875de
diff --git a/.clang-format b/.clang-format
new file mode 100644
index 000000000..ad70f0798
--- /dev/null
+++ b/.clang-format
@@ -0,0 +1,26 @@
+BasedOnStyle: LLVM
+IndentWidth: 2
+ContinuationIndentWidth: 4
+ColumnLimit: 80
+
+# Keep return type with function name
+AlwaysBreakAfterReturnType: None
+PenaltyReturnTypeOnItsOwnLine: 1000000
+
+# One parameter/argument per line; indent as block under '('
+BinPackParameters: false
+BinPackArguments: false
+AllowAllParametersOfDeclarationOnNextLine: false
+AllowAllArgumentsOnNextLine: false
+AlignAfterOpenBracket: BlockIndent
+
+# Template closing bracket on own line
+BreakTemplateDeclarations: Yes
+Cpp11BracedListStyle: true
+
+# Brace and pointer/reference style
+BreakBeforeBraces: Attach
+DerivePointerAlignment: false
+PointerAlignment: Left
+
+SortIncludes: false
diff --git a/.vscode/extensions.json b/.vscode/extensions.json
index 2d402c4e5..69837c6b5 100644
--- a/.vscode/extensions.json
+++ b/.vscode/extensions.json
@@ -3,6 +3,7 @@
"vadimcn.vscode-lldb",
"tamasfe.even-better-toml",
"doublebuffer.metal-shader",
- "rust-lang.rust-analyzer"
+ "rust-lang.rust-analyzer",
+ "xaver.clang-format"
]
}
\ No newline at end of file
diff --git a/.vscode/settings.json b/.vscode/settings.json
index 953b78551..d767a8a34 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -19,4 +19,23 @@
"[toml]": {
"editor.defaultFormatter": "tamasfe.even-better-toml"
}
+ // Treat Metal headers/sources as C++ for clang-format
+ "files.associations": {
+ "*.metal": "cpp",
+ "*.h": "cpp",
+ "*.hpp": "cpp"
+ },
+ // Use repository .clang-format
+ "clang-format.style": "file",
+ "C_Cpp.formatting": "clangFormat",
+ "C_Cpp.clang_format_style": "file",
+ // Default to MS C/C++ extension formatter for C/C++ (clang-format backend)
+ "[cpp]": {
+ "editor.defaultFormatter": "ms-vscode.cpptools",
+ "editor.formatOnSave": true
+ },
+ "[c]": {
+ "editor.defaultFormatter": "ms-vscode.cpptools",
+ "editor.formatOnSave": true
+ }
}
\ No newline at end of file
diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention.metal b/crates/uzu/src/backends/metal/kernel/attention/attention.metal
index 1b2aac913..271b97341 100644
--- a/crates/uzu/src/backends/metal/kernel/attention/attention.metal
+++ b/crates/uzu/src/backends/metal/kernel/attention/attention.metal
@@ -30,138 +30,146 @@ void attention_single_pass_impl(
const constant int& mask_q_seq_stride,
const constant int& mask_head_stride,
const device float* sinks,
- uint3 tid, // threadgroup position in grid
- uint3 tpg, // threadgroups per grid
+ uint3 tid, // threadgroup position in grid
+ uint3 tpg, // threadgroups per grid
uint simd_gid, // simdgroup index in threadgroup
uint simd_lid, // thread index in simdgroup
threadgroup float* shared_max_scores,
threadgroup float* shared_sum_exp_scores,
threadgroup float* shared_outputs
) {
- constexpr int sequence_block_size = 32;
- constexpr int head_block_size = 32;
- constexpr int qk_elements_per_thread = head_dim / head_block_size;
- constexpr int value_elements_per_thread = value_dim / head_block_size;
- int inner_k_stride = sequence_block_size * int(k_seq_stride);
- int inner_v_stride = sequence_block_size * int(v_seq_stride);
-
- typedef float U;
-
- thread U q[qk_elements_per_thread];
- thread U k[qk_elements_per_thread];
- thread U o[value_elements_per_thread];
-
- const int head_idx = tid.x;
- const int q_seq_idx = tid.y;
- const int kv_head_idx = head_idx / gqa_factor;
- const int o_offset = q_seq_idx * tpg.x + head_idx;
- const int q_offset = query_transposed ? tpg.x * q_seq_idx + head_idx : head_idx * tpg.y + q_seq_idx;
-
- queries += q_offset * head_dim + simd_lid * qk_elements_per_thread;
- keys += kv_head_idx * k_head_stride + simd_gid * k_seq_stride + simd_lid * qk_elements_per_thread;
- values += kv_head_idx * v_head_stride + simd_gid * v_seq_stride + simd_lid * value_elements_per_thread;
-
- if (bool_mask) {
- bmask += head_idx * mask_head_stride + simd_gid * mask_kv_seq_stride + q_seq_idx * mask_q_seq_stride;
- }
- if (float_mask) {
- fmask += head_idx * mask_head_stride + simd_gid * mask_kv_seq_stride + q_seq_idx * mask_q_seq_stride;
+ constexpr int sequence_block_size = 32;
+ constexpr int head_block_size = 32;
+ constexpr int qk_elements_per_thread = head_dim / head_block_size;
+ constexpr int value_elements_per_thread = value_dim / head_block_size;
+ int inner_k_stride = sequence_block_size * int(k_seq_stride);
+ int inner_v_stride = sequence_block_size * int(v_seq_stride);
+
+ typedef float U;
+
+ thread U q[qk_elements_per_thread];
+ thread U k[qk_elements_per_thread];
+ thread U o[value_elements_per_thread];
+
+ const int head_idx = tid.x;
+ const int q_seq_idx = tid.y;
+ const int kv_head_idx = head_idx / gqa_factor;
+ const int o_offset = q_seq_idx * tpg.x + head_idx;
+ const int q_offset = query_transposed ? tpg.x * q_seq_idx + head_idx
+ : head_idx * tpg.y + q_seq_idx;
+
+ queries += q_offset * head_dim + simd_lid * qk_elements_per_thread;
+ keys += kv_head_idx * k_head_stride + simd_gid * k_seq_stride +
+ simd_lid * qk_elements_per_thread;
+ values += kv_head_idx * v_head_stride + simd_gid * v_seq_stride +
+ simd_lid * value_elements_per_thread;
+
+ if (bool_mask) {
+ bmask += head_idx * mask_head_stride + simd_gid * mask_kv_seq_stride +
+ q_seq_idx * mask_q_seq_stride;
+ }
+ if (float_mask) {
+ fmask += head_idx * mask_head_stride + simd_gid * mask_kv_seq_stride +
+ q_seq_idx * mask_q_seq_stride;
+ }
+
+ out += o_offset * value_dim + simd_gid * value_elements_per_thread;
+
+ // Read the query and 0 the output accumulator
+ for (int i = 0; i < qk_elements_per_thread; i++) {
+ q[i] = static_cast<U>(scale) * queries[i];
+ }
+ for (int i = 0; i < value_elements_per_thread; i++) {
+ o[i] = 0;
+ }
+
+ U max_score = -INFINITY;
+ U sum_exp_score = 0;
+ if (has_sinks && simd_gid == 0) {
+ const int num_q_heads = static_cast<int>(tpg.x);
+ int q_head_idx = head_idx % num_q_heads;
+ max_score = static_cast<U>(sinks[q_head_idx]);
+ sum_exp_score = 1;
+ }
+
+ // For each key
+ for (int i = simd_gid; i < sequence_length; i += sequence_block_size) {
+ bool use_key = true;
+ if (do_causal) {
+ use_key = i <= (sequence_length - int(tpg.y) + int(q_seq_idx));
+ } else if (bool_mask) {
+ use_key = bmask[0];
}
- out += o_offset * value_dim + simd_gid * value_elements_per_thread;
-
- // Read the query and 0 the output accumulator
- for (int i = 0; i < qk_elements_per_thread; i++) {
- q[i] = static_cast<U>(scale) * queries[i];
- }
- for (int i = 0; i < value_elements_per_thread; i++) {
- o[i] = 0;
+ if (use_key) {
+ // Read the key
+ for (int j = 0; j < qk_elements_per_thread; j++) {
+ k[j] = keys[j];
+ }
+
+ // Compute the i-th score
+ U score = 0;
+ for (int j = 0; j < qk_elements_per_thread; j++) {
+ score += q[j] * k[j];
+ }
+ score = simd_sum(score);
+ if (float_mask) {
+ score += max(-1e9f, static_cast<U>(fmask[0]));
+ }
+
+ // Update the accumulators
+ U new_max = max(max_score, score);
+ U factor = fast::exp(max_score - new_max);
+ U exp_score = fast::exp(score - new_max);
+
+ max_score = new_max;
+ sum_exp_score = sum_exp_score * factor + exp_score;
+
+ // Update the output accumulator
+ for (int j = 0; j < value_elements_per_thread; j++) {
+ o[j] = o[j] * factor + exp_score * values[j];
+ }
}
- U max_score = -INFINITY;
- U sum_exp_score = 0;
- if (has_sinks && simd_gid == 0) {
- const int num_q_heads = static_cast<int>(tpg.x);
- int q_head_idx = head_idx % num_q_heads;
- max_score = static_cast<U>(sinks[q_head_idx]);
- sum_exp_score = 1;
- }
-
- // For each key
- for (int i = simd_gid; i < sequence_length; i += sequence_block_size) {
- bool use_key = true;
- if (do_causal) {
- use_key = i <= (sequence_length - int(tpg.y) + int(q_seq_idx));
- } else if (bool_mask) {
- use_key = bmask[0];
- }
-
- if (use_key) {
- // Read the key
- for (int j = 0; j < qk_elements_per_thread; j++) {
- k[j] = keys[j];
- }
-
- // Compute the i-th score
- U score = 0;
- for (int j = 0; j < qk_elements_per_thread; j++) {
- score += q[j] * k[j];
- }
- score = simd_sum(score);
- if (float_mask) {
- score += max(-1e9f, static_cast<U>(fmask[0]));
- }
-
- // Update the accumulators
- U new_max = max(max_score, score);
- U factor = fast::exp(max_score - new_max);
- U exp_score = fast::exp(score - new_max);
-
- max_score = new_max;
- sum_exp_score = sum_exp_score * factor + exp_score;
-
- // Update the output accumulator
- for (int j = 0; j < value_elements_per_thread; j++) {
- o[j] = o[j] * factor + exp_score * values[j];
- }
- }
-
- // Move the pointers to the next kv
- keys += inner_k_stride;
- values += inner_v_stride;
- if (bool_mask) {
- bmask += sequence_block_size * mask_kv_seq_stride;
- }
- if (float_mask) {
- fmask += sequence_block_size * mask_kv_seq_stride;
- }
+ // Move the pointers to the next kv
+ keys += inner_k_stride;
+ values += inner_v_stride;
+ if (bool_mask) {
+ bmask += sequence_block_size * mask_kv_seq_stride;
}
-
- // Each thread has a partial part of the output so we need to combine them.
- if (simd_lid == 0) {
- shared_max_scores[simd_gid] = max_score;
- shared_sum_exp_scores[simd_gid] = sum_exp_score;
+ if (float_mask) {
+ fmask += sequence_block_size * mask_kv_seq_stride;
}
+ }
+
+ // Each thread has a partial part of the output so we need to combine them.
+ if (simd_lid == 0) {
+ shared_max_scores[simd_gid] = max_score;
+ shared_sum_exp_scores[simd_gid] = sum_exp_score;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ max_score = shared_max_scores[simd_lid];
+ U new_max = simd_max(max_score);
+ U factor = fast::exp(max_score - new_max);
+ sum_exp_score = simd_sum(shared_sum_exp_scores[simd_lid] * factor);
+
+ // Now we need to aggregate all the outputs
+ for (int i = 0; i < value_elements_per_thread; i++) {
+ shared_outputs[simd_lid * head_block_size + simd_gid] = o[i];
threadgroup_barrier(mem_flags::mem_threadgroup);
- max_score = shared_max_scores[simd_lid];
- U new_max = simd_max(max_score);
- U factor = fast::exp(max_score - new_max);
- sum_exp_score = simd_sum(shared_sum_exp_scores[simd_lid] * factor);
+ o[i] = simd_sum(
+ shared_outputs[simd_gid * head_block_size + simd_lid] * factor
+ ) /
+ sum_exp_score;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
- // Now we need to aggregate all the outputs
+ // And write the output
+ if (simd_lid == 0) {
for (int i = 0; i < value_elements_per_thread; i++) {
- shared_outputs[simd_lid * head_block_size + simd_gid] = o[i];
- threadgroup_barrier(mem_flags::mem_threadgroup);
- o[i] = simd_sum(shared_outputs[simd_gid * head_block_size + simd_lid] * factor) / sum_exp_score;
- threadgroup_barrier(mem_flags::mem_threadgroup);
- }
-
- // And write the output
- if (simd_lid == 0) {
- for (int i = 0; i < value_elements_per_thread; i++) {
- out[i] = static_cast<T>(o[i]);
- }
+ out[i] = static_cast<T>(o[i]);
}
+ }
}
template <typename T, int head_dim, int value_dim = head_dim>
@@ -185,149 +193,165 @@ void attention_2pass_1_impl(
const constant int& mask_q_seq_stride,
const constant int& mask_head_stride,
const device float* sinks,
- uint3 tid, // threadgroup position in grid
- uint3 tpg, // threadgroups per grid
+ uint3 tid, // threadgroup position in grid
+ uint3 tpg, // threadgroups per grid
uint simd_gid, // simdgroup index in threadgroup
uint simd_lid, // thread index in simdgroup
threadgroup float* shared_max_scores,
threadgroup float* shared_sum_exp_scores,
threadgroup float* shared_outputs
) {
- constexpr int sequence_block_size = 8;
- constexpr int head_block_size = 32;
- constexpr int qk_elements_per_thread = head_dim / head_block_size;
- constexpr int value_elements_per_thread = value_dim / head_block_size;
- int inner_k_stride = sequence_block_size * int(k_seq_stride);
- int inner_v_stride = sequence_block_size * int(v_seq_stride);
- constexpr int total_blocks_count = 32;
-
- typedef float U;
-
- thread U q[qk_elements_per_thread];
- thread U k[qk_elements_per_thread];
- thread U o[value_elements_per_thread];
-
- const int block_idx = tid.z;
- const int head_idx = tid.x;
- const int q_seq_idx = tid.y;
- const int o_offset = q_seq_idx * tpg.x + head_idx; // Our custom layout
- const int q_offset = query_transposed ? tpg.x * q_seq_idx + head_idx : head_idx * tpg.y + q_seq_idx; // Consistent with single-pass
- const int kv_head_idx = head_idx / gqa_factor;
-
- queries += q_offset * head_dim + simd_lid * qk_elements_per_thread;
- keys += kv_head_idx * k_head_stride + (block_idx * sequence_block_size + simd_gid) * k_seq_stride + simd_lid * qk_elements_per_thread;
- values += kv_head_idx * v_head_stride + (block_idx * sequence_block_size + simd_gid) * v_seq_stride + simd_lid * value_elements_per_thread;
- out += o_offset * total_blocks_count * value_dim + block_idx * value_dim + simd_lid * value_elements_per_thread;
-
- if (bool_mask) {
- bmask += head_idx * mask_head_stride + (block_idx * sequence_block_size + simd_gid) * mask_kv_seq_stride + q_seq_idx * mask_q_seq_stride;
- }
- if (float_mask) {
- fmask += head_idx * mask_head_stride + (block_idx * sequence_block_size + simd_gid) * mask_kv_seq_stride + q_seq_idx * mask_q_seq_stride;
- }
- sums += o_offset * total_blocks_count + block_idx;
- maxs += o_offset * total_blocks_count + block_idx;
-
- // Read the query and 0 the output accumulator
- for (int i = 0; i < qk_elements_per_thread; i++) {
- q[i] = static_cast<U>(scale) * queries[i];
- }
- for (int i = 0; i < value_elements_per_thread; i++) {
- o[i] = 0;
+ constexpr int sequence_block_size = 8;
+ constexpr int head_block_size = 32;
+ constexpr int qk_elements_per_thread = head_dim / head_block_size;
+ constexpr int value_elements_per_thread = value_dim / head_block_size;
+ int inner_k_stride = sequence_block_size * int(k_seq_stride);
+ int inner_v_stride = sequence_block_size * int(v_seq_stride);
+ constexpr int total_blocks_count = 32;
+
+ typedef float U;
+
+ thread U q[qk_elements_per_thread];
+ thread U k[qk_elements_per_thread];
+ thread U o[value_elements_per_thread];
+
+ const int block_idx = tid.z;
+ const int head_idx = tid.x;
+ const int q_seq_idx = tid.y;
+ const int o_offset = q_seq_idx * tpg.x + head_idx; // Our custom layout
+ const int q_offset =
+ query_transposed
+ ? tpg.x * q_seq_idx + head_idx
+ : head_idx * tpg.y + q_seq_idx; // Consistent with single-pass
+ const int kv_head_idx = head_idx / gqa_factor;
+
+ queries += q_offset * head_dim + simd_lid * qk_elements_per_thread;
+ keys += kv_head_idx * k_head_stride +
+ (block_idx * sequence_block_size + simd_gid) * k_seq_stride +
+ simd_lid * qk_elements_per_thread;
+ values += kv_head_idx * v_head_stride +
+ (block_idx * sequence_block_size + simd_gid) * v_seq_stride +
+ simd_lid * value_elements_per_thread;
+ out += o_offset * total_blocks_count * value_dim + block_idx * value_dim +
+ simd_lid * value_elements_per_thread;
+
+ if (bool_mask) {
+ bmask += head_idx * mask_head_stride +
+ (block_idx * sequence_block_size + simd_gid) * mask_kv_seq_stride +
+ q_seq_idx * mask_q_seq_stride;
+ }
+ if (float_mask) {
+ fmask += head_idx * mask_head_stride +
+ (block_idx * sequence_block_size + simd_gid) * mask_kv_seq_stride +
+ q_seq_idx * mask_q_seq_stride;
+ }
+ sums += o_offset * total_blocks_count + block_idx;
+ maxs += o_offset * total_blocks_count + block_idx;
+
+ // Read the query and 0 the output accumulator
+ for (int i = 0; i < qk_elements_per_thread; i++) {
+ q[i] = static_cast<U>(scale) * queries[i];
+ }
+ for (int i = 0; i < value_elements_per_thread; i++) {
+ o[i] = 0;
+ }
+
+ U max_score = -1e9;
+ U sum_exp_score = 0;
+ if (has_sinks && block_idx == 0 && simd_gid == 0) {
+ const int num_q_heads = static_cast<int>(tpg.x);
+ int q_head_idx = head_idx % num_q_heads;
+ max_score = static_cast<U>(sinks[q_head_idx]);
+ sum_exp_score = 1;
+ }
+
+ // For each key
+ for (int i = block_idx * sequence_block_size + simd_gid; i < sequence_length;
+ i += total_blocks_count * sequence_block_size) {
+ bool use_key = true;
+ if (do_causal) {
+ use_key = i <= (sequence_length - int(tpg.y) + int(q_seq_idx));
+ } else if (bool_mask) {
+ use_key = bmask[0];
}
- U max_score = -1e9;
- U sum_exp_score = 0;
- if (has_sinks && block_idx == 0 && simd_gid == 0) {
- const int num_q_heads = static_cast<int>(tpg.x);
- int q_head_idx = head_idx % num_q_heads;
- max_score = static_cast<U>(sinks[q_head_idx]);
- sum_exp_score = 1;
+ if (use_key) {
+ // Read the key
+ for (int j = 0; j < qk_elements_per_thread; j++) {
+ k[j] = keys[j];
+ }
+
+ // Compute the i-th score
+ U score = 0;
+ for (int j = 0; j < qk_elements_per_thread; j++) {
+ score += q[j] * k[j];
+ }
+ score = simd_sum(score);
+ if (float_mask) {
+ score += max(-1e9f, static_cast<U>(fmask[0]));
+ }
+
+ // Update the accumulators
+ U new_max = max(max_score, score);
+ U factor = fast::exp(max_score - new_max);
+ U exp_score = fast::exp(score - new_max);
+
+ max_score = new_max;
+ sum_exp_score = sum_exp_score * factor + exp_score;
+
+ // Update the output accumulator
+ for (int j = 0; j < value_elements_per_thread; j++) {
+ o[j] = o[j] * factor + exp_score * values[j];
+ }
}
- // For each key
- for (int i = block_idx * sequence_block_size + simd_gid; i < sequence_length; i += total_blocks_count * sequence_block_size) {
- bool use_key = true;
- if (do_causal) {
- use_key = i <= (sequence_length - int(tpg.y) + int(q_seq_idx));
- } else if (bool_mask) {
- use_key = bmask[0];
- }
-
- if (use_key) {
- // Read the key
- for (int j = 0; j < qk_elements_per_thread; j++) {
- k[j] = keys[j];
- }
-
- // Compute the i-th score
- U score = 0;
- for (int j = 0; j < qk_elements_per_thread; j++) {
- score += q[j] * k[j];
- }
- score = simd_sum(score);
- if (float_mask) {
- score += max(-1e9f, static_cast<U>(fmask[0]));
- }
-
- // Update the accumulators
- U new_max = max(max_score, score);
- U factor = fast::exp(max_score - new_max);
- U exp_score = fast::exp(score - new_max);
-
- max_score = new_max;
- sum_exp_score = sum_exp_score * factor + exp_score;
-
- // Update the output accumulator
- for (int j = 0; j < value_elements_per_thread; j++) {
- o[j] = o[j] * factor + exp_score * values[j];
- }
- }
-
- // Move the pointers to the next kv
- keys += total_blocks_count * inner_k_stride;
- values += total_blocks_count * inner_v_stride;
- if (bool_mask) {
- bmask += sequence_block_size * total_blocks_count * mask_kv_seq_stride;
- }
- if (float_mask) {
- fmask += sequence_block_size * total_blocks_count * mask_kv_seq_stride;
- }
+ // Move the pointers to the next kv
+ keys += total_blocks_count * inner_k_stride;
+ values += total_blocks_count * inner_v_stride;
+ if (bool_mask) {
+ bmask += sequence_block_size * total_blocks_count * mask_kv_seq_stride;
}
-
- // Each thread has a partial part of the output so we need to combine them.
- if (simd_lid == 0) {
- shared_max_scores[simd_gid] = max_score;
- shared_sum_exp_scores[simd_gid] = sum_exp_score;
+ if (float_mask) {
+ fmask += sequence_block_size * total_blocks_count * mask_kv_seq_stride;
}
+ }
+
+ // Each thread has a partial part of the output so we need to combine them.
+ if (simd_lid == 0) {
+ shared_max_scores[simd_gid] = max_score;
+ shared_sum_exp_scores[simd_gid] = sum_exp_score;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ max_score =
+ (simd_lid < sequence_block_size) ? shared_max_scores[simd_lid] : -1e9;
+ U new_max = simd_max(max_score);
+ U factor = fast::exp(max_score - new_max);
+ sum_exp_score =
+ (simd_lid < sequence_block_size) ? shared_sum_exp_scores[simd_lid] : 0;
+ sum_exp_score = simd_sum(sum_exp_score * factor);
+
+ // Write the sum and new max
+ if (simd_gid == 0) {
+ sums[0] = sum_exp_score;
+ maxs[0] = new_max;
+ }
+
+ // Now we need to aggregate all the outputs
+ for (int i = 0; i < value_elements_per_thread; i++) {
+ shared_outputs[simd_lid * sequence_block_size + simd_gid] =
+ o[i] * fast::exp(shared_max_scores[simd_gid] - new_max);
threadgroup_barrier(mem_flags::mem_threadgroup);
- max_score = (simd_lid < sequence_block_size) ? shared_max_scores[simd_lid] : -1e9;
- U new_max = simd_max(max_score);
- U factor = fast::exp(max_score - new_max);
- sum_exp_score = (simd_lid < sequence_block_size) ? shared_sum_exp_scores[simd_lid] : 0;
- sum_exp_score = simd_sum(sum_exp_score * factor);
- // Write the sum and new max
+ // And write the output
if (simd_gid == 0) {
- sums[0] = sum_exp_score;
- maxs[0] = new_max;
- }
-
- // Now we need to aggregate all the outputs
- for (int i = 0; i < value_elements_per_thread; i++) {
- shared_outputs[simd_lid * sequence_block_size + simd_gid] = o[i] * fast::exp(shared_max_scores[simd_gid] - new_max);
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // And write the output
- if (simd_gid == 0) {
- U output = shared_outputs[simd_lid * sequence_block_size];
- for (int j = 1; j < sequence_block_size; j++) {
- output += shared_outputs[simd_lid * sequence_block_size + j];
- }
- out[i] = static_cast<T>(output);
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
+ U output = shared_outputs[simd_lid * sequence_block_size];
+ for (int j = 1; j < sequence_block_size; j++) {
+ output += shared_outputs[simd_lid * sequence_block_size + j];
+ }
+ out[i] = static_cast<T>(output);
}
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
}
template <typename T, int head_dim>
@@ -342,48 +366,57 @@ void attention_2pass_2_impl(
uint simd_lid,
threadgroup float* shared_outputs
) {
- constexpr int sequence_block_size = 32;
- constexpr int head_block_size = 32;
- constexpr int elements_per_thread = head_dim / head_block_size;
- constexpr int total_blocks_count = 32;
-
- typedef float U;
-
- thread U o[elements_per_thread];
-
- const int head_idx = tid.x;
- const int q_seq_idx = tid.y;
- const int o_offset = q_seq_idx * tpg.x + head_idx; // Our custom layout
- const int q_offset = query_transposed ? tpg.x * q_seq_idx + head_idx : head_idx * tpg.y + q_seq_idx; // Consistent with single-pass
-
- partials += o_offset * total_blocks_count * head_dim + simd_gid * head_dim + simd_lid * elements_per_thread;
- sums += o_offset * total_blocks_count;
- maxs += o_offset * total_blocks_count;
- out += o_offset * head_dim + simd_gid * elements_per_thread; // Our custom output layout
-
- // First everybody reads the max and sum_exp
- U max_score = maxs[simd_lid];
- U new_max = simd_max(max_score);
- U factor = fast::exp(max_score - new_max);
- U sum_exp_score = simd_sum(sums[simd_lid] * factor);
-
- // Now read the block into registers and then use shared memory to transpose it
- for (int i = 0; i < elements_per_thread; i++) {
- o[i] = partials[i];
- }
- for (int i = 0; i < elements_per_thread; i++) {
- shared_outputs[simd_lid * head_block_size + simd_gid] = o[i];
- threadgroup_barrier(mem_flags::mem_threadgroup);
- o[i] = simd_sum(shared_outputs[simd_gid * head_block_size + simd_lid] * factor) / sum_exp_score;
- threadgroup_barrier(mem_flags::mem_threadgroup);
- }
+ constexpr int sequence_block_size = 32;
+ constexpr int head_block_size = 32;
+ constexpr int elements_per_thread = head_dim / head_block_size;
+ constexpr int total_blocks_count = 32;
+
+ typedef float U;
+
+ thread U o[elements_per_thread];
+
+ const int head_idx = tid.x;
+ const int q_seq_idx = tid.y;
+ const int o_offset = q_seq_idx * tpg.x + head_idx; // Our custom layout
+ const int q_offset =
+ query_transposed
+ ? tpg.x * q_seq_idx + head_idx
+ : head_idx * tpg.y + q_seq_idx; // Consistent with single-pass
+
+ partials += o_offset * total_blocks_count * head_dim + simd_gid * head_dim +
+ simd_lid * elements_per_thread;
+ sums += o_offset * total_blocks_count;
+ maxs += o_offset * total_blocks_count;
+ out += o_offset * head_dim +
+ simd_gid * elements_per_thread; // Our custom output layout
+
+ // First everybody reads the max and sum_exp
+ U max_score = maxs[simd_lid];
+ U new_max = simd_max(max_score);
+ U factor = fast::exp(max_score - new_max);
+ U sum_exp_score = simd_sum(sums[simd_lid] * factor);
+
+ // Now read the block into registers and then use shared memory to transpose
+ // it
+ for (int i = 0; i < elements_per_thread; i++) {
+ o[i] = partials[i];
+ }
+ for (int i = 0; i < elements_per_thread; i++) {
+ shared_outputs[simd_lid * head_block_size + simd_gid] = o[i];
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ o[i] = simd_sum(
+ shared_outputs[simd_gid * head_block_size + simd_lid] * factor
+ ) /
+ sum_exp_score;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
- // And write the output
- if (simd_lid == 0) {
- for (int i = 0; i < elements_per_thread; i++) {
- out[i] = static_cast<T>(o[i]);
- }
+ // And write the output
+ if (simd_lid == 0) {
+ for (int i = 0; i < elements_per_thread; i++) {
+ out[i] = static_cast<T>(o[i]);
}
+ }
}
template <typename T, int head_dim, int value_dim = head_dim>
@@ -413,14 +446,32 @@ void attention_single_pass(
threadgroup float* shared_sum_exp_scores,
threadgroup float* shared_outputs
) {
- attention_single_pass_impl<T, head_dim, value_dim>(
- queries, keys, values, out, gqa_factor, sequence_length,
- k_head_stride, k_seq_stride, v_head_stride, v_seq_stride,
- scale, bmask, fmask, mask_kv_seq_stride, mask_q_seq_stride, mask_head_stride,
- sinks,
- tid, tpg, simd_gid, simd_lid,
- shared_max_scores, shared_sum_exp_scores, shared_outputs
- );
+ attention_single_pass_impl<T, head_dim, value_dim>(
+ queries,
+ keys,
+ values,
+ out,
+ gqa_factor,
+ sequence_length,
+ k_head_stride,
+ k_seq_stride,
+ v_head_stride,
+ v_seq_stride,
+ scale,
+ bmask,
+ fmask,
+ mask_kv_seq_stride,
+ mask_q_seq_stride,
+ mask_head_stride,
+ sinks,
+ tid,
+ tpg,
+ simd_gid,
+ simd_lid,
+ shared_max_scores,
+ shared_sum_exp_scores,
+ shared_outputs
+ );
}
template <typename T, int head_dim, int value_dim = head_dim>
@@ -452,14 +503,34 @@ void attention_2pass_1(
threadgroup float* shared_sum_exp_scores,
threadgroup float* shared_outputs
) {
- attention_2pass_1_impl<T, head_dim, value_dim>(
- queries, keys, values, out, sums, maxs, gqa_factor, sequence_length,
- k_head_stride, k_seq_stride, v_head_stride, v_seq_stride,
- scale, bmask, fmask, mask_kv_seq_stride, mask_q_seq_stride, mask_head_stride,
- sinks,
- tid, tpg, simd_gid, simd_lid,
- shared_max_scores, shared_sum_exp_scores, shared_outputs
- );
+ attention_2pass_1_impl<T, head_dim, value_dim>(
+ queries,
+ keys,
+ values,
+ out,
+ sums,
+ maxs,
+ gqa_factor,
+ sequence_length,
+ k_head_stride,
+ k_seq_stride,
+ v_head_stride,
+ v_seq_stride,
+ scale,
+ bmask,
+ fmask,
+ mask_kv_seq_stride,
+ mask_q_seq_stride,
+ mask_head_stride,
+ sinks,
+ tid,
+ tpg,
+ simd_gid,
+ simd_lid,
+ shared_max_scores,
+ shared_sum_exp_scores,
+ shared_outputs
+ );
}
template <typename T, int head_dim>
@@ -474,66 +545,106 @@ void attention_2pass_2(
uint simd_lid,
threadgroup float* shared_outputs
) {
- attention_2pass_2_impl<T, head_dim>(
- partials, sums, maxs, out, tid, tpg, simd_gid, simd_lid, shared_outputs
- );
+ attention_2pass_2_impl<T, head_dim>(
+ partials,
+ sums,
+ maxs,
+ out,
+ tid,
+ tpg,
+ simd_gid,
+ simd_lid,
+ shared_outputs
+ );
}
-#define outerArguments(T) \
-(const device T* queries [[ buffer(0) ]], \
- const device T* keys [[ buffer(1) ]], \
- const device T* values [[ buffer(2) ]], \
- device T* out [[ buffer(3) ]], \
- const constant int& gqa_factor [[ buffer(4) ]], \
- const constant int& sequence_length [[ buffer(5) ]], \
- const constant int& k_head_stride [[ buffer(6) ]], \
- const constant int& k_seq_stride [[ buffer(7) ]], \
- const constant int& v_head_stride [[ buffer(8) ]], \
- const constant int& v_seq_stride [[ buffer(9) ]], \
- const constant float& scale [[ buffer(10) ]], \
- const device bool* bmask [[ buffer(11), function_constant(bool_mask) ]], \
- const device T* fmask [[ buffer(12), function_constant(float_mask) ]], \
- const constant int& mask_kv_seq_stride [[ buffer(13), function_constant(has_mask) ]], \
- const constant int& mask_q_seq_stride [[ buffer(14), function_constant(has_mask) ]], \
- const constant int& mask_head_stride [[ buffer(15), function_constant(has_mask) ]], \
- const device float* sinks [[ buffer(16), function_constant(has_sinks) ]], \
- uint3 tid [[ threadgroup_position_in_grid ]], \
- uint3 tpg [[ threadgroups_per_grid ]], \
- uint simd_gid [[ simdgroup_index_in_threadgroup ]], \
- uint simd_lid [[ thread_index_in_simdgroup ]]) \
-
-#define innerArguments \
-(queries, keys, values, out, gqa_factor, sequence_length, k_head_stride, k_seq_stride, \
- v_head_stride, v_seq_stride, scale, bmask, fmask, mask_kv_seq_stride, \
- mask_q_seq_stride, mask_head_stride, sinks, tid, tpg, simd_gid, simd_lid, \
- shared_max_scores, shared_sum_exp_scores, shared_outputs) \
+#define outerArguments(T) \
+ (const device T* queries [[buffer(0)]], \
+ const device T* keys [[buffer(1)]], \
+ const device T* values [[buffer(2)]], \
+ device T* out [[buffer(3)]], \
+ const constant int& gqa_factor [[buffer(4)]], \
+ const constant int& sequence_length [[buffer(5)]], \
+ const constant int& k_head_stride [[buffer(6)]], \
+ const constant int& k_seq_stride [[buffer(7)]], \
+ const constant int& v_head_stride [[buffer(8)]], \
+ const constant int& v_seq_stride [[buffer(9)]], \
+ const constant float& scale [[buffer(10)]], \
+ const device bool* bmask [[buffer(11), function_constant(bool_mask)]], \
+ const device T* fmask [[buffer(12), function_constant(float_mask)]], \
+ const constant int& mask_kv_seq_stride \
+ [[buffer(13), function_constant(has_mask)]], \
+ const constant int& mask_q_seq_stride \
+ [[buffer(14), function_constant(has_mask)]], \
+ const constant int& mask_head_stride \
+ [[buffer(15), function_constant(has_mask)]], \
+ const device float* sinks [[buffer(16), function_constant(has_sinks)]], \
+ uint3 tid [[threadgroup_position_in_grid]], \
+ uint3 tpg [[threadgroups_per_grid]], \
+ uint simd_gid [[simdgroup_index_in_threadgroup]], \
+ uint simd_lid [[thread_index_in_simdgroup]])
+
+#define innerArguments \
+ (queries, \
+ keys, \
+ values, \
+ out, \
+ gqa_factor, \
+ sequence_length, \
+ k_head_stride, \
+ k_seq_stride, \
+ v_head_stride, \
+ v_seq_stride, \
+ scale, \
+ bmask, \
+ fmask, \
+ mask_kv_seq_stride, \
+ mask_q_seq_stride, \
+ mask_head_stride, \
+ sinks, \
+ tid, \
+ tpg, \
+ simd_gid, \
+ simd_lid, \
+ shared_max_scores, \
+ shared_sum_exp_scores, \
+ shared_outputs)
// Generate single-pass kernels for different head dimensions
-#define GENERATE_SINGLE_PASS_KERNELS(head_dim_value) \
-[[max_total_threads_per_threadgroup(1024)]] kernel void attention_single_pass_half_##head_dim_value outerArguments(half) { \
- constexpr int sequence_block_size = 32; \
- constexpr int head_block_size = 32; \
- threadgroup float shared_max_scores[sequence_block_size]; \
- threadgroup float shared_sum_exp_scores[sequence_block_size]; \
- threadgroup float shared_outputs[sequence_block_size * head_block_size]; \
- attention_single_pass<half, head_dim_value> innerArguments; \
-} \
-[[max_total_threads_per_threadgroup(1024)]] kernel void attention_single_pass_bfloat_##head_dim_value outerArguments(bfloat) { \
- constexpr int sequence_block_size = 32; \
- constexpr int head_block_size = 32; \
- threadgroup float shared_max_scores[sequence_block_size]; \
- threadgroup float shared_sum_exp_scores[sequence_block_size]; \
- threadgroup float shared_outputs[sequence_block_size * head_block_size]; \
- attention_single_pass<bfloat, head_dim_value> innerArguments; \
-} \
-[[max_total_threads_per_threadgroup(1024)]] kernel void attention_single_pass_float_##head_dim_value outerArguments(float) { \
- constexpr int sequence_block_size = 32; \
- constexpr int head_block_size = 32; \
- threadgroup float shared_max_scores[sequence_block_size]; \
- threadgroup float shared_sum_exp_scores[sequence_block_size]; \
- threadgroup float shared_outputs[sequence_block_size * head_block_size]; \
- attention_single_pass<float, head_dim_value> innerArguments; \
-}
+#define GENERATE_SINGLE_PASS_KERNELS(head_dim_value) \
+ [[max_total_threads_per_threadgroup( \
+ 1024 \
+ )]] kernel void attention_single_pass_half_##head_dim_value \
+ outerArguments(half) { \
+ constexpr int sequence_block_size = 32; \
+ constexpr int head_block_size = 32; \
+ threadgroup float shared_max_scores[sequence_block_size]; \
+ threadgroup float shared_sum_exp_scores[sequence_block_size]; \
+ threadgroup float shared_outputs[sequence_block_size * head_block_size]; \
+ attention_single_pass<half, head_dim_value> innerArguments; \
+ } \
+ [[max_total_threads_per_threadgroup( \
+ 1024 \
+ )]] kernel void attention_single_pass_bfloat_##head_dim_value \
+ outerArguments(bfloat) { \
+ constexpr int sequence_block_size = 32; \
+ constexpr int head_block_size = 32; \
+ threadgroup float shared_max_scores[sequence_block_size]; \
+ threadgroup float shared_sum_exp_scores[sequence_block_size]; \
+ threadgroup float shared_outputs[sequence_block_size * head_block_size]; \
+ attention_single_pass<bfloat, head_dim_value> innerArguments; \
+ } \
+ [[max_total_threads_per_threadgroup( \
+ 1024 \
+ )]] kernel void attention_single_pass_float_##head_dim_value \
+ outerArguments(float) { \
+ constexpr int sequence_block_size = 32; \
+ constexpr int head_block_size = 32; \
+ threadgroup float shared_max_scores[sequence_block_size]; \
+ threadgroup float shared_sum_exp_scores[sequence_block_size]; \
+ threadgroup float shared_outputs[sequence_block_size * head_block_size]; \
+ attention_single_pass<float, head_dim_value> innerArguments; \
+ }
GENERATE_SINGLE_PASS_KERNELS(64)
GENERATE_SINGLE_PASS_KERNELS(128)
@@ -542,63 +653,97 @@ GENERATE_SINGLE_PASS_KERNELS(256)
#undef outerArguments
#undef innerArguments
-#define outerArguments(T) \
-(const device T* queries [[ buffer(0) ]], \
- const device T* keys [[ buffer(1) ]], \
- const device T* values [[ buffer(2) ]], \
- device float* out [[ buffer(3) ]], \
- device float* sums [[ buffer(4) ]], \
- device float* maxs [[ buffer(5) ]], \
- const constant int& gqa_factor [[ buffer(6) ]], \
- const constant int& sequence_length [[ buffer(7) ]], \
- const constant int& k_head_stride [[ buffer(8) ]], \
- const constant int& k_seq_stride [[ buffer(9) ]], \
- const constant int& v_head_stride [[ buffer(10) ]], \
- const constant int& v_seq_stride [[ buffer(11) ]], \
- const constant float& scale [[ buffer(12) ]], \
- const device bool* bmask [[ buffer(13), function_constant(bool_mask) ]], \
- const device T* fmask [[ buffer(14), function_constant(float_mask) ]], \
- const constant int& mask_kv_seq_stride [[ buffer(15), function_constant(has_mask) ]], \
- const constant int& mask_q_seq_stride [[ buffer(16), function_constant(has_mask) ]], \
- const constant int& mask_head_stride [[ buffer(17), function_constant(has_mask) ]], \
- const device float* sinks [[ buffer(18), function_constant(has_sinks) ]], \
- uint3 tid [[ threadgroup_position_in_grid ]], \
- uint3 tpg [[ threadgroups_per_grid ]], \
- uint simd_gid [[ simdgroup_index_in_threadgroup ]], \
- uint simd_lid [[ thread_index_in_simdgroup ]]) \
-
-#define innerArguments \
-(queries, keys, values, out, sums, maxs, gqa_factor, sequence_length, k_head_stride, k_seq_stride, \
- v_head_stride, v_seq_stride, scale, bmask, fmask, mask_kv_seq_stride, \
- mask_q_seq_stride, mask_head_stride, sinks, tid, tpg, simd_gid, simd_lid, \
- shared_max_scores, shared_sum_exp_scores, shared_outputs) \
+#define outerArguments(T) \
+ (const device T* queries [[buffer(0)]], \
+ const device T* keys [[buffer(1)]], \
+ const device T* values [[buffer(2)]], \
+ device float* out [[buffer(3)]], \
+ device float* sums [[buffer(4)]], \
+ device float* maxs [[buffer(5)]], \
+ const constant int& gqa_factor [[buffer(6)]], \
+ const constant int& sequence_length [[buffer(7)]], \
+ const constant int& k_head_stride [[buffer(8)]], \
+ const constant int& k_seq_stride [[buffer(9)]], \
+ const constant int& v_head_stride [[buffer(10)]], \
+ const constant int& v_seq_stride [[buffer(11)]], \
+ const constant float& scale [[buffer(12)]], \
+ const device bool* bmask [[buffer(13), function_constant(bool_mask)]], \
+ const device T* fmask [[buffer(14), function_constant(float_mask)]], \
+ const constant int& mask_kv_seq_stride \
+ [[buffer(15), function_constant(has_mask)]], \
+ const constant int& mask_q_seq_stride \
+ [[buffer(16), function_constant(has_mask)]], \
+ const constant int& mask_head_stride \
+ [[buffer(17), function_constant(has_mask)]], \
+ const device float* sinks [[buffer(18), function_constant(has_sinks)]], \
+ uint3 tid [[threadgroup_position_in_grid]], \
+ uint3 tpg [[threadgroups_per_grid]], \
+ uint simd_gid [[simdgroup_index_in_threadgroup]], \
+ uint simd_lid [[thread_index_in_simdgroup]])
+
+#define innerArguments \
+ (queries, \
+ keys, \
+ values, \
+ out, \
+ sums, \
+ maxs, \
+ gqa_factor, \
+ sequence_length, \
+ k_head_stride, \
+ k_seq_stride, \
+ v_head_stride, \
+ v_seq_stride, \
+ scale, \
+ bmask, \
+ fmask, \
+ mask_kv_seq_stride, \
+ mask_q_seq_stride, \
+ mask_head_stride, \
+ sinks, \
+ tid, \
+ tpg, \
+ simd_gid, \
+ simd_lid, \
+ shared_max_scores, \
+ shared_sum_exp_scores, \
+ shared_outputs)
// Generate 2-pass Pass 1 kernels for different head dimensions
-#define GENERATE_2PASS_1_KERNELS(head_dim_value) \
-[[max_total_threads_per_threadgroup(256)]] kernel void attention_2pass_1_half_##head_dim_value outerArguments(half) { \
- constexpr int sequence_block_size = 8; \
- constexpr int head_block_size = 32; \
- threadgroup float shared_max_scores[sequence_block_size]; \
- threadgroup float shared_sum_exp_scores[sequence_block_size]; \
- threadgroup float shared_outputs[sequence_block_size * head_block_size]; \
- attention_2pass_1<half, head_dim_value> innerArguments; \
-} \
-[[max_total_threads_per_threadgroup(256)]] kernel void attention_2pass_1_bfloat_##head_dim_value outerArguments(bfloat) { \
- constexpr int sequence_block_size = 8; \
- constexpr int head_block_size = 32; \
- threadgroup float shared_max_scores[sequence_block_size]; \
- threadgroup float shared_sum_exp_scores[sequence_block_size]; \
- threadgroup float shared_outputs[sequence_block_size * head_block_size]; \
- attention_2pass_1<bfloat, head_dim_value> innerArguments; \
-} \
-[[max_total_threads_per_threadgroup(256)]] kernel void attention_2pass_1_float_##head_dim_value outerArguments(float) { \
- constexpr int sequence_block_size = 8; \
- constexpr int head_block_size = 32; \
- threadgroup float shared_max_scores[sequence_block_size]; \
- threadgroup float shared_sum_exp_scores[sequence_block_size]; \
- threadgroup float shared_outputs[sequence_block_size * head_block_size]; \
- attention_2pass_1<float, head_dim_value> innerArguments; \
-}
+#define GENERATE_2PASS_1_KERNELS(head_dim_value) \
+ [[max_total_threads_per_threadgroup( \
+ 256 \
+ )]] kernel void attention_2pass_1_half_##head_dim_value \
+ outerArguments(half) { \
+ constexpr int sequence_block_size = 8; \
+ constexpr int head_block_size = 32; \
+ threadgroup float shared_max_scores[sequence_block_size]; \
+ threadgroup float shared_sum_exp_scores[sequence_block_size]; \
+ threadgroup float shared_outputs[sequence_block_size * head_block_size]; \
+ attention_2pass_1<half, head_dim_value> innerArguments; \
+ } \
+ [[max_total_threads_per_threadgroup( \
+ 256 \
+ )]] kernel void attention_2pass_1_bfloat_##head_dim_value \
+ outerArguments(bfloat) { \
+ constexpr int sequence_block_size = 8; \
+ constexpr int head_block_size = 32; \
+ threadgroup float shared_max_scores[sequence_block_size]; \
+ threadgroup float shared_sum_exp_scores[sequence_block_size]; \
+ threadgroup float shared_outputs[sequence_block_size * head_block_size]; \
+ attention_2pass_1<bfloat, head_dim_value> innerArguments; \
+ } \
+ [[max_total_threads_per_threadgroup( \
+ 256 \
+ )]] kernel void attention_2pass_1_float_##head_dim_value \
+ outerArguments(float) { \
+ constexpr int sequence_block_size = 8; \
+ constexpr int head_block_size = 32; \
+ threadgroup float shared_max_scores[sequence_block_size]; \
+ threadgroup float shared_sum_exp_scores[sequence_block_size]; \
+ threadgroup float shared_outputs[sequence_block_size * head_block_size]; \
+ attention_2pass_1<float, head_dim_value> innerArguments; \
+ }
GENERATE_2PASS_1_KERNELS(64)
GENERATE_2PASS_1_KERNELS(128)
@@ -607,33 +752,39 @@ GENERATE_2PASS_1_KERNELS(256)
#undef outerArguments
#undef innerArguments
-#define outerArguments(T) \
-(const device float* partials [[ buffer(0) ]], \
- const device float* sums [[ buffer(1) ]], \
- const device float* maxs [[ buffer(2) ]], \
- device T* out [[ buffer(3) ]], \
- uint3 tid [[ threadgroup_position_in_grid ]], \
- uint3 tpg [[ threadgroups_per_grid ]], \
- uint simd_gid [[ simdgroup_index_in_threadgroup ]], \
- uint simd_lid [[ thread_index_in_simdgroup ]]) \
+#define outerArguments(T) \
+ (const device float* partials [[buffer(0)]], \
+ const device float* sums [[buffer(1)]], \
+ const device float* maxs [[buffer(2)]], \
+ device T* out [[buffer(3)]], \
+ uint3 tid [[threadgroup_position_in_grid]], \
+ uint3 tpg [[threadgroups_per_grid]], \
+ uint simd_gid [[simdgroup_index_in_threadgroup]], \
+ uint simd_lid [[thread_index_in_simdgroup]])
-#define innerArguments \
-(partials, sums, maxs, out, tid, tpg, simd_gid, simd_lid, shared_outputs) \
+#define innerArguments \
+ (partials, sums, maxs, out, tid, tpg, simd_gid, simd_lid, shared_outputs)
// Generate 2-pass Pass 2 kernels for different head dimensions
-#define GENERATE_2PASS_2_KERNELS(head_dim_value) \
-[[max_total_threads_per_threadgroup(1024)]] kernel void attention_2pass_2_half_##head_dim_value outerArguments(half) { \
- constexpr int sequence_block_size = 32; \
- constexpr int head_block_size = 32; \
- threadgroup float shared_outputs[sequence_block_size * head_block_size]; \
- attention_2pass_2<half, head_dim_value> innerArguments; \
-} \
-[[max_total_threads_per_threadgroup(1024)]] kernel void attention_2pass_2_float_##head_dim_value outerArguments(float) { \
- constexpr int sequence_block_size = 32; \
- constexpr int head_block_size = 32; \
- threadgroup float shared_outputs[sequence_block_size * head_block_size]; \
- attention_2pass_2<float, head_dim_value> innerArguments; \
-}
+#define GENERATE_2PASS_2_KERNELS(head_dim_value) \
+ [[max_total_threads_per_threadgroup( \
+ 1024 \
+ )]] kernel void attention_2pass_2_half_##head_dim_value \
+ outerArguments(half) { \
+ constexpr int sequence_block_size = 32; \
+ constexpr int head_block_size = 32; \
+ threadgroup float shared_outputs[sequence_block_size * head_block_size]; \
+ attention_2pass_2<half, head_dim_value> innerArguments; \
+ } \
+ [[max_total_threads_per_threadgroup( \
+ 1024 \
+ )]] kernel void attention_2pass_2_float_##head_dim_value \
+ outerArguments(float) { \
+ constexpr int sequence_block_size = 32; \
+ constexpr int head_block_size = 32; \
+ threadgroup float shared_outputs[sequence_block_size * head_block_size]; \
+ attention_2pass_2<float, head_dim_value> innerArguments; \
+ }
GENERATE_2PASS_2_KERNELS(64)
GENERATE_2PASS_2_KERNELS(128)
@@ -644,66 +795,85 @@ GENERATE_2PASS_2_KERNELS(256)
template <typename T>
void update_kv_cache(
- const device T* rotated_keys, // [num_groups, suffix_length, head_dim]
- const device T* qkv, // [suffix_length, (num_heads + 2*num_groups) * head_dim]
- device T* key_cache, // [num_groups, max_sequence_length, head_dim]
- device T* value_cache, // [num_groups, max_sequence_length, head_dim]
+ const device T* rotated_keys, // [num_groups, suffix_length, head_dim]
+ const device T*
+ qkv, // [suffix_length, (num_heads + 2*num_groups) * head_dim]
+ device T* key_cache, // [num_groups, max_sequence_length, head_dim]
+ device T* value_cache, // [num_groups, max_sequence_length, head_dim]
const constant int& num_groups,
const constant int& num_heads,
const constant int& head_dim,
const constant int& suffix_length,
const constant int& prefix_segment_length,
const constant int& max_sequence_length,
- uint3 position // x: group_idx, y: token_idx, z: dim_idx
+ uint3 position // x: group_idx, y: token_idx, z: dim_idx
) {
- const uint groupIndex = position.x;
- const uint tokenIndex = position.y;
- const uint dimIndex = position.z;
-
- if (groupIndex >= num_groups || tokenIndex >= suffix_length || dimIndex >= head_dim) {
- return;
- }
-
- TensorView3D<const T> rotatedKeysTensorView = TensorView3D<const T>(rotated_keys, num_groups, suffix_length, head_dim);
- TensorView3D<T> keyCacheTensorView = TensorView3D<T>(key_cache, num_groups, max_sequence_length, head_dim);
- TensorView3D<T> valueCacheTensorView = TensorView3D<T>(value_cache, num_groups, max_sequence_length, head_dim);
-
- // Copy rotated key to cache
- keyCacheTensorView(groupIndex, prefix_segment_length + tokenIndex, dimIndex) = rotatedKeysTensorView(groupIndex, tokenIndex, dimIndex);
-
- // Update value cache (only first thread in each token processes values to avoid redundant work)
- if (dimIndex == 0) {
- const uint totalQueryDim = num_heads * head_dim;
- const uint totalKeyValueDim = num_groups * head_dim;
-
- const int qkvStride = totalQueryDim + 2 * totalKeyValueDim;
- TensorView2D<const T> qkvTensorView = TensorView2D<const T>(qkv, suffix_length, qkvStride);
-
- // Extract values from QKV tensor
- // Values start at offset: total_query_dim + total_key_value_dim
- for (uint d = 0; d < head_dim; d++) {
- const uint valueOffset = totalQueryDim + totalKeyValueDim + groupIndex * head_dim + d;
- valueCacheTensorView(groupIndex, prefix_segment_length + tokenIndex, d) = qkvTensorView(tokenIndex, valueOffset);
- }
+ const uint groupIndex = position.x;
+ const uint tokenIndex = position.y;
+ const uint dimIndex = position.z;
+
+ if (groupIndex >= num_groups || tokenIndex >= suffix_length ||
+ dimIndex >= head_dim) {
+ return;
+ }
+
+ TensorView3D<const T> rotatedKeysTensorView =
+ TensorView3D<const T>(rotated_keys, num_groups, suffix_length, head_dim);
+ TensorView3D<T> keyCacheTensorView =
+ TensorView3D<T>(key_cache, num_groups, max_sequence_length, head_dim);
+ TensorView3D<T> valueCacheTensorView =
+ TensorView3D<T>(value_cache, num_groups, max_sequence_length, head_dim);
+
+ // Copy rotated key to cache
+ keyCacheTensorView(groupIndex, prefix_segment_length + tokenIndex, dimIndex) =
+ rotatedKeysTensorView(groupIndex, tokenIndex, dimIndex);
+
+ // Update value cache (only first thread in each token processes values to
+ // avoid redundant work)
+ if (dimIndex == 0) {
+ const uint totalQueryDim = num_heads * head_dim;
+ const uint totalKeyValueDim = num_groups * head_dim;
+
+ const int qkvStride = totalQueryDim + 2 * totalKeyValueDim;
+ TensorView2D<const T> qkvTensorView =
+ TensorView2D<const T>(qkv, suffix_length, qkvStride);
+
+ // Extract values from QKV tensor
+ // Values start at offset: total_query_dim + total_key_value_dim
+ for (uint d = 0; d < head_dim; d++) {
+ const uint valueOffset =
+ totalQueryDim + totalKeyValueDim + groupIndex * head_dim + d;
+ valueCacheTensorView(groupIndex, prefix_segment_length + tokenIndex, d) =
+ qkvTensorView(tokenIndex, valueOffset);
}
+ }
}
-#define outerArguments(T) \
-(const device T* rotated_keys [[ buffer(0) ]], \
- const device T* qkv [[ buffer(1) ]], \
- device T* key_cache [[ buffer(2) ]], \
- device T* value_cache [[ buffer(3) ]], \
- const constant int& num_groups [[ buffer(4) ]], \
- const constant int& num_heads [[ buffer(5) ]], \
- const constant int& head_dim [[ buffer(6) ]], \
- const constant int& suffix_length [[ buffer(7) ]], \
- const constant int& prefix_segment_length [[ buffer(8) ]], \
- const constant int& max_sequence_length [[ buffer(9) ]], \
- uint3 position [[ thread_position_in_grid ]]) \
-
-#define innerArguments \
-(rotated_keys, qkv, key_cache, value_cache, num_groups, num_heads, \
- head_dim, suffix_length, prefix_segment_length, max_sequence_length, position) \
+#define outerArguments(T) \
+ (const device T* rotated_keys [[buffer(0)]], \
+ const device T* qkv [[buffer(1)]], \
+ device T* key_cache [[buffer(2)]], \
+ device T* value_cache [[buffer(3)]], \
+ const constant int& num_groups [[buffer(4)]], \
+ const constant int& num_heads [[buffer(5)]], \
+ const constant int& head_dim [[buffer(6)]], \
+ const constant int& suffix_length [[buffer(7)]], \
+ const constant int& prefix_segment_length [[buffer(8)]], \
+ const constant int& max_sequence_length [[buffer(9)]], \
+ uint3 position [[thread_position_in_grid]])
+
+#define innerArguments \
+ (rotated_keys, \
+ qkv, \
+ key_cache, \
+ value_cache, \
+ num_groups, \
+ num_heads, \
+ head_dim, \
+ suffix_length, \
+ prefix_segment_length, \
+ max_sequence_length, \
+ position)
generateKernels(update_kv_cache)
diff --git a/crates/uzu/src/backends/metal/kernel/definitions.metal b/crates/uzu/src/backends/metal/kernel/definitions.metal
index 399c0ae51..972ccd1c7 100644
--- a/crates/uzu/src/backends/metal/kernel/definitions.metal
+++ b/crates/uzu/src/backends/metal/kernel/definitions.metal
@@ -7,426 +7,458 @@ using namespace metal;
// MARK: - Tensor Access Templates
-template<typename T>
+template <typename T>
struct TensorView2D {
- device T* data;
- uint dim0, dim1;
- uint stride0, stride1;
-
- TensorView2D(device T* ptr) : data(ptr) {}
-
- TensorView2D(device T* ptr, int d0, int d1)
- : data(ptr), dim0(d0), dim1(d1) {
- stride0 = d1;
- stride1 = 1;
- }
-
- // Custom stride constructor
- TensorView2D(device T* ptr, int d0, int d1, int s0, int s1)
- : data(ptr), dim0(d0), dim1(d1), stride0(s0), stride1(s1) {}
-
- thread TensorView2D& shaped(int d0, int d1) {
- dim0 = d0;
- dim1 = d1;
- stride0 = d1;
- stride1 = 1;
- return *this;
- }
-
- device T& at(uint i, uint j) const {
- return data[i * stride0 + j * stride1];
- }
-
- device T& operator()(uint i, uint j) const {
- return at(i, j);
- }
+ device T* data;
+ uint dim0, dim1;
+ uint stride0, stride1;
+
+ TensorView2D(device T* ptr) : data(ptr) {}
+
+ TensorView2D(device T* ptr, int d0, int d1) : data(ptr), dim0(d0), dim1(d1) {
+ stride0 = d1;
+ stride1 = 1;
+ }
+
+ // Custom stride constructor
+ TensorView2D(device T* ptr, int d0, int d1, int s0, int s1)
+ : data(ptr), dim0(d0), dim1(d1), stride0(s0), stride1(s1) {}
+
+ thread TensorView2D& shaped(int d0, int d1) {
+ dim0 = d0;
+ dim1 = d1;
+ stride0 = d1;
+ stride1 = 1;
+ return *this;
+ }
+
+ device T& at(uint i, uint j) const { return data[i * stride0 + j * stride1]; }
+
+ device T& operator()(uint i, uint j) const { return at(i, j); }
};
-template<typename T>
+template <typename T>
struct TensorView3D {
- device T* data;
- uint dim0, dim1, dim2;
- uint stride0, stride1, stride2;
-
- TensorView3D(device T* ptr) : data(ptr) {}
-
- TensorView3D(device T* ptr, int d0, int d1, int d2)
- : data(ptr), dim0(d0), dim1(d1), dim2(d2) {
- stride0 = d1 * d2;
- stride1 = d2;
- stride2 = 1;
- }
-
- // Custom stride constructor
- TensorView3D(device T* ptr, int d0, int d1, int d2, int s0, int s1, int s2)
- : data(ptr), dim0(d0), dim1(d1), dim2(d2), stride0(s0), stride1(s1), stride2(s2) {}
-
- thread TensorView3D& shaped(int d0, int d1, int d2) {
- dim0 = d0;
- dim1 = d1;
- dim2 = d2;
- stride0 = d1 * d2;
- stride1 = d2;
- stride2 = 1;
- return *this;
- }
-
- device T& at(uint i, uint j, uint k) const {
- return data[i * stride0 + j * stride1 + k * stride2];
- }
-
- device T& operator()(uint i, uint j, uint k) const {
- return at(i, j, k);
- }
+ device T* data;
+ uint dim0, dim1, dim2;
+ uint stride0, stride1, stride2;
+
+ TensorView3D(device T* ptr) : data(ptr) {}
+
+ TensorView3D(device T* ptr, int d0, int d1, int d2)
+ : data(ptr), dim0(d0), dim1(d1), dim2(d2) {
+ stride0 = d1 * d2;
+ stride1 = d2;
+ stride2 = 1;
+ }
+
+ // Custom stride constructor
+ TensorView3D(device T* ptr, int d0, int d1, int d2, int s0, int s1, int s2)
+ : data(ptr), dim0(d0), dim1(d1), dim2(d2), stride0(s0), stride1(s1),
+ stride2(s2) {}
+
+ thread TensorView3D& shaped(int d0, int d1, int d2) {
+ dim0 = d0;
+ dim1 = d1;
+ dim2 = d2;
+ stride0 = d1 * d2;
+ stride1 = d2;
+ stride2 = 1;
+ return *this;
+ }
+
+ device T& at(uint i, uint j, uint k) const {
+ return data[i * stride0 + j * stride1 + k * stride2];
+ }
+
+ device T& operator()(uint i, uint j, uint k) const { return at(i, j, k); }
};
-template<typename T>
+template <typename T>
struct TensorView4D {
- device T* data;
- uint dim0, dim1, dim2, dim3;
- uint stride0, stride1, stride2, stride3;
-
- TensorView4D(device T* ptr) : data(ptr) {}
-
- TensorView4D(device T* ptr, int d0, int d1, int d2, int d3)
- : data(ptr), dim0(d0), dim1(d1), dim2(d2), dim3(d3) {
- stride0 = d1 * d2 * d3;
- stride1 = d2 * d3;
- stride2 = d3;
- stride3 = 1;
- }
-
- // Custom stride constructor
- TensorView4D(device T* ptr, int d0, int d1, int d2, int d3, int s0, int s1, int s2, int s3)
- : data(ptr), dim0(d0), dim1(d1), dim2(d2), dim3(d3), stride0(s0), stride1(s1), stride2(s2), stride3(s3) {}
-
- thread TensorView4D& shaped(int d0, int d1, int d2, int d3) {
- dim0 = d0;
- dim1 = d1;
- dim2 = d2;
- dim3 = d3;
- stride0 = d1 * d2 * d3;
- stride1 = d2 * d3;
- stride2 = d3;
- stride3 = 1;
- return *this;
- }
-
- device T& at(uint i, uint j, uint k, uint l) const {
- return data[i * stride0 + j * stride1 + k * stride2 + l * stride3];
- }
-
- device T& operator()(uint i, uint j, uint k, uint l) const {
- return at(i, j, k, l);
- }
+ device T* data;
+ uint dim0, dim1, dim2, dim3;
+ uint stride0, stride1, stride2, stride3;
+
+ TensorView4D(device T* ptr) : data(ptr) {}
+
+ TensorView4D(device T* ptr, int d0, int d1, int d2, int d3)
+ : data(ptr), dim0(d0), dim1(d1), dim2(d2), dim3(d3) {
+ stride0 = d1 * d2 * d3;
+ stride1 = d2 * d3;
+ stride2 = d3;
+ stride3 = 1;
+ }
+
+ // Custom stride constructor
+ TensorView4D(
+ device T* ptr,
+ int d0,
+ int d1,
+ int d2,
+ int d3,
+ int s0,
+ int s1,
+ int s2,
+ int s3
+ )
+ : data(ptr), dim0(d0), dim1(d1), dim2(d2), dim3(d3), stride0(s0),
+ stride1(s1), stride2(s2), stride3(s3) {}
+
+ thread TensorView4D& shaped(int d0, int d1, int d2, int d3) {
+ dim0 = d0;
+ dim1 = d1;
+ dim2 = d2;
+ dim3 = d3;
+ stride0 = d1 * d2 * d3;
+ stride1 = d2 * d3;
+ stride2 = d3;
+ stride3 = 1;
+ return *this;
+ }
+
+ device T& at(uint i, uint j, uint k, uint l) const {
+ return data[i * stride0 + j * stride1 + k * stride2 + l * stride3];
+ }
+
+ device T& operator()(uint i, uint j, uint k, uint l) const {
+ return at(i, j, k, l);
+ }
};
///////////////////////////////////////////////////////////////////////////////
// MARK: - Thread Functions
///////////////////////////////////////////////////////////////////////////////
-template<ushort LENGTH, typename T>
-static inline T thread_prefix_inclusive_sum(thread T (&values)[LENGTH]){
- for (ushort i = 1; i < LENGTH; i++){
- values[i] += values[i - 1];
- }
- return values[LENGTH - 1];
+template <ushort LENGTH, typename T>
+static inline T thread_prefix_inclusive_sum(thread T (&values)[LENGTH]) {
+ for (ushort i = 1; i < LENGTH; i++) {
+ values[i] += values[i - 1];
+ }
+ return values[LENGTH - 1];
}
-template<ushort LENGTH, typename T>
-static inline T thread_prefix_inclusive_sum(threadgroup T* values){
- for (ushort i = 1; i < LENGTH; i++){
- values[i] += values[i - 1];
- }
- return values[LENGTH - 1];
+template <ushort LENGTH, typename T>
+static inline T thread_prefix_inclusive_sum(threadgroup T* values) {
+ for (ushort i = 1; i < LENGTH; i++) {
+ values[i] += values[i - 1];
+ }
+ return values[LENGTH - 1];
}
-
-template<ushort LENGTH, typename T>
-static inline T thread_prefix_exclusive_sum(thread T (&values)[LENGTH]){
- // do as an inclusive scan first
- T inclusive_prefix = thread_prefix_inclusive_sum<LENGTH>(values);
- // convert to an exclusive scan in the reverse direction
- for (ushort i = LENGTH - 1; i > 0; i--){
- values[i] = values[i - 1];
- }
- values[0] = 0;
- return inclusive_prefix;
+template <ushort LENGTH, typename T>
+static inline T thread_prefix_exclusive_sum(thread T (&values)[LENGTH]) {
+ // do as an inclusive scan first
+ T inclusive_prefix = thread_prefix_inclusive_sum<LENGTH>(values);
+ // convert to an exclusive scan in the reverse direction
+ for (ushort i = LENGTH - 1; i > 0; i--) {
+ values[i] = values[i - 1];
+ }
+ values[0] = 0;
+ return inclusive_prefix;
}
-template<ushort LENGTH, typename T>
-static inline T thread_prefix_exclusive_sum(threadgroup T* values){
- // do as an inclusive scan first
- T inclusive_prefix = thread_prefix_inclusive_sum<LENGTH>(values);
- // convert to an exclusive scan in the reverse direction
- for (ushort i = LENGTH - 1; i > 0; i--){
- values[i] = values[i - 1];
- }
- values[0] = 0;
- return inclusive_prefix;
+template <ushort LENGTH, typename T>
+static inline T thread_prefix_exclusive_sum(threadgroup T* values) {
+ // do as an inclusive scan first
+ T inclusive_prefix = thread_prefix_inclusive_sum<LENGTH>(values);
+ // convert to an exclusive scan in the reverse direction
+ for (ushort i = LENGTH - 1; i > 0; i--) {
+ values[i] = values[i - 1];
+ }
+ values[0] = 0;
+ return inclusive_prefix;
}
-
-template<ushort LENGTH, typename T> static inline void
-thread_uniform_add(thread T (&values)[LENGTH], T uni){
- for (ushort i = 0; i < LENGTH; i++){
- values[i] += uni;
- }
+template <ushort LENGTH, typename T>
+static inline void thread_uniform_add(thread T (&values)[LENGTH], T uni) {
+ for (ushort i = 0; i < LENGTH; i++) {
+ values[i] += uni;
+ }
}
-template<ushort LENGTH, typename T> static inline void
-thread_uniform_add(threadgroup T* values, T uni){
- for (ushort i = 0; i < LENGTH; i++){
- values[i] += uni;
- }
+template <ushort LENGTH, typename T>
+static inline void thread_uniform_add(threadgroup T* values, T uni) {
+ for (ushort i = 0; i < LENGTH; i++) {
+ values[i] += uni;
+ }
}
-
///////////////////////////////////////////////////////////////////////////////
// MARK: - Threadgroup Functions
///////////////////////////////////////////////////////////////////////////////
//------------------------------------------------------------------------------------------------//
// Raking threadgroup scan
-template<ushort BLOCK_SIZE, typename T> static T
-threadgroup_raking_prefix_exclusive_sum(T value, threadgroup T* shared, const ushort lid) {
- // load values into shared memory
- shared[lid] = value;
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // only the first 32 threads form the rake
- if (lid < 32) {
- // scan by thread in shared mem
- const short values_per_thread = BLOCK_SIZE / 32;
- const short first_index = lid * values_per_thread;
- for (short i = first_index + 1; i < first_index + values_per_thread; i++){
- shared[i] += shared[i - 1];
- }
- T partial_sum = shared[first_index + values_per_thread - 1];
- for (short i = first_index + values_per_thread - 1; i > first_index; i--){
- shared[i] = shared[i - 1];
- }
- shared[first_index] = 0;
-
- // scan the partial sums
- T prefix = simd_prefix_exclusive_sum(partial_sum);
-
- // add back the prefix
- for (short i = first_index; i < first_index + values_per_thread; i++){
- shared[i] += prefix;
- }
-
+template <ushort BLOCK_SIZE, typename T>
+static T threadgroup_raking_prefix_exclusive_sum(
+ T value,
+ threadgroup T* shared,
+ const ushort lid
+) {
+ // load values into shared memory
+ shared[lid] = value;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // only the first 32 threads form the rake
+ if (lid < 32) {
+ // scan by thread in shared mem
+ const short values_per_thread = BLOCK_SIZE / 32;
+ const short first_index = lid * values_per_thread;
+ for (short i = first_index + 1; i < first_index + values_per_thread; i++) {
+ shared[i] += shared[i - 1];
+ }
+ T partial_sum = shared[first_index + values_per_thread - 1];
+ for (short i = first_index + values_per_thread - 1; i > first_index; i--) {
+ shared[i] = shared[i - 1];
}
- threadgroup_barrier(mem_flags::mem_threadgroup);
+ shared[first_index] = 0;
- return shared[lid];
+ // scan the partial sums
+ T prefix = simd_prefix_exclusive_sum(partial_sum);
+
+ // add back the prefix
+ for (short i = first_index; i < first_index + values_per_thread; i++) {
+ shared[i] += prefix;
+ }
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ return shared[lid];
}
//------------------------------------------------------------------------------------------------//
// Raking threadgroup sum reduction
-template<ushort BLOCK_SIZE, typename T> static T
-threadgroup_raking_reduce_sum(T value, threadgroup T* shared, const ushort lid) {
- // load values into shared memory
- shared[lid] = value;
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // only the first 32 threads form the rake
- if (lid < 32) {
- // reduce by thread in shared mem
- const short values_per_thread = BLOCK_SIZE / 32;
- const short first_index = lid * values_per_thread;
-
- T thread_sum = shared[first_index];
- for (short i = first_index + 1; i < first_index + values_per_thread; i++){
- thread_sum += shared[i];
- }
-
- // reduce the partial sums using SIMD
- T total_sum = simd_sum(thread_sum);
-
- // broadcast result back to shared memory
- if (lid == 0) {
- shared[0] = total_sum;
- }
+template <ushort BLOCK_SIZE, typename T>
+static T threadgroup_raking_reduce_sum(
+ T value,
+ threadgroup T* shared,
+ const ushort lid
+) {
+ // load values into shared memory
+ shared[lid] = value;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // only the first 32 threads form the rake
+ if (lid < 32) {
+ // reduce by thread in shared mem
+ const short values_per_thread = BLOCK_SIZE / 32;
+ const short first_index = lid * values_per_thread;
+
+ T thread_sum = shared[first_index];
+ for (short i = first_index + 1; i < first_index + values_per_thread; i++) {
+ thread_sum += shared[i];
+ }
+
+ // reduce the partial sums using SIMD
+ T total_sum = simd_sum(thread_sum);
+
+ // broadcast result back to shared memory
+ if (lid == 0) {
+ shared[0] = total_sum;
}
+ }
- threadgroup_barrier(mem_flags::mem_threadgroup);
+ threadgroup_barrier(mem_flags::mem_threadgroup);
- return shared[0];
+ return shared[0];
}
//------------------------------------------------------------------------------------------------//
-// Raking threadgroup max reduction
-template<ushort BLOCK_SIZE, typename T> static T
-threadgroup_raking_reduce_max(T value, threadgroup T* shared, const ushort lid) {
- // load values into shared memory
- shared[lid] = value;
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // only the first 32 threads form the rake
- if (lid < 32) {
- // reduce by thread in shared mem
- const short values_per_thread = BLOCK_SIZE / 32;
- const short first_index = lid * values_per_thread;
-
- T thread_max = shared[first_index];
- for (short i = first_index + 1; i < first_index + values_per_thread; i++){
- thread_max = max(thread_max, shared[i]);
- }
-
- // reduce the partial maxes using SIMD
- T total_max = simd_max(thread_max);
-
- // broadcast result back to shared memory
- if (lid == 0) {
- shared[0] = total_max;
- }
+// Raking threadgroup max reduction
+template <ushort BLOCK_SIZE, typename T>
+static T threadgroup_raking_reduce_max(
+ T value,
+ threadgroup T* shared,
+ const ushort lid
+) {
+ // load values into shared memory
+ shared[lid] = value;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // only the first 32 threads form the rake
+ if (lid < 32) {
+ // reduce by thread in shared mem
+ const short values_per_thread = BLOCK_SIZE / 32;
+ const short first_index = lid * values_per_thread;
+
+ T thread_max = shared[first_index];
+ for (short i = first_index + 1; i < first_index + values_per_thread; i++) {
+ thread_max = max(thread_max, shared[i]);
}
- threadgroup_barrier(mem_flags::mem_threadgroup);
+ // reduce the partial maxes using SIMD
+ T total_max = simd_max(thread_max);
- return shared[0];
+ // broadcast result back to shared memory
+ if (lid == 0) {
+ shared[0] = total_max;
+ }
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ return shared[0];
}
//------------------------------------------------------------------------------------------------//
-// Raking threadgroup min reduction
-template<ushort BLOCK_SIZE, typename T> static T
-threadgroup_raking_reduce_min(T value, threadgroup T* shared, const ushort lid) {
- // load values into shared memory
- shared[lid] = value;
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // only the first 32 threads form the rake
- if (lid < 32) {
- // reduce by thread in shared mem
- const short values_per_thread = BLOCK_SIZE / 32;
- const short first_index = lid * values_per_thread;
-
- T thread_min = shared[first_index];
- for (short i = first_index + 1; i < first_index + values_per_thread; i++){
- thread_min = min(thread_min, shared[i]);
- }
-
- // reduce the partial mins using SIMD
- T total_min = simd_min(thread_min);
-
- // broadcast result back to shared memory
- if (lid == 0) {
- shared[0] = total_min;
- }
+// Raking threadgroup min reduction
+template <ushort BLOCK_SIZE, typename T>
+static T threadgroup_raking_reduce_min(
+ T value,
+ threadgroup T* shared,
+ const ushort lid
+) {
+ // load values into shared memory
+ shared[lid] = value;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // only the first 32 threads form the rake
+ if (lid < 32) {
+ // reduce by thread in shared mem
+ const short values_per_thread = BLOCK_SIZE / 32;
+ const short first_index = lid * values_per_thread;
+
+ T thread_min = shared[first_index];
+ for (short i = first_index + 1; i < first_index + values_per_thread; i++) {
+ thread_min = min(thread_min, shared[i]);
}
- threadgroup_barrier(mem_flags::mem_threadgroup);
- return shared[0];
+ // reduce the partial mins using SIMD
+ T total_min = simd_min(thread_min);
+
+ // broadcast result back to shared memory
+ if (lid == 0) {
+ shared[0] = total_min;
+ }
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ return shared[0];
}
//------------------------------------------------------------------------------------------------//
// Cooperative threadgroup sum reduction (2-level hierarchical)
-template<ushort BLOCK_SIZE, typename T> static T
-threadgroup_cooperative_reduce_sum(T value, threadgroup T* shared, const ushort lid) {
- const ushort simd_group_id = lid / 32;
- const ushort simd_lane_id = lid % 32;
-
- // Reduce within simdgroup
- T local_sum = simd_sum(value);
-
- // First thread in each simdgroup writes to shared memory
- if (simd_lane_id == 0) {
- shared[simd_group_id] = local_sum;
- }
-
- // Synchronize across the threadgroup
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // Reduce across simdgroups
- T total_sum = T(0);
- const ushort num_simd_groups = (BLOCK_SIZE + 31) / 32;
- if (lid < num_simd_groups) {
- total_sum = shared[lid];
- }
- total_sum = simd_sum(total_sum);
-
- // Broadcast the result to all threads
- if (lid == 0) {
- shared[0] = total_sum;
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- return shared[0];
+template <ushort BLOCK_SIZE, typename T>
+static T threadgroup_cooperative_reduce_sum(
+ T value,
+ threadgroup T* shared,
+ const ushort lid
+) {
+ const ushort simd_group_id = lid / 32;
+ const ushort simd_lane_id = lid % 32;
+
+ // Reduce within simdgroup
+ T local_sum = simd_sum(value);
+
+ // First thread in each simdgroup writes to shared memory
+ if (simd_lane_id == 0) {
+ shared[simd_group_id] = local_sum;
+ }
+
+ // Synchronize across the threadgroup
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // Reduce across simdgroups
+ T total_sum = T(0);
+ const ushort num_simd_groups = (BLOCK_SIZE + 31) / 32;
+ if (lid < num_simd_groups) {
+ total_sum = shared[lid];
+ }
+ total_sum = simd_sum(total_sum);
+
+ // Broadcast the result to all threads
+ if (lid == 0) {
+ shared[0] = total_sum;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ return shared[0];
}
//------------------------------------------------------------------------------------------------//
// Cooperative threadgroup max reduction (2-level hierarchical)
-template<ushort BLOCK_SIZE, typename T> static T
-threadgroup_cooperative_reduce_max(T value, threadgroup T* shared, const ushort lid) {
- const ushort simd_group_id = lid / 32;
- const ushort simd_lane_id = lid % 32;
-
- // Reduce within simdgroup
- T local_max = simd_max(value);
-
- // First thread in each simdgroup writes to shared memory
- if (simd_lane_id == 0) {
- shared[simd_group_id] = local_max;
- }
-
- // Synchronize across the threadgroup
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // Reduce across simdgroups
- T total_max = (lid < ((BLOCK_SIZE + 31) / 32)) ? shared[lid] : T(-INFINITY);
- total_max = simd_max(total_max);
-
- // Broadcast the result to all threads
- if (lid == 0) {
- shared[0] = total_max;
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- return shared[0];
+template <ushort BLOCK_SIZE, typename T>
+static T threadgroup_cooperative_reduce_max(
+ T value,
+ threadgroup T* shared,
+ const ushort lid
+) {
+ const ushort simd_group_id = lid / 32;
+ const ushort simd_lane_id = lid % 32;
+
+ // Reduce within simdgroup
+ T local_max = simd_max(value);
+
+ // First thread in each simdgroup writes to shared memory
+ if (simd_lane_id == 0) {
+ shared[simd_group_id] = local_max;
+ }
+
+ // Synchronize across the threadgroup
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // Reduce across simdgroups
+ T total_max = (lid < ((BLOCK_SIZE + 31) / 32)) ? shared[lid] : T(-INFINITY);
+ total_max = simd_max(total_max);
+
+ // Broadcast the result to all threads
+ if (lid == 0) {
+ shared[0] = total_max;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ return shared[0];
}
//------------------------------------------------------------------------------------------------//
// Cooperative threadgroup min reduction (2-level hierarchical)
-template<ushort BLOCK_SIZE, typename T> static T
-threadgroup_cooperative_reduce_min(T value, threadgroup T* shared, const ushort lid) {
- const ushort simd_group_id = lid / 32;
- const ushort simd_lane_id = lid % 32;
-
- // Reduce within simdgroup
- T local_min = simd_min(value);
-
- // First thread in each simdgroup writes to shared memory
- if (simd_lane_id == 0) {
- shared[simd_group_id] = local_min;
- }
-
- // Synchronize across the threadgroup
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // Reduce across simdgroups
- T total_min = (lid < ((BLOCK_SIZE + 31) / 32)) ? shared[lid] : T(INFINITY);
- total_min = simd_min(total_min);
-
- // Broadcast the result to all threads
- if (lid == 0) {
- shared[0] = total_min;
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- return shared[0];
+template <ushort BLOCK_SIZE, typename T>
+static T threadgroup_cooperative_reduce_min(
+ T value,
+ threadgroup T* shared,
+ const ushort lid
+) {
+ const ushort simd_group_id = lid / 32;
+ const ushort simd_lane_id = lid % 32;
+
+ // Reduce within simdgroup
+ T local_min = simd_min(value);
+
+ // First thread in each simdgroup writes to shared memory
+ if (simd_lane_id == 0) {
+ shared[simd_group_id] = local_min;
+ }
+
+ // Synchronize across the threadgroup
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // Reduce across simdgroups
+ T total_min = (lid < ((BLOCK_SIZE + 31) / 32)) ? shared[lid] : T(INFINITY);
+ total_min = simd_min(total_min);
+
+ // Broadcast the result to all threads
+ if (lid == 0) {
+ shared[0] = total_min;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ return shared[0];
}
// MARK: - Generate Template Kernels
-#define generateKernel(functionName, scalarType, outerArgs, innerArgs) \
-kernel void functionName##_##scalarType outerArgs { \
- functionName innerArgs; \
-}
-
-#define generateKernels(functionName) \
-generateKernel(functionName, float, outerArguments(float), innerArguments); \
-generateKernel(functionName, bfloat, outerArguments(bfloat), innerArguments); \
-generateKernel(functionName, half, outerArguments(half), innerArguments);
+#define generateKernel(functionName, scalarType, outerArgs, innerArgs) \
+ kernel void functionName##_##scalarType outerArgs { functionName innerArgs; }
+
+#define generateKernels(functionName) \
+ generateKernel(functionName, float, outerArguments(float), innerArguments); \
+ generateKernel( \
+ functionName, \
+ bfloat, \
+ outerArguments(bfloat), \
+ innerArguments \
+ ); \
+ generateKernel(functionName, half, outerArguments(half), innerArguments);
#endif /* definitions_metal */
diff --git a/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal b/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal
index 76b5b3612..a1e2f9e8a 100644
--- a/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal
+++ b/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal
@@ -1,63 +1,63 @@
#include <metal_stdlib>
using namespace metal;
-template<typename T, uint PACKING_DIVISOR, bool SIGNED_STORAGE>
+template <typename T, uint PACKING_DIVISOR, bool SIGNED_STORAGE>
[[kernel]] void quantized_embedding_lookup(
- const device uint64_t* token_ids [[buffer(0)]], // [batch_size]
- const device uint8_t* weights [[buffer(1)]], // [vocab_size, model_dim/packing_divisor] packed
- const device T* scales [[buffer(2)]], // [vocab_size, num_groups]
- const device T* biases [[buffer(3)]], // [vocab_size, num_groups]
- device T* output [[buffer(4)]], // [batch_size, model_dim]
+ const device uint64_t* token_ids [[buffer(0)]], // [batch_size]
+ const device uint8_t* weights
+ [[buffer(1)]], // [vocab_size, model_dim/packing_divisor] packed
+ const device T* scales [[buffer(2)]], // [vocab_size, num_groups]
+ const device T* biases [[buffer(3)]], // [vocab_size, num_groups]
+ device T* output [[buffer(4)]], // [batch_size, model_dim]
constant uint32_t& batch_size [[buffer(5)]],
constant uint32_t& vocab_size [[buffer(6)]],
constant uint32_t& model_dim [[buffer(7)]],
constant uint32_t& group_size [[buffer(8)]],
uint thread_position_in_grid [[thread_position_in_grid]]
) {
- const uint batch_idx = thread_position_in_grid / model_dim;
- const uint dim_idx = thread_position_in_grid % model_dim;
-
- if (batch_idx >= batch_size) return;
-
- const uint64_t token_id = token_ids[batch_idx];
- if (token_id >= vocab_size) {
- output[thread_position_in_grid] = T(0);
- return;
- }
-
- const uint group_idx = dim_idx / group_size;
- const uint num_groups = (model_dim + group_size - 1) / group_size;
-
- const T scale = scales[token_id * num_groups + group_idx];
- const T bias = biases[token_id * num_groups + group_idx];
-
- const uint weights_stride = model_dim / PACKING_DIVISOR;
- const device uint8_t* weights_u8 = weights;
- const device int8_t* weights_i8 =
- reinterpret_cast<const device int8_t*>(weights);
-
- int quantized_value = 0;
- if (PACKING_DIVISOR == 2) {
- const uint byte_idx =
- token_id * weights_stride + (dim_idx / 2);
- const uint8_t packed = weights_u8[byte_idx];
- if ((dim_idx & 1) == 0) {
- quantized_value = int(packed & 0x0F);
- } else {
- quantized_value = int((packed >> 4) & 0x0F);
- }
+ const uint batch_idx = thread_position_in_grid / model_dim;
+ const uint dim_idx = thread_position_in_grid % model_dim;
+
+ if (batch_idx >= batch_size)
+ return;
+
+ const uint64_t token_id = token_ids[batch_idx];
+ if (token_id >= vocab_size) {
+ output[thread_position_in_grid] = T(0);
+ return;
+ }
+
+ const uint group_idx = dim_idx / group_size;
+ const uint num_groups = (model_dim + group_size - 1) / group_size;
+
+ const T scale = scales[token_id * num_groups + group_idx];
+ const T bias = biases[token_id * num_groups + group_idx];
+
+ const uint weights_stride = model_dim / PACKING_DIVISOR;
+ const device uint8_t* weights_u8 = weights;
+ const device int8_t* weights_i8 =
+ reinterpret_cast<const device int8_t*>(weights);
+
+ int quantized_value = 0;
+ if (PACKING_DIVISOR == 2) {
+ const uint byte_idx = token_id * weights_stride + (dim_idx / 2);
+ const uint8_t packed = weights_u8[byte_idx];
+ if ((dim_idx & 1) == 0) {
+ quantized_value = int(packed & 0x0F);
} else {
- const uint elem_idx = token_id * weights_stride + dim_idx;
- quantized_value =
- SIGNED_STORAGE ? int(weights_i8[elem_idx])
- : int(weights_u8[elem_idx]);
+ quantized_value = int((packed >> 4) & 0x0F);
}
+ } else {
+ const uint elem_idx = token_id * weights_stride + dim_idx;
+ quantized_value =
+ SIGNED_STORAGE ? int(weights_i8[elem_idx]) : int(weights_u8[elem_idx]);
+ }
- output[thread_position_in_grid] = scale * T(quantized_value) + bias;
+ output[thread_position_in_grid] = scale * T(quantized_value) + bias;
}
-template [[host_name("quantized_embedding_lookup_f32_uint4")]]
-[[kernel]] void quantized_embedding_lookup<float, 2, false>(
+template [[host_name("quantized_embedding_lookup_f32_uint4")]] [[kernel]] void
+quantized_embedding_lookup<float, 2, false>(
const device uint64_t* token_ids [[buffer(0)]],
const device uint8_t* weights [[buffer(1)]],
const device float* scales [[buffer(2)]],
@@ -70,8 +70,8 @@ template [[host_name("quantized_embedding_lookup_f32_uint4")]]
uint thread_position_in_grid [[thread_position_in_grid]]
);
-template [[host_name("quantized_embedding_lookup_f16_uint4")]]
-[[kernel]] void quantized_embedding_lookup<half, 2, false>(
+template [[host_name("quantized_embedding_lookup_f16_uint4")]] [[kernel]] void
+quantized_embedding_lookup<half, 2, false>(
const device uint64_t* token_ids [[buffer(0)]],
const device uint8_t* weights [[buffer(1)]],
const device half* scales [[buffer(2)]],
@@ -84,8 +84,8 @@ template [[host_name("quantized_embedding_lookup_f16_uint4")]]
uint thread_position_in_grid [[thread_position_in_grid]]
);
-template [[host_name("quantized_embedding_lookup_bf16_uint4")]]
-[[kernel]] void quantized_embedding_lookup<bfloat, 2, false>(
+template [[host_name("quantized_embedding_lookup_bf16_uint4")]] [[kernel]] void
+quantized_embedding_lookup<bfloat, 2, false>(
const device uint64_t* token_ids [[buffer(0)]],
const device uint8_t* weights [[buffer(1)]],
const device bfloat* scales [[buffer(2)]],
@@ -98,8 +98,8 @@ template [[host_name("quantized_embedding_lookup_bf16_uint4")]]
uint thread_position_in_grid [[thread_position_in_grid]]
);
-template [[host_name("quantized_embedding_lookup_f32_int8")]]
-[[kernel]] void quantized_embedding_lookup<float, 1, true>(
+template [[host_name("quantized_embedding_lookup_f32_int8")]] [[kernel]] void
+quantized_embedding_lookup<float, 1, true>(
const device uint64_t* token_ids [[buffer(0)]],
const device uint8_t* weights [[buffer(1)]],
const device float* scales [[buffer(2)]],
@@ -112,8 +112,8 @@ template [[host_name("quantized_embedding_lookup_f32_int8")]]
uint thread_position_in_grid [[thread_position_in_grid]]
);
-template [[host_name("quantized_embedding_lookup_f16_int8")]]
-[[kernel]] void quantized_embedding_lookup<half, 1, true>(
+template [[host_name("quantized_embedding_lookup_f16_int8")]] [[kernel]] void
+quantized_embedding_lookup<half, 1, true>(
const device uint64_t* token_ids [[buffer(0)]],
const device uint8_t* weights [[buffer(1)]],
const device half* scales [[buffer(2)]],
@@ -126,8 +126,8 @@ template [[host_name("quantized_embedding_lookup_f16_int8")]]
uint thread_position_in_grid [[thread_position_in_grid]]
);
-template [[host_name("quantized_embedding_lookup_bf16_int8")]]
-[[kernel]] void quantized_embedding_lookup<bfloat, 1, true>(
+template [[host_name("quantized_embedding_lookup_bf16_int8")]] [[kernel]] void
+quantized_embedding_lookup<bfloat, 1, true>(
const device uint64_t* token_ids [[buffer(0)]],
const device uint8_t* weights [[buffer(1)]],
const device bfloat* scales [[buffer(2)]],
@@ -140,8 +140,8 @@ template [[host_name("quantized_embedding_lookup_bf16_int8")]]
uint thread_position_in_grid [[thread_position_in_grid]]
);
-template [[host_name("quantized_embedding_lookup_f32_uint8")]]
-[[kernel]] void quantized_embedding_lookup<float, 1, false>(
+template [[host_name("quantized_embedding_lookup_f32_uint8")]] [[kernel]] void
+quantized_embedding_lookup<float, 1, false>(
const device uint64_t* token_ids [[buffer(0)]],
const device uint8_t* weights [[buffer(1)]],
const device float* scales [[buffer(2)]],
@@ -154,8 +154,8 @@ template [[host_name("quantized_embedding_lookup_f32_uint8")]]
uint thread_position_in_grid [[thread_position_in_grid]]
);
-template [[host_name("quantized_embedding_lookup_f16_uint8")]]
-[[kernel]] void quantized_embedding_lookup<half, 1, false>(
+template [[host_name("quantized_embedding_lookup_f16_uint8")]] [[kernel]] void
+quantized_embedding_lookup<half, 1, false>(
const device uint64_t* token_ids [[buffer(0)]],
const device uint8_t* weights [[buffer(1)]],
const device half* scales [[buffer(2)]],
@@ -168,8 +168,8 @@ template [[host_name("quantized_embedding_lookup_f16_uint8")]]
uint thread_position_in_grid [[thread_position_in_grid]]
);
-template [[host_name("quantized_embedding_lookup_bf16_uint8")]]
-[[kernel]] void quantized_embedding_lookup<bfloat, 1, false>(
+template [[host_name("quantized_embedding_lookup_bf16_uint8")]] [[kernel]] void
+quantized_embedding_lookup<bfloat, 1, false>(
const device uint64_t* token_ids [[buffer(0)]],
const device uint8_t* weights [[buffer(1)]],
const device bfloat* scales [[buffer(2)]],
diff --git a/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal b/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal
index 110921510..c4b0b52f6 100644
--- a/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal
+++ b/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal
@@ -2,14 +2,10 @@
#include "../definitions.metal"
template <typename T>
-void swap(
- device T* buffer,
- const int sourceIdx,
- const int destIdx
-) {
- const T temp = buffer[sourceIdx];
- buffer[sourceIdx] = buffer[destIdx];
- buffer[destIdx] = temp;
+void swap(device T* buffer, const int sourceIdx, const int destIdx) {
+ const T temp = buffer[sourceIdx];
+ buffer[sourceIdx] = buffer[destIdx];
+ buffer[destIdx] = temp;
}
template <typename T>
@@ -23,37 +19,38 @@ void updateKVCache(
constant int& headDim,
const uint2 position
) {
- for (int i = 0; i < swapCount; ++i) {
- // [headIdx: 0..numHeads, tokenIdx: 0..maxSequenceLength, channelIdx: 0..headDim]
- const int headOffset = position.x * maxSequenceLength * headDim;
- const int channelOffset = position.y;
- const int sourceIdx = headOffset + swaps[i].x * headDim + channelOffset;
- const int destIdx = headOffset + swaps[i].y * headDim + channelOffset;
+ for (int i = 0; i < swapCount; ++i) {
+ // [headIdx: 0..numHeads, tokenIdx: 0..maxSequenceLength, channelIdx:
+ // 0..headDim]
+ const int headOffset = position.x * maxSequenceLength * headDim;
+ const int channelOffset = position.y;
+ const int sourceIdx = headOffset + swaps[i].x * headDim + channelOffset;
+ const int destIdx = headOffset + swaps[i].y * headDim + channelOffset;
- swap(inPlaceKeys, sourceIdx, destIdx);
- swap(inPlaceValues, sourceIdx, destIdx);
- }
+ swap(inPlaceKeys, sourceIdx, destIdx);
+ swap(inPlaceValues, sourceIdx, destIdx);
+ }
}
-#define outerArguments(T) \
-(device T* inPlaceKeys [[ buffer(0) ]], \
-device T* inPlaceValues [[ buffer(1) ]], \
-device int2* swaps [[ buffer(2) ]], \
-constant int& swapCount [[ buffer(3) ]], \
-constant int& numHeads [[ buffer(4) ]], \
-constant int& maxSequenceLength [[ buffer(5) ]], \
-constant int& headDim [[ buffer(6) ]], \
-const uint2 position [[ thread_position_in_grid ]]) \
+#define outerArguments(T) \
+ (device T * inPlaceKeys [[buffer(0)]], \
+ device T * inPlaceValues [[buffer(1)]], \
+ device int2 * swaps [[buffer(2)]], \
+ constant int& swapCount [[buffer(3)]], \
+ constant int& numHeads [[buffer(4)]], \
+ constant int& maxSequenceLength [[buffer(5)]], \
+ constant int& headDim [[buffer(6)]], \
+ const uint2 position [[thread_position_in_grid]])
-#define innerArguments \
-(inPlaceKeys, \
-inPlaceValues, \
-swaps, \
-swapCount, \
-numHeads, \
-maxSequenceLength, \
-headDim, \
-position) \
+#define innerArguments \
+ (inPlaceKeys, \
+ inPlaceValues, \
+ swaps, \
+ swapCount, \
+ numHeads, \
+ maxSequenceLength, \
+ headDim, \
+ position)
generateKernels(updateKVCache)
diff --git a/crates/uzu/src/backends/metal/kernel/media_kernels/image_ops.metal b/crates/uzu/src/backends/metal/kernel/media_kernels/image_ops.metal
index db94fe047..1ef155c1c 100644
--- a/crates/uzu/src/backends/metal/kernel/media_kernels/image_ops.metal
+++ b/crates/uzu/src/backends/metal/kernel/media_kernels/image_ops.metal
@@ -4,110 +4,126 @@ using namespace metal;
constexpr sampler bicubicSampler(filter::linear, address::clamp_to_edge);
struct ImageParameters {
- uint2 inputDimensions;
- float3 imageMean;
- float3 imageStd;
- float paddingValueRgb;
+ uint2 inputDimensions;
+ float3 imageMean;
+ float3 imageStd;
+ float paddingValueRgb;
};
struct PatchParameters {
- uint2 paddedDimensions;
- uint patchSize;
- uint numChannels;
- uint temporalSlices;
+ uint2 paddedDimensions;
+ uint patchSize;
+ uint numChannels;
+ uint temporalSlices;
};
kernel void scalePadNormalizeImage(
texture2d<float, access::sample> inputTexture [[texture(0)]],
texture2d<float, access::write> outputTexture [[texture(1)]],
constant ImageParameters& params [[buffer(0)]],
- uint2 gid [[thread_position_in_grid]]) {
-
- uint2 outputDims = uint2(outputTexture.get_width(), outputTexture.get_height());
-
- if (gid.x >= outputDims.x || gid.y >= outputDims.y) {
- return;
- }
-
- float inputAspectRatio = float(params.inputDimensions.x) / float(params.inputDimensions.y);
- float outputAspectRatio = float(outputDims.x) / float(outputDims.y);
-
- float scaleFactor;
- if (inputAspectRatio > outputAspectRatio) {
- scaleFactor = float(outputDims.x) / float(params.inputDimensions.x);
- } else {
- scaleFactor = float(outputDims.y) / float(params.inputDimensions.y);
- }
-
- float scaledInputWidth = float(params.inputDimensions.x) * scaleFactor;
- float scaledInputHeight = float(params.inputDimensions.y) * scaleFactor;
-
- float offsetX = (float(outputDims.x) - scaledInputWidth) / 2.0f;
- float offsetY = (float(outputDims.y) - scaledInputHeight) / 2.0f;
-
- float targetXInScaledArea = float(gid.x) - offsetX;
- float targetYInScaledArea = float(gid.y) - offsetY;
-
- float4 pixelColor;
-
- if (targetXInScaledArea >= 0.0f && targetXInScaledArea < scaledInputWidth &&
- targetYInScaledArea >= 0.0f && targetYInScaledArea < scaledInputHeight) {
-
- float uNorm = targetXInScaledArea / scaledInputWidth;
- float vNorm = targetYInScaledArea / scaledInputHeight;
-
- pixelColor = inputTexture.sample(bicubicSampler, float2(uNorm, vNorm));
- } else {
- pixelColor = float4(params.paddingValueRgb, params.paddingValueRgb, params.paddingValueRgb, 1.0f);
- }
-
- float3 normalizedRgb;
- float epsilon = 1e-6f;
- normalizedRgb.r = (pixelColor.r - params.imageMean.r) / (params.imageStd.r + epsilon);
- normalizedRgb.g = (pixelColor.g - params.imageMean.g) / (params.imageStd.g + epsilon);
- normalizedRgb.b = (pixelColor.b - params.imageMean.b) / (params.imageStd.b + epsilon);
-
- outputTexture.write(float4(normalizedRgb, 1.0f), gid);
+ uint2 gid [[thread_position_in_grid]]
+) {
+
+ uint2 outputDims =
+ uint2(outputTexture.get_width(), outputTexture.get_height());
+
+ if (gid.x >= outputDims.x || gid.y >= outputDims.y) {
+ return;
+ }
+
+ float inputAspectRatio =
+ float(params.inputDimensions.x) / float(params.inputDimensions.y);
+ float outputAspectRatio = float(outputDims.x) / float(outputDims.y);
+
+ float scaleFactor;
+ if (inputAspectRatio > outputAspectRatio) {
+ scaleFactor = float(outputDims.x) / float(params.inputDimensions.x);
+ } else {
+ scaleFactor = float(outputDims.y) / float(params.inputDimensions.y);
+ }
+
+ float scaledInputWidth = float(params.inputDimensions.x) * scaleFactor;
+ float scaledInputHeight = float(params.inputDimensions.y) * scaleFactor;
+
+ float offsetX = (float(outputDims.x) - scaledInputWidth) / 2.0f;
+ float offsetY = (float(outputDims.y) - scaledInputHeight) / 2.0f;
+
+ float targetXInScaledArea = float(gid.x) - offsetX;
+ float targetYInScaledArea = float(gid.y) - offsetY;
+
+ float4 pixelColor;
+
+ if (targetXInScaledArea >= 0.0f && targetXInScaledArea < scaledInputWidth &&
+ targetYInScaledArea >= 0.0f && targetYInScaledArea < scaledInputHeight) {
+
+ float uNorm = targetXInScaledArea / scaledInputWidth;
+ float vNorm = targetYInScaledArea / scaledInputHeight;
+
+ pixelColor = inputTexture.sample(bicubicSampler, float2(uNorm, vNorm));
+ } else {
+ pixelColor = float4(
+ params.paddingValueRgb,
+ params.paddingValueRgb,
+ params.paddingValueRgb,
+ 1.0f
+ );
+ }
+
+ float3 normalizedRgb;
+ float epsilon = 1e-6f;
+ normalizedRgb.r =
+ (pixelColor.r - params.imageMean.r) / (params.imageStd.r + epsilon);
+ normalizedRgb.g =
+ (pixelColor.g - params.imageMean.g) / (params.imageStd.g + epsilon);
+ normalizedRgb.b =
+ (pixelColor.b - params.imageMean.b) / (params.imageStd.b + epsilon);
+
+ outputTexture.write(float4(normalizedRgb, 1.0f), gid);
}
kernel void extractImagePatches(
texture2d<float, access::read> paddedNormalizedTexture [[texture(0)]],
device float* outputBuffer [[buffer(0)]],
constant PatchParameters& params [[buffer(1)]],
- uint2 gid [[thread_position_in_grid]]) {
+ uint2 gid [[thread_position_in_grid]]
+) {
- if (gid.x >= params.paddedDimensions.x || gid.y >= params.paddedDimensions.y) {
- return;
- }
+ if (gid.x >= params.paddedDimensions.x ||
+ gid.y >= params.paddedDimensions.y) {
+ return;
+ }
+
+ uint numPatchesX = params.paddedDimensions.x / params.patchSize;
+
+ uint patchCol = gid.x / params.patchSize;
+ uint patchRow = gid.y / params.patchSize;
+
+ uint spatialPatchIdxFlat = patchRow * numPatchesX + patchCol;
- uint numPatchesX = params.paddedDimensions.x / params.patchSize;
+ uint pxInPatch = gid.x % params.patchSize;
+ uint pyInPatch = gid.y % params.patchSize;
- uint patchCol = gid.x / params.patchSize;
- uint patchRow = gid.y / params.patchSize;
+ float4 normalizedPixelRgba = paddedNormalizedTexture.read(gid);
- uint spatialPatchIdxFlat = patchRow * numPatchesX + patchCol;
+ uint patchPixelFlatIdx = pyInPatch * params.patchSize + pxInPatch;
+ uint singlePatchChannelTemporalSliceSize =
+ params.patchSize * params.patchSize;
+ uint singlePatchChannelSize =
+ params.temporalSlices * singlePatchChannelTemporalSliceSize;
+ uint singlePatchSize = params.numChannels * singlePatchChannelSize;
- uint pxInPatch = gid.x % params.patchSize;
- uint pyInPatch = gid.y % params.patchSize;
+ uint outputPatchBaseIdx = spatialPatchIdxFlat * singlePatchSize;
- float4 normalizedPixelRgba = paddedNormalizedTexture.read(gid);
+ for (uint c = 0; c < params.numChannels; ++c) {
+ float valueToStore = normalizedPixelRgba[c];
+ for (uint t = 0; t < params.temporalSlices; ++t) {
+ uint channelBlockOffset = c * singlePatchChannelSize;
+ uint temporalBlockOffset = t * singlePatchChannelTemporalSliceSize;
- uint patchPixelFlatIdx = pyInPatch * params.patchSize + pxInPatch;
- uint singlePatchChannelTemporalSliceSize = params.patchSize * params.patchSize;
- uint singlePatchChannelSize = params.temporalSlices * singlePatchChannelTemporalSliceSize;
- uint singlePatchSize = params.numChannels * singlePatchChannelSize;
-
- uint outputPatchBaseIdx = spatialPatchIdxFlat * singlePatchSize;
+ uint outputBufferIdx = outputPatchBaseIdx + channelBlockOffset +
+ temporalBlockOffset + patchPixelFlatIdx;
- for (uint c = 0; c < params.numChannels; ++c) {
- float valueToStore = normalizedPixelRgba[c];
- for (uint t = 0; t < params.temporalSlices; ++t) {
- uint channelBlockOffset = c * singlePatchChannelSize;
- uint temporalBlockOffset = t * singlePatchChannelTemporalSliceSize;
-
- uint outputBufferIdx = outputPatchBaseIdx + channelBlockOffset + temporalBlockOffset + patchPixelFlatIdx;
-
- outputBuffer[outputBufferIdx] = valueToStore;
- }
+ outputBuffer[outputBufferIdx] = valueToStore;
}
-}
\ No newline at end of file
+ }
+}
\ No newline at end of file
diff --git a/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal b/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal
index 8d08af8fc..54048bbfc 100644
--- a/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal
+++ b/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal
@@ -25,7 +25,8 @@ inline T gelu_approx(T x) {
template <typename T>
inline T activate(T x, ushort act) {
- if (act == ACT_SILU) return silu(x);
+ if (act == ACT_SILU)
+ return silu(x);
return gelu_approx(x);
}
@@ -36,10 +37,12 @@ template <typename T>
const constant int& H [[buffer(2)]],
const constant int& M [[buffer(3)]],
const constant ushort& act_type [[buffer(4)]],
- uint2 tid [[thread_position_in_grid]]) {
+ uint2 tid [[thread_position_in_grid]]
+) {
int row = tid.y;
int j = tid.x;
- if (row >= M || j >= H) return;
+ if (row >= M || j >= H)
+ return;
int base = row * (2 * H);
T up = fused_up[base + j];
T gate = fused_up[base + H + j];
@@ -48,31 +51,32 @@ template <typename T>
}
// Explicit instantiations with stable host names
-template [[host_name("mlp_activation_mul_f16")]]
-[[kernel]] void mlp_activation_mul<half>(
+template [[host_name("mlp_activation_mul_f16")]] [[kernel]] void
+mlp_activation_mul<half>(
const device half* fused_up [[buffer(0)]],
device half* hidden [[buffer(1)]],
const constant int& H [[buffer(2)]],
const constant int& M [[buffer(3)]],
const constant ushort& act_type [[buffer(4)]],
- uint2 tid [[thread_position_in_grid]]);
+ uint2 tid [[thread_position_in_grid]]
+);
-template [[host_name("mlp_activation_mul_f32")]]
-[[kernel]] void mlp_activation_mul<float>(
+template [[host_name("mlp_activation_mul_f32")]] [[kernel]] void
+mlp_activation_mul<float>(
const device float* fused_up [[buffer(0)]],
device float* hidden [[buffer(1)]],
const constant int& H [[buffer(2)]],
const constant int& M [[buffer(3)]],
const constant ushort& act_type [[buffer(4)]],
- uint2 tid [[thread_position_in_grid]]);
+ uint2 tid [[thread_position_in_grid]]
+);
-template [[host_name("mlp_activation_mul_bf16")]]
-[[kernel]] void mlp_activation_mul<bfloat>(
+template [[host_name("mlp_activation_mul_bf16")]] [[kernel]] void
+mlp_activation_mul<bfloat>(
const device bfloat* fused_up [[buffer(0)]],
device bfloat* hidden [[buffer(1)]],
const constant int& H [[buffer(2)]],
const constant int& M [[buffer(3)]],
const constant ushort& act_type [[buffer(4)]],
- uint2 tid [[thread_position_in_grid]]);
-
-
+ uint2 tid [[thread_position_in_grid]]
+);
diff --git a/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal b/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal
index d8d5a9aca..21ec0fccc 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal
@@ -11,93 +11,108 @@ using namespace metal;
// This kernel is launched with SINGLE threadgroup
kernel void moe_counts_offsets_fused(
device const int* topk_ids [[buffer(0)]],
- device uint* offsets [[buffer(1)]], // output: exclusive scan [E+1]
- device uint* sum_k_out [[buffer(2)]], // output: total count [1]
- device uint* partials [[buffer(3)]], // output: partials [num_tiles * TILE_E] (for block_bases)
+ device uint* offsets [[buffer(1)]], // output: exclusive scan [E+1]
+ device uint* sum_k_out [[buffer(2)]], // output: total count [1]
+ device uint* partials
+ [[buffer(3)]], // output: partials [num_tiles * TILE_E] (for block_bases)
constant uint& T [[buffer(4)]],
constant uint& E [[buffer(5)]],
constant uint& K [[buffer(6)]],
- uint lid [[thread_index_in_threadgroup]])
-{
- if (E == 0) {
- if (lid == 0) {
- offsets[0] = 0u;
- sum_k_out[0] = 0u;
- }
- return;
+ uint lid [[thread_index_in_threadgroup]]
+) {
+ if (E == 0) {
+ if (lid == 0) {
+ offsets[0] = 0u;
+ sum_k_out[0] = 0u;
}
+ return;
+ }
- threadgroup _atomic<uint> tg_hist[TILE_E];
- threadgroup uint scan_shared[BLOCK_SIZE];
- threadgroup uint reduce_shared[BLOCK_SIZE];
- threadgroup uint counts_shared[BLOCK_SIZE]; // Cache counts in threadgroup memory
-
- // ═══════════════════════════════════════════════════════════
- // PHASE 1: Count tokens per expert using tiled histogram
- // ═══════════════════════════════════════════════════════════
- // Tile over E dimension
- for (uint e0 = 0; e0 < E; e0 += TILE_E) {
- const uint tile_e = (e0 + TILE_E <= E) ? TILE_E : (E - e0);
+ threadgroup _atomic<uint> tg_hist[TILE_E];
+ threadgroup uint scan_shared[BLOCK_SIZE];
+ threadgroup uint reduce_shared[BLOCK_SIZE];
+ threadgroup uint
+ counts_shared[BLOCK_SIZE]; // Cache counts in threadgroup memory
- // Zero threadgroup histogram
- for (uint e = lid; e < tile_e; e += BLOCK_SIZE) {
- atomic_store_explicit(&tg_hist[e], 0u, memory_order_relaxed);
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
+ // ═══════════════════════════════════════════════════════════
+ // PHASE 1: Count tokens per expert using tiled histogram
+ // ═══════════════════════════════════════════════════════════
+ // Tile over E dimension
+ for (uint e0 = 0; e0 < E; e0 += TILE_E) {
+ const uint tile_e = (e0 + TILE_E <= E) ? TILE_E : (E - e0);
- // Process all tokens in strided fashion
- // Since we have only 1 threadgroup, we need to cover all T tokens
- for (uint t = lid; t < T; t += BLOCK_SIZE) {
- const uint base = t * K;
- for (uint k = 0; k < K; ++k) {
- int eid = topk_ids[base + k];
- if (eid >= 0) {
- uint ue = uint(eid);
- if (ue >= e0 && ue < e0 + tile_e) {
- uint te = ue - e0;
- atomic_fetch_add_explicit(&tg_hist[te], 1u, memory_order_relaxed);
- }
- }
- }
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
+ // Zero threadgroup histogram
+ for (uint e = lid; e < tile_e; e += BLOCK_SIZE) {
+ atomic_store_explicit(&tg_hist[e], 0u, memory_order_relaxed);
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
- // Write final counts and partials for this tile
- for (uint e = lid; e < tile_e; e += BLOCK_SIZE) {
- const uint count_val = atomic_load_explicit(&tg_hist[e], memory_order_relaxed);
- counts_shared[e0 + e] = count_val;
- partials[e0 + e] = count_val;
+ // Process all tokens in strided fashion
+ // Since we have only 1 threadgroup, we need to cover all T tokens
+ for (uint t = lid; t < T; t += BLOCK_SIZE) {
+ const uint base = t * K;
+ for (uint k = 0; k < K; ++k) {
+ int eid = topk_ids[base + k];
+ if (eid >= 0) {
+ uint ue = uint(eid);
+ if (ue >= e0 && ue < e0 + tile_e) {
+ uint te = ue - e0;
+ atomic_fetch_add_explicit(&tg_hist[te], 1u, memory_order_relaxed);
+ }
}
- threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
}
+ threadgroup_barrier(mem_flags::mem_threadgroup);
- // ═══════════════════════════════════════════════════════════
- // PHASE 2: Compute exclusive prefix scan on counts
- // ═══════════════════════════════════════════════════════════
- threadgroup uint carry;
- if (lid == 0) { carry = 0u; }
+ // Write final counts and partials for this tile
+ for (uint e = lid; e < tile_e; e += BLOCK_SIZE) {
+ const uint count_val =
+ atomic_load_explicit(&tg_hist[e], memory_order_relaxed);
+ counts_shared[e0 + e] = count_val;
+ partials[e0 + e] = count_val;
+ }
threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
- for (uint base = 0; base < E; base += BLOCK_SIZE) {
- uint remaining = E - base;
- uint chunk_n = remaining < BLOCK_SIZE ? remaining : BLOCK_SIZE;
+ // ═══════════════════════════════════════════════════════════
+ // PHASE 2: Compute exclusive prefix scan on counts
+ // ═══════════════════════════════════════════════════════════
+ threadgroup uint carry;
+ if (lid == 0) {
+ carry = 0u;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
- uint v = (lid < chunk_n) ? counts_shared[lid] : 0u;
+ for (uint base = 0; base < E; base += BLOCK_SIZE) {
+ uint remaining = E - base;
+ uint chunk_n = remaining < BLOCK_SIZE ? remaining : BLOCK_SIZE;
- uint prefix_local = threadgroup_raking_prefix_exclusive_sum<BLOCK_SIZE>(v, scan_shared, (ushort)lid);
- uint prefix_global = prefix_local + carry;
- if (lid < chunk_n) {
- offsets[base + lid] = prefix_global;
- }
+ uint v = (lid < chunk_n) ? counts_shared[lid] : 0u;
- uint block_sum = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(v, reduce_shared, (ushort)lid);
- if (lid == 0) { carry += block_sum; }
- threadgroup_barrier(mem_flags::mem_threadgroup);
+ uint prefix_local = threadgroup_raking_prefix_exclusive_sum<BLOCK_SIZE>(
+ v,
+ scan_shared,
+ (ushort)lid
+ );
+ uint prefix_global = prefix_local + carry;
+ if (lid < chunk_n) {
+ offsets[base + lid] = prefix_global;
}
- // Write final offset and total
+ uint block_sum = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(
+ v,
+ reduce_shared,
+ (ushort)lid
+ );
if (lid == 0) {
- offsets[E] = carry;
- sum_k_out[0] = carry;
+ carry += block_sum;
}
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+
+ // Write final offset and total
+ if (lid == 0) {
+ offsets[E] = carry;
+ sum_k_out[0] = carry;
+ }
}
diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal
index edf485ceb..67d95608e 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal
@@ -2,31 +2,36 @@
#include <metal_simdgroup>
using namespace metal;
-constant uint GATING_SEL [[function_constant(30)]]; // 0=GELU,1=SiLU,2=SwiGLU,3=GEGLU
-constant uint TILE_H [[function_constant(32)]]; // tile size for hidden dimension in Pass A
+constant uint GATING_SEL
+ [[function_constant(30)]]; // 0=GELU,1=SiLU,2=SwiGLU,3=GEGLU
+constant uint TILE_H
+ [[function_constant(32)]]; // tile size for hidden dimension in Pass A
static inline float gelu_approx(float x) {
- const float k0 = 0.7978845608f;
- const float k1 = 0.044715f;
- if (x > 10.0f) return x;
- if (x < -10.0f) return 0.0f;
- return 0.5f * x * (1.0f + tanh(clamp(k0 * (x + k1 * x * x * x), -10.0f, 10.0f)));
+ const float k0 = 0.7978845608f;
+ const float k1 = 0.044715f;
+ if (x > 10.0f)
+ return x;
+ if (x < -10.0f)
+ return 0.0f;
+ return 0.5f * x *
+ (1.0f + tanh(clamp(k0 * (x + k1 * x * x * x), -10.0f, 10.0f)));
}
static inline float silu(float x, float alpha) {
- return x / (1.0f + exp(-alpha * x));
+ return x / (1.0f + exp(-alpha * x));
}
#define MTL_CONST static constant constexpr const
#define MTL_PRAGMA_UNROLL _Pragma("clang loop unroll(full)")
// Tiling configuration (tunable)
-MTL_CONST uint BM = 1; // Simdgroups in M dimension within threadgroup
-MTL_CONST uint BN = 4; // Simdgroups in N dimension within threadgroup
-MTL_CONST uint SM = 1; // Thread rows per simdgroup
-MTL_CONST uint SN = 32; // Thread cols per simdgroup (full warp width)
-MTL_CONST uint TM = 4; // Output elements per thread in M
-MTL_CONST uint TN = 4; // Input elements per thread in N (per iteration)
+MTL_CONST uint BM = 1; // Simdgroups in M dimension within threadgroup
+MTL_CONST uint BN = 4; // Simdgroups in N dimension within threadgroup
+MTL_CONST uint SM = 1; // Thread rows per simdgroup
+MTL_CONST uint SN = 32; // Thread cols per simdgroup (full warp width)
+MTL_CONST uint TM = 4; // Output elements per thread in M
+MTL_CONST uint TN = 4; // Input elements per thread in N (per iteration)
MTL_CONST uint THREADS_M = BM * SM;
MTL_CONST uint THREADS_N = BN * SN;
@@ -34,18 +39,22 @@ MTL_CONST uint BLOCK_M = THREADS_M * TM;
MTL_CONST uint BLOCK_N = THREADS_N * TN;
static_assert(SM * SN == 32, "simdgroup must have 32 threads");
-static_assert(SN == 4 || SN == 8 || SN == 16 || SN == 32, "SN must be 4, 8, 16, or 32");
+static_assert(
+ SN == 4 || SN == 8 || SN == 16 || SN == 32,
+ "SN must be 4, 8, 16, or 32"
+);
// === Pass A: Optimized GEMV with hierarchical tiling ===
// Computes: hidden[row, h] = activation(x[row, d] @ W13[d, 2*h])
// AccumT is used for internal accumulations, output still stored as T
-template<typename T, typename AccumT>
+template <typename T, typename AccumT>
void moe_experts_decode_pass_a_impl(
- device const T* X_perm, // [total_rows, d_model]
- device const uint* expert_offsets, // [E + 1]
- device const T* W13_all, // weights in layout [E, 2*d_ff, d_model]
- device const T* up_biases, // [E, 2*d_ff]
- device float* hidden_out, // [total_rows, d_ff] - f32 for activation precision
+ device const T* X_perm, // [total_rows, d_model]
+ device const uint* expert_offsets, // [E + 1]
+ device const T* W13_all, // weights in layout [E, 2*d_ff, d_model]
+ device const T* up_biases, // [E, 2*d_ff]
+ device float*
+ hidden_out, // [total_rows, d_ff] - f32 for activation precision
uint d_model,
uint d_ff,
uint E,
@@ -61,227 +70,254 @@ void moe_experts_decode_pass_a_impl(
uint simd_lid,
threadgroup float* tgp_memory
) {
- // Thread position within simdgroup
- const uint thrM = SN != 32 ? simd_lid / SN : 0;
- const uint thrN = SN != 32 ? simd_lid % SN : uint(simd_lid);
-
- // Simdgroup position within threadgroup
- const uint sgN = BN != 1 ? (simd_gid % BN) : 0;
- const uint simdM = BN != 1 ? SM * (simd_gid / BN) : uint(SM * simd_gid);
- const uint simdN = BN != 1 ? SN * (simd_gid % BN) : 0;
-
- // Thread's work block
- int bm = (simdM + thrM) * TM;
- int bn = (simdN + thrN) * TN;
-
- // Output row position
- const uint seg_start = expert_offsets[expert_idx];
- const uint seg_end = expert_offsets[expert_idx + 1];
- uint global_row = seg_start + row_in_expert;
-
- if (global_row >= seg_end) return;
-
- // Calculate output h index
- int h_row = h_block_idx * BLOCK_M + bm;
-
- // Adjust tail block to ensure in-bounds reads
- if (h_row + TM > d_ff) {
- h_row = d_ff > TM ? int(d_ff - TM) : 0;
- bm = h_row - int(h_block_idx * BLOCK_M);
+ // Thread position within simdgroup
+ const uint thrM = SN != 32 ? simd_lid / SN : 0;
+ const uint thrN = SN != 32 ? simd_lid % SN : uint(simd_lid);
+
+ // Simdgroup position within threadgroup
+ const uint sgN = BN != 1 ? (simd_gid % BN) : 0;
+ const uint simdM = BN != 1 ? SM * (simd_gid / BN) : uint(SM * simd_gid);
+ const uint simdN = BN != 1 ? SN * (simd_gid % BN) : 0;
+
+ // Thread's work block
+ int bm = (simdM + thrM) * TM;
+ int bn = (simdN + thrN) * TN;
+
+ // Output row position
+ const uint seg_start = expert_offsets[expert_idx];
+ const uint seg_end = expert_offsets[expert_idx + 1];
+ uint global_row = seg_start + row_in_expert;
+
+ if (global_row >= seg_end)
+ return;
+
+ // Calculate output h index
+ int h_row = h_block_idx * BLOCK_M + bm;
+
+ // Adjust tail block to ensure in-bounds reads
+ if (h_row + TM > d_ff) {
+ h_row = d_ff > TM ? int(d_ff - TM) : 0;
+ bm = h_row - int(h_block_idx * BLOCK_M);
+ }
+
+ if (h_row >= int(d_ff))
+ return;
+
+ // Accumulation arrays for up and gate projections
+ thread AccumT result_up[TM] = {0};
+ thread AccumT result_gate[TM] = {0};
+ thread T weights_up[TN];
+ thread T weights_gate[TN];
+ thread AccumT x_vals[TN];
+
+ // Expert weight base addresses
+ const ulong w13_base = (ulong)expert_idx * (ulong)d_model * (ulong)(2 * d_ff);
+ const ulong bias_base = (ulong)expert_idx * (ulong)(2 * d_ff);
+ const ulong x_row_base = (ulong)global_row * (ulong)d_model;
+
+ // Main accumulation loop over d_model in blocks of BLOCK_N
+ const uint n_iter = d_model / BLOCK_N;
+ const uint leftover = d_model - (n_iter * BLOCK_N);
+
+ for (uint i = 0; i < n_iter; ++i) {
+ // Load x values for this block
+ MTL_PRAGMA_UNROLL
+ for (uint tn = 0; tn < TN; tn++) {
+ uint d_idx = bn + tn;
+ x_vals[tn] = AccumT(X_perm[x_row_base + d_idx]);
}
- if (h_row >= int(d_ff)) return;
-
- // Accumulation arrays for up and gate projections
- thread AccumT result_up[TM] = {0};
- thread AccumT result_gate[TM] = {0};
- thread T weights_up[TN];
- thread T weights_gate[TN];
- thread AccumT x_vals[TN];
-
- // Expert weight base addresses
- const ulong w13_base = (ulong)expert_idx * (ulong)d_model * (ulong)(2 * d_ff);
- const ulong bias_base = (ulong)expert_idx * (ulong)(2 * d_ff);
- const ulong x_row_base = (ulong)global_row * (ulong)d_model;
-
- // Main accumulation loop over d_model in blocks of BLOCK_N
- const uint n_iter = d_model / BLOCK_N;
- const uint leftover = d_model - (n_iter * BLOCK_N);
-
- for (uint i = 0; i < n_iter; ++i) {
- // Load x values for this block
+ // Accumulate for each output row
+ MTL_PRAGMA_UNROLL
+ for (uint tm = 0; tm < TM; tm++) {
+ uint h_idx = h_row + tm;
+
+ // Load weights for up projection
+ MTL_PRAGMA_UNROLL
+ for (uint tn = 0; tn < TN; tn++) {
+ uint d_idx = bn + tn;
+ ulong w_idx = w13_base + (ulong)h_idx * (ulong)d_model + (ulong)d_idx;
+ weights_up[tn] = W13_all[w_idx];
+ }
+
+ // Accumulate up projection
+ MTL_PRAGMA_UNROLL
+ for (uint tn = 0; tn < TN; tn++) {
+ result_up[tm] += x_vals[tn] * AccumT(weights_up[tn]);
+ }
+
+ // Load and accumulate gate projection if needed
+ if (GATING_SEL > 1) {
MTL_PRAGMA_UNROLL
for (uint tn = 0; tn < TN; tn++) {
- uint d_idx = bn + tn;
- x_vals[tn] = AccumT(X_perm[x_row_base + d_idx]);
+ uint d_idx = bn + tn;
+ ulong w_gate_idx =
+ w13_base + (ulong)(d_ff + h_idx) * (ulong)d_model + (ulong)d_idx;
+ weights_gate[tn] = W13_all[w_gate_idx];
}
- // Accumulate for each output row
MTL_PRAGMA_UNROLL
- for (uint tm = 0; tm < TM; tm++) {
- uint h_idx = h_row + tm;
-
- // Load weights for up projection
- MTL_PRAGMA_UNROLL
- for (uint tn = 0; tn < TN; tn++) {
- uint d_idx = bn + tn;
- ulong w_idx = w13_base + (ulong)h_idx * (ulong)d_model + (ulong)d_idx;
- weights_up[tn] = W13_all[w_idx];
- }
-
- // Accumulate up projection
- MTL_PRAGMA_UNROLL
- for (uint tn = 0; tn < TN; tn++) {
- result_up[tm] += x_vals[tn] * AccumT(weights_up[tn]);
- }
-
- // Load and accumulate gate projection if needed
- if (GATING_SEL > 1) {
- MTL_PRAGMA_UNROLL
- for (uint tn = 0; tn < TN; tn++) {
- uint d_idx = bn + tn;
- ulong w_gate_idx = w13_base + (ulong)(d_ff + h_idx) * (ulong)d_model + (ulong)d_idx;
- weights_gate[tn] = W13_all[w_gate_idx];
- }
-
- MTL_PRAGMA_UNROLL
- for (uint tn = 0; tn < TN; tn++) {
- result_gate[tm] += x_vals[tn] * AccumT(weights_gate[tn]);
- }
- }
+ for (uint tn = 0; tn < TN; tn++) {
+ result_gate[tm] += x_vals[tn] * AccumT(weights_gate[tn]);
}
-
- bn += BLOCK_N;
+ }
}
- // Handle leftover elements
- if (leftover > 0) {
- MTL_PRAGMA_UNROLL
- for (uint tn = 0; tn < TN; tn++) {
- uint d_idx = bn + tn;
- x_vals[tn] = (d_idx < d_model) ? AccumT(X_perm[x_row_base + d_idx]) : AccumT(0.0);
- }
+ bn += BLOCK_N;
+ }
- MTL_PRAGMA_UNROLL
- for (uint tm = 0; tm < TM; tm++) {
- uint h_idx = h_row + tm;
-
- MTL_PRAGMA_UNROLL
- for (uint tn = 0; tn < TN; tn++) {
- uint d_idx = bn + tn;
- if (d_idx < d_model) {
- ulong w_idx = w13_base + (ulong)h_idx * (ulong)d_model + (ulong)d_idx;
- weights_up[tn] = W13_all[w_idx];
- result_up[tm] += x_vals[tn] * AccumT(weights_up[tn]);
-
- if (GATING_SEL > 1) {
- ulong w_gate_idx = w13_base + (ulong)(d_ff + h_idx) * (ulong)d_model + (ulong)d_idx;
- weights_gate[tn] = W13_all[w_gate_idx];
- result_gate[tm] += x_vals[tn] * AccumT(weights_gate[tn]);
- }
- }
- }
- }
+ // Handle leftover elements
+ if (leftover > 0) {
+ MTL_PRAGMA_UNROLL
+ for (uint tn = 0; tn < TN; tn++) {
+ uint d_idx = bn + tn;
+ x_vals[tn] =
+ (d_idx < d_model) ? AccumT(X_perm[x_row_base + d_idx]) : AccumT(0.0);
}
- // Simdgroup reduction across thrN dimension
MTL_PRAGMA_UNROLL
for (uint tm = 0; tm < TM; tm++) {
- result_up[tm] = simd_sum(result_up[tm]);
- if (GATING_SEL > 1) {
- result_gate[tm] = simd_sum(result_gate[tm]);
+ uint h_idx = h_row + tm;
+
+ MTL_PRAGMA_UNROLL
+ for (uint tn = 0; tn < TN; tn++) {
+ uint d_idx = bn + tn;
+ if (d_idx < d_model) {
+ ulong w_idx = w13_base + (ulong)h_idx * (ulong)d_model + (ulong)d_idx;
+ weights_up[tn] = W13_all[w_idx];
+ result_up[tm] += x_vals[tn] * AccumT(weights_up[tn]);
+
+ if (GATING_SEL > 1) {
+ ulong w_gate_idx = w13_base +
+ (ulong)(d_ff + h_idx) * (ulong)d_model +
+ (ulong)d_idx;
+ weights_gate[tn] = W13_all[w_gate_idx];
+ result_gate[tm] += x_vals[tn] * AccumT(weights_gate[tn]);
+ }
}
+ }
+ }
+ }
+
+ // Simdgroup reduction across thrN dimension
+ MTL_PRAGMA_UNROLL
+ for (uint tm = 0; tm < TM; tm++) {
+ result_up[tm] = simd_sum(result_up[tm]);
+ if (GATING_SEL > 1) {
+ result_gate[tm] = simd_sum(result_gate[tm]);
}
+ }
- // Threadgroup reduction if BN > 1
- if (BN > 1) {
- threadgroup float* tgp_up = tgp_memory;
- threadgroup float* tgp_gate = tgp_memory + BN * (BLOCK_M + TM);
-
- if (thrN == 0) {
- MTL_PRAGMA_UNROLL
- for (uint tm = 0; tm < TM; tm++) {
- tgp_up[sgN * (BLOCK_M + TM) + bm + tm] = float(result_up[tm]);
- if (GATING_SEL > 1) {
- tgp_gate[sgN * (BLOCK_M + TM) + bm + tm] = float(result_gate[tm]);
- }
- }
+ // Threadgroup reduction if BN > 1
+ if (BN > 1) {
+ threadgroup float* tgp_up = tgp_memory;
+ threadgroup float* tgp_gate = tgp_memory + BN * (BLOCK_M + TM);
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- if (sgN == 0) {
- MTL_PRAGMA_UNROLL
- for (uint sgn = 1; sgn < BN; sgn++) {
- MTL_PRAGMA_UNROLL
- for (uint tm = 0; tm < TM; tm++) {
- result_up[tm] += AccumT(tgp_up[sgn * (BLOCK_M + TM) + bm + tm]);
- if (GATING_SEL > 1) {
- result_gate[tm] += AccumT(tgp_gate[sgn * (BLOCK_M + TM) + bm + tm]);
- }
- }
- }
- }
+ if (thrN == 0) {
+ MTL_PRAGMA_UNROLL
+ for (uint tm = 0; tm < TM; tm++) {
+ tgp_up[sgN * (BLOCK_M + TM) + bm + tm] = float(result_up[tm]);
+ if (GATING_SEL > 1) {
+ tgp_gate[sgN * (BLOCK_M + TM) + bm + tm] = float(result_gate[tm]);
}
- }
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
- // Apply activation and write output (only first thread in simdgroup width)
- if (simdN == 0 && thrN == 0) {
+ if (sgN == 0) {
MTL_PRAGMA_UNROLL
- for (uint tm = 0; tm < TM; tm++) {
- uint h_idx = h_row + tm;
- if (h_idx < d_ff) {
- // Add bias and clip
- AccumT up_val = result_up[tm] + AccumT(up_biases[bias_base + h_idx]);
- up_val = AccumT(clamp(float(up_val), up_clip_min, up_clip_max));
-
- AccumT activated_val;
- if (GATING_SEL <= 1) {
- activated_val = (GATING_SEL == 0) ? AccumT(gelu_approx(float(up_val))) : AccumT(silu(float(up_val), silu_alpha));
- } else {
- AccumT gate_val = result_gate[tm] + AccumT(up_biases[bias_base + d_ff + h_idx]);
- gate_val = AccumT(clamp(float(gate_val), gate_clip_min, gate_clip_max));
- AccumT gate_activated = (GATING_SEL == 2) ? AccumT(silu(float(gate_val), silu_alpha)) : AccumT(gelu_approx(float(gate_val)));
- activated_val = gate_activated * up_val;
- }
-
- ulong hidden_idx = (ulong)global_row * (ulong)d_ff + (ulong)h_idx;
- hidden_out[hidden_idx] = float(activated_val);
+ for (uint sgn = 1; sgn < BN; sgn++) {
+ MTL_PRAGMA_UNROLL
+ for (uint tm = 0; tm < TM; tm++) {
+ result_up[tm] += AccumT(tgp_up[sgn * (BLOCK_M + TM) + bm + tm]);
+ if (GATING_SEL > 1) {
+ result_gate[tm] +=
+ AccumT(tgp_gate[sgn * (BLOCK_M + TM) + bm + tm]);
}
+ }
}
+ }
}
-}
+ }
-#define MOE_PASS_A_KERNEL(DTYPE, SUFFIX) \
-kernel void moe_experts_decode_pass_a_##SUFFIX( \
- device const DTYPE* X_perm [[buffer(0)]], \
- device const uint* expert_offsets [[buffer(1)]], \
- device const DTYPE* W13_all [[buffer(2)]], \
- device float* hidden_out [[buffer(3)]], \
- device const DTYPE* up_biases [[buffer(4)]], \
- constant uint& d_model [[buffer(5)]], \
- constant uint& d_ff [[buffer(6)]], \
- constant uint& E [[buffer(7)]], \
- constant float& gate_clip_min [[buffer(8)]], \
- constant float& gate_clip_max [[buffer(9)]], \
- constant float& up_clip_min [[buffer(10)]], \
- constant float& up_clip_max [[buffer(11)]], \
- constant float& silu_alpha [[buffer(12)]], \
- uint3 tgpig [[threadgroup_position_in_grid]], \
- uint simd_gid [[simdgroup_index_in_threadgroup]], \
- uint simd_lid [[thread_index_in_simdgroup]]) \
-{ \
- constexpr uint tgp_mem_size = (BN > 1) ? 2 * BN * (BLOCK_M + TM) : 1; \
- threadgroup float tgp_memory[tgp_mem_size]; \
- \
- moe_experts_decode_pass_a_impl<DTYPE, float>( \
- X_perm, expert_offsets, W13_all, up_biases, hidden_out, \
- d_model, d_ff, E, gate_clip_min, gate_clip_max, \
- up_clip_min, up_clip_max, silu_alpha, \
- tgpig.y, tgpig.z, tgpig.x, \
- simd_gid, simd_lid, \
- (BN > 1) ? tgp_memory : nullptr); \
+ // Apply activation and write output (only first thread in simdgroup width)
+ if (simdN == 0 && thrN == 0) {
+ MTL_PRAGMA_UNROLL
+ for (uint tm = 0; tm < TM; tm++) {
+ uint h_idx = h_row + tm;
+ if (h_idx < d_ff) {
+ // Add bias and clip
+ AccumT up_val = result_up[tm] + AccumT(up_biases[bias_base + h_idx]);
+ up_val = AccumT(clamp(float(up_val), up_clip_min, up_clip_max));
+
+ AccumT activated_val;
+ if (GATING_SEL <= 1) {
+ activated_val = (GATING_SEL == 0)
+ ? AccumT(gelu_approx(float(up_val)))
+ : AccumT(silu(float(up_val), silu_alpha));
+ } else {
+ AccumT gate_val =
+ result_gate[tm] + AccumT(up_biases[bias_base + d_ff + h_idx]);
+ gate_val =
+ AccumT(clamp(float(gate_val), gate_clip_min, gate_clip_max));
+ AccumT gate_activated =
+ (GATING_SEL == 2) ? AccumT(silu(float(gate_val), silu_alpha))
+ : AccumT(gelu_approx(float(gate_val)));
+ activated_val = gate_activated * up_val;
+ }
+
+ ulong hidden_idx = (ulong)global_row * (ulong)d_ff + (ulong)h_idx;
+ hidden_out[hidden_idx] = float(activated_val);
+ }
+ }
+ }
}
+#define MOE_PASS_A_KERNEL(DTYPE, SUFFIX) \
+ kernel void moe_experts_decode_pass_a_##SUFFIX( \
+ device const DTYPE* X_perm [[buffer(0)]], \
+ device const uint* expert_offsets [[buffer(1)]], \
+ device const DTYPE* W13_all [[buffer(2)]], \
+ device float* hidden_out [[buffer(3)]], \
+ device const DTYPE* up_biases [[buffer(4)]], \
+ constant uint& d_model [[buffer(5)]], \
+ constant uint& d_ff [[buffer(6)]], \
+ constant uint& E [[buffer(7)]], \
+ constant float& gate_clip_min [[buffer(8)]], \
+ constant float& gate_clip_max [[buffer(9)]], \
+ constant float& up_clip_min [[buffer(10)]], \
+ constant float& up_clip_max [[buffer(11)]], \
+ constant float& silu_alpha [[buffer(12)]], \
+ uint3 tgpig [[threadgroup_position_in_grid]], \
+ uint simd_gid [[simdgroup_index_in_threadgroup]], \
+ uint simd_lid [[thread_index_in_simdgroup]] \
+ ) { \
+ constexpr uint tgp_mem_size = (BN > 1) ? 2 * BN * (BLOCK_M + TM) : 1; \
+ threadgroup float tgp_memory[tgp_mem_size]; \
+ \
+ moe_experts_decode_pass_a_impl<DTYPE, float>( \
+ X_perm, \
+ expert_offsets, \
+ W13_all, \
+ up_biases, \
+ hidden_out, \
+ d_model, \
+ d_ff, \
+ E, \
+ gate_clip_min, \
+ gate_clip_max, \
+ up_clip_min, \
+ up_clip_max, \
+ silu_alpha, \
+ tgpig.y, \
+ tgpig.z, \
+ tgpig.x, \
+ simd_gid, \
+ simd_lid, \
+ (BN > 1) ? tgp_memory : nullptr \
+ ); \
+ }
+
MOE_PASS_A_KERNEL(bfloat, bf16)
MOE_PASS_A_KERNEL(half, f16)
MOE_PASS_A_KERNEL(float, f32)
@@ -290,143 +326,150 @@ MOE_PASS_A_KERNEL(float, f32)
// Count tiles per expert: tiles = (num_rows > 0) ? num_rows * h_blocks : 0
kernel void moe_pass_a_tile_counts(
- device const uint* expert_offsets [[buffer(0)]], // [E+1]
- device uint* tile_counts [[buffer(1)]], // [E]
+ device const uint* expert_offsets [[buffer(0)]], // [E+1]
+ device uint* tile_counts [[buffer(1)]], // [E]
constant uint& E [[buffer(2)]],
constant uint& h_blocks [[buffer(3)]],
uint tid [[thread_position_in_grid]]
) {
- if (tid >= E) return;
- const uint start = expert_offsets[tid];
- const uint end = expert_offsets[tid + 1];
- const uint num_rows = end - start;
- tile_counts[tid] = (num_rows > 0) ? (num_rows * h_blocks) : 0;
+ if (tid >= E)
+ return;
+ const uint start = expert_offsets[tid];
+ const uint end = expert_offsets[tid + 1];
+ const uint num_rows = end - start;
+ tile_counts[tid] = (num_rows > 0) ? (num_rows * h_blocks) : 0;
}
// Exclusive scan of tile_counts to get tile_offsets and total_tiles
kernel void moe_pass_a_tile_scan(
- device const uint* tile_counts [[buffer(0)]], // [E]
- device uint* tile_offsets [[buffer(1)]], // [E+1]
- device uint* total_tiles [[buffer(2)]], // [1]
+ device const uint* tile_counts [[buffer(0)]], // [E]
+ device uint* tile_offsets [[buffer(1)]], // [E+1]
+ device uint* total_tiles [[buffer(2)]], // [1]
constant uint& E [[buffer(3)]],
uint lid [[thread_index_in_threadgroup]],
threadgroup uint* scratch [[threadgroup(0)]]
) {
- // Simple single-threadgroup scan (works for E <= 1024)
- const uint idx = lid;
-
- // Load into threadgroup memory
- if (idx < E) {
- scratch[idx] = tile_counts[idx];
- } else {
- scratch[idx] = 0;
+ // Simple single-threadgroup scan (works for E <= 1024)
+ const uint idx = lid;
+
+ // Load into threadgroup memory
+ if (idx < E) {
+ scratch[idx] = tile_counts[idx];
+ } else {
+ scratch[idx] = 0;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // Kogge-Stone scan
+ uint val = scratch[idx];
+ MTL_PRAGMA_UNROLL
+ for (uint offset = 1; offset < 1024; offset *= 2) {
+ uint temp = 0;
+ if (idx >= offset && idx < E) {
+ temp = scratch[idx - offset];
}
threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // Kogge-Stone scan
- uint val = scratch[idx];
- MTL_PRAGMA_UNROLL
- for (uint offset = 1; offset < 1024; offset *= 2) {
- uint temp = 0;
- if (idx >= offset && idx < E) {
- temp = scratch[idx - offset];
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
- if (idx >= offset && idx < E) {
- val += temp;
- scratch[idx] = val;
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (idx >= offset && idx < E) {
+ val += temp;
+ scratch[idx] = val;
}
-
- // Write exclusive scan (shift right by 1)
- if (idx == 0) {
- tile_offsets[0] = 0;
- }
- if (idx < E) {
- tile_offsets[idx + 1] = scratch[idx];
- if (idx == E - 1) {
- total_tiles[0] = scratch[idx];
- }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+
+ // Write exclusive scan (shift right by 1)
+ if (idx == 0) {
+ tile_offsets[0] = 0;
+ }
+ if (idx < E) {
+ tile_offsets[idx + 1] = scratch[idx];
+ if (idx == E - 1) {
+ total_tiles[0] = scratch[idx];
}
+ }
}
// Build row→expert map: one thread per routed row
kernel void moe_pass_a_build_row_map(
- device const uint* expert_offsets [[buffer(0)]] , // [E+1]
- device uint* row_expert_map [[buffer(1)]] , // [total_rows]
+ device const uint* expert_offsets [[buffer(0)]], // [E+1]
+ device uint* row_expert_map [[buffer(1)]], // [total_rows]
constant uint& total_rows [[buffer(2)]],
constant uint& E [[buffer(3)]],
uint tid [[thread_position_in_grid]]
) {
- if (tid >= total_rows) return;
+ if (tid >= total_rows)
+ return;
- uint left = 0u;
- uint right = E;
- const uint row = tid;
+ uint left = 0u;
+ uint right = E;
+ const uint row = tid;
- while (left + 1u < right) {
- const uint mid = (left + right) >> 1;
- if (row < expert_offsets[mid]) {
- right = mid;
- } else {
- left = mid;
- }
+ while (left + 1u < right) {
+ const uint mid = (left + right) >> 1;
+ if (row < expert_offsets[mid]) {
+ right = mid;
+ } else {
+ left = mid;
}
+ }
- row_expert_map[row] = left;
+ row_expert_map[row] = left;
}
// Build tile map entries from row→expert map
kernel void moe_pass_a_build_tile_map(
- device const uint* expert_offsets [[buffer(0)]], // [E+1]
- device const uint* tile_offsets [[buffer(1)]], // [E+1]
- device const uint* row_expert_map [[buffer(2)]], // [total_rows]
- device uint* tile_map [[buffer(3)]], // [total_tiles * 3]
+ device const uint* expert_offsets [[buffer(0)]], // [E+1]
+ device const uint* tile_offsets [[buffer(1)]], // [E+1]
+ device const uint* row_expert_map [[buffer(2)]], // [total_rows]
+ device uint* tile_map [[buffer(3)]], // [total_tiles * 3]
constant uint& total_rows [[buffer(4)]],
constant uint& h_blocks [[buffer(5)]],
uint tid [[thread_position_in_grid]]
) {
- const uint total_tiles = total_rows * h_blocks;
- if (tid >= total_tiles) return;
+ const uint total_tiles = total_rows * h_blocks;
+ if (tid >= total_tiles)
+ return;
- const uint row_idx = tid / h_blocks;
- const uint h_block = tid % h_blocks;
+ const uint row_idx = tid / h_blocks;
+ const uint h_block = tid % h_blocks;
- if (row_idx >= total_rows) return;
+ if (row_idx >= total_rows)
+ return;
- const uint expert_idx = row_expert_map[row_idx];
- const uint row_start = expert_offsets[expert_idx];
- const uint row_in_expert = row_idx - row_start;
- const uint tile_base = tile_offsets[expert_idx] + row_in_expert * h_blocks + h_block;
+ const uint expert_idx = row_expert_map[row_idx];
+ const uint row_start = expert_offsets[expert_idx];
+ const uint row_in_expert = row_idx - row_start;
+ const uint tile_base =
+ tile_offsets[expert_idx] + row_in_expert * h_blocks + h_block;
- tile_map[tile_base * 3 + 0] = h_block;
- tile_map[tile_base * 3 + 1] = expert_idx;
- tile_map[tile_base * 3 + 2] = row_in_expert;
+ tile_map[tile_base * 3 + 0] = h_block;
+ tile_map[tile_base * 3 + 1] = expert_idx;
+ tile_map[tile_base * 3 + 2] = row_in_expert;
}
// Write dispatch args for indirect dispatch (reusable from tiled version)
kernel void moe_pass_a_write_dispatch_args(
- device const uint* total_tiles [[buffer(0)]], // [1]
- device uint* dispatch_args [[buffer(1)]], // [3] - MTLDispatchThreadgroupsIndirectArguments
- constant uint& num_tiles_y [[buffer(2)]], // usually 1 for Pass A
+ device const uint* total_tiles [[buffer(0)]], // [1]
+ device uint* dispatch_args
+ [[buffer(1)]], // [3] - MTLDispatchThreadgroupsIndirectArguments
+ constant uint& num_tiles_y [[buffer(2)]], // usually 1 for Pass A
uint tid [[thread_position_in_grid]]
) {
- if (tid > 0) return;
- dispatch_args[0] = total_tiles[0]; // x dimension = total tiles
- dispatch_args[1] = num_tiles_y; // y dimension
- dispatch_args[2] = 1; // z dimension
+ if (tid > 0)
+ return;
+ dispatch_args[0] = total_tiles[0]; // x dimension = total tiles
+ dispatch_args[1] = num_tiles_y; // y dimension
+ dispatch_args[2] = 1; // z dimension
}
// Modified Pass A that reads from tile map for indirect dispatch
-template<typename T>
+template <typename T>
void moe_experts_decode_pass_a_indirect_impl(
device const T* X_perm,
device const uint* expert_offsets,
device const T* W13_all,
device const T* up_biases,
device float* hidden_out,
- device const uint* tile_map, // [total_tiles * 3]
+ device const uint* tile_map, // [total_tiles * 3]
uint d_model,
uint d_ff,
uint E,
@@ -435,73 +478,101 @@ void moe_experts_decode_pass_a_indirect_impl(
float up_clip_min,
float up_clip_max,
float silu_alpha,
- uint tile_idx, // flat threadgroup index
+ uint tile_idx, // flat threadgroup index
uint simd_gid,
uint simd_lid,
threadgroup float* tgp_memory
) {
- // Read tile descriptor
- const uint h_block_idx = tile_map[tile_idx * 3 + 0];
- const uint expert_idx = tile_map[tile_idx * 3 + 1];
- const uint row_in_expert = tile_map[tile_idx * 3 + 2];
-
- // Call original implementation
- moe_experts_decode_pass_a_impl<T, float>(
- X_perm, expert_offsets, W13_all, up_biases, hidden_out,
- d_model, d_ff, E,
- gate_clip_min, gate_clip_max,
- up_clip_min, up_clip_max, silu_alpha,
- expert_idx, row_in_expert, h_block_idx,
- simd_gid, simd_lid, tgp_memory
- );
+ // Read tile descriptor
+ const uint h_block_idx = tile_map[tile_idx * 3 + 0];
+ const uint expert_idx = tile_map[tile_idx * 3 + 1];
+ const uint row_in_expert = tile_map[tile_idx * 3 + 2];
+
+ // Call original implementation
+ moe_experts_decode_pass_a_impl<T, float>(
+ X_perm,
+ expert_offsets,
+ W13_all,
+ up_biases,
+ hidden_out,
+ d_model,
+ d_ff,
+ E,
+ gate_clip_min,
+ gate_clip_max,
+ up_clip_min,
+ up_clip_max,
+ silu_alpha,
+ expert_idx,
+ row_in_expert,
+ h_block_idx,
+ simd_gid,
+ simd_lid,
+ tgp_memory
+ );
}
-#define MOE_PASS_A_INDIRECT_KERNEL(DTYPE, SUFFIX) \
-kernel void moe_experts_decode_pass_a_indirect_##SUFFIX( \
- device const DTYPE* X_perm [[buffer(0)]], \
- device const uint* expert_offsets [[buffer(1)]], \
- device const DTYPE* W13_all [[buffer(2)]], \
- device float* hidden_out [[buffer(3)]], \
- device const DTYPE* up_biases [[buffer(4)]], \
- constant uint& d_model [[buffer(5)]], \
- constant uint& d_ff [[buffer(6)]], \
- constant uint& E [[buffer(7)]], \
- constant float& gate_clip_min [[buffer(8)]], \
- constant float& gate_clip_max [[buffer(9)]], \
- constant float& up_clip_min [[buffer(10)]], \
- constant float& up_clip_max [[buffer(11)]], \
- constant float& silu_alpha [[buffer(12)]], \
- device const uint* tile_map [[buffer(13)]], \
- uint3 tgpig [[threadgroup_position_in_grid]], \
- uint simd_gid [[simdgroup_index_in_threadgroup]], \
- uint simd_lid [[thread_index_in_simdgroup]]) \
-{ \
- constexpr uint tgp_mem_size = (BN > 1) ? 2 * BN * (BLOCK_M + TM) : 1; \
- threadgroup float tgp_memory[tgp_mem_size]; \
- \
- moe_experts_decode_pass_a_indirect_impl<DTYPE>( \
- X_perm, expert_offsets, W13_all, up_biases, hidden_out, tile_map, \
- d_model, d_ff, E, gate_clip_min, gate_clip_max, \
- up_clip_min, up_clip_max, silu_alpha, \
- tgpig.x, simd_gid, simd_lid, \
- (BN > 1) ? tgp_memory : nullptr); \
-}
+#define MOE_PASS_A_INDIRECT_KERNEL(DTYPE, SUFFIX) \
+ kernel void moe_experts_decode_pass_a_indirect_##SUFFIX( \
+ device const DTYPE* X_perm [[buffer(0)]], \
+ device const uint* expert_offsets [[buffer(1)]], \
+ device const DTYPE* W13_all [[buffer(2)]], \
+ device float* hidden_out [[buffer(3)]], \
+ device const DTYPE* up_biases [[buffer(4)]], \
+ constant uint& d_model [[buffer(5)]], \
+ constant uint& d_ff [[buffer(6)]], \
+ constant uint& E [[buffer(7)]], \
+ constant float& gate_clip_min [[buffer(8)]], \
+ constant float& gate_clip_max [[buffer(9)]], \
+ constant float& up_clip_min [[buffer(10)]], \
+ constant float& up_clip_max [[buffer(11)]], \
+ constant float& silu_alpha [[buffer(12)]], \
+ device const uint* tile_map [[buffer(13)]], \
+ uint3 tgpig [[threadgroup_position_in_grid]], \
+ uint simd_gid [[simdgroup_index_in_threadgroup]], \
+ uint simd_lid [[thread_index_in_simdgroup]] \
+ ) { \
+ constexpr uint tgp_mem_size = (BN > 1) ? 2 * BN * (BLOCK_M + TM) : 1; \
+ threadgroup float tgp_memory[tgp_mem_size]; \
+ \
+ moe_experts_decode_pass_a_indirect_impl<DTYPE>( \
+ X_perm, \
+ expert_offsets, \
+ W13_all, \
+ up_biases, \
+ hidden_out, \
+ tile_map, \
+ d_model, \
+ d_ff, \
+ E, \
+ gate_clip_min, \
+ gate_clip_max, \
+ up_clip_min, \
+ up_clip_max, \
+ silu_alpha, \
+ tgpig.x, \
+ simd_gid, \
+ simd_lid, \
+ (BN > 1) ? tgp_memory : nullptr \
+ ); \
+ }
MOE_PASS_A_INDIRECT_KERNEL(bfloat, bf16)
MOE_PASS_A_INDIRECT_KERNEL(half, f16)
MOE_PASS_A_INDIRECT_KERNEL(float, f32)
-
// === Pass B: Simdgroup cooperation along K for coalescing ===
-// W2 layout [E, d_model, d_ff] - 32 threads cooperate on one output, reading consecutive K elements
+// W2 layout [E, d_model, d_ff] - 32 threads cooperate on one output, reading
+// consecutive K elements
-template<typename T, typename AccumT>
+template <typename T, typename AccumT>
void moe_experts_decode_down_fused_2d_impl(
- device const float* hidden, // [total_rows, d_ff] - f32 from Pass A
- device const uint* row_expert_map, // [total_rows] - direct row->expert lookup
- device const T* w2_all, // [E, d_model, d_ff] - layout
- device const T* down_biases, // [E, d_model]
- device T* y_out, // [total_rows, d_model]
+ device const float* hidden, // [total_rows, d_ff] - f32 from Pass A
+ device const uint*
+ row_expert_map, // [total_rows] - direct row->expert lookup
+ device const T* w2_all, // [E, d_model, d_ff] - layout
+ device const T* down_biases, // [E, d_model]
+ device T* y_out, // [total_rows, d_model]
uint total_rows,
uint d_model,
uint d_ff,
@@ -510,116 +581,143 @@ void moe_experts_decode_down_fused_2d_impl(
uint simd_gid,
uint simd_lid
) {
- constexpr uint THREADS_PER_SIMD = 32;
- constexpr uint SIMDGROUPS_PER_TG = 8;
-
- const uint row_idx = tgpig.y;
- if (row_idx >= total_rows) return;
-
- // Each simdgroup computes one output column
- const uint my_col = tgpig.x * SIMDGROUPS_PER_TG + simd_gid;
- if (my_col >= d_model) return;
-
- const uint expert_idx = row_expert_map[row_idx];
-
- // Base addresses for this output column
- const ulong hidden_base = (ulong)row_idx * (ulong)d_ff;
- const ulong w2_col_base = (ulong)expert_idx * (ulong)d_model * (ulong)d_ff + (ulong)my_col * (ulong)d_ff;
-
- // Dual accumulators for ILP: breaks FMA dependency chains
- AccumT acc0 = AccumT(0.0);
- AccumT acc1 = AccumT(0.0);
- AccumT acc = AccumT(0.0);
-
- // Main loop: stride-32 with vec8 loads for ILP
- const uint k_iters = d_ff / THREADS_PER_SIMD;
- const uint k_vec_iters = k_iters / 8;
-
- MTL_PRAGMA_UNROLL
- for (uint iter = 0; iter < k_vec_iters; ++iter) {
- const uint k_base = iter * (8 * THREADS_PER_SIMD) + simd_lid;
-
- // hidden: stride-32 per lane, already f32 from Pass A
- const AccumT h0 = hidden[hidden_base + k_base + 0 * THREADS_PER_SIMD];
- const AccumT h1 = hidden[hidden_base + k_base + 1 * THREADS_PER_SIMD];
- const AccumT h2 = hidden[hidden_base + k_base + 2 * THREADS_PER_SIMD];
- const AccumT h3 = hidden[hidden_base + k_base + 3 * THREADS_PER_SIMD];
- const AccumT h4 = hidden[hidden_base + k_base + 4 * THREADS_PER_SIMD];
- const AccumT h5 = hidden[hidden_base + k_base + 5 * THREADS_PER_SIMD];
- const AccumT h6 = hidden[hidden_base + k_base + 6 * THREADS_PER_SIMD];
- const AccumT h7 = hidden[hidden_base + k_base + 7 * THREADS_PER_SIMD];
-
- // W2: lane-coalesced
- const AccumT w0 = AccumT(w2_all[w2_col_base + k_base + 0 * THREADS_PER_SIMD]);
- const AccumT w1 = AccumT(w2_all[w2_col_base + k_base + 1 * THREADS_PER_SIMD]);
- const AccumT w2 = AccumT(w2_all[w2_col_base + k_base + 2 * THREADS_PER_SIMD]);
- const AccumT w3 = AccumT(w2_all[w2_col_base + k_base + 3 * THREADS_PER_SIMD]);
- const AccumT w4 = AccumT(w2_all[w2_col_base + k_base + 4 * THREADS_PER_SIMD]);
- const AccumT w5 = AccumT(w2_all[w2_col_base + k_base + 5 * THREADS_PER_SIMD]);
- const AccumT w6 = AccumT(w2_all[w2_col_base + k_base + 6 * THREADS_PER_SIMD]);
- const AccumT w7 = AccumT(w2_all[w2_col_base + k_base + 7 * THREADS_PER_SIMD]);
-
- // dual trees for ILP
- acc0 = fma(h0, w0, acc0);
- acc1 = fma(h1, w1, acc1);
- acc0 = fma(h2, w2, acc0);
- acc1 = fma(h3, w3, acc1);
- acc0 = fma(h4, w4, acc0);
- acc1 = fma(h5, w5, acc1);
- acc0 = fma(h6, w6, acc0);
- acc1 = fma(h7, w7, acc1);
- }
-
- // fold the vectorized contribution once
- acc += (acc0 + acc1);
-
- // Handle remaining full iterations
- MTL_PRAGMA_UNROLL
- for (uint iter = k_vec_iters * 8; iter < k_iters; ++iter) {
- const uint k = iter * THREADS_PER_SIMD + simd_lid;
- acc = fma(AccumT(hidden[hidden_base + k]), AccumT(w2_all[w2_col_base + k]), acc);
- }
-
- // Handle leftover elements (d_ff % 32)
- const uint leftover_start = k_iters * THREADS_PER_SIMD;
- if (leftover_start + simd_lid < d_ff) {
- const uint k = leftover_start + simd_lid;
- acc = fma(AccumT(hidden[hidden_base + k]), AccumT(w2_all[w2_col_base + k]), acc);
- }
-
- // Simdgroup reduction
- AccumT result = simd_sum(acc);
-
- // Lane 0 writes result
- if (simd_lid == 0) {
- const ulong bias_idx = (ulong)expert_idx * (ulong)d_model + (ulong)my_col;
- result += AccumT(down_biases[bias_idx]);
-
- const ulong out_idx = (ulong)row_idx * (ulong)d_model + (ulong)my_col;
- y_out[out_idx] = T(result);
- }
+ constexpr uint THREADS_PER_SIMD = 32;
+ constexpr uint SIMDGROUPS_PER_TG = 8;
+
+ const uint row_idx = tgpig.y;
+ if (row_idx >= total_rows)
+ return;
+
+ // Each simdgroup computes one output column
+ const uint my_col = tgpig.x * SIMDGROUPS_PER_TG + simd_gid;
+ if (my_col >= d_model)
+ return;
+
+ const uint expert_idx = row_expert_map[row_idx];
+
+ // Base addresses for this output column
+ const ulong hidden_base = (ulong)row_idx * (ulong)d_ff;
+ const ulong w2_col_base = (ulong)expert_idx * (ulong)d_model * (ulong)d_ff +
+ (ulong)my_col * (ulong)d_ff;
+
+ // Dual accumulators for ILP: breaks FMA dependency chains
+ AccumT acc0 = AccumT(0.0);
+ AccumT acc1 = AccumT(0.0);
+ AccumT acc = AccumT(0.0);
+
+ // Main loop: stride-32 with vec8 loads for ILP
+ const uint k_iters = d_ff / THREADS_PER_SIMD;
+ const uint k_vec_iters = k_iters / 8;
+
+ MTL_PRAGMA_UNROLL
+ for (uint iter = 0; iter < k_vec_iters; ++iter) {
+ const uint k_base = iter * (8 * THREADS_PER_SIMD) + simd_lid;
+
+ // hidden: stride-32 per lane, already f32 from Pass A
+ const AccumT h0 = hidden[hidden_base + k_base + 0 * THREADS_PER_SIMD];
+ const AccumT h1 = hidden[hidden_base + k_base + 1 * THREADS_PER_SIMD];
+ const AccumT h2 = hidden[hidden_base + k_base + 2 * THREADS_PER_SIMD];
+ const AccumT h3 = hidden[hidden_base + k_base + 3 * THREADS_PER_SIMD];
+ const AccumT h4 = hidden[hidden_base + k_base + 4 * THREADS_PER_SIMD];
+ const AccumT h5 = hidden[hidden_base + k_base + 5 * THREADS_PER_SIMD];
+ const AccumT h6 = hidden[hidden_base + k_base + 6 * THREADS_PER_SIMD];
+ const AccumT h7 = hidden[hidden_base + k_base + 7 * THREADS_PER_SIMD];
+
+ // W2: lane-coalesced
+ const AccumT w0 =
+ AccumT(w2_all[w2_col_base + k_base + 0 * THREADS_PER_SIMD]);
+ const AccumT w1 =
+ AccumT(w2_all[w2_col_base + k_base + 1 * THREADS_PER_SIMD]);
+ const AccumT w2 =
+ AccumT(w2_all[w2_col_base + k_base + 2 * THREADS_PER_SIMD]);
+ const AccumT w3 =
+ AccumT(w2_all[w2_col_base + k_base + 3 * THREADS_PER_SIMD]);
+ const AccumT w4 =
+ AccumT(w2_all[w2_col_base + k_base + 4 * THREADS_PER_SIMD]);
+ const AccumT w5 =
+ AccumT(w2_all[w2_col_base + k_base + 5 * THREADS_PER_SIMD]);
+ const AccumT w6 =
+ AccumT(w2_all[w2_col_base + k_base + 6 * THREADS_PER_SIMD]);
+ const AccumT w7 =
+ AccumT(w2_all[w2_col_base + k_base + 7 * THREADS_PER_SIMD]);
+
+ // dual trees for ILP
+ acc0 = fma(h0, w0, acc0);
+ acc1 = fma(h1, w1, acc1);
+ acc0 = fma(h2, w2, acc0);
+ acc1 = fma(h3, w3, acc1);
+ acc0 = fma(h4, w4, acc0);
+ acc1 = fma(h5, w5, acc1);
+ acc0 = fma(h6, w6, acc0);
+ acc1 = fma(h7, w7, acc1);
+ }
+
+ // fold the vectorized contribution once
+ acc += (acc0 + acc1);
+
+ // Handle remaining full iterations
+ MTL_PRAGMA_UNROLL
+ for (uint iter = k_vec_iters * 8; iter < k_iters; ++iter) {
+ const uint k = iter * THREADS_PER_SIMD + simd_lid;
+ acc =
+ fma(AccumT(hidden[hidden_base + k]),
+ AccumT(w2_all[w2_col_base + k]),
+ acc);
+ }
+
+ // Handle leftover elements (d_ff % 32)
+ const uint leftover_start = k_iters * THREADS_PER_SIMD;
+ if (leftover_start + simd_lid < d_ff) {
+ const uint k = leftover_start + simd_lid;
+ acc =
+ fma(AccumT(hidden[hidden_base + k]),
+ AccumT(w2_all[w2_col_base + k]),
+ acc);
+ }
+
+ // Simdgroup reduction
+ AccumT result = simd_sum(acc);
+
+ // Lane 0 writes result
+ if (simd_lid == 0) {
+ const ulong bias_idx = (ulong)expert_idx * (ulong)d_model + (ulong)my_col;
+ result += AccumT(down_biases[bias_idx]);
+
+ const ulong out_idx = (ulong)row_idx * (ulong)d_model + (ulong)my_col;
+ y_out[out_idx] = T(result);
+ }
}
-#define MOE_PASS_B_FUSED_2D_KERNEL(DTYPE, SUFFIX) \
-kernel void moe_experts_decode_down_fused_2d_##SUFFIX( \
- device const float* hidden [[buffer(0)]], \
- device const uint* row_expert_map [[buffer(1)]], \
- device const DTYPE* w2_all [[buffer(2)]], \
- device const DTYPE* down_biases [[buffer(3)]], \
- device DTYPE* y_out [[buffer(4)]], \
- constant uint& total_rows [[buffer(5)]], \
- constant uint& d_model [[buffer(6)]], \
- constant uint& d_ff [[buffer(7)]], \
- constant uint& E [[buffer(8)]], \
- uint2 tgpig [[threadgroup_position_in_grid]], \
- uint simd_gid [[simdgroup_index_in_threadgroup]], \
- uint simd_lid [[thread_index_in_simdgroup]]) \
-{ \
- moe_experts_decode_down_fused_2d_impl<DTYPE, float>( \
- hidden, row_expert_map, w2_all, down_biases, y_out, \
- total_rows, d_model, d_ff, E, \
- tgpig, simd_gid, simd_lid); \
-}
+#define MOE_PASS_B_FUSED_2D_KERNEL(DTYPE, SUFFIX) \
+ kernel void moe_experts_decode_down_fused_2d_##SUFFIX( \
+ device const float* hidden [[buffer(0)]], \
+ device const uint* row_expert_map [[buffer(1)]], \
+ device const DTYPE* w2_all [[buffer(2)]], \
+ device const DTYPE* down_biases [[buffer(3)]], \
+ device DTYPE* y_out [[buffer(4)]], \
+ constant uint& total_rows [[buffer(5)]], \
+ constant uint& d_model [[buffer(6)]], \
+ constant uint& d_ff [[buffer(7)]], \
+ constant uint& E [[buffer(8)]], \
+ uint2 tgpig [[threadgroup_position_in_grid]], \
+ uint simd_gid [[simdgroup_index_in_threadgroup]], \
+ uint simd_lid [[thread_index_in_simdgroup]] \
+ ) { \
+ moe_experts_decode_down_fused_2d_impl<DTYPE, float>( \
+ hidden, \
+ row_expert_map, \
+ w2_all, \
+ down_biases, \
+ y_out, \
+ total_rows, \
+ d_model, \
+ d_ff, \
+ E, \
+ tgpig, \
+ simd_gid, \
+ simd_lid \
+ ); \
+ }
MOE_PASS_B_FUSED_2D_KERNEL(bfloat, bf16)
MOE_PASS_B_FUSED_2D_KERNEL(half, f16)
diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal
index db64af685..03aad6517 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal
@@ -19,7 +19,7 @@
#include <metal_stdlib>
#include <metal_simdgroup>
#include <metal_simdgroup_matrix>
-#include "../quant_matmul/mma.h"
+#include "../quant_matmul/mma.h"
using namespace metal;
@@ -27,50 +27,52 @@ using namespace metal;
static inline uint ceil_div(uint a, uint b) { return (a + b - 1u) / b; }
static inline uint linear_tid(uint3 tid, uint3 tpg) {
- return tid.z * (tpg.x * tpg.y) + tid.y * tpg.x + tid.x;
+ return tid.z * (tpg.x * tpg.y) + tid.y * tpg.x + tid.x;
}
static inline float gelu_approx(float x) {
- const float k0 = 0.7978845608f; // sqrt(2/pi)
- const float k1 = 0.044715f;
- if (x > 10.0f) return x;
- if (x < -10.0f) return 0.0f;
- float x3 = x * x * x;
- float t = clamp(k0 * (x + k1 * x3), -10.0f, 10.0f);
- return 0.5f * x * (1.0f + tanh(t));
+ const float k0 = 0.7978845608f; // sqrt(2/pi)
+ const float k1 = 0.044715f;
+ if (x > 10.0f)
+ return x;
+ if (x < -10.0f)
+ return 0.0f;
+ float x3 = x * x * x;
+ float t = clamp(k0 * (x + k1 * x3), -10.0f, 10.0f);
+ return 0.5f * x * (1.0f + tanh(t));
}
static inline float silu(float x, float alpha) {
- // alpha=1 gives standard SiLU; keep alpha param for parity with your API
- return x / (1.0f + exp(-alpha * x));
+ // alpha=1 gives standard SiLU; keep alpha param for parity with your API
+ return x / (1.0f + exp(-alpha * x));
}
-template<typename T>
+template <typename T>
inline void store_vec4(device T* dst, ulong base, float4 vals) {
- dst[base + 0] = T(vals.x);
- dst[base + 1] = T(vals.y);
- dst[base + 2] = T(vals.z);
- dst[base + 3] = T(vals.w);
+ dst[base + 0] = T(vals.x);
+ dst[base + 1] = T(vals.y);
+ dst[base + 2] = T(vals.z);
+ dst[base + 3] = T(vals.w);
}
-template<>
+template <>
inline void store_vec4<float>(device float* dst, ulong base, float4 vals) {
- *reinterpret_cast<device float4*>(dst + base) = vals;
+ *reinterpret_cast<device float4*>(dst + base) = vals;
}
-template<typename T>
+template <typename T>
struct StageStorage {
- using type = float;
+ using type = float;
};
-template<>
+template <>
struct StageStorage<half> {
- using type = half;
+ using type = half;
};
-template<>
+template <>
struct StageStorage<bfloat> {
- using type = bfloat;
+ using type = bfloat;
};
// 0=GELU(up), 1=SiLU(up), 2=SwiGLU(gate)*up, 3=GEGLU(gate)*up
@@ -84,694 +86,917 @@ constant uint PASSA_BK = 64;
constant uint PASSA_SG_BM = 8;
constant uint PASSA_SG_BN = 16;
-template<typename T>
+template <typename T>
inline void pass_a_impl(
- device const T* X_perm, // [total_rows, D]
- device const uint* expert_offsets, // [E+1]
- device const T* W13_all, // [E, 2*FF, D]
- device const T* up_biases, // [E, 2*FF] (up | gate)
- device float* hidden_out, // [total_rows, FF]
- uint d_model, uint d_ff, uint E,
- float gate_clip_min, float gate_clip_max,
- float up_clip_min, float up_clip_max,
+ device const T* X_perm, // [total_rows, D]
+ device const uint* expert_offsets, // [E+1]
+ device const T* W13_all, // [E, 2*FF, D]
+ device const T* up_biases, // [E, 2*FF] (up | gate)
+ device float* hidden_out, // [total_rows, FF]
+ uint d_model,
+ uint d_ff,
+ uint E,
+ float gate_clip_min,
+ float gate_clip_max,
+ float up_clip_min,
+ float up_clip_max,
float silu_alpha,
// work partition
- uint tile_seg_start, uint expert_idx, uint tile_m, uint tile_n,
+ uint tile_seg_start,
+ uint expert_idx,
+ uint tile_m,
+ uint tile_n,
// per-thread info
- uint sg_id, uint3 threads_per_tg, uint3 local_tid,
+ uint sg_id,
+ uint3 threads_per_tg,
+ uint3 local_tid,
// TG storage
- threadgroup typename StageStorage<T>::type* Xs, // [BM,BK]
- threadgroup typename StageStorage<T>::type* Wk_up, // [BN,BK]
- threadgroup typename StageStorage<T>::type* Wk_gate, // [BN,BK]
- threadgroup float* bias_up, // [BN]
- threadgroup float* bias_gate // [BN]
+ threadgroup typename StageStorage<T>::type* Xs, // [BM,BK]
+ threadgroup typename StageStorage<T>::type* Wk_up, // [BN,BK]
+ threadgroup typename StageStorage<T>::type* Wk_gate, // [BN,BK]
+ threadgroup float* bias_up, // [BN]
+ threadgroup float* bias_gate // [BN]
) {
- using StageT = typename StageStorage<T>::type;
- using StageVec4 = metal::vec<StageT, 4>;
-
- const uint Bm = PASSA_BM, Bn = PASSA_BN, Bk = PASSA_BK;
- const uint SgBm = PASSA_SG_BM, SgBn = PASSA_SG_BN;
-
- if (expert_idx >= E) return;
-
- const uint seg_start = expert_offsets[expert_idx];
- const uint seg_end = expert_offsets[expert_idx + 1];
- const uint seg_len = (seg_end > seg_start) ? (seg_end - seg_start) : 0u;
- if (seg_len == 0) return;
-
- const uint row_tg_off = tile_m * Bm;
- const uint col_tg_off = tile_n * Bn;
- if (row_tg_off >= seg_len || col_tg_off >= d_ff) return;
-
- const uint m_rows = min(Bm, seg_len - row_tg_off);
- const uint n_cols = min(Bn, d_ff - col_tg_off);
-
- const ulong x_base = ((ulong)tile_seg_start + (ulong)row_tg_off) * (ulong)d_model;
- const ulong w13_expert_base = (ulong)expert_idx * (ulong)(2 * d_ff) * (ulong)d_model;
- const ulong bias_base = (ulong)expert_idx * (ulong)(2 * d_ff);
-
- // simdgroup tile mapping
- const uint sg_col_count = Bn / SgBn;
- const uint row_sg = sg_id / sg_col_count;
- const uint col_sg = sg_id % sg_col_count;
- const uint row_sg_off = row_sg * SgBm;
- const uint col_sg_off = col_sg * SgBn;
-
- // Guard against misconfigured TG sizes
- constexpr uint SG_TILE = 8; // simdgroup fragment dimension (8x8)
- constexpr uint SG_EXPECTED = (PASSA_BM / SG_TILE) * (PASSA_BN / SG_TILE);
- if (sg_id >= SG_EXPECTED) return;
-
- // partial accumulators (8x8 tiles per simdgroup)
- constexpr uint TEMP = (PASSA_SG_BM / SG_TILE) * (PASSA_SG_BN / SG_TILE);
- metal::simdgroup_float8x8 OutUp[TEMP];
- metal::simdgroup_float8x8 OutGate[TEMP];
- for (uint i = 0; i < TEMP; ++i) {
- OutUp[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
- if (GATING_SEL > 1u) {
- OutGate[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
- }
+ using StageT = typename StageStorage<T>::type;
+ using StageVec4 = metal::vec<StageT, 4>;
+
+ const uint Bm = PASSA_BM, Bn = PASSA_BN, Bk = PASSA_BK;
+ const uint SgBm = PASSA_SG_BM, SgBn = PASSA_SG_BN;
+
+ if (expert_idx >= E)
+ return;
+
+ const uint seg_start = expert_offsets[expert_idx];
+ const uint seg_end = expert_offsets[expert_idx + 1];
+ const uint seg_len = (seg_end > seg_start) ? (seg_end - seg_start) : 0u;
+ if (seg_len == 0)
+ return;
+
+ const uint row_tg_off = tile_m * Bm;
+ const uint col_tg_off = tile_n * Bn;
+ if (row_tg_off >= seg_len || col_tg_off >= d_ff)
+ return;
+
+ const uint m_rows = min(Bm, seg_len - row_tg_off);
+ const uint n_cols = min(Bn, d_ff - col_tg_off);
+
+ const ulong x_base =
+ ((ulong)tile_seg_start + (ulong)row_tg_off) * (ulong)d_model;
+ const ulong w13_expert_base =
+ (ulong)expert_idx * (ulong)(2 * d_ff) * (ulong)d_model;
+ const ulong bias_base = (ulong)expert_idx * (ulong)(2 * d_ff);
+
+ // simdgroup tile mapping
+ const uint sg_col_count = Bn / SgBn;
+ const uint row_sg = sg_id / sg_col_count;
+ const uint col_sg = sg_id % sg_col_count;
+ const uint row_sg_off = row_sg * SgBm;
+ const uint col_sg_off = col_sg * SgBn;
+
+ // Guard against misconfigured TG sizes
+ constexpr uint SG_TILE = 8; // simdgroup fragment dimension (8x8)
+ constexpr uint SG_EXPECTED = (PASSA_BM / SG_TILE) * (PASSA_BN / SG_TILE);
+ if (sg_id >= SG_EXPECTED)
+ return;
+
+ // partial accumulators (8x8 tiles per simdgroup)
+ constexpr uint TEMP = (PASSA_SG_BM / SG_TILE) * (PASSA_SG_BN / SG_TILE);
+ metal::simdgroup_float8x8 OutUp[TEMP];
+ metal::simdgroup_float8x8 OutGate[TEMP];
+ for (uint i = 0; i < TEMP; ++i) {
+ OutUp[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
+ if (GATING_SEL > 1u) {
+ OutGate[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
}
+ }
- const uint threads_total = threads_per_tg.x * threads_per_tg.y * threads_per_tg.z;
- const uint lin = linear_tid(local_tid, threads_per_tg);
-
- // K-loop (tile along D)
- for (uint k_off = 0; k_off < d_model; k_off += Bk) {
- const uint valid_k = min(Bk, d_model - k_off);
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // ---- Stage LHS: Xs [Bm,Bk] as float, vectorized 4 ----
- {
- constexpr uint row_stride = PASSA_BK;
- constexpr uint vec_cols = PASSA_BK / 4;
- const uint total_vecs = PASSA_BM * vec_cols;
- const uint iters = ceil_div(total_vecs, threads_total);
-
-UZU_PRAGMA_UNROLL
- for (uint t = 0; t < iters; ++t) {
- const uint i = t * threads_total + lin;
- if (i < total_vecs) {
- const uint r = i / vec_cols;
- const uint c4 = i % vec_cols;
-
- threadgroup StageVec4* dst4 =
- reinterpret_cast<threadgroup StageVec4*>(Xs + r * row_stride + (c4 << 2));
-
- StageVec4 packed = StageVec4(StageT(0.0f));
- if (r < m_rows) {
- const ulong base = x_base + (ulong)r * (ulong)d_model + (ulong)(k_off + (c4 << 2));
- const uint base_vec = (c4 << 2);
- float v0 = 0.0f, v1 = 0.0f, v2 = 0.0f, v3 = 0.0f;
- if (base_vec + 0 < valid_k) v0 = float(X_perm[base + 0]);
- if (base_vec + 1 < valid_k) v1 = float(X_perm[base + 1]);
- if (base_vec + 2 < valid_k) v2 = float(X_perm[base + 2]);
- if (base_vec + 3 < valid_k) v3 = float(X_perm[base + 3]);
- packed = StageVec4(StageT(v0), StageT(v1), StageT(v2), StageT(v3));
- }
- *dst4 = packed;
- }
- }
- }
+ const uint threads_total =
+ threads_per_tg.x * threads_per_tg.y * threads_per_tg.z;
+ const uint lin = linear_tid(local_tid, threads_per_tg);
- // ---- Stage RHS: W_up [Bn,Bk], W_gate [Bn,Bk], vectorized 4 ----
- {
- constexpr uint row_stride = PASSA_BK;
- constexpr uint vec_cols = PASSA_BK / 4;
- const uint total_vecs = PASSA_BN * vec_cols;
- const uint iters = ceil_div(total_vecs, threads_total);
-
-UZU_PRAGMA_UNROLL
- for (uint t = 0; t < iters; ++t) {
- const uint i = t * threads_total + lin;
- if (i < total_vecs) {
- const uint n = i / vec_cols; // column within the BN tile
- const uint c4 = i % vec_cols;
-
- threadgroup StageVec4* up4 =
- reinterpret_cast<threadgroup StageVec4*>(Wk_up + n * row_stride + (c4 << 2));
- StageVec4 upv = StageVec4(StageT(0.0f));
-
- threadgroup StageVec4* gt4 =
- reinterpret_cast<threadgroup StageVec4*>(Wk_gate + n * row_stride + (c4 << 2));
- StageVec4 gtv = StageVec4(StageT(0.0f));
-
- if (n < n_cols) {
- const uint gcol = col_tg_off + n;
- const ulong up_base = w13_expert_base + (ulong)gcol * (ulong)d_model + (ulong)(k_off + (c4 << 2));
- const ulong gate_base = w13_expert_base + (ulong)(d_ff + gcol) * (ulong)d_model + (ulong)(k_off + (c4 << 2));
- const uint base_vec = (c4 << 2);
- float up0 = 0.0f, up1 = 0.0f, up2 = 0.0f, up3 = 0.0f;
- float gt0 = 0.0f, gt1 = 0.0f, gt2 = 0.0f, gt3 = 0.0f;
- if (base_vec + 0 < valid_k) { up0 = float(W13_all[up_base + 0]); gt0 = float(W13_all[gate_base + 0]); }
- if (base_vec + 1 < valid_k) { up1 = float(W13_all[up_base + 1]); gt1 = float(W13_all[gate_base + 1]); }
- if (base_vec + 2 < valid_k) { up2 = float(W13_all[up_base + 2]); gt2 = float(W13_all[gate_base + 2]); }
- if (base_vec + 3 < valid_k) { up3 = float(W13_all[up_base + 3]); gt3 = float(W13_all[gate_base + 3]); }
- upv = StageVec4(StageT(up0), StageT(up1), StageT(up2), StageT(up3));
- gtv = StageVec4(StageT(gt0), StageT(gt1), StageT(gt2), StageT(gt3));
- }
-
- *up4 = upv;
- if (GATING_SEL > 1u) *gt4 = gtv;
- }
- }
- }
+ // K-loop (tile along D)
+ for (uint k_off = 0; k_off < d_model; k_off += Bk) {
+ const uint valid_k = min(Bk, d_model - k_off);
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // ---- MMA across the BK tile (in simdgroup-sized chunks) ----
-UZU_PRAGMA_UNROLL
- for (uint kk = 0; kk < PASSA_BK; kk += SG_TILE) {
-UZU_PRAGMA_UNROLL
- for (uint m_sub = 0; m_sub < PASSA_SG_BM; m_sub += SG_TILE) {
- const uint r_idx = m_sub / SG_TILE;
- metal::simdgroup_matrix<StageT, 8, 8> lhs_frag;
- simdgroup_load(lhs_frag, Xs, PASSA_BK, ulong2(kk, row_sg_off + m_sub));
-
-UZU_PRAGMA_UNROLL
- for (uint n_sub = 0; n_sub < PASSA_SG_BN; n_sub += SG_TILE) {
- const uint c_idx = n_sub / SG_TILE;
- const uint tile = r_idx * (PASSA_SG_BN / SG_TILE) + c_idx;
- metal::simdgroup_matrix<StageT, 8, 8> rhs_up;
- simdgroup_load(rhs_up, Wk_up, PASSA_BK, ulong2(kk, col_sg_off + n_sub), true);
- simdgroup_multiply_accumulate(OutUp[tile], lhs_frag, rhs_up, OutUp[tile]);
-
- if (GATING_SEL > 1u) {
- metal::simdgroup_matrix<StageT, 8, 8> rhs_gate;
- simdgroup_load(rhs_gate, Wk_gate, PASSA_BK, ulong2(kk, col_sg_off + n_sub), true);
- simdgroup_multiply_accumulate(OutGate[tile], lhs_frag, rhs_gate, OutGate[tile]);
- }
- }
- }
- }
- } // K loop
-
- // ---- Bias tile [Bn] ----
- // Note: bias layout is [up | gate] in contiguous FF chunks.
- for (uint c_local = lin; c_local < Bn; c_local += threads_total) {
- const uint c_global = col_tg_off + c_local;
- bias_up[c_local] = (c_global < d_ff) ? float(up_biases[bias_base + c_global]) : 0.0f;
- if (GATING_SEL > 1u) {
- bias_gate[c_local] = (c_global < d_ff) ? float(up_biases[bias_base + d_ff + c_global]) : 0.0f;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // ---- Stage LHS: Xs [Bm,Bk] as float, vectorized 4 ----
+ {
+ constexpr uint row_stride = PASSA_BK;
+ constexpr uint vec_cols = PASSA_BK / 4;
+ const uint total_vecs = PASSA_BM * vec_cols;
+ const uint iters = ceil_div(total_vecs, threads_total);
+
+ UZU_PRAGMA_UNROLL
+ for (uint t = 0; t < iters; ++t) {
+ const uint i = t * threads_total + lin;
+ if (i < total_vecs) {
+ const uint r = i / vec_cols;
+ const uint c4 = i % vec_cols;
+
+ threadgroup StageVec4* dst4 =
+ reinterpret_cast<threadgroup StageVec4*>(
+ Xs + r * row_stride + (c4 << 2)
+ );
+
+ StageVec4 packed = StageVec4(StageT(0.0f));
+ if (r < m_rows) {
+ const ulong base =
+ x_base + (ulong)r * (ulong)d_model + (ulong)(k_off + (c4 << 2));
+ const uint base_vec = (c4 << 2);
+ float v0 = 0.0f, v1 = 0.0f, v2 = 0.0f, v3 = 0.0f;
+ if (base_vec + 0 < valid_k)
+ v0 = float(X_perm[base + 0]);
+ if (base_vec + 1 < valid_k)
+ v1 = float(X_perm[base + 1]);
+ if (base_vec + 2 < valid_k)
+ v2 = float(X_perm[base + 2]);
+ if (base_vec + 3 < valid_k)
+ v3 = float(X_perm[base + 3]);
+ packed = StageVec4(StageT(v0), StageT(v1), StageT(v2), StageT(v3));
+ }
+ *dst4 = packed;
}
+ }
}
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // ---- Epilogue: activation/gating → hidden_out (f32) ----
- const uint sg_lane_start = sg_id * 32;
- const uint sg_lane_end = sg_lane_start + 32;
- if (lin < sg_lane_start || lin >= sg_lane_end) return;
- const uint sg_lin = lin - sg_lane_start;
- // Lane→(row,col) mapping for 8x8 simdgroup fragments, matches the fragment layout used above.
- const uint lane_qid = sg_lin >> 2;
- const uint lane_row = (lane_qid & 4u) + ((sg_lin >> 1) & 3u);
- const uint lane_col_base = ((lane_qid & 2u) << 1) + ((sg_lin & 1u) << 1);
-
-UZU_PRAGMA_UNROLL
- for (uint n_sub = 0; n_sub < PASSA_SG_BN; n_sub += SG_TILE) {
- const uint c_idx = n_sub / SG_TILE;
-UZU_PRAGMA_UNROLL
- for (uint m_sub = 0; m_sub < PASSA_SG_BM; m_sub += SG_TILE) {
- const uint r_idx = m_sub / SG_TILE;
- const uint tile = r_idx * (PASSA_SG_BN / SG_TILE) + c_idx;
-
- const uint tile_row_base = row_sg_off + m_sub;
- const uint tile_col_base = col_sg_off + n_sub;
- const uint local_row = tile_row_base + lane_row;
- if (local_row >= m_rows || tile_col_base >= n_cols) {
- continue;
+ // ---- Stage RHS: W_up [Bn,Bk], W_gate [Bn,Bk], vectorized 4 ----
+ {
+ constexpr uint row_stride = PASSA_BK;
+ constexpr uint vec_cols = PASSA_BK / 4;
+ const uint total_vecs = PASSA_BN * vec_cols;
+ const uint iters = ceil_div(total_vecs, threads_total);
+
+ UZU_PRAGMA_UNROLL
+ for (uint t = 0; t < iters; ++t) {
+ const uint i = t * threads_total + lin;
+ if (i < total_vecs) {
+ const uint n = i / vec_cols; // column within the BN tile
+ const uint c4 = i % vec_cols;
+
+ threadgroup StageVec4* up4 = reinterpret_cast<threadgroup StageVec4*>(
+ Wk_up + n * row_stride + (c4 << 2)
+ );
+ StageVec4 upv = StageVec4(StageT(0.0f));
+
+ threadgroup StageVec4* gt4 = reinterpret_cast<threadgroup StageVec4*>(
+ Wk_gate + n * row_stride + (c4 << 2)
+ );
+ StageVec4 gtv = StageVec4(StageT(0.0f));
+
+ if (n < n_cols) {
+ const uint gcol = col_tg_off + n;
+ const ulong up_base = w13_expert_base +
+ (ulong)gcol * (ulong)d_model +
+ (ulong)(k_off + (c4 << 2));
+ const ulong gate_base = w13_expert_base +
+ (ulong)(d_ff + gcol) * (ulong)d_model +
+ (ulong)(k_off + (c4 << 2));
+ const uint base_vec = (c4 << 2);
+ float up0 = 0.0f, up1 = 0.0f, up2 = 0.0f, up3 = 0.0f;
+ float gt0 = 0.0f, gt1 = 0.0f, gt2 = 0.0f, gt3 = 0.0f;
+ if (base_vec + 0 < valid_k) {
+ up0 = float(W13_all[up_base + 0]);
+ gt0 = float(W13_all[gate_base + 0]);
}
-
- const auto up_frag = OutUp[tile].thread_elements();
- float gate_frag_0 = 0.0f;
- float gate_frag_1 = 0.0f;
- if (GATING_SEL > 1u) {
- const auto gate_frag = OutGate[tile].thread_elements();
- gate_frag_0 = gate_frag[0];
- gate_frag_1 = gate_frag[1];
+ if (base_vec + 1 < valid_k) {
+ up1 = float(W13_all[up_base + 1]);
+ gt1 = float(W13_all[gate_base + 1]);
}
-
- const uint col0 = tile_col_base + lane_col_base;
- const uint col1 = col0 + 1;
- const ulong out_row = seg_start + row_tg_off + local_row;
-
- if (col0 < n_cols) {
- float up_v = clamp(up_frag[0] + bias_up[col0], up_clip_min, up_clip_max);
- float out_val;
- if (GATING_SEL <= 1u) {
- out_val =
- (GATING_SEL == 0u) ? gelu_approx(up_v) : silu(up_v, silu_alpha);
- } else {
- float gate_v =
- clamp(gate_frag_0 + bias_gate[col0], gate_clip_min, gate_clip_max);
- const float gate_act =
- (GATING_SEL == 2u) ? silu(gate_v, silu_alpha) : gelu_approx(gate_v);
- out_val = gate_act * up_v;
- }
- const ulong out_col = col_tg_off + col0;
- hidden_out[out_row * (ulong)d_ff + out_col] = out_val;
+ if (base_vec + 2 < valid_k) {
+ up2 = float(W13_all[up_base + 2]);
+ gt2 = float(W13_all[gate_base + 2]);
}
-
- if (col1 < n_cols) {
- float up_v = clamp(up_frag[1] + bias_up[col1], up_clip_min, up_clip_max);
- float out_val;
- if (GATING_SEL <= 1u) {
- out_val =
- (GATING_SEL == 0u) ? gelu_approx(up_v) : silu(up_v, silu_alpha);
- } else {
- float gate_v =
- clamp(gate_frag_1 + bias_gate[col1], gate_clip_min, gate_clip_max);
- const float gate_act =
- (GATING_SEL == 2u) ? silu(gate_v, silu_alpha) : gelu_approx(gate_v);
- out_val = gate_act * up_v;
- }
- const ulong out_col = col_tg_off + col1;
- hidden_out[out_row * (ulong)d_ff + out_col] = out_val;
+ if (base_vec + 3 < valid_k) {
+ up3 = float(W13_all[up_base + 3]);
+ gt3 = float(W13_all[gate_base + 3]);
}
+ upv = StageVec4(StageT(up0), StageT(up1), StageT(up2), StageT(up3));
+ gtv = StageVec4(StageT(gt0), StageT(gt1), StageT(gt2), StageT(gt3));
+ }
+
+ *up4 = upv;
+ if (GATING_SEL > 1u)
+ *gt4 = gtv;
+ }
+ }
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // ---- MMA across the BK tile (in simdgroup-sized chunks) ----
+ UZU_PRAGMA_UNROLL
+ for (uint kk = 0; kk < PASSA_BK; kk += SG_TILE) {
+ UZU_PRAGMA_UNROLL
+ for (uint m_sub = 0; m_sub < PASSA_SG_BM; m_sub += SG_TILE) {
+ const uint r_idx = m_sub / SG_TILE;
+ metal::simdgroup_matrix<StageT, 8, 8> lhs_frag;
+ simdgroup_load(lhs_frag, Xs, PASSA_BK, ulong2(kk, row_sg_off + m_sub));
+
+ UZU_PRAGMA_UNROLL
+ for (uint n_sub = 0; n_sub < PASSA_SG_BN; n_sub += SG_TILE) {
+ const uint c_idx = n_sub / SG_TILE;
+ const uint tile = r_idx * (PASSA_SG_BN / SG_TILE) + c_idx;
+ metal::simdgroup_matrix<StageT, 8, 8> rhs_up;
+ simdgroup_load(
+ rhs_up,
+ Wk_up,
+ PASSA_BK,
+ ulong2(kk, col_sg_off + n_sub),
+ true
+ );
+ simdgroup_multiply_accumulate(
+ OutUp[tile],
+ lhs_frag,
+ rhs_up,
+ OutUp[tile]
+ );
+
+ if (GATING_SEL > 1u) {
+ metal::simdgroup_matrix<StageT, 8, 8> rhs_gate;
+ simdgroup_load(
+ rhs_gate,
+ Wk_gate,
+ PASSA_BK,
+ ulong2(kk, col_sg_off + n_sub),
+ true
+ );
+ simdgroup_multiply_accumulate(
+ OutGate[tile],
+ lhs_frag,
+ rhs_gate,
+ OutGate[tile]
+ );
+ }
+ }
+ }
+ }
+ } // K loop
+
+ // ---- Bias tile [Bn] ----
+ // Note: bias layout is [up | gate] in contiguous FF chunks.
+ for (uint c_local = lin; c_local < Bn; c_local += threads_total) {
+ const uint c_global = col_tg_off + c_local;
+ bias_up[c_local] =
+ (c_global < d_ff) ? float(up_biases[bias_base + c_global]) : 0.0f;
+ if (GATING_SEL > 1u) {
+ bias_gate[c_local] = (c_global < d_ff)
+ ? float(up_biases[bias_base + d_ff + c_global])
+ : 0.0f;
+ }
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // ---- Epilogue: activation/gating → hidden_out (f32) ----
+ const uint sg_lane_start = sg_id * 32;
+ const uint sg_lane_end = sg_lane_start + 32;
+ if (lin < sg_lane_start || lin >= sg_lane_end)
+ return;
+ const uint sg_lin = lin - sg_lane_start;
+ // Lane→(row,col) mapping for 8x8 simdgroup fragments, matches the fragment
+ // layout used above.
+ const uint lane_qid = sg_lin >> 2;
+ const uint lane_row = (lane_qid & 4u) + ((sg_lin >> 1) & 3u);
+ const uint lane_col_base = ((lane_qid & 2u) << 1) + ((sg_lin & 1u) << 1);
+
+ UZU_PRAGMA_UNROLL
+ for (uint n_sub = 0; n_sub < PASSA_SG_BN; n_sub += SG_TILE) {
+ const uint c_idx = n_sub / SG_TILE;
+ UZU_PRAGMA_UNROLL
+ for (uint m_sub = 0; m_sub < PASSA_SG_BM; m_sub += SG_TILE) {
+ const uint r_idx = m_sub / SG_TILE;
+ const uint tile = r_idx * (PASSA_SG_BN / SG_TILE) + c_idx;
+
+ const uint tile_row_base = row_sg_off + m_sub;
+ const uint tile_col_base = col_sg_off + n_sub;
+ const uint local_row = tile_row_base + lane_row;
+ if (local_row >= m_rows || tile_col_base >= n_cols) {
+ continue;
+ }
+
+ const auto up_frag = OutUp[tile].thread_elements();
+ float gate_frag_0 = 0.0f;
+ float gate_frag_1 = 0.0f;
+ if (GATING_SEL > 1u) {
+ const auto gate_frag = OutGate[tile].thread_elements();
+ gate_frag_0 = gate_frag[0];
+ gate_frag_1 = gate_frag[1];
+ }
+
+ const uint col0 = tile_col_base + lane_col_base;
+ const uint col1 = col0 + 1;
+ const ulong out_row = seg_start + row_tg_off + local_row;
+
+ if (col0 < n_cols) {
+ float up_v =
+ clamp(up_frag[0] + bias_up[col0], up_clip_min, up_clip_max);
+ float out_val;
+ if (GATING_SEL <= 1u) {
+ out_val =
+ (GATING_SEL == 0u) ? gelu_approx(up_v) : silu(up_v, silu_alpha);
+ } else {
+ float gate_v = clamp(
+ gate_frag_0 + bias_gate[col0],
+ gate_clip_min,
+ gate_clip_max
+ );
+ const float gate_act = (GATING_SEL == 2u) ? silu(gate_v, silu_alpha)
+ : gelu_approx(gate_v);
+ out_val = gate_act * up_v;
}
+ const ulong out_col = col_tg_off + col0;
+ hidden_out[out_row * (ulong)d_ff + out_col] = out_val;
+ }
+
+ if (col1 < n_cols) {
+ float up_v =
+ clamp(up_frag[1] + bias_up[col1], up_clip_min, up_clip_max);
+ float out_val;
+ if (GATING_SEL <= 1u) {
+ out_val =
+ (GATING_SEL == 0u) ? gelu_approx(up_v) : silu(up_v, silu_alpha);
+ } else {
+ float gate_v = clamp(
+ gate_frag_1 + bias_gate[col1],
+ gate_clip_min,
+ gate_clip_max
+ );
+ const float gate_act = (GATING_SEL == 2u) ? silu(gate_v, silu_alpha)
+ : gelu_approx(gate_v);
+ out_val = gate_act * up_v;
+ }
+ const ulong out_col = col_tg_off + col1;
+ hidden_out[out_row * (ulong)d_ff + out_col] = out_val;
+ }
}
+ }
}
// ----- Kernel entry points (Pass A) -----
-#define MOE_PASS_A_KERNEL(DTYPE, SUFFIX) \
-kernel void moe_two_pass_prefill_pass_a_##SUFFIX( \
- device const DTYPE* X_perm [[buffer(0)]], \
- device const uint* expert_offsets [[buffer(1)]], \
- device const DTYPE* W13_all [[buffer(2)]], \
- device const DTYPE* up_biases [[buffer(3)]], \
- device float* hidden_out [[buffer(4)]], \
- constant uint& d_model [[buffer(5)]], \
- constant uint& d_ff [[buffer(6)]], \
- constant uint& E [[buffer(7)]], \
- constant float& gate_clip_min [[buffer(8)]], \
- constant float& gate_clip_max [[buffer(9)]], \
- constant float& up_clip_min [[buffer(10)]], \
- constant float& up_clip_max [[buffer(11)]], \
- constant float& silu_alpha [[buffer(12)]], \
- uint sg_id [[simdgroup_index_in_threadgroup]], \
- uint3 threads_per_tg [[threads_per_threadgroup]], \
- uint3 tg_pos [[threadgroup_position_in_grid]], \
- uint3 local_tid [[thread_position_in_threadgroup]] ) \
-{ \
- using StageT = typename StageStorage<DTYPE>::type; \
- threadgroup StageT Xs [PASSA_BM * PASSA_BK]; \
- threadgroup StageT Wk_up [PASSA_BN * PASSA_BK]; \
- threadgroup StageT Wk_gate [PASSA_BN * PASSA_BK]; \
- threadgroup float bias_up [PASSA_BN]; \
- threadgroup float bias_gate [PASSA_BN]; \
- pass_a_impl<DTYPE>( \
- X_perm, expert_offsets, W13_all, up_biases, hidden_out, \
- d_model, d_ff, E, \
- gate_clip_min, gate_clip_max, up_clip_min, up_clip_max, silu_alpha, \
- /*tile_seg_start*/ expert_offsets[tg_pos.z], /*expert*/ tg_pos.z, /*tile_m*/ tg_pos.y, /*tile_n*/ tg_pos.x, \
- sg_id, threads_per_tg, local_tid, \
- Xs, Wk_up, Wk_gate, bias_up, bias_gate); \
-}
+#define MOE_PASS_A_KERNEL(DTYPE, SUFFIX) \
+ kernel void moe_two_pass_prefill_pass_a_##SUFFIX( \
+ device const DTYPE* X_perm [[buffer(0)]], \
+ device const uint* expert_offsets [[buffer(1)]], \
+ device const DTYPE* W13_all [[buffer(2)]], \
+ device const DTYPE* up_biases [[buffer(3)]], \
+ device float* hidden_out [[buffer(4)]], \
+ constant uint& d_model [[buffer(5)]], \
+ constant uint& d_ff [[buffer(6)]], \
+ constant uint& E [[buffer(7)]], \
+ constant float& gate_clip_min [[buffer(8)]], \
+ constant float& gate_clip_max [[buffer(9)]], \
+ constant float& up_clip_min [[buffer(10)]], \
+ constant float& up_clip_max [[buffer(11)]], \
+ constant float& silu_alpha [[buffer(12)]], \
+ uint sg_id [[simdgroup_index_in_threadgroup]], \
+ uint3 threads_per_tg [[threads_per_threadgroup]], \
+ uint3 tg_pos [[threadgroup_position_in_grid]], \
+ uint3 local_tid [[thread_position_in_threadgroup]] \
+ ) { \
+ using StageT = typename StageStorage<DTYPE>::type; \
+ threadgroup StageT Xs[PASSA_BM * PASSA_BK]; \
+ threadgroup StageT Wk_up[PASSA_BN * PASSA_BK]; \
+ threadgroup StageT Wk_gate[PASSA_BN * PASSA_BK]; \
+ threadgroup float bias_up[PASSA_BN]; \
+ threadgroup float bias_gate[PASSA_BN]; \
+ pass_a_impl<DTYPE>( \
+ X_perm, \
+ expert_offsets, \
+ W13_all, \
+ up_biases, \
+ hidden_out, \
+ d_model, \
+ d_ff, \
+ E, \
+ gate_clip_min, \
+ gate_clip_max, \
+ up_clip_min, \
+ up_clip_max, \
+ silu_alpha, \
+ /*tile_seg_start*/ expert_offsets[tg_pos.z], \
+ /*expert*/ tg_pos.z, \
+ /*tile_m*/ tg_pos.y, \
+ /*tile_n*/ tg_pos.x, \
+ sg_id, \
+ threads_per_tg, \
+ local_tid, \
+ Xs, \
+ Wk_up, \
+ Wk_gate, \
+ bias_up, \
+ bias_gate \
+ ); \
+ }
MOE_PASS_A_KERNEL(bfloat, bf16)
-MOE_PASS_A_KERNEL(half, f16)
-MOE_PASS_A_KERNEL(float, f32)
+MOE_PASS_A_KERNEL(half, f16)
+MOE_PASS_A_KERNEL(float, f32)
// Indirect variant (consumes [expert, seg_start, tile_row_offset] triples)
-template<typename T>
+template <typename T>
inline void pass_a_indirect_impl(
- device const T* X_perm,
+ device const T* X_perm,
device const uint* expert_offsets,
- device const T* W13_all,
- device const T* up_biases,
- device float* hidden_out,
- device const uint* tile_map, // [tiles * 3]: (expert, seg_start, row_offset_elems)
- uint d_model, uint d_ff, uint E,
- float gate_clip_min, float gate_clip_max,
- float up_clip_min, float up_clip_max,
+ device const T* W13_all,
+ device const T* up_biases,
+ device float* hidden_out,
+ device const uint*
+ tile_map, // [tiles * 3]: (expert, seg_start, row_offset_elems)
+ uint d_model,
+ uint d_ff,
+ uint E,
+ float gate_clip_min,
+ float gate_clip_max,
+ float up_clip_min,
+ float up_clip_max,
float silu_alpha,
- uint row_tile_idx, uint n_tile_idx,
- uint sg_id, uint3 threads_per_tg, uint3 local_tid,
+ uint row_tile_idx,
+ uint n_tile_idx,
+ uint sg_id,
+ uint3 threads_per_tg,
+ uint3 local_tid,
threadgroup typename StageStorage<T>::type* Xs,
threadgroup typename StageStorage<T>::type* Wk_up,
threadgroup typename StageStorage<T>::type* Wk_gate,
threadgroup float* bias_up,
- threadgroup float* bias_gate)
-{
- constexpr uint ROW_TILE = PASSA_BM;
- const uint base = row_tile_idx * 3u;
- const uint expert_idx = tile_map[base + 0u];
- if (expert_idx >= E) return;
- const uint tile_seg_start = tile_map[base + 1u];
- const uint row_off_elems = tile_map[base + 2u];
- const uint tile_m = row_off_elems / ROW_TILE;
-
- pass_a_impl<T>(
- X_perm, expert_offsets, W13_all, up_biases, hidden_out,
- d_model, d_ff, E,
- gate_clip_min, gate_clip_max, up_clip_min, up_clip_max, silu_alpha,
- tile_seg_start, expert_idx, tile_m, n_tile_idx,
- sg_id, threads_per_tg, local_tid,
- Xs, Wk_up, Wk_gate, bias_up, bias_gate);
+ threadgroup float* bias_gate
+) {
+ constexpr uint ROW_TILE = PASSA_BM;
+ const uint base = row_tile_idx * 3u;
+ const uint expert_idx = tile_map[base + 0u];
+ if (expert_idx >= E)
+ return;
+ const uint tile_seg_start = tile_map[base + 1u];
+ const uint row_off_elems = tile_map[base + 2u];
+ const uint tile_m = row_off_elems / ROW_TILE;
+
+ pass_a_impl<T>(
+ X_perm,
+ expert_offsets,
+ W13_all,
+ up_biases,
+ hidden_out,
+ d_model,
+ d_ff,
+ E,
+ gate_clip_min,
+ gate_clip_max,
+ up_clip_min,
+ up_clip_max,
+ silu_alpha,
+ tile_seg_start,
+ expert_idx,
+ tile_m,
+ n_tile_idx,
+ sg_id,
+ threads_per_tg,
+ local_tid,
+ Xs,
+ Wk_up,
+ Wk_gate,
+ bias_up,
+ bias_gate
+ );
}
-#define MOE_PASS_A_INDIRECT_KERNEL(DTYPE, SUFFIX) \
-kernel void moe_two_pass_prefill_pass_a_indirect_##SUFFIX( \
- device const DTYPE* X_perm [[buffer(0)]], \
- device const uint* expert_offsets [[buffer(1)]], \
- device const DTYPE* W13_all [[buffer(2)]], \
- device const DTYPE* up_biases [[buffer(3)]], \
- device float* hidden_out [[buffer(4)]], \
- constant uint& d_model [[buffer(5)]], \
- constant uint& d_ff [[buffer(6)]], \
- constant uint& E [[buffer(7)]], \
- constant float& gate_clip_min [[buffer(8)]], \
- constant float& gate_clip_max [[buffer(9)]], \
- constant float& up_clip_min [[buffer(10)]], \
- constant float& up_clip_max [[buffer(11)]], \
- constant float& silu_alpha [[buffer(12)]], \
- device const uint* tile_map [[buffer(13)]], \
- uint sg_id [[simdgroup_index_in_threadgroup]], \
- uint3 threads_per_tg [[threads_per_threadgroup]], \
- uint3 tg_pos [[threadgroup_position_in_grid]], \
- uint3 local_tid [[thread_position_in_threadgroup]]) \
-{ \
- using StageT = typename StageStorage<DTYPE>::type; \
- threadgroup StageT Xs [PASSA_BM * PASSA_BK]; \
- threadgroup StageT Wk_up [PASSA_BN * PASSA_BK]; \
- threadgroup StageT Wk_gate [PASSA_BN * PASSA_BK]; \
- threadgroup float bias_up [PASSA_BN]; \
- threadgroup float bias_gate [PASSA_BN]; \
- pass_a_indirect_impl<DTYPE>( \
- X_perm, expert_offsets, W13_all, up_biases, hidden_out, tile_map, \
- d_model, d_ff, E, \
- gate_clip_min, gate_clip_max, up_clip_min, up_clip_max, silu_alpha, \
- /*row_tile_idx*/ tg_pos.y, /*n_tile_idx*/ tg_pos.x, \
- sg_id, threads_per_tg, local_tid, \
- Xs, Wk_up, Wk_gate, bias_up, bias_gate); \
-}
+#define MOE_PASS_A_INDIRECT_KERNEL(DTYPE, SUFFIX) \
+ kernel void moe_two_pass_prefill_pass_a_indirect_##SUFFIX( \
+ device const DTYPE* X_perm [[buffer(0)]], \
+ device const uint* expert_offsets [[buffer(1)]], \
+ device const DTYPE* W13_all [[buffer(2)]], \
+ device const DTYPE* up_biases [[buffer(3)]], \
+ device float* hidden_out [[buffer(4)]], \
+ constant uint& d_model [[buffer(5)]], \
+ constant uint& d_ff [[buffer(6)]], \
+ constant uint& E [[buffer(7)]], \
+ constant float& gate_clip_min [[buffer(8)]], \
+ constant float& gate_clip_max [[buffer(9)]], \
+ constant float& up_clip_min [[buffer(10)]], \
+ constant float& up_clip_max [[buffer(11)]], \
+ constant float& silu_alpha [[buffer(12)]], \
+ device const uint* tile_map [[buffer(13)]], \
+ uint sg_id [[simdgroup_index_in_threadgroup]], \
+ uint3 threads_per_tg [[threads_per_threadgroup]], \
+ uint3 tg_pos [[threadgroup_position_in_grid]], \
+ uint3 local_tid [[thread_position_in_threadgroup]] \
+ ) { \
+ using StageT = typename StageStorage<DTYPE>::type; \
+ threadgroup StageT Xs[PASSA_BM * PASSA_BK]; \
+ threadgroup StageT Wk_up[PASSA_BN * PASSA_BK]; \
+ threadgroup StageT Wk_gate[PASSA_BN * PASSA_BK]; \
+ threadgroup float bias_up[PASSA_BN]; \
+ threadgroup float bias_gate[PASSA_BN]; \
+ pass_a_indirect_impl<DTYPE>( \
+ X_perm, \
+ expert_offsets, \
+ W13_all, \
+ up_biases, \
+ hidden_out, \
+ tile_map, \
+ d_model, \
+ d_ff, \
+ E, \
+ gate_clip_min, \
+ gate_clip_max, \
+ up_clip_min, \
+ up_clip_max, \
+ silu_alpha, \
+ /*row_tile_idx*/ tg_pos.y, \
+ /*n_tile_idx*/ tg_pos.x, \
+ sg_id, \
+ threads_per_tg, \
+ local_tid, \
+ Xs, \
+ Wk_up, \
+ Wk_gate, \
+ bias_up, \
+ bias_gate \
+ ); \
+ }
MOE_PASS_A_INDIRECT_KERNEL(bfloat, bf16)
-MOE_PASS_A_INDIRECT_KERNEL(half, f16)
-MOE_PASS_A_INDIRECT_KERNEL(float, f32)
+MOE_PASS_A_INDIRECT_KERNEL(half, f16)
+MOE_PASS_A_INDIRECT_KERNEL(float, f32)
-// ------------------------ Pass B (hidden @ W2 → output) ------------------------
+// ------------------------ Pass B (hidden @ W2 → output)
+// ------------------------
constant uint PASSB_BM = 16;
constant uint PASSB_BN = 64;
constant uint PASSB_BK = 64;
constant uint PASSB_SG_BM = 8;
constant uint PASSB_SG_BN = 32;
-template<typename T>
+template <typename T>
inline void pass_b_impl(
- device const float* hidden, // [total_rows, FF]
- device const uint* expert_offsets, // [E+1]
- device const T* W2_all, // [E, D, FF] (FF contiguous)
- device const T* down_biases, // [E, D]
- device T* output, // [total_rows, D]
- uint d_model, uint d_ff, uint E,
- uint expert_idx, uint tile_m, uint tile_n,
- uint sg_id, uint3 threads_per_tg, uint3 local_tid,
- threadgroup float* Hs, // [BM,BK]
- threadgroup typename StageStorage<T>::type* Wk, // [BN,BK]
- threadgroup float* bias_tile // [BN]
+ device const float* hidden, // [total_rows, FF]
+ device const uint* expert_offsets, // [E+1]
+ device const T* W2_all, // [E, D, FF] (FF contiguous)
+ device const T* down_biases, // [E, D]
+ device T* output, // [total_rows, D]
+ uint d_model,
+ uint d_ff,
+ uint E,
+ uint expert_idx,
+ uint tile_m,
+ uint tile_n,
+ uint sg_id,
+ uint3 threads_per_tg,
+ uint3 local_tid,
+ threadgroup float* Hs, // [BM,BK]
+ threadgroup typename StageStorage<T>::type* Wk, // [BN,BK]
+ threadgroup float* bias_tile // [BN]
) {
- using StageT = typename StageStorage<T>::type;
- using StageVec4 = metal::vec<StageT, 4>;
-
- const uint Bm = PASSB_BM, Bn = PASSB_BN, Bk = PASSB_BK;
- const uint SgBm = PASSB_SG_BM, SgBn = PASSB_SG_BN;
+ using StageT = typename StageStorage<T>::type;
+ using StageVec4 = metal::vec<StageT, 4>;
+
+ const uint Bm = PASSB_BM, Bn = PASSB_BN, Bk = PASSB_BK;
+ const uint SgBm = PASSB_SG_BM, SgBn = PASSB_SG_BN;
+
+ if (expert_idx >= E)
+ return;
+
+ const uint seg_start = expert_offsets[expert_idx];
+ const uint seg_end = expert_offsets[expert_idx + 1];
+ const uint seg_len = (seg_end > seg_start) ? (seg_end - seg_start) : 0u;
+ if (seg_len == 0)
+ return;
+
+ const uint row_tg_off = tile_m * Bm;
+ const uint col_tg_off = tile_n * Bn;
+ if (row_tg_off >= seg_len || col_tg_off >= d_model)
+ return;
+
+ const uint m_rows = min(Bm, seg_len - row_tg_off);
+ const uint n_cols = min(Bn, d_model - col_tg_off);
+
+ const ulong h_base = (ulong)(seg_start + row_tg_off) * (ulong)d_ff;
+ const ulong w2_expert_base = (ulong)expert_idx * (ulong)d_model * (ulong)d_ff;
+ const ulong bias_base = (ulong)expert_idx * (ulong)d_model;
+
+ const uint sg_col_count = Bn / SgBn;
+ const uint row_sg = sg_id / sg_col_count;
+ const uint col_sg = sg_id % sg_col_count;
+ const uint row_sg_off = row_sg * SgBm;
+ const uint col_sg_off = col_sg * SgBn;
+
+ constexpr uint SG_TILE = 8;
+ constexpr uint TEMP = (PASSB_SG_BM / SG_TILE) * (PASSB_SG_BN / SG_TILE);
+ metal::simdgroup_float8x8 Out[TEMP];
+ for (uint i = 0; i < TEMP; ++i) {
+ Out[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
+ }
+
+ const uint threads_total =
+ threads_per_tg.x * threads_per_tg.y * threads_per_tg.z;
+ const uint lin = linear_tid(local_tid, threads_per_tg);
+
+ // K-loop across FF
+ for (uint k_off = 0; k_off < d_ff; k_off += Bk) {
+ const uint valid_k = min(Bk, d_ff - k_off);
- if (expert_idx >= E) return;
-
- const uint seg_start = expert_offsets[expert_idx];
- const uint seg_end = expert_offsets[expert_idx + 1];
- const uint seg_len = (seg_end > seg_start) ? (seg_end - seg_start) : 0u;
- if (seg_len == 0) return;
-
- const uint row_tg_off = tile_m * Bm;
- const uint col_tg_off = tile_n * Bn;
- if (row_tg_off >= seg_len || col_tg_off >= d_model) return;
-
- const uint m_rows = min(Bm, seg_len - row_tg_off);
- const uint n_cols = min(Bn, d_model - col_tg_off);
-
- const ulong h_base = (ulong)(seg_start + row_tg_off) * (ulong)d_ff;
- const ulong w2_expert_base = (ulong)expert_idx * (ulong)d_model * (ulong)d_ff;
- const ulong bias_base = (ulong)expert_idx * (ulong)d_model;
-
- const uint sg_col_count = Bn / SgBn;
- const uint row_sg = sg_id / sg_col_count;
- const uint col_sg = sg_id % sg_col_count;
- const uint row_sg_off = row_sg * SgBm;
- const uint col_sg_off = col_sg * SgBn;
-
- constexpr uint SG_TILE = 8;
- constexpr uint TEMP = (PASSB_SG_BM / SG_TILE) * (PASSB_SG_BN / SG_TILE);
- metal::simdgroup_float8x8 Out[TEMP];
- for (uint i = 0; i < TEMP; ++i) {
- Out[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
- }
-
- const uint threads_total = threads_per_tg.x * threads_per_tg.y * threads_per_tg.z;
- const uint lin = linear_tid(local_tid, threads_per_tg);
-
- // K-loop across FF
- for (uint k_off = 0; k_off < d_ff; k_off += Bk) {
- const uint valid_k = min(Bk, d_ff - k_off);
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // ---- Stage hidden [Bm,Bk] as float4 ----
- {
- constexpr uint row_stride = PASSB_BK;
- constexpr uint vec_cols = PASSB_BK / 4;
- const uint total_vecs = PASSB_BM * vec_cols;
- const uint iters = ceil_div(total_vecs, threads_total);
-
-UZU_PRAGMA_UNROLL
- for (uint t = 0; t < iters; ++t) {
- const uint i = t * threads_total + lin;
- if (i < total_vecs) {
- const uint r = i / vec_cols;
- const uint c4 = i % vec_cols;
-
- threadgroup float4* dst4 =
- reinterpret_cast<threadgroup float4*>(Hs + r * row_stride + (c4 << 2));
-
- float4 v = float4(0.0f);
- if (r < m_rows) {
- const uint base_vec = (c4 << 2);
- const ulong base = h_base + (ulong)r * (ulong)d_ff + (ulong)(k_off + base_vec);
- if (base_vec + 0u < valid_k) v.x = hidden[base + 0];
- if (base_vec + 1u < valid_k) v.y = hidden[base + 1];
- if (base_vec + 2u < valid_k) v.z = hidden[base + 2];
- if (base_vec + 3u < valid_k) v.w = hidden[base + 3];
- }
- *dst4 = v;
- }
- }
- }
-
- // ---- Stage W2 [Bn,Bk] as StageT (column-major for simd transpose) ----
- {
- constexpr uint row_stride = PASSB_BK;
- constexpr uint vec_cols = PASSB_BK / 4;
- const uint total_vecs = PASSB_BN * vec_cols;
- const uint iters = ceil_div(total_vecs, threads_total);
-
-UZU_PRAGMA_UNROLL
- for (uint t = 0; t < iters; ++t) {
- const uint i = t * threads_total + lin;
- if (i < total_vecs) {
- const uint n = i / vec_cols;
- const uint c4 = i % vec_cols;
-
- threadgroup StageVec4* dst4 =
- reinterpret_cast<threadgroup StageVec4*>(Wk + n * row_stride + (c4 << 2));
-
- StageVec4 packed = StageVec4(StageT(0.0f));
- if (n < n_cols) {
- const uint gcol = col_tg_off + n;
- const ulong base = w2_expert_base + (ulong)gcol * (ulong)d_ff + (ulong)(k_off + (c4 << 2));
- const uint base_vec = (c4 << 2);
- float v0 = 0.0f, v1 = 0.0f, v2 = 0.0f, v3 = 0.0f;
- if (base_vec + 0u < valid_k) v0 = float(W2_all[base + 0]);
- if (base_vec + 1u < valid_k) v1 = float(W2_all[base + 1]);
- if (base_vec + 2u < valid_k) v2 = float(W2_all[base + 2]);
- if (base_vec + 3u < valid_k) v3 = float(W2_all[base + 3]);
- packed = StageVec4(StageT(v0), StageT(v1), StageT(v2), StageT(v3));
- }
- *dst4 = packed;
- }
- }
- }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // ---- MMA ----
-UZU_PRAGMA_UNROLL
- for (uint kk = 0; kk < PASSB_BK; kk += SG_TILE) {
-UZU_PRAGMA_UNROLL
- for (uint m_sub = 0; m_sub < PASSB_SG_BM; m_sub += SG_TILE) {
- const uint r_idx = m_sub / SG_TILE;
- metal::simdgroup_float8x8 lhs;
- simdgroup_load(lhs, Hs, PASSB_BK, ulong2(kk, row_sg_off + m_sub));
-UZU_PRAGMA_UNROLL
- for (uint n_sub = 0; n_sub < PASSB_SG_BN; n_sub += SG_TILE) {
- const uint c_idx = n_sub / SG_TILE;
- const uint tile = r_idx * (PASSB_SG_BN / SG_TILE) + c_idx;
- metal::simdgroup_matrix<StageT, 8, 8> rhs;
- simdgroup_load(rhs, Wk, PASSB_BK, ulong2(kk, col_sg_off + n_sub), true);
- simdgroup_multiply_accumulate(Out[tile], lhs, rhs, Out[tile]);
- }
- }
+ // ---- Stage hidden [Bm,Bk] as float4 ----
+ {
+ constexpr uint row_stride = PASSB_BK;
+ constexpr uint vec_cols = PASSB_BK / 4;
+ const uint total_vecs = PASSB_BM * vec_cols;
+ const uint iters = ceil_div(total_vecs, threads_total);
+
+ UZU_PRAGMA_UNROLL
+ for (uint t = 0; t < iters; ++t) {
+ const uint i = t * threads_total + lin;
+ if (i < total_vecs) {
+ const uint r = i / vec_cols;
+ const uint c4 = i % vec_cols;
+
+ threadgroup float4* dst4 = reinterpret_cast<threadgroup float4*>(
+ Hs + r * row_stride + (c4 << 2)
+ );
+
+ float4 v = float4(0.0f);
+ if (r < m_rows) {
+ const uint base_vec = (c4 << 2);
+ const ulong base =
+ h_base + (ulong)r * (ulong)d_ff + (ulong)(k_off + base_vec);
+ if (base_vec + 0u < valid_k)
+ v.x = hidden[base + 0];
+ if (base_vec + 1u < valid_k)
+ v.y = hidden[base + 1];
+ if (base_vec + 2u < valid_k)
+ v.z = hidden[base + 2];
+ if (base_vec + 3u < valid_k)
+ v.w = hidden[base + 3];
+ }
+ *dst4 = v;
}
+ }
}
- // ---- Bias tile ----
- for (uint c_local = lin; c_local < Bn; c_local += threads_total) {
- const uint c_global = col_tg_off + c_local;
- bias_tile[c_local] = (c_global < d_model) ? float(down_biases[bias_base + c_global]) : 0.0f;
+ // ---- Stage W2 [Bn,Bk] as StageT (column-major for simd transpose) ----
+ {
+ constexpr uint row_stride = PASSB_BK;
+ constexpr uint vec_cols = PASSB_BK / 4;
+ const uint total_vecs = PASSB_BN * vec_cols;
+ const uint iters = ceil_div(total_vecs, threads_total);
+
+ UZU_PRAGMA_UNROLL
+ for (uint t = 0; t < iters; ++t) {
+ const uint i = t * threads_total + lin;
+ if (i < total_vecs) {
+ const uint n = i / vec_cols;
+ const uint c4 = i % vec_cols;
+
+ threadgroup StageVec4* dst4 =
+ reinterpret_cast<threadgroup StageVec4*>(
+ Wk + n * row_stride + (c4 << 2)
+ );
+
+ StageVec4 packed = StageVec4(StageT(0.0f));
+ if (n < n_cols) {
+ const uint gcol = col_tg_off + n;
+ const ulong base = w2_expert_base + (ulong)gcol * (ulong)d_ff +
+ (ulong)(k_off + (c4 << 2));
+ const uint base_vec = (c4 << 2);
+ float v0 = 0.0f, v1 = 0.0f, v2 = 0.0f, v3 = 0.0f;
+ if (base_vec + 0u < valid_k)
+ v0 = float(W2_all[base + 0]);
+ if (base_vec + 1u < valid_k)
+ v1 = float(W2_all[base + 1]);
+ if (base_vec + 2u < valid_k)
+ v2 = float(W2_all[base + 2]);
+ if (base_vec + 3u < valid_k)
+ v3 = float(W2_all[base + 3]);
+ packed = StageVec4(StageT(v0), StageT(v1), StageT(v2), StageT(v3));
+ }
+ *dst4 = packed;
+ }
+ }
}
threadgroup_barrier(mem_flags::mem_threadgroup);
- // ---- Writeout ----
- const uint sg_lane_start = sg_id * 32;
- const uint sg_lane_end = sg_lane_start + 32;
- if (lin < sg_lane_start || lin >= sg_lane_end) return;
- const uint sg_lin = lin - sg_lane_start;
-
- // Lane→(row,col) mapping for 8x8 simdgroup fragments, matches PASSB fragment packing.
- const uint lane_qid = sg_lin >> 2;
- const uint lane_row = (lane_qid & 4u) + ((sg_lin >> 1) & 3u);
- const uint lane_col_base = ((lane_qid & 2u) << 1) + ((sg_lin & 1u) << 1);
-
-UZU_PRAGMA_UNROLL
- for (uint n_sub = 0; n_sub < PASSB_SG_BN; n_sub += SG_TILE) {
- const uint c_idx = n_sub / SG_TILE;
-UZU_PRAGMA_UNROLL
- for (uint m_sub = 0; m_sub < PASSB_SG_BM; m_sub += SG_TILE) {
- const uint r_idx = m_sub / SG_TILE;
- const uint tile = r_idx * (PASSB_SG_BN / SG_TILE) + c_idx;
-
- const uint tile_row_base = row_sg_off + m_sub;
- const uint tile_col_base = col_sg_off + n_sub;
- const uint local_row = tile_row_base + lane_row;
- if (local_row >= m_rows || tile_col_base >= n_cols) {
- continue;
- }
-
- const auto accum = Out[tile].thread_elements();
-
- const uint col0 = tile_col_base + lane_col_base;
- const uint col1 = col0 + 1u;
- const ulong out_row = seg_start + row_tg_off + local_row;
-
- if (col0 < n_cols) {
- const ulong out_col = col_tg_off + col0;
- const float val = accum[0] + bias_tile[col0];
- output[out_row * (ulong)d_model + out_col] = T(val);
- }
-
- if (col1 < n_cols) {
- const ulong out_col = col_tg_off + col1;
- const float val = accum[1] + bias_tile[col1];
- output[out_row * (ulong)d_model + out_col] = T(val);
- }
+ // ---- MMA ----
+ UZU_PRAGMA_UNROLL
+ for (uint kk = 0; kk < PASSB_BK; kk += SG_TILE) {
+ UZU_PRAGMA_UNROLL
+ for (uint m_sub = 0; m_sub < PASSB_SG_BM; m_sub += SG_TILE) {
+ const uint r_idx = m_sub / SG_TILE;
+ metal::simdgroup_float8x8 lhs;
+ simdgroup_load(lhs, Hs, PASSB_BK, ulong2(kk, row_sg_off + m_sub));
+ UZU_PRAGMA_UNROLL
+ for (uint n_sub = 0; n_sub < PASSB_SG_BN; n_sub += SG_TILE) {
+ const uint c_idx = n_sub / SG_TILE;
+ const uint tile = r_idx * (PASSB_SG_BN / SG_TILE) + c_idx;
+ metal::simdgroup_matrix<StageT, 8, 8> rhs;
+ simdgroup_load(
+ rhs,
+ Wk,
+ PASSB_BK,
+ ulong2(kk, col_sg_off + n_sub),
+ true
+ );
+ simdgroup_multiply_accumulate(Out[tile], lhs, rhs, Out[tile]);
}
+ }
}
+ }
+
+ // ---- Bias tile ----
+ for (uint c_local = lin; c_local < Bn; c_local += threads_total) {
+ const uint c_global = col_tg_off + c_local;
+ bias_tile[c_local] =
+ (c_global < d_model) ? float(down_biases[bias_base + c_global]) : 0.0f;
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // ---- Writeout ----
+ const uint sg_lane_start = sg_id * 32;
+ const uint sg_lane_end = sg_lane_start + 32;
+ if (lin < sg_lane_start || lin >= sg_lane_end)
+ return;
+ const uint sg_lin = lin - sg_lane_start;
+
+ // Lane→(row,col) mapping for 8x8 simdgroup fragments, matches PASSB fragment
+ // packing.
+ const uint lane_qid = sg_lin >> 2;
+ const uint lane_row = (lane_qid & 4u) + ((sg_lin >> 1) & 3u);
+ const uint lane_col_base = ((lane_qid & 2u) << 1) + ((sg_lin & 1u) << 1);
+
+ UZU_PRAGMA_UNROLL
+ for (uint n_sub = 0; n_sub < PASSB_SG_BN; n_sub += SG_TILE) {
+ const uint c_idx = n_sub / SG_TILE;
+ UZU_PRAGMA_UNROLL
+ for (uint m_sub = 0; m_sub < PASSB_SG_BM; m_sub += SG_TILE) {
+ const uint r_idx = m_sub / SG_TILE;
+ const uint tile = r_idx * (PASSB_SG_BN / SG_TILE) + c_idx;
+
+ const uint tile_row_base = row_sg_off + m_sub;
+ const uint tile_col_base = col_sg_off + n_sub;
+ const uint local_row = tile_row_base + lane_row;
+ if (local_row >= m_rows || tile_col_base >= n_cols) {
+ continue;
+ }
+
+ const auto accum = Out[tile].thread_elements();
+
+ const uint col0 = tile_col_base + lane_col_base;
+ const uint col1 = col0 + 1u;
+ const ulong out_row = seg_start + row_tg_off + local_row;
+
+ if (col0 < n_cols) {
+ const ulong out_col = col_tg_off + col0;
+ const float val = accum[0] + bias_tile[col0];
+ output[out_row * (ulong)d_model + out_col] = T(val);
+ }
+
+ if (col1 < n_cols) {
+ const ulong out_col = col_tg_off + col1;
+ const float val = accum[1] + bias_tile[col1];
+ output[out_row * (ulong)d_model + out_col] = T(val);
+ }
+ }
+ }
}
// ----- Kernel entry points (Pass B) -----
-#define MOE_PASS_B_KERNEL(DTYPE, SUFFIX) \
-kernel void moe_two_pass_prefill_pass_b_##SUFFIX( \
- device const float* hidden [[buffer(0)]], \
- device const uint* expert_offsets [[buffer(1)]], \
- device const DTYPE* W2_all [[buffer(2)]], \
- device const DTYPE* down_biases [[buffer(3)]], \
- device DTYPE* output [[buffer(4)]], \
- constant uint& d_model [[buffer(5)]], \
- constant uint& d_ff [[buffer(6)]], \
- constant uint& E [[buffer(7)]], \
- uint sg_id [[simdgroup_index_in_threadgroup]], \
- uint3 threads_per_tg [[threads_per_threadgroup]], \
- uint3 tg_pos [[threadgroup_position_in_grid]], \
- uint3 local_tid [[thread_position_in_threadgroup]]) \
-{ \
- using StageT = typename StageStorage<DTYPE>::type; \
- threadgroup float Hs [PASSB_BM * PASSB_BK]; \
- threadgroup StageT Wk [PASSB_BN * PASSB_BK]; \
- threadgroup float bias [PASSB_BN]; \
- pass_b_impl<DTYPE>( \
- hidden, expert_offsets, W2_all, down_biases, output, \
- d_model, d_ff, E, \
- /*expert*/ tg_pos.z, /*tile_m*/ tg_pos.y, /*tile_n*/ tg_pos.x, \
- sg_id, threads_per_tg, local_tid, \
- Hs, Wk, bias); \
-}
+#define MOE_PASS_B_KERNEL(DTYPE, SUFFIX) \
+ kernel void moe_two_pass_prefill_pass_b_##SUFFIX( \
+ device const float* hidden [[buffer(0)]], \
+ device const uint* expert_offsets [[buffer(1)]], \
+ device const DTYPE* W2_all [[buffer(2)]], \
+ device const DTYPE* down_biases [[buffer(3)]], \
+ device DTYPE* output [[buffer(4)]], \
+ constant uint& d_model [[buffer(5)]], \
+ constant uint& d_ff [[buffer(6)]], \
+ constant uint& E [[buffer(7)]], \
+ uint sg_id [[simdgroup_index_in_threadgroup]], \
+ uint3 threads_per_tg [[threads_per_threadgroup]], \
+ uint3 tg_pos [[threadgroup_position_in_grid]], \
+ uint3 local_tid [[thread_position_in_threadgroup]] \
+ ) { \
+ using StageT = typename StageStorage<DTYPE>::type; \
+ threadgroup float Hs[PASSB_BM * PASSB_BK]; \
+ threadgroup StageT Wk[PASSB_BN * PASSB_BK]; \
+ threadgroup float bias[PASSB_BN]; \
+ pass_b_impl<DTYPE>( \
+ hidden, \
+ expert_offsets, \
+ W2_all, \
+ down_biases, \
+ output, \
+ d_model, \
+ d_ff, \
+ E, \
+ /*expert*/ tg_pos.z, \
+ /*tile_m*/ tg_pos.y, \
+ /*tile_n*/ tg_pos.x, \
+ sg_id, \
+ threads_per_tg, \
+ local_tid, \
+ Hs, \
+ Wk, \
+ bias \
+ ); \
+ }
MOE_PASS_B_KERNEL(bfloat, bf16)
-MOE_PASS_B_KERNEL(half, f16)
-MOE_PASS_B_KERNEL(float, f32)
+MOE_PASS_B_KERNEL(half, f16)
+MOE_PASS_B_KERNEL(float, f32)
// Indirect variant
-template<typename T>
+template <typename T>
inline void pass_b_indirect_impl(
device const float* hidden,
- device const uint* expert_offsets,
- device const T* W2_all,
- device const T* down_biases,
- device T* output,
- device const uint* tile_map, // [tiles * 3]
- uint d_model, uint d_ff, uint E,
- uint row_tile_idx, uint n_tile_idx,
- uint sg_id, uint3 threads_per_tg, uint3 local_tid,
+ device const uint* expert_offsets,
+ device const T* W2_all,
+ device const T* down_biases,
+ device T* output,
+ device const uint* tile_map, // [tiles * 3]
+ uint d_model,
+ uint d_ff,
+ uint E,
+ uint row_tile_idx,
+ uint n_tile_idx,
+ uint sg_id,
+ uint3 threads_per_tg,
+ uint3 local_tid,
threadgroup float* Hs,
threadgroup typename StageStorage<T>::type* Wk,
- threadgroup float* bias_tile)
-{
- constexpr uint ROW_TILE = PASSB_BM;
- const uint base = row_tile_idx * 3u;
- const uint expert_idx = tile_map[base + 0u];
- if (expert_idx >= E) return;
- const uint row_off_elems = tile_map[base + 2u];
- const uint tile_m = row_off_elems / ROW_TILE;
-
- pass_b_impl<T>(
- hidden, expert_offsets, W2_all, down_biases, output,
- d_model, d_ff, E,
- expert_idx, tile_m, n_tile_idx,
- sg_id, threads_per_tg, local_tid,
- Hs, Wk, bias_tile);
+ threadgroup float* bias_tile
+) {
+ constexpr uint ROW_TILE = PASSB_BM;
+ const uint base = row_tile_idx * 3u;
+ const uint expert_idx = tile_map[base + 0u];
+ if (expert_idx >= E)
+ return;
+ const uint row_off_elems = tile_map[base + 2u];
+ const uint tile_m = row_off_elems / ROW_TILE;
+
+ pass_b_impl<T>(
+ hidden,
+ expert_offsets,
+ W2_all,
+ down_biases,
+ output,
+ d_model,
+ d_ff,
+ E,
+ expert_idx,
+ tile_m,
+ n_tile_idx,
+ sg_id,
+ threads_per_tg,
+ local_tid,
+ Hs,
+ Wk,
+ bias_tile
+ );
}
-#define MOE_PASS_B_INDIRECT_KERNEL(DTYPE, SUFFIX) \
-kernel void moe_two_pass_prefill_pass_b_indirect_##SUFFIX( \
- device const float* hidden [[buffer(0)]], \
- device const uint* expert_offsets [[buffer(1)]], \
- device const DTYPE* W2_all [[buffer(2)]], \
- device const DTYPE* down_biases [[buffer(3)]], \
- device DTYPE* output [[buffer(4)]], \
- constant uint& d_model [[buffer(5)]], \
- constant uint& d_ff [[buffer(6)]], \
- constant uint& E [[buffer(7)]], \
- device const uint* tile_map [[buffer(8)]], \
- uint sg_id [[simdgroup_index_in_threadgroup]], \
- uint3 threads_per_tg [[threads_per_threadgroup]], \
- uint3 tg_pos [[threadgroup_position_in_grid]], \
- uint3 local_tid [[thread_position_in_threadgroup]]) \
-{ \
- using StageT = typename StageStorage<DTYPE>::type; \
- threadgroup float Hs [PASSB_BM * PASSB_BK]; \
- threadgroup StageT Wk [PASSB_BN * PASSB_BK]; \
- threadgroup float bias [PASSB_BN]; \
- pass_b_indirect_impl<DTYPE>( \
- hidden, expert_offsets, W2_all, down_biases, output, tile_map, \
- d_model, d_ff, E, \
- /*row_tile_idx*/ tg_pos.y, /*n_tile_idx*/ tg_pos.x, \
- sg_id, threads_per_tg, local_tid, \
- Hs, Wk, bias); \
-}
+#define MOE_PASS_B_INDIRECT_KERNEL(DTYPE, SUFFIX) \
+ kernel void moe_two_pass_prefill_pass_b_indirect_##SUFFIX( \
+ device const float* hidden [[buffer(0)]], \
+ device const uint* expert_offsets [[buffer(1)]], \
+ device const DTYPE* W2_all [[buffer(2)]], \
+ device const DTYPE* down_biases [[buffer(3)]], \
+ device DTYPE* output [[buffer(4)]], \
+ constant uint& d_model [[buffer(5)]], \
+ constant uint& d_ff [[buffer(6)]], \
+ constant uint& E [[buffer(7)]], \
+ device const uint* tile_map [[buffer(8)]], \
+ uint sg_id [[simdgroup_index_in_threadgroup]], \
+ uint3 threads_per_tg [[threads_per_threadgroup]], \
+ uint3 tg_pos [[threadgroup_position_in_grid]], \
+ uint3 local_tid [[thread_position_in_threadgroup]] \
+ ) { \
+ using StageT = typename StageStorage<DTYPE>::type; \
+ threadgroup float Hs[PASSB_BM * PASSB_BK]; \
+ threadgroup StageT Wk[PASSB_BN * PASSB_BK]; \
+ threadgroup float bias[PASSB_BN]; \
+ pass_b_indirect_impl<DTYPE>( \
+ hidden, \
+ expert_offsets, \
+ W2_all, \
+ down_biases, \
+ output, \
+ tile_map, \
+ d_model, \
+ d_ff, \
+ E, \
+ /*row_tile_idx*/ tg_pos.y, \
+ /*n_tile_idx*/ tg_pos.x, \
+ sg_id, \
+ threads_per_tg, \
+ local_tid, \
+ Hs, \
+ Wk, \
+ bias \
+ ); \
+ }
MOE_PASS_B_INDIRECT_KERNEL(bfloat, bf16)
-MOE_PASS_B_INDIRECT_KERNEL(half, f16)
-MOE_PASS_B_INDIRECT_KERNEL(float, f32)
+MOE_PASS_B_INDIRECT_KERNEL(half, f16)
+MOE_PASS_B_INDIRECT_KERNEL(float, f32)
diff --git a/crates/uzu/src/backends/metal/kernel/moe/finalize.metal b/crates/uzu/src/backends/metal/kernel/moe/finalize.metal
index 978cec9ba..168394b91 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/finalize.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/finalize.metal
@@ -5,69 +5,72 @@ using namespace metal;
#define BM 32
#define BN 64
-template<typename T>
+template <typename T>
inline void moe_finalize_impl(
- device const int* tok2row, // [T*K], -1 if dropped
- device const T* probs, // [T*K]
- device const T* Y_partial, // [sum_k, d_model]
- device T* Y, // [T, d_model]
+ device const int* tok2row, // [T*K], -1 if dropped
+ device const T* probs, // [T*K]
+ device const T* Y_partial, // [sum_k, d_model]
+ device T* Y, // [T, d_model]
constant uint& T_count,
constant uint& d_model,
constant uint& K,
uint lid,
- uint3 tgpig)
-{
- if (T_count == 0u || d_model == 0u) return;
- const uint tile_m0 = tgpig.y * BM;
- const uint tile_n0 = tgpig.x * BN;
- if (tile_m0 >= T_count || tile_n0 >= d_model) return;
- const uint m_rows = min((uint)BM, T_count - tile_m0);
- const uint n_cols = min((uint)BN, d_model - tile_n0);
+ uint3 tgpig
+) {
+ if (T_count == 0u || d_model == 0u)
+ return;
+ const uint tile_m0 = tgpig.y * BM;
+ const uint tile_n0 = tgpig.x * BN;
+ if (tile_m0 >= T_count || tile_n0 >= d_model)
+ return;
+ const uint m_rows = min((uint)BM, T_count - tile_m0);
+ const uint n_cols = min((uint)BN, d_model - tile_n0);
- // 128 threads per TG expected
- for (uint idx = lid; idx < m_rows * n_cols; idx += 128u) {
- const uint mi = idx / n_cols;
- const uint nj = idx % n_cols;
- const uint t = tile_m0 + mi;
- const uint f = tile_n0 + nj;
- float acc = 0.0f;
- const uint base = t * K;
- for (uint k = 0; k < K; ++k) {
- const int row = tok2row[base + k];
- if (row >= 0) {
- const ulong yidx = (ulong)(uint)row * (ulong)d_model + (ulong)f;
- float prob = (float)probs[base + k];
- if (!isfinite(prob)) {
- prob = 0.0f;
- }
- float val = (float)Y_partial[yidx];
- if (!isfinite(val)) {
- val = 0.0f;
- }
- acc = fma(prob, val, acc);
- }
+ // 128 threads per TG expected
+ for (uint idx = lid; idx < m_rows * n_cols; idx += 128u) {
+ const uint mi = idx / n_cols;
+ const uint nj = idx % n_cols;
+ const uint t = tile_m0 + mi;
+ const uint f = tile_n0 + nj;
+ float acc = 0.0f;
+ const uint base = t * K;
+ for (uint k = 0; k < K; ++k) {
+ const int row = tok2row[base + k];
+ if (row >= 0) {
+ const ulong yidx = (ulong)(uint)row * (ulong)d_model + (ulong)f;
+ float prob = (float)probs[base + k];
+ if (!isfinite(prob)) {
+ prob = 0.0f;
}
- if (!isfinite(acc)) {
- acc = 0.0f;
+ float val = (float)Y_partial[yidx];
+ if (!isfinite(val)) {
+ val = 0.0f;
}
- Y[t * d_model + f] = T(acc);
+ acc = fma(prob, val, acc);
+ }
}
+ if (!isfinite(acc)) {
+ acc = 0.0f;
+ }
+ Y[t * d_model + f] = T(acc);
+ }
}
-#define DEFINE_MOE_FINALIZE_KERNEL(SUFFIX, T) \
-kernel void moe_finalize_##SUFFIX( \
- device const int* tok2row [[buffer(0)]], \
- device const T* probs [[buffer(1)]], \
- device const T* Y_partial [[buffer(2)]], \
- device T* Y [[buffer(3)]], \
- constant uint& T_count [[buffer(4)]], \
- constant uint& d_model [[buffer(5)]], \
- constant uint& K [[buffer(6)]], \
- uint lid [[thread_index_in_threadgroup]], \
- uint3 tgpig [[threadgroup_position_in_grid]]) \
-{ \
- moe_finalize_impl<T>(tok2row, probs, Y_partial, Y, T_count, d_model, K, lid, tgpig); \
-}
+#define DEFINE_MOE_FINALIZE_KERNEL(SUFFIX, T) \
+ kernel void moe_finalize_##SUFFIX( \
+ device const int* tok2row [[buffer(0)]], \
+ device const T* probs [[buffer(1)]], \
+ device const T* Y_partial [[buffer(2)]], \
+ device T* Y [[buffer(3)]], \
+ constant uint& T_count [[buffer(4)]], \
+ constant uint& d_model [[buffer(5)]], \
+ constant uint& K [[buffer(6)]], \
+ uint lid [[thread_index_in_threadgroup]], \
+ uint3 tgpig [[threadgroup_position_in_grid]] \
+ ) { \
+ moe_finalize_impl< \
+ T>(tok2row, probs, Y_partial, Y, T_count, d_model, K, lid, tgpig); \
+ }
DEFINE_MOE_FINALIZE_KERNEL(f16, half)
DEFINE_MOE_FINALIZE_KERNEL(bf16, bfloat)
diff --git a/crates/uzu/src/backends/metal/kernel/moe/prepare.metal b/crates/uzu/src/backends/metal/kernel/moe/prepare.metal
index 2499930b2..fb9589771 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/prepare.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/prepare.metal
@@ -3,11 +3,11 @@ using namespace metal;
// Tiling config for 2D gather
constant uint ROWS_PER_TG = 8u;
-constant uint COL_VECS = 32u; // 256 threads = 8×32
-constant uint ELEMS_PER_THREAD = 8u; // each thread copies 8 elements (2× vec4)
+constant uint COL_VECS = 32u; // 256 threads = 8×32
+constant uint ELEMS_PER_THREAD = 8u; // each thread copies 8 elements (2× vec4)
// ========== 2D Tiled Gather Template ==========
-template<typename T>
+template <typename T>
void moe_gather_x_perm_2d_impl(
device const T* X,
device const int* bucketed_ids,
@@ -15,119 +15,133 @@ void moe_gather_x_perm_2d_impl(
device const uint* sumk_buf,
uint d_model,
uint3 tgid,
- uint lid)
-{
- const uint total_rows = sumk_buf[0];
- if (total_rows == 0u || d_model == 0u) return;
-
- // Thread organization: lid = row_in_tg * COL_VECS + col_vec
- const uint row_in_tg = lid / COL_VECS;
- const uint col_vec = lid % COL_VECS;
+ uint lid
+) {
+ const uint total_rows = sumk_buf[0];
+ if (total_rows == 0u || d_model == 0u)
+ return;
+
+ // Thread organization: lid = row_in_tg * COL_VECS + col_vec
+ const uint row_in_tg = lid / COL_VECS;
+ const uint col_vec = lid % COL_VECS;
+
+ // Global row assignment
+ const uint base_row = tgid.x * ROWS_PER_TG;
+ const uint row = base_row + row_in_tg;
+
+ if (row >= total_rows)
+ return;
+
+ int token = bucketed_ids[row];
+ if (token < 0)
+ return;
+
+ const device T* src = X + (ulong)(uint)token * (ulong)d_model;
+ device T* dst = x_perm + (ulong)row * (ulong)d_model;
+
+ // Each thread copies ELEMS_PER_THREAD elements
+ // Thread i processes: col_vec*8, col_vec*8+256, col_vec*8+512, ...
+ uint col = col_vec * ELEMS_PER_THREAD;
+ const uint stride = COL_VECS * ELEMS_PER_THREAD;
+
+ // Vectorized copy: 8 elements = 2× vec4
+ using Vec4 = typename metal::vec<T, 4>;
+ while (col + 7u < d_model) {
+ Vec4 v0 = *(reinterpret_cast<const device Vec4*>(src + col));
+ Vec4 v1 = *(reinterpret_cast<const device Vec4*>(src + col + 4u));
+ *(reinterpret_cast<device Vec4*>(dst + col)) = v0;
+ *(reinterpret_cast<device Vec4*>(dst + col + 4u)) = v1;
+ col += stride;
+ }
+
+ // Scalar remainder
+ if (col < d_model) {
+ const uint limit = min(d_model, col + ELEMS_PER_THREAD);
+ for (uint tail = col; tail < limit; ++tail) {
+ dst[tail] = src[tail];
+ }
+ }
+}
- // Global row assignment
- const uint base_row = tgid.x * ROWS_PER_TG;
- const uint row = base_row + row_in_tg;
+// ========== 1D Gather Template ==========
+template <typename T>
+void moe_gather_x_perm_1d_impl(
+ device const T* X,
+ device const int* bucketed_ids,
+ device T* x_perm,
+ device const uint* sumk_buf,
+ uint d_model,
+ uint lid,
+ uint threads_x
+) {
+ const uint total_rows = sumk_buf[0];
+ if (total_rows == 0u || d_model == 0u)
+ return;
- if (row >= total_rows) return;
+ const uint num_vec4 = d_model / 4u;
+ // Each thread processes multiple rows sequentially
+ for (uint row = lid; row < total_rows; row += threads_x) {
int token = bucketed_ids[row];
- if (token < 0) return;
+ if (token < 0)
+ continue;
const device T* src = X + (ulong)(uint)token * (ulong)d_model;
device T* dst = x_perm + (ulong)row * (ulong)d_model;
- // Each thread copies ELEMS_PER_THREAD elements
- // Thread i processes: col_vec*8, col_vec*8+256, col_vec*8+512, ...
- uint col = col_vec * ELEMS_PER_THREAD;
- const uint stride = COL_VECS * ELEMS_PER_THREAD;
-
- // Vectorized copy: 8 elements = 2× vec4
+ // Vectorized copy: 4 elements at a time
using Vec4 = typename metal::vec<T, 4>;
- while (col + 7u < d_model) {
- Vec4 v0 = *(reinterpret_cast<const device Vec4*>(src + col));
- Vec4 v1 = *(reinterpret_cast<const device Vec4*>(src + col + 4u));
- *(reinterpret_cast<device Vec4*>(dst + col)) = v0;
- *(reinterpret_cast<device Vec4*>(dst + col + 4u)) = v1;
- col += stride;
+ for (uint vec_idx = 0u; vec_idx < num_vec4; vec_idx++) {
+ uint col = vec_idx * 4u;
+ Vec4 v = *(reinterpret_cast<const device Vec4*>(src + col));
+ *(reinterpret_cast<device Vec4*>(dst + col)) = v;
}
// Scalar remainder
- if (col < d_model) {
- const uint limit = min(d_model, col + ELEMS_PER_THREAD);
- for (uint tail = col; tail < limit; ++tail) {
- dst[tail] = src[tail];
- }
- }
-}
-
-// ========== 1D Gather Template ==========
-template<typename T>
-void moe_gather_x_perm_1d_impl(
- device const T* X,
- device const int* bucketed_ids,
- device T* x_perm,
- device const uint* sumk_buf,
- uint d_model,
- uint lid,
- uint threads_x)
-{
- const uint total_rows = sumk_buf[0];
- if (total_rows == 0u || d_model == 0u) return;
-
- const uint num_vec4 = d_model / 4u;
-
- // Each thread processes multiple rows sequentially
- for (uint row = lid; row < total_rows; row += threads_x) {
- int token = bucketed_ids[row];
- if (token < 0) continue;
-
- const device T* src = X + (ulong)(uint)token * (ulong)d_model;
- device T* dst = x_perm + (ulong)row * (ulong)d_model;
-
- // Vectorized copy: 4 elements at a time
- using Vec4 = typename metal::vec<T, 4>;
- for (uint vec_idx = 0u; vec_idx < num_vec4; vec_idx++) {
- uint col = vec_idx * 4u;
- Vec4 v = *(reinterpret_cast<const device Vec4*>(src + col));
- *(reinterpret_cast<device Vec4*>(dst + col)) = v;
- }
-
- // Scalar remainder
- for (uint col = num_vec4 * 4u; col < d_model; col++) {
- dst[col] = src[col];
- }
+ for (uint col = num_vec4 * 4u; col < d_model; col++) {
+ dst[col] = src[col];
}
+ }
}
// ========== Kernel Entry Points ==========
// 2D Tiled kernels
-#define GATHER_2D_KERNEL(DTYPE, SUFFIX) \
-kernel void moe_gather_x_perm_##SUFFIX##_2d( \
- device const DTYPE* X [[buffer(0)]], \
- device const int* bucketed_ids [[buffer(1)]], \
- device DTYPE* x_perm [[buffer(2)]], \
- device const uint* sumk_buf [[buffer(3)]], \
- constant uint& d_model [[buffer(4)]], \
- uint3 tgid [[threadgroup_position_in_grid]], \
- uint lid [[thread_index_in_threadgroup]]) \
-{ \
- moe_gather_x_perm_2d_impl<DTYPE>(X, bucketed_ids, x_perm, sumk_buf, d_model, tgid, lid); \
-}
+#define GATHER_2D_KERNEL(DTYPE, SUFFIX) \
+ kernel void moe_gather_x_perm_##SUFFIX##_2d( \
+ device const DTYPE* X [[buffer(0)]], \
+ device const int* bucketed_ids [[buffer(1)]], \
+ device DTYPE* x_perm [[buffer(2)]], \
+ device const uint* sumk_buf [[buffer(3)]], \
+ constant uint& d_model [[buffer(4)]], \
+ uint3 tgid [[threadgroup_position_in_grid]], \
+ uint lid [[thread_index_in_threadgroup]] \
+ ) { \
+ moe_gather_x_perm_2d_impl< \
+ DTYPE>(X, bucketed_ids, x_perm, sumk_buf, d_model, tgid, lid); \
+ }
// 1D kernels
-#define GATHER_1D_KERNEL(DTYPE, SUFFIX) \
-kernel void moe_gather_x_perm_##SUFFIX( \
- device const DTYPE* X [[buffer(0)]], \
- device const int* bucketed_ids [[buffer(1)]], \
- device DTYPE* x_perm [[buffer(2)]], \
- device const uint* sumk_buf [[buffer(3)]], \
- constant uint& d_model [[buffer(4)]], \
- uint lid [[thread_index_in_threadgroup]], \
- uint3 threads_per_threadgroup [[threads_per_threadgroup]]) \
-{ \
- moe_gather_x_perm_1d_impl<DTYPE>(X, bucketed_ids, x_perm, sumk_buf, d_model, lid, threads_per_threadgroup.x); \
-}
+#define GATHER_1D_KERNEL(DTYPE, SUFFIX) \
+ kernel void moe_gather_x_perm_##SUFFIX( \
+ device const DTYPE* X [[buffer(0)]], \
+ device const int* bucketed_ids [[buffer(1)]], \
+ device DTYPE* x_perm [[buffer(2)]], \
+ device const uint* sumk_buf [[buffer(3)]], \
+ constant uint& d_model [[buffer(4)]], \
+ uint lid [[thread_index_in_threadgroup]], \
+ uint3 threads_per_threadgroup [[threads_per_threadgroup]] \
+ ) { \
+ moe_gather_x_perm_1d_impl<DTYPE>( \
+ X, \
+ bucketed_ids, \
+ x_perm, \
+ sumk_buf, \
+ d_model, \
+ lid, \
+ threads_per_threadgroup.x \
+ ); \
+ }
// Instantiate for all types
GATHER_2D_KERNEL(bfloat, bf16)
diff --git a/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal b/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal
index 25b41db03..7b19cf3ce 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal
@@ -10,11 +10,11 @@ constant float NEG_INF = -INFINITY;
template <typename Vec4T, typename ScalarT, typename ProbT>
inline void moe_router_topk_impl(
- const device Vec4T* input, // [T, d_model/4]
- const device Vec4T* weight, // [E, d_model/4]
- const device ScalarT* bias, // [E]
- device int* topk_ids, // [T, K]
- device ProbT* topk_probs, // [T, K]
+ const device Vec4T* input, // [T, d_model/4]
+ const device Vec4T* weight, // [E, d_model/4]
+ const device ScalarT* bias, // [E]
+ device int* topk_ids, // [T, K]
+ device ProbT* topk_probs, // [T, K]
uint T,
uint d_model,
uint E,
@@ -31,152 +31,168 @@ inline void moe_router_topk_impl(
threadgroup float* reduce_tmp,
threadgroup uint* reduce_tmp_u,
threadgroup uint* shared_best_idx,
- threadgroup float* shared_best_val)
-{
- const uint token_idx = tgpig.y;
- if (token_idx >= T || d_model == 0 || E == 0 || K == 0) {
- return;
- }
+ threadgroup float* shared_best_val
+) {
+ const uint token_idx = tgpig.y;
+ if (token_idx >= T || d_model == 0 || E == 0 || K == 0) {
+ return;
+ }
- const uint vecs = d_model / 4u;
- const device Vec4T* x_vec =
- input + (ulong)token_idx * (ulong)vecs;
+ const uint vecs = d_model / 4u;
+ const device Vec4T* x_vec = input + (ulong)token_idx * (ulong)vecs;
- for (uint c = lid; c < vecs; c += THREADS_PER_TG) {
- x_cache[c] = float4(x_vec[c]);
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
+ for (uint c = lid; c < vecs; c += THREADS_PER_TG) {
+ x_cache[c] = float4(x_vec[c]);
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
- for (uint row = simdgroup_idx; row < E; row += simdgroups_per_tg) {
- const device Vec4T* w_vec =
- weight + (ulong)row * (ulong)vecs;
+ for (uint row = simdgroup_idx; row < E; row += simdgroups_per_tg) {
+ const device Vec4T* w_vec = weight + (ulong)row * (ulong)vecs;
- float4 accum4 = float4(0.0f);
- for (uint c = simd_lane; c < vecs; c += 32u) {
- const float4 wv = float4(w_vec[c]);
- const float4 xv = x_cache[c];
- accum4 = fma(wv, xv, accum4);
- }
- float sum = (accum4.x + accum4.y) + (accum4.z + accum4.w);
- sum = simd_sum(sum);
- if (simd_is_first()) {
- sum += float(bias[row]);
- logits_shared[row] = sum;
- idx_shared[row] = row;
- }
+ float4 accum4 = float4(0.0f);
+ for (uint c = simd_lane; c < vecs; c += 32u) {
+ const float4 wv = float4(w_vec[c]);
+ const float4 xv = x_cache[c];
+ accum4 = fma(wv, xv, accum4);
}
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- for (uint row = lid + E; row < MAX_EXPERTS; row += THREADS_PER_TG) {
- logits_shared[row] = NEG_INF;
- idx_shared[row] = row;
+ float sum = (accum4.x + accum4.y) + (accum4.z + accum4.w);
+ sum = simd_sum(sum);
+ if (simd_is_first()) {
+ sum += float(bias[row]);
+ logits_shared[row] = sum;
+ idx_shared[row] = row;
}
- threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
- const uint effective_k = min(K, MAX_TOPK);
- for (uint sel = 0; sel < effective_k; ++sel) {
- float local_best = NEG_INF;
- uint local_idx = 0xFFFFFFFFu;
- for (uint row = lid; row < E; row += THREADS_PER_TG) {
- float candidate = logits_shared[row];
- if (candidate > local_best ||
- (candidate == local_best && row < local_idx)) {
- local_best = candidate;
- local_idx = row;
- }
- }
+ for (uint row = lid + E; row < MAX_EXPERTS; row += THREADS_PER_TG) {
+ logits_shared[row] = NEG_INF;
+ idx_shared[row] = row;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
- float max_val =
- threadgroup_cooperative_reduce_max<THREADS_PER_TG>(
- local_best,
- reduce_tmp,
- lid);
+ const uint effective_k = min(K, MAX_TOPK);
+ for (uint sel = 0; sel < effective_k; ++sel) {
+ float local_best = NEG_INF;
+ uint local_idx = 0xFFFFFFFFu;
+ for (uint row = lid; row < E; row += THREADS_PER_TG) {
+ float candidate = logits_shared[row];
+ if (candidate > local_best ||
+ (candidate == local_best && row < local_idx)) {
+ local_best = candidate;
+ local_idx = row;
+ }
+ }
- uint candidate_id =
- (local_best == max_val) ? local_idx : 0xFFFFFFFFu;
- uint best_idx =
- threadgroup_cooperative_reduce_min<THREADS_PER_TG>(
- candidate_id,
- reduce_tmp_u,
- lid);
+ float max_val = threadgroup_cooperative_reduce_max<THREADS_PER_TG>(
+ local_best,
+ reduce_tmp,
+ lid
+ );
- if (lid == 0) {
- *shared_best_idx = best_idx;
- *shared_best_val = max_val;
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
+ uint candidate_id = (local_best == max_val) ? local_idx : 0xFFFFFFFFu;
+ uint best_idx = threadgroup_cooperative_reduce_min<THREADS_PER_TG>(
+ candidate_id,
+ reduce_tmp_u,
+ lid
+ );
- uint winner_idx = *shared_best_idx;
- float winner_val = *shared_best_val;
- if (winner_idx < MAX_EXPERTS) {
- if (lid == 0) {
- logits_shared[winner_idx] = NEG_INF;
- idx_shared[winner_idx] = 0xFFFFFFFFu;
- device int* out_ids = topk_ids + token_idx * K;
- device ProbT* out_probs = topk_probs + token_idx * K;
- out_ids[sel] = int(winner_idx);
- out_probs[sel] = ProbT(winner_val);
- }
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (lid == 0) {
+ *shared_best_idx = best_idx;
+ *shared_best_val = max_val;
}
+ threadgroup_barrier(mem_flags::mem_threadgroup);
- if (lid == 0 && renorm != 0 && effective_k > 0) {
+ uint winner_idx = *shared_best_idx;
+ float winner_val = *shared_best_val;
+ if (winner_idx < MAX_EXPERTS) {
+ if (lid == 0) {
+ logits_shared[winner_idx] = NEG_INF;
+ idx_shared[winner_idx] = 0xFFFFFFFFu;
+ device int* out_ids = topk_ids + token_idx * K;
device ProbT* out_probs = topk_probs + token_idx * K;
- float max_logit = -INFINITY;
- for (uint i = 0; i < effective_k; ++i) {
- max_logit = fmax(max_logit, float(out_probs[i]));
- }
- float sum_exp = 0.0f;
- for (uint i = 0; i < effective_k; ++i) {
- sum_exp += exp(float(out_probs[i]) - max_logit);
- }
- float default_prob = 1.0f / float(effective_k);
- float inv_sum = (sum_exp > 0.0f) ? (1.0f / sum_exp) : default_prob;
- for (uint i = 0; i < effective_k; ++i) {
- float prob = (sum_exp > 0.0f)
- ? exp(float(out_probs[i]) - max_logit) * inv_sum
- : default_prob;
- out_probs[i] = ProbT(prob);
- }
- for (uint i = effective_k; i < K; ++i) {
- out_probs[i] = ProbT(0.0f);
- }
+ out_ids[sel] = int(winner_idx);
+ out_probs[sel] = ProbT(winner_val);
+ }
}
-}
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
-#define DEFINE_ROUTER_TOPK_KERNEL(SUFFIX, VEC4, SCALAR, PROB) \
-kernel void moe_router_topk_##SUFFIX( \
- const device VEC4* input [[buffer(0)]], \
- const device VEC4* weight [[buffer(1)]], \
- const device SCALAR* bias [[buffer(2)]], \
- device int* topk_ids [[buffer(3)]], \
- device PROB* topk_probs [[buffer(4)]], \
- constant uint& T [[buffer(5)]], \
- constant uint& d_model [[buffer(6)]], \
- constant uint& E [[buffer(7)]], \
- constant uint& K [[buffer(8)]], \
- constant uint& renorm [[buffer(9)]], \
- uint2 tgpig [[threadgroup_position_in_grid]], \
- ushort lid [[thread_index_in_threadgroup]], \
- ushort simd_lane [[thread_index_in_simdgroup]], \
- ushort simdgroup_idx [[simdgroup_index_in_threadgroup]], \
- ushort simdgroups_per_tg [[simdgroups_per_threadgroup]]) \
-{ \
- threadgroup float4 x_cache[1024]; \
- threadgroup float logits_shared[MAX_EXPERTS]; \
- threadgroup uint idx_shared[MAX_EXPERTS]; \
- threadgroup float reduce_tmp[THREADS_PER_TG]; \
- threadgroup uint reduce_tmp_u[THREADS_PER_TG]; \
- threadgroup uint shared_best_idx_mem; \
- threadgroup float shared_best_val_mem; \
- moe_router_topk_impl<VEC4, SCALAR, PROB>( \
- input, weight, bias, topk_ids, topk_probs, \
- T, d_model, E, K, renorm, tgpig, lid, simd_lane, simdgroup_idx, simdgroups_per_tg, \
- x_cache, logits_shared, idx_shared, reduce_tmp, reduce_tmp_u, \
- &shared_best_idx_mem, &shared_best_val_mem); \
+ if (lid == 0 && renorm != 0 && effective_k > 0) {
+ device ProbT* out_probs = topk_probs + token_idx * K;
+ float max_logit = -INFINITY;
+ for (uint i = 0; i < effective_k; ++i) {
+ max_logit = fmax(max_logit, float(out_probs[i]));
+ }
+ float sum_exp = 0.0f;
+ for (uint i = 0; i < effective_k; ++i) {
+ sum_exp += exp(float(out_probs[i]) - max_logit);
+ }
+ float default_prob = 1.0f / float(effective_k);
+ float inv_sum = (sum_exp > 0.0f) ? (1.0f / sum_exp) : default_prob;
+ for (uint i = 0; i < effective_k; ++i) {
+ float prob = (sum_exp > 0.0f)
+ ? exp(float(out_probs[i]) - max_logit) * inv_sum
+ : default_prob;
+ out_probs[i] = ProbT(prob);
+ }
+ for (uint i = effective_k; i < K; ++i) {
+ out_probs[i] = ProbT(0.0f);
+ }
+ }
}
+#define DEFINE_ROUTER_TOPK_KERNEL(SUFFIX, VEC4, SCALAR, PROB) \
+ kernel void moe_router_topk_##SUFFIX( \
+ const device VEC4* input [[buffer(0)]], \
+ const device VEC4* weight [[buffer(1)]], \
+ const device SCALAR* bias [[buffer(2)]], \
+ device int* topk_ids [[buffer(3)]], \
+ device PROB* topk_probs [[buffer(4)]], \
+ constant uint& T [[buffer(5)]], \
+ constant uint& d_model [[buffer(6)]], \
+ constant uint& E [[buffer(7)]], \
+ constant uint& K [[buffer(8)]], \
+ constant uint& renorm [[buffer(9)]], \
+ uint2 tgpig [[threadgroup_position_in_grid]], \
+ ushort lid [[thread_index_in_threadgroup]], \
+ ushort simd_lane [[thread_index_in_simdgroup]], \
+ ushort simdgroup_idx [[simdgroup_index_in_threadgroup]], \
+ ushort simdgroups_per_tg [[simdgroups_per_threadgroup]] \
+ ) { \
+ threadgroup float4 x_cache[1024]; \
+ threadgroup float logits_shared[MAX_EXPERTS]; \
+ threadgroup uint idx_shared[MAX_EXPERTS]; \
+ threadgroup float reduce_tmp[THREADS_PER_TG]; \
+ threadgroup uint reduce_tmp_u[THREADS_PER_TG]; \
+ threadgroup uint shared_best_idx_mem; \
+ threadgroup float shared_best_val_mem; \
+ moe_router_topk_impl<VEC4, SCALAR, PROB>( \
+ input, \
+ weight, \
+ bias, \
+ topk_ids, \
+ topk_probs, \
+ T, \
+ d_model, \
+ E, \
+ K, \
+ renorm, \
+ tgpig, \
+ lid, \
+ simd_lane, \
+ simdgroup_idx, \
+ simdgroups_per_tg, \
+ x_cache, \
+ logits_shared, \
+ idx_shared, \
+ reduce_tmp, \
+ reduce_tmp_u, \
+ &shared_best_idx_mem, \
+ &shared_best_val_mem \
+ ); \
+ }
+
DEFINE_ROUTER_TOPK_KERNEL(f16, half4, half, half)
DEFINE_ROUTER_TOPK_KERNEL(bf16, bfloat4, bfloat, bfloat)
DEFINE_ROUTER_TOPK_KERNEL(f32, float4, float, float)
diff --git a/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal b/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal
index d55ca893b..3a2245045 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal
@@ -17,34 +17,39 @@ kernel void moe_block_bases_from_partials(
constant uint& num_blocks [[buffer(4)]],
constant uint& num_tiles [[buffer(5)]],
constant uint& capacity_per_expert [[buffer(6)]], // 0 => no clamp
- uint gid [[thread_position_in_grid]])
-{
- const uint entries = num_tiles * TILE_E;
- if (gid >= entries) return;
- const uint tile_id = gid / TILE_E;
- const uint te = gid % TILE_E;
- const uint e = tile_id * TILE_E + te;
- if (e >= E) return;
+ uint gid [[thread_position_in_grid]]
+) {
+ const uint entries = num_tiles * TILE_E;
+ if (gid >= entries)
+ return;
+ const uint tile_id = gid / TILE_E;
+ const uint te = gid % TILE_E;
+ const uint e = tile_id * TILE_E + te;
+ if (e >= E)
+ return;
- uint running = 0u;
- const bool clamp = (capacity_per_expert > 0u);
- for (uint b = 0; b < num_blocks; ++b) {
- const uint idx = (b * num_tiles + tile_id) * TILE_E + te;
- const uint p = partials[idx];
- const uint alloc = clamp ? (running < capacity_per_expert ? min(p, capacity_per_expert - running) : 0u) : p;
- block_bases[idx] = running;
- block_alloc[idx] = alloc;
- running += alloc;
- if (clamp && running >= capacity_per_expert) {
- // Remainder allocate zero and base stays at capacity
- for (uint bb = b + 1; bb < num_blocks; ++bb) {
- const uint idx2 = (bb * num_tiles + tile_id) * TILE_E + te;
- block_bases[idx2] = running;
- block_alloc[idx2] = 0u;
- }
- break;
- }
+ uint running = 0u;
+ const bool clamp = (capacity_per_expert > 0u);
+ for (uint b = 0; b < num_blocks; ++b) {
+ const uint idx = (b * num_tiles + tile_id) * TILE_E + te;
+ const uint p = partials[idx];
+ const uint alloc = clamp ? (running < capacity_per_expert
+ ? min(p, capacity_per_expert - running)
+ : 0u)
+ : p;
+ block_bases[idx] = running;
+ block_alloc[idx] = alloc;
+ running += alloc;
+ if (clamp && running >= capacity_per_expert) {
+ // Remainder allocate zero and base stays at capacity
+ for (uint bb = b + 1; bb < num_blocks; ++bb) {
+ const uint idx2 = (bb * num_tiles + tile_id) * TILE_E + te;
+ block_bases[idx2] = running;
+ block_alloc[idx2] = 0u;
+ }
+ break;
}
+ }
}
// 4B: Scatter into expert-major buckets using atomic-free device writes
@@ -58,155 +63,219 @@ inline void moe_scatter_buckets_impl(
device int* out_ids,
device ProbT* out_probs,
device int* tok2row_opt,
- uint T, uint E, uint K,
- uint num_blocks, uint num_tiles,
- uint3 tid3, uint lid, uint3 tgpig,
+ uint T,
+ uint E,
+ uint K,
+ uint num_blocks,
+ uint num_tiles,
+ uint3 tid3,
+ uint lid,
+ uint3 tgpig,
threadgroup _atomic<uint>* sg_counts,
- threadgroup uint* sg_base)
-{
- if (E == 0 || T == 0 || K == 0) return;
- const uint block_id = tgpig.x;
- const uint t_start = block_id * BLOCK_SIZE;
- const uint t_end = min(t_start + BLOCK_SIZE, T);
- const uint sg_id = lid / SIMD_WIDTH;
- const uint lane = lid % SIMD_WIDTH;
+ threadgroup uint* sg_base
+) {
+ if (E == 0 || T == 0 || K == 0)
+ return;
+ const uint block_id = tgpig.x;
+ const uint t_start = block_id * BLOCK_SIZE;
+ const uint t_end = min(t_start + BLOCK_SIZE, T);
+ const uint sg_id = lid / SIMD_WIDTH;
+ const uint lane = lid % SIMD_WIDTH;
- // Per-tile processing
- for (uint e0 = 0; e0 < E; e0 += TILE_E) {
- const uint tile_e = (e0 + TILE_E <= E) ? TILE_E : (E - e0);
+ // Per-tile processing
+ for (uint e0 = 0; e0 < E; e0 += TILE_E) {
+ const uint tile_e = (e0 + TILE_E <= E) ? TILE_E : (E - e0);
- // Zero counts and bases for active tile
- for (uint te = lid; te < tile_e; te += BLOCK_SIZE) {
- for (uint sg = 0; sg < NUM_SG; ++sg) {
- atomic_store_explicit(&sg_counts[sg * TILE_E + te], 0u, memory_order_relaxed);
- sg_base[sg * TILE_E + te] = 0u;
- }
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
+ // Zero counts and bases for active tile
+ for (uint te = lid; te < tile_e; te += BLOCK_SIZE) {
+ for (uint sg = 0; sg < NUM_SG; ++sg) {
+ atomic_store_explicit(
+ &sg_counts[sg * TILE_E + te],
+ 0u,
+ memory_order_relaxed
+ );
+ sg_base[sg * TILE_E + te] = 0u;
+ }
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
- // Phase 1: per-sg counting using threadgroup atomics (order-independent)
- for (uint t = t_start + lid; t < t_end; t += BLOCK_SIZE) {
- const uint base = t * K;
- for (uint k = 0; k < K; ++k) {
- int eid = topk_ids[base + k];
- if (eid >= 0) {
- const uint ue = (uint)eid;
- if (ue >= e0 && ue < e0 + tile_e) {
- const uint te = ue - e0;
- atomic_fetch_add_explicit(&sg_counts[sg_id * TILE_E + te], 1u, memory_order_relaxed);
- }
- }
- }
+ // Phase 1: per-sg counting using threadgroup atomics (order-independent)
+ for (uint t = t_start + lid; t < t_end; t += BLOCK_SIZE) {
+ const uint base = t * K;
+ for (uint k = 0; k < K; ++k) {
+ int eid = topk_ids[base + k];
+ if (eid >= 0) {
+ const uint ue = (uint)eid;
+ if (ue >= e0 && ue < e0 + tile_e) {
+ const uint te = ue - e0;
+ atomic_fetch_add_explicit(
+ &sg_counts[sg_id * TILE_E + te],
+ 1u,
+ memory_order_relaxed
+ );
+ }
}
- threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
- // Phase 2: compute per-sg bases via prefix across sgs with capacity clamp
- for (uint te = lid; te < tile_e; te += BLOCK_SIZE) {
- const uint tile_id = e0 / TILE_E;
- const uint base_idx_block = (block_id * num_tiles + tile_id) * TILE_E + te;
- const uint alloc_total = block_alloc[base_idx_block];
- uint prefix = 0u;
- for (uint sg = 0; sg < NUM_SG; ++sg) {
- const uint idx = sg * TILE_E + te;
- const uint c = atomic_load_explicit(&sg_counts[idx], memory_order_relaxed);
- const uint room = (prefix < alloc_total) ? (alloc_total - prefix) : 0u;
- const uint take = min(c, room);
- sg_base[idx] = prefix;
- prefix += take;
- }
- // Reuse sg_counts as write indices: reset to zero
- for (uint sg = 0; sg < NUM_SG; ++sg) {
- atomic_store_explicit(&sg_counts[sg * TILE_E + te], 0u, memory_order_relaxed);
- }
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
+ // Phase 2: compute per-sg bases via prefix across sgs with capacity clamp
+ for (uint te = lid; te < tile_e; te += BLOCK_SIZE) {
+ const uint tile_id = e0 / TILE_E;
+ const uint base_idx_block =
+ (block_id * num_tiles + tile_id) * TILE_E + te;
+ const uint alloc_total = block_alloc[base_idx_block];
+ uint prefix = 0u;
+ for (uint sg = 0; sg < NUM_SG; ++sg) {
+ const uint idx = sg * TILE_E + te;
+ const uint c =
+ atomic_load_explicit(&sg_counts[idx], memory_order_relaxed);
+ const uint room = (prefix < alloc_total) ? (alloc_total - prefix) : 0u;
+ const uint take = min(c, room);
+ sg_base[idx] = prefix;
+ prefix += take;
+ }
+ // Reuse sg_counts as write indices: reset to zero
+ for (uint sg = 0; sg < NUM_SG; ++sg) {
+ atomic_store_explicit(
+ &sg_counts[sg * TILE_E + te],
+ 0u,
+ memory_order_relaxed
+ );
+ }
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
- // Phase 3: deterministic write with 32-lane sequencing, no device atomics
- for (uint step = 0; step < SIMD_WIDTH; ++step) {
- if (lane == step) {
- const uint t = t_start + lid;
- if (t < t_end) {
- const uint base = t * K;
- for (uint k = 0; k < K; ++k) {
- int eid = topk_ids[base + k];
- if (eid >= 0) {
- const uint ue = (uint)eid;
- if (ue >= e0 && ue < e0 + tile_e) {
- const uint te = ue - e0;
- const uint tile_id = e0 / TILE_E;
- const uint base_idx_block = (block_id * num_tiles + tile_id) * TILE_E + te;
- const uint idx_sg_te = sg_id * TILE_E + te;
- const uint local = atomic_fetch_add_explicit(&sg_counts[idx_sg_te], 1u, memory_order_relaxed);
- const uint base_s = sg_base[idx_sg_te];
+ // Phase 3: deterministic write with 32-lane sequencing, no device atomics
+ for (uint step = 0; step < SIMD_WIDTH; ++step) {
+ if (lane == step) {
+ const uint t = t_start + lid;
+ if (t < t_end) {
+ const uint base = t * K;
+ for (uint k = 0; k < K; ++k) {
+ int eid = topk_ids[base + k];
+ if (eid >= 0) {
+ const uint ue = (uint)eid;
+ if (ue >= e0 && ue < e0 + tile_e) {
+ const uint te = ue - e0;
+ const uint tile_id = e0 / TILE_E;
+ const uint base_idx_block =
+ (block_id * num_tiles + tile_id) * TILE_E + te;
+ const uint idx_sg_te = sg_id * TILE_E + te;
+ const uint local = atomic_fetch_add_explicit(
+ &sg_counts[idx_sg_te],
+ 1u,
+ memory_order_relaxed
+ );
+ const uint base_s = sg_base[idx_sg_te];
- const uint slot = offsets[ue] + block_bases[base_idx_block] + base_s + local;
- out_ids[slot] = (int)t;
- out_probs[slot] = topk_probs[base + k];
- if (tok2row_opt != nullptr) {
- tok2row_opt[base + k] = (int)slot;
- }
- }
- }
- }
+ const uint slot =
+ offsets[ue] + block_bases[base_idx_block] + base_s + local;
+ out_ids[slot] = (int)t;
+ out_probs[slot] = topk_probs[base + k];
+ if (tok2row_opt != nullptr) {
+ tok2row_opt[base + k] = (int)slot;
}
+ }
}
- threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
}
- threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
}
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
}
-#define DEFINE_MOE_SCATTER_BUCKETS_KERNEL(SUFFIX, ProbT) \
-kernel void moe_scatter_buckets_##SUFFIX( \
- device const int* topk_ids [[buffer(0)]], \
- device const ProbT* topk_probs [[buffer(1)]], \
- device const uint* offsets [[buffer(2)]], \
- device const uint* block_bases [[buffer(3)]], \
- device const uint* block_alloc [[buffer(4)]], \
- device int* out_ids [[buffer(5)]], \
- device ProbT* out_probs [[buffer(6)]], \
- constant uint& T [[buffer(7)]], \
- constant uint& E [[buffer(8)]], \
- constant uint& K [[buffer(9)]], \
- constant uint& num_blocks [[buffer(10)]], \
- constant uint& num_tiles [[buffer(11)]], \
- uint3 tid3 [[thread_position_in_grid]], \
- uint lid [[thread_index_in_threadgroup]], \
- uint3 tgpig [[threadgroup_position_in_grid]]) \
-{ \
- threadgroup _atomic<uint> sg_counts[NUM_SG * TILE_E]; \
- threadgroup uint sg_base [NUM_SG * TILE_E]; \
- moe_scatter_buckets_impl<ProbT>(topk_ids, topk_probs, offsets, block_bases, block_alloc, out_ids, out_probs, (device int*)nullptr, T, E, K, num_blocks, num_tiles, tid3, lid, tgpig, sg_counts, sg_base); \
-}
+#define DEFINE_MOE_SCATTER_BUCKETS_KERNEL(SUFFIX, ProbT) \
+ kernel void moe_scatter_buckets_##SUFFIX( \
+ device const int* topk_ids [[buffer(0)]], \
+ device const ProbT* topk_probs [[buffer(1)]], \
+ device const uint* offsets [[buffer(2)]], \
+ device const uint* block_bases [[buffer(3)]], \
+ device const uint* block_alloc [[buffer(4)]], \
+ device int* out_ids [[buffer(5)]], \
+ device ProbT* out_probs [[buffer(6)]], \
+ constant uint& T [[buffer(7)]], \
+ constant uint& E [[buffer(8)]], \
+ constant uint& K [[buffer(9)]], \
+ constant uint& num_blocks [[buffer(10)]], \
+ constant uint& num_tiles [[buffer(11)]], \
+ uint3 tid3 [[thread_position_in_grid]], \
+ uint lid [[thread_index_in_threadgroup]], \
+ uint3 tgpig [[threadgroup_position_in_grid]] \
+ ) { \
+ threadgroup _atomic<uint> sg_counts[NUM_SG * TILE_E]; \
+ threadgroup uint sg_base[NUM_SG * TILE_E]; \
+ moe_scatter_buckets_impl<ProbT>( \
+ topk_ids, \
+ topk_probs, \
+ offsets, \
+ block_bases, \
+ block_alloc, \
+ out_ids, \
+ out_probs, \
+ (device int*)nullptr, \
+ T, \
+ E, \
+ K, \
+ num_blocks, \
+ num_tiles, \
+ tid3, \
+ lid, \
+ tgpig, \
+ sg_counts, \
+ sg_base \
+ ); \
+ }
DEFINE_MOE_SCATTER_BUCKETS_KERNEL(f16, half)
DEFINE_MOE_SCATTER_BUCKETS_KERNEL(f32, float)
DEFINE_MOE_SCATTER_BUCKETS_KERNEL(bf16, bfloat)
// Map variants
-#define DEFINE_MOE_SCATTER_BUCKETS_MAP_KERNEL(SUFFIX, ProbT) \
-kernel void moe_scatter_buckets_map_##SUFFIX( \
- device const int* topk_ids [[buffer(0)]], \
- device const ProbT* topk_probs [[buffer(1)]], \
- device const uint* offsets [[buffer(2)]], \
- device const uint* block_bases [[buffer(3)]], \
- device const uint* block_alloc [[buffer(4)]], \
- device int* out_ids [[buffer(5)]], \
- device ProbT* out_probs [[buffer(6)]], \
- constant uint& T [[buffer(7)]], \
- constant uint& E [[buffer(8)]], \
- constant uint& K [[buffer(9)]], \
- constant uint& num_blocks [[buffer(10)]], \
- constant uint& num_tiles [[buffer(11)]], \
- device int* tok2row [[buffer(12)]], \
- uint3 tid3 [[thread_position_in_grid]], \
- uint lid [[thread_index_in_threadgroup]], \
- uint3 tgpig [[threadgroup_position_in_grid]]) \
-{ \
- threadgroup _atomic<uint> sg_counts[NUM_SG * TILE_E]; \
- threadgroup uint sg_base [NUM_SG * TILE_E]; \
- moe_scatter_buckets_impl<ProbT>(topk_ids, topk_probs, offsets, block_bases, block_alloc, out_ids, out_probs, tok2row, T, E, K, num_blocks, num_tiles, tid3, lid, tgpig, sg_counts, sg_base); \
-}
+#define DEFINE_MOE_SCATTER_BUCKETS_MAP_KERNEL(SUFFIX, ProbT) \
+ kernel void moe_scatter_buckets_map_##SUFFIX( \
+ device const int* topk_ids [[buffer(0)]], \
+ device const ProbT* topk_probs [[buffer(1)]], \
+ device const uint* offsets [[buffer(2)]], \
+ device const uint* block_bases [[buffer(3)]], \
+ device const uint* block_alloc [[buffer(4)]], \
+ device int* out_ids [[buffer(5)]], \
+ device ProbT* out_probs [[buffer(6)]], \
+ constant uint& T [[buffer(7)]], \
+ constant uint& E [[buffer(8)]], \
+ constant uint& K [[buffer(9)]], \
+ constant uint& num_blocks [[buffer(10)]], \
+ constant uint& num_tiles [[buffer(11)]], \
+ device int* tok2row [[buffer(12)]], \
+ uint3 tid3 [[thread_position_in_grid]], \
+ uint lid [[thread_index_in_threadgroup]], \
+ uint3 tgpig [[threadgroup_position_in_grid]] \
+ ) { \
+ threadgroup _atomic<uint> sg_counts[NUM_SG * TILE_E]; \
+ threadgroup uint sg_base[NUM_SG * TILE_E]; \
+ moe_scatter_buckets_impl<ProbT>( \
+ topk_ids, \
+ topk_probs, \
+ offsets, \
+ block_bases, \
+ block_alloc, \
+ out_ids, \
+ out_probs, \
+ tok2row, \
+ T, \
+ E, \
+ K, \
+ num_blocks, \
+ num_tiles, \
+ tid3, \
+ lid, \
+ tgpig, \
+ sg_counts, \
+ sg_base \
+ ); \
+ }
DEFINE_MOE_SCATTER_BUCKETS_MAP_KERNEL(f16, half)
DEFINE_MOE_SCATTER_BUCKETS_MAP_KERNEL(f32, float)
diff --git a/crates/uzu/src/backends/metal/kernel/moe/tiles.metal b/crates/uzu/src/backends/metal/kernel/moe/tiles.metal
index f57eff6f8..ccda7f661 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/tiles.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/tiles.metal
@@ -6,104 +6,109 @@ using namespace metal;
// Compute per-expert tile counts: tiles_e = ceil((seg_len)/BM)
kernel void moe_tile_counts(
- device const uint* offsets [[buffer(0)]], // [E+1]
- device uint* tile_counts [[buffer(1)]], // [E]
+ device const uint* offsets [[buffer(0)]], // [E+1]
+ device uint* tile_counts [[buffer(1)]], // [E]
constant uint& E [[buffer(2)]],
- uint gid [[thread_position_in_grid]])
-{
- if (gid >= E) return;
- const uint start = offsets[gid];
- const uint end = offsets[gid + 1];
- const uint seg_len = (end > start) ? (end - start) : 0u;
- const uint tiles = (seg_len == 0u) ? 0u : ((seg_len + BM - 1u) / BM);
- tile_counts[gid] = tiles;
+ uint gid [[thread_position_in_grid]]
+) {
+ if (gid >= E)
+ return;
+ const uint start = offsets[gid];
+ const uint end = offsets[gid + 1];
+ const uint seg_len = (end > start) ? (end - start) : 0u;
+ const uint tiles = (seg_len == 0u) ? 0u : ((seg_len + BM - 1u) / BM);
+ tile_counts[gid] = tiles;
}
-// Single-threadgroup exclusive scan over tile_counts -> tile_row_offsets, also writes total_tiles
+// Single-threadgroup exclusive scan over tile_counts -> tile_row_offsets, also
+// writes total_tiles
kernel void moe_tile_scan(
- device const uint* tile_counts [[buffer(0)]], // [E]
- device uint* tile_row_offsets [[buffer(1)]], // [E+1]
- device uint* total_tiles_buf [[buffer(2)]], // [>=2]
+ device const uint* tile_counts [[buffer(0)]], // [E]
+ device uint* tile_row_offsets [[buffer(1)]], // [E+1]
+ device uint* total_tiles_buf [[buffer(2)]], // [>=2]
constant uint& E [[buffer(3)]],
- uint tid [[thread_index_in_threadgroup]])
-{
- threadgroup uint scratch[1024]; // supports up to E<=1023
+ uint tid [[thread_index_in_threadgroup]]
+) {
+ threadgroup uint scratch[1024]; // supports up to E<=1023
+ if (tid < E) {
+ scratch[tid] = tile_counts[tid];
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // Inclusive scan in-place over scratch
+ // Simple O(log E) scan; E is small
+ for (uint offset = 1u; offset < E; offset <<= 1u) {
+ uint val = 0u;
if (tid < E) {
- scratch[tid] = tile_counts[tid];
+ val = scratch[tid];
+ if (tid >= offset) {
+ val += scratch[tid - offset];
+ }
}
threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // Inclusive scan in-place over scratch
- // Simple O(log E) scan; E is small
- for (uint offset = 1u; offset < E; offset <<= 1u) {
- uint val = 0u;
- if (tid < E) {
- val = scratch[tid];
- if (tid >= offset) {
- val += scratch[tid - offset];
- }
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
- if (tid < E) {
- scratch[tid] = val;
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
- }
-
- if (tid == 0u) {
- tile_row_offsets[0] = 0u;
- }
if (tid < E) {
- // exclusive: shift right by one
- const uint prefix = (tid == 0u) ? 0u : scratch[tid - 1u];
- tile_row_offsets[tid] = prefix;
- }
- if (tid == 0u) {
- tile_row_offsets[E] = (E == 0u) ? 0u : scratch[E - 1u];
- total_tiles_buf[0] = tile_row_offsets[E];
- total_tiles_buf[1] = 0u;
- total_tiles_buf[2] = 0u;
- total_tiles_buf[3] = 0u;
- total_tiles_buf[4] = 0u;
- total_tiles_buf[5] = 0u;
- total_tiles_buf[6] = 0u;
- total_tiles_buf[7] = 0u;
+ scratch[tid] = val;
}
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+
+ if (tid == 0u) {
+ tile_row_offsets[0] = 0u;
+ }
+ if (tid < E) {
+ // exclusive: shift right by one
+ const uint prefix = (tid == 0u) ? 0u : scratch[tid - 1u];
+ tile_row_offsets[tid] = prefix;
+ }
+ if (tid == 0u) {
+ tile_row_offsets[E] = (E == 0u) ? 0u : scratch[E - 1u];
+ total_tiles_buf[0] = tile_row_offsets[E];
+ total_tiles_buf[1] = 0u;
+ total_tiles_buf[2] = 0u;
+ total_tiles_buf[3] = 0u;
+ total_tiles_buf[4] = 0u;
+ total_tiles_buf[5] = 0u;
+ total_tiles_buf[6] = 0u;
+ total_tiles_buf[7] = 0u;
+ }
}
// Build flattened tile_map of length total_tiles; each entry is 3 uints:
// [3*i+0]=expert_idx, [3*i+1]=seg_start, [3*i+2]=tile_m0
kernel void moe_build_tile_map(
- device const uint* offsets [[buffer(0)]], // [E+1]
- device const uint* tile_row_offsets [[buffer(1)]], // [E+1]
- device const uint* tile_counts [[buffer(2)]], // [E]
- device uint* tile_map [[buffer(3)]], // [3*total_tiles]
+ device const uint* offsets [[buffer(0)]], // [E+1]
+ device const uint* tile_row_offsets [[buffer(1)]], // [E+1]
+ device const uint* tile_counts [[buffer(2)]], // [E]
+ device uint* tile_map [[buffer(3)]], // [3*total_tiles]
constant uint& E [[buffer(4)]],
- uint gid [[thread_position_in_grid]])
-{
- if (gid >= E) return;
- const uint start = offsets[gid];
- const uint tiles = tile_counts[gid];
- const uint base = tile_row_offsets[gid];
- for (uint t = 0u; t < tiles; ++t) {
- const uint idx = base + t;
- tile_map[3u * idx + 0u] = gid;
- tile_map[3u * idx + 1u] = start;
- tile_map[3u * idx + 2u] = t * BM;
- }
+ uint gid [[thread_position_in_grid]]
+) {
+ if (gid >= E)
+ return;
+ const uint start = offsets[gid];
+ const uint tiles = tile_counts[gid];
+ const uint base = tile_row_offsets[gid];
+ for (uint t = 0u; t < tiles; ++t) {
+ const uint idx = base + t;
+ tile_map[3u * idx + 0u] = gid;
+ tile_map[3u * idx + 1u] = start;
+ tile_map[3u * idx + 2u] = t * BM;
+ }
}
// Write MTLDispatchThreadgroupsIndirectArguments {x, y, z} where:
// x = num_tiles_n (computed on CPU and passed in), y = total_tiles, z = 1
kernel void moe_write_dispatch_args(
- device const uint* total_tiles_buf [[buffer(0)]], // [>=1], total_tiles_buf[0] = total_rows
- device uint* dispatch_args [[buffer(1)]], // [3] u32
- constant uint& num_tiles_n [[buffer(2)]], // x dimension
- uint gid [[thread_position_in_grid]])
-{
- if (gid > 0u) return;
- const uint total_rows = total_tiles_buf[0];
- dispatch_args[0] = num_tiles_n; // x
- dispatch_args[1] = total_rows; // y
- dispatch_args[2] = 1u; // z
+ device const uint* total_tiles_buf
+ [[buffer(0)]], // [>=1], total_tiles_buf[0] = total_rows
+ device uint* dispatch_args [[buffer(1)]], // [3] u32
+ constant uint& num_tiles_n [[buffer(2)]], // x dimension
+ uint gid [[thread_position_in_grid]]
+) {
+ if (gid > 0u)
+ return;
+ const uint total_rows = total_tiles_buf[0];
+ dispatch_args[0] = num_tiles_n; // x
+ dispatch_args[1] = total_rows; // y
+ dispatch_args[2] = 1u; // z
}
diff --git a/crates/uzu/src/backends/metal/kernel/moe/topk_select.metal b/crates/uzu/src/backends/metal/kernel/moe/topk_select.metal
index a0f1c85f6..68cf8b73c 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/topk_select.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/topk_select.metal
@@ -7,7 +7,7 @@ constant uint MAX_EXPERTS = 512;
constant uint MAX_TOPK = 128;
constant float NEG_INF = -INFINITY;
-template<typename LogitT, typename ProbT>
+template <typename LogitT, typename ProbT>
inline void moe_topk_select_impl(
device const LogitT* logits,
device int* topk_ids,
@@ -23,133 +23,148 @@ inline void moe_topk_select_impl(
threadgroup float* reduce_tmp,
threadgroup uint* reduce_tmp_u,
threadgroup float* selected_vals,
- threadgroup uint* selected_ids)
-{
- if (token >= T || K == 0) {
- return;
+ threadgroup uint* selected_ids
+) {
+ if (token >= T || K == 0) {
+ return;
+ }
+
+ const uint effective_k = min(K, MAX_TOPK);
+
+ for (uint i = lid; i < MAX_EXPERTS; i += THREADS_PER_TG) {
+ if (i < E) {
+ tg_logits[i] = float(logits[token * E + i]);
+ } else {
+ tg_logits[i] = NEG_INF;
+ }
+ tg_indices[i] = i;
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ const uint selections = min(effective_k, E);
+ for (uint sel = 0; sel < selections; ++sel) {
+ float local_best_val = NEG_INF;
+ uint local_best_idx = 0xFFFFFFFFu;
+
+ for (uint i = lid; i < E; i += THREADS_PER_TG) {
+ float candidate = tg_logits[i];
+ if (candidate > local_best_val ||
+ (candidate == local_best_val && i < local_best_idx)) {
+ local_best_val = candidate;
+ local_best_idx = i;
+ }
}
- const uint effective_k = min(K, MAX_TOPK);
+ float max_val = threadgroup_cooperative_reduce_max<THREADS_PER_TG>(
+ local_best_val,
+ reduce_tmp,
+ lid
+ );
- for (uint i = lid; i < MAX_EXPERTS; i += THREADS_PER_TG) {
- if (i < E) {
- tg_logits[i] = float(logits[token * E + i]);
- } else {
- tg_logits[i] = NEG_INF;
- }
- tg_indices[i] = i;
- }
+ uint candidate_id =
+ (local_best_val == max_val) ? local_best_idx : 0xFFFFFFFFu;
+ uint best_idx = threadgroup_cooperative_reduce_min<THREADS_PER_TG>(
+ candidate_id,
+ reduce_tmp_u,
+ lid
+ );
+ if (lid == 0) {
+ selected_vals[sel] = max_val;
+ selected_ids[sel] = best_idx;
+ tg_logits[best_idx] = NEG_INF;
+ }
threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
- const uint selections = min(effective_k, E);
- for (uint sel = 0; sel < selections; ++sel) {
- float local_best_val = NEG_INF;
- uint local_best_idx = 0xFFFFFFFFu;
-
- for (uint i = lid; i < E; i += THREADS_PER_TG) {
- float candidate = tg_logits[i];
- if (candidate > local_best_val ||
- (candidate == local_best_val && i < local_best_idx)) {
- local_best_val = candidate;
- local_best_idx = i;
- }
- }
-
- float max_val = threadgroup_cooperative_reduce_max<THREADS_PER_TG>(
- local_best_val,
- reduce_tmp,
- lid);
-
- uint candidate_id =
- (local_best_val == max_val) ? local_best_idx : 0xFFFFFFFFu;
- uint best_idx = threadgroup_cooperative_reduce_min<THREADS_PER_TG>(
- candidate_id,
- reduce_tmp_u,
- lid);
-
- if (lid == 0) {
- selected_vals[sel] = max_val;
- selected_ids[sel] = best_idx;
- tg_logits[best_idx] = NEG_INF;
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
- }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
- threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (lid == 0) {
+ device int* out_ids = topk_ids + token * K;
+ device ProbT* out_probs = topk_probs + token * K;
- if (lid == 0) {
- device int* out_ids = topk_ids + token * K;
- device ProbT* out_probs = topk_probs + token * K;
-
- const uint written = selections;
- for (uint i = 0; i < written; ++i) {
- out_ids[i] = int(selected_ids[i]);
- }
- for (uint i = written; i < K; ++i) {
- out_ids[i] = 0;
- }
-
- if (renorm != 0 && written > 0) {
- float max_logit = -INFINITY;
- for (uint i = 0; i < written; ++i) {
- max_logit = fmax(max_logit, selected_vals[i]);
- }
-
- float exp_values[MAX_TOPK];
- float sum_exp = 0.0f;
- for (uint i = 0; i < written; ++i) {
- float e_val = exp(selected_vals[i] - max_logit);
- exp_values[i] = e_val;
- sum_exp += e_val;
- }
-
- float default_prob = 1.0f / float(written);
- float inv_sum =
- (sum_exp > 0.0f) ? (1.0f / sum_exp) : default_prob;
-
- for (uint i = 0; i < written; ++i) {
- float prob = (sum_exp > 0.0f) ? (exp_values[i] * inv_sum)
- : default_prob;
- out_probs[i] = ProbT(prob);
- }
- for (uint i = written; i < K; ++i) {
- out_probs[i] = ProbT(0.0f);
- }
- } else {
- for (uint i = 0; i < written; ++i) {
- out_probs[i] = ProbT(selected_vals[i]);
- }
- for (uint i = written; i < K; ++i) {
- out_probs[i] = ProbT(0.0f);
- }
- }
+ const uint written = selections;
+ for (uint i = 0; i < written; ++i) {
+ out_ids[i] = int(selected_ids[i]);
+ }
+ for (uint i = written; i < K; ++i) {
+ out_ids[i] = 0;
}
-}
-#define DEFINE_MOE_TOPK_KERNEL(LOGIT_T, PROB_T, SUFFIX) \
-kernel void moe_topk_select_##SUFFIX( \
- device const LOGIT_T* logits [[buffer(0)]], \
- device int* topk_ids [[buffer(1)]], \
- device PROB_T* topk_probs [[buffer(2)]], \
- constant uint& T [[buffer(3)]], \
- constant uint& E [[buffer(4)]], \
- constant uint& K [[buffer(5)]], \
- constant uint& renorm [[buffer(6)]], \
- ushort lid [[thread_index_in_threadgroup]], \
- uint3 tgpig [[threadgroup_position_in_grid]]) \
-{ \
- threadgroup float tg_logits[MAX_EXPERTS]; \
- threadgroup uint tg_indices[MAX_EXPERTS]; \
- threadgroup float reduce_tmp[THREADS_PER_TG]; \
- threadgroup uint reduce_tmp_u[THREADS_PER_TG]; \
- threadgroup float selected_vals[MAX_TOPK]; \
- threadgroup uint selected_ids[MAX_TOPK]; \
- moe_topk_select_impl<LOGIT_T, PROB_T>( \
- logits, topk_ids, topk_probs, T, E, K, renorm, lid, tgpig.x, \
- tg_logits, tg_indices, reduce_tmp, reduce_tmp_u, selected_vals, selected_ids); \
+ if (renorm != 0 && written > 0) {
+ float max_logit = -INFINITY;
+ for (uint i = 0; i < written; ++i) {
+ max_logit = fmax(max_logit, selected_vals[i]);
+ }
+
+ float exp_values[MAX_TOPK];
+ float sum_exp = 0.0f;
+ for (uint i = 0; i < written; ++i) {
+ float e_val = exp(selected_vals[i] - max_logit);
+ exp_values[i] = e_val;
+ sum_exp += e_val;
+ }
+
+ float default_prob = 1.0f / float(written);
+ float inv_sum = (sum_exp > 0.0f) ? (1.0f / sum_exp) : default_prob;
+
+ for (uint i = 0; i < written; ++i) {
+ float prob =
+ (sum_exp > 0.0f) ? (exp_values[i] * inv_sum) : default_prob;
+ out_probs[i] = ProbT(prob);
+ }
+ for (uint i = written; i < K; ++i) {
+ out_probs[i] = ProbT(0.0f);
+ }
+ } else {
+ for (uint i = 0; i < written; ++i) {
+ out_probs[i] = ProbT(selected_vals[i]);
+ }
+ for (uint i = written; i < K; ++i) {
+ out_probs[i] = ProbT(0.0f);
+ }
+ }
+ }
}
+#define DEFINE_MOE_TOPK_KERNEL(LOGIT_T, PROB_T, SUFFIX) \
+ kernel void moe_topk_select_##SUFFIX( \
+ device const LOGIT_T* logits [[buffer(0)]], \
+ device int* topk_ids [[buffer(1)]], \
+ device PROB_T* topk_probs [[buffer(2)]], \
+ constant uint& T [[buffer(3)]], \
+ constant uint& E [[buffer(4)]], \
+ constant uint& K [[buffer(5)]], \
+ constant uint& renorm [[buffer(6)]], \
+ ushort lid [[thread_index_in_threadgroup]], \
+ uint3 tgpig [[threadgroup_position_in_grid]] \
+ ) { \
+ threadgroup float tg_logits[MAX_EXPERTS]; \
+ threadgroup uint tg_indices[MAX_EXPERTS]; \
+ threadgroup float reduce_tmp[THREADS_PER_TG]; \
+ threadgroup uint reduce_tmp_u[THREADS_PER_TG]; \
+ threadgroup float selected_vals[MAX_TOPK]; \
+ threadgroup uint selected_ids[MAX_TOPK]; \
+ moe_topk_select_impl<LOGIT_T, PROB_T>( \
+ logits, \
+ topk_ids, \
+ topk_probs, \
+ T, \
+ E, \
+ K, \
+ renorm, \
+ lid, \
+ tgpig.x, \
+ tg_logits, \
+ tg_indices, \
+ reduce_tmp, \
+ reduce_tmp_u, \
+ selected_vals, \
+ selected_ids \
+ ); \
+ }
+
DEFINE_MOE_TOPK_KERNEL(half, half, f16)
DEFINE_MOE_TOPK_KERNEL(float, half, f32)
DEFINE_MOE_TOPK_KERNEL(bfloat, bfloat, bf16)
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/mma.h b/crates/uzu/src/backends/metal/kernel/quant_matmul/mma.h
index b75b3241c..56000c057 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/mma.h
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/mma.h
@@ -12,37 +12,37 @@ using namespace metal;
namespace matmul_utils {
-template<typename T>
+template <typename T>
struct pointer_element_t_impl {
- using type = T;
+ using type = T;
};
-template<typename T>
+template <typename T>
struct pointer_element_t_impl<device T*> {
- using type = T;
+ using type = T;
};
-template<typename T>
+template <typename T>
struct pointer_element_t_impl<threadgroup T*> {
- using type = T;
+ using type = T;
};
-template<typename T>
+template <typename T>
using pointer_element_t = typename pointer_element_t_impl<T>::type;
template <typename T, typename U = T>
struct TransformNone {
- METAL_FUNC static U apply(U val) {
- return val;
- }
+ METAL_FUNC static U apply(U val) { return val; }
};
template <typename T, int kFragRows_, int kFragCols_>
struct BaseMMAFrag {
static_assert(
kFragRows_ == 8,
- "Only 8 x 8 fragment matrices are currently supported");
+ "Only 8 x 8 fragment matrices are currently supported"
+ );
static_assert(
kFragCols_ == 8,
- "Only 8 x 8 fragment matrices are currently supported");
+ "Only 8 x 8 fragment matrices are currently supported"
+ );
};
template <typename T>
@@ -55,13 +55,15 @@ struct BaseMMAFrag<T, 8, 8> {
static_assert(
kElemRows * kElemCols == kElemsPerFrag,
- "MMAFrag shape is not consistent with MMAFrag size");
+ "MMAFrag shape is not consistent with MMAFrag size"
+ );
typedef metal::simdgroup_matrix<T, kFragRows, kFragCols> mat_type;
typedef metal::vec<T, kElemsPerFrag> frag_type;
- METAL_FUNC static constexpr short2 get_coord(ushort simd_lane_id
- [[thread_index_in_simdgroup]]) {
+ METAL_FUNC static constexpr short2 get_coord(
+ ushort simd_lane_id [[thread_index_in_simdgroup]]
+ ) {
const short qid = simd_lane_id / 4;
const short fm = (qid & 4) + ((simd_lane_id / 2) % 4);
const short fn = (qid & 2) * 2 + (simd_lane_id % 2) * 2;
@@ -69,8 +71,12 @@ struct BaseMMAFrag<T, 8, 8> {
}
template <typename SrcPtrType, typename StrX, typename StrY>
- METAL_FUNC static constexpr void
- load(thread frag_type& dst, SrcPtrType src, StrX str_x, StrY str_y) {
+ METAL_FUNC static constexpr void load(
+ thread frag_type& dst,
+ SrcPtrType src,
+ StrX str_x,
+ StrY str_y
+ ) {
UZU_PRAGMA_UNROLL
for (short i = 0; i < kElemRows; i++) {
UZU_PRAGMA_UNROLL
@@ -80,10 +86,13 @@ struct BaseMMAFrag<T, 8, 8> {
}
}
-
template <typename DstPtrType, typename StrX, typename StrY>
- METAL_FUNC static constexpr void
- store(const thread frag_type& src, DstPtrType dst, StrX str_x, StrY str_y) {
+ METAL_FUNC static constexpr void store(
+ const thread frag_type& src,
+ DstPtrType dst,
+ StrX str_x,
+ StrY str_y
+ ) {
using U = pointer_element_t<DstPtrType>;
UZU_PRAGMA_UNROLL
@@ -111,7 +120,8 @@ struct BaseMMAFrag<T, 8, 8> {
LimX lim_x,
LimY lim_y,
OffX off_x = 0,
- OffY off_y = 0) {
+ OffY off_y = 0
+ ) {
using U = pointer_element_t<DstPtrType>;
UZU_PRAGMA_UNROLL
@@ -126,12 +136,12 @@ struct BaseMMAFrag<T, 8, 8> {
}
}
-
METAL_FUNC static constexpr void mma(
thread frag_type& D,
thread frag_type& A,
thread frag_type& B,
- thread frag_type& C) {
+ thread frag_type& C
+ ) {
mat_type D_mat;
mat_type A_mat;
mat_type B_mat;
@@ -150,7 +160,8 @@ struct BaseMMAFrag<T, 8, 8> {
thread mat_type& D,
thread mat_type& A,
thread mat_type& B,
- thread mat_type& C) {
+ thread mat_type& C
+ ) {
simdgroup_multiply_accumulate(D, A, B, C);
}
};
@@ -196,11 +207,11 @@ struct MMATile {
METAL_FUNC constexpr const thread frag_type& frag_at(
const short i,
- const short j) const {
+ const short j
+ ) const {
return val_frags[i * kTileCols + j];
}
-
METAL_FUNC thread elem_type* elems() {
return reinterpret_cast<thread elem_type*>(val_frags);
}
@@ -219,14 +230,15 @@ struct MMATile {
frag_at(i, j),
&(
src[(i * kFragRows) * w_x * str_x +
- (j * kFragCols) * w_y * str_y]),
+ (j * kFragCols) * w_y * str_y]
+ ),
str_x,
- str_y);
+ str_y
+ );
}
}
}
-
template <typename U, int w_x, int w_y>
METAL_FUNC void store(device U* dst, const int ld) const {
UZU_PRAGMA_UNROLL
@@ -237,7 +249,8 @@ struct MMATile {
frag_at(i, j),
&(dst[(i * kFragRows) * w_x * ld + (j * kFragCols) * w_y]),
ld,
- 1);
+ 1
+ );
}
}
}
@@ -253,14 +266,18 @@ struct MMATile {
frag_at(i, j),
&(dst[(i * kFragRows) * w_x * ld + (j * kFragCols) * w_y]),
ld,
- 1);
+ 1
+ );
}
}
}
template <typename U, int w_x, int w_y>
- METAL_FUNC void
- store_safe(device U* dst, const int ld, const short2 dst_tile_dims) const {
+ METAL_FUNC void store_safe(
+ device U* dst,
+ const int ld,
+ const short2 dst_tile_dims
+ ) const {
UZU_PRAGMA_UNROLL
for (int i = 0; i < kTileRows; ++i) {
UZU_PRAGMA_UNROLL
@@ -273,11 +290,11 @@ struct MMATile {
dst_tile_dims.y,
dst_tile_dims.x,
(i * kFragRows) * w_x,
- (j * kFragCols) * w_y);
+ (j * kFragCols) * w_y
+ );
}
}
}
-
};
template <typename T, typename U, int M, int N, int K>
@@ -285,7 +302,8 @@ METAL_FUNC void tile_matmad(
thread MMATile<T, M, N>& D,
thread MMATile<U, M, K>& A,
thread MMATile<U, K, N>& B,
- thread MMATile<T, M, N>& C) {
+ thread MMATile<T, M, N>& C
+) {
UZU_PRAGMA_UNROLL
for (short m = 0; m < M; ++m) {
UZU_PRAGMA_UNROLL
@@ -297,7 +315,8 @@ METAL_FUNC void tile_matmad(
D.frag_at(m, n_serp),
A.frag_at(m, k),
B.frag_at(k, n_serp),
- C.frag_at(m, n_serp));
+ C.frag_at(m, n_serp)
+ );
}
}
}
@@ -359,7 +378,8 @@ struct BlockMMA {
/* Constructor */
METAL_FUNC BlockMMA(
ushort simd_group_id [[simdgroup_index_in_threadgroup]],
- ushort simd_lane_id [[thread_index_in_simdgroup]]) {
+ ushort simd_lane_id [[thread_index_in_simdgroup]]
+ ) {
// Determine thread position in simdgroup matrix
short tm = kFragSize * (simd_group_id / WN);
short tn = kFragSize * (simd_group_id % WN);
@@ -431,9 +451,11 @@ struct BlockMMA {
Ctile.template store_tg<U, WM, WN>(D, ldd);
}
-
- METAL_FUNC void
- store_result_safe(device U* D, const int ldd, short2 dst_tile_dims) {
+ METAL_FUNC void store_result_safe(
+ device U* D,
+ const int ldd,
+ short2 dst_tile_dims
+ ) {
// Apply epilogue
UZU_PRAGMA_UNROLL
for (short i = 0; i < decltype(Ctile)::kElemsPerTile; i++) {
@@ -449,72 +471,69 @@ struct BlockMMA {
Ctile.template store_safe<U, WM, WN>(D, ldd, dst_tile_dims);
}
-
};
template <
typename T,
int BM,
- int BK,
+ int BK,
int lda_tgp,
int reduction_dim,
int tgp_size,
int align_M = 1,
int align_K = 4>
struct BlockLoader {
- UZU_MTL_CONST short n_reads = (BM * BK < tgp_size) ? 1 : (BM * BK) / tgp_size;
-
- const int src_ld;
- const short thread_idx;
- const short bi;
- const short bj;
-
- threadgroup T* dst;
- const device T* src;
-
- BlockLoader(
- const device T* src_,
- const int src_ld_,
- threadgroup T* dst_,
- ushort simd_group_id [[simdgroup_index_in_threadgroup]],
- ushort simd_lane_id [[thread_index_in_simdgroup]])
- : src_ld(src_ld_),
- thread_idx(simd_group_id * 32 + simd_lane_id),
- bi(n_reads * thread_idx / BK),
- bj((n_reads * thread_idx) % BK),
- dst(dst_ + bi * lda_tgp + bj),
- src(src_ + bi * src_ld + bj) {}
-
- void load_unsafe() const {
- if (BM * BK < tgp_size && bi >= BM) {
- return;
- }
-
- for (int i = 0; i < n_reads; i++) {
- dst[i] = src[i];
- }
+ UZU_MTL_CONST short n_reads = (BM * BK < tgp_size) ? 1 : (BM * BK) / tgp_size;
+
+ const int src_ld;
+ const short thread_idx;
+ const short bi;
+ const short bj;
+
+ threadgroup T* dst;
+ const device T* src;
+
+ BlockLoader(
+ const device T* src_,
+ const int src_ld_,
+ threadgroup T* dst_,
+ ushort simd_group_id [[simdgroup_index_in_threadgroup]],
+ ushort simd_lane_id [[thread_index_in_simdgroup]]
+ )
+ : src_ld(src_ld_), thread_idx(simd_group_id * 32 + simd_lane_id),
+ bi(n_reads * thread_idx / BK), bj((n_reads * thread_idx) % BK),
+ dst(dst_ + bi * lda_tgp + bj), src(src_ + bi * src_ld + bj) {}
+
+ void load_unsafe() const {
+ if (BM * BK < tgp_size && bi >= BM) {
+ return;
}
-
- void load_safe(short2 src_tile_dim) const {
- if (BM * BK < tgp_size && bi >= BM) {
- return;
- }
-
- if (bi >= src_tile_dim.y || bj >= src_tile_dim.x) {
- for (int i = 0; i < n_reads; i++) {
- dst[i] = T(0);
- }
- return;
- }
-
- for (int i = 0; i < n_reads; i++) {
- dst[i] = src[i];
- }
+
+ for (int i = 0; i < n_reads; i++) {
+ dst[i] = src[i];
+ }
+ }
+
+ void load_safe(short2 src_tile_dim) const {
+ if (BM * BK < tgp_size && bi >= BM) {
+ return;
+ }
+
+ if (bi >= src_tile_dim.y || bj >= src_tile_dim.x) {
+ for (int i = 0; i < n_reads; i++) {
+ dst[i] = T(0);
+ }
+ return;
}
-
- void next() {
- src += BK; // Advance by tile size
+
+ for (int i = 0; i < n_reads; i++) {
+ dst[i] = src[i];
}
+ }
+
+ void next() {
+ src += BK; // Advance by tile size
+ }
};
-}
\ No newline at end of file
+} // namespace matmul_utils
\ No newline at end of file
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal
index dc90432d4..a79e2a67a 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal
@@ -5,11 +5,11 @@ using namespace metal;
#include "mma.h"
-// Function constant: true = MLX-style (pre-computed biases), false = AWQ-style (zero-points)
+// Function constant: true = MLX-style (pre-computed biases), false = AWQ-style
+// (zero-points)
constant bool kUseMlxQuant [[function_constant(40)]];
constant bool kUseZeroPoints = !kUseMlxQuant;
-
template <int bits, int wsize = 8>
inline constexpr short get_pack_factor() {
return (bits == 3 || bits == 5) ? 8 : (bits == 6 ? 4 : wsize / bits);
@@ -50,10 +50,10 @@ inline U load_vector_safe(const device T* x, thread U* x_thread, int N) {
U sum = 0;
if (bits == 4) {
const U scale_lut[4] = {
- static_cast<U>(1.0f),
- static_cast<U>(1.0f / 16.0f),
- static_cast<U>(1.0f / 256.0f),
- static_cast<U>(1.0f / 4096.0f)
+ static_cast<U>(1.0f),
+ static_cast<U>(1.0f / 16.0f),
+ static_cast<U>(1.0f / 256.0f),
+ static_cast<U>(1.0f / 4096.0f)
};
for (int i = 0; i < values_per_thread; ++i) {
@@ -78,8 +78,13 @@ inline U load_vector_safe(const device T* x, thread U* x_thread, int N) {
}
template <typename U, int values_per_thread, int bits>
-inline void
-qouter(const thread uint8_t* w, U x, U scale, U bias, thread U* result) {
+inline void qouter(
+ const thread uint8_t* w,
+ U x,
+ U scale,
+ U bias,
+ thread U* result
+) {
static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");
if (bits == 4) {
@@ -88,8 +93,7 @@ qouter(const thread uint8_t* w, U x, U scale, U bias, thread U* result) {
result[2 * i] += x * (s[0] * (w[i] & 0x0f) + bias);
result[2 * i + 1] += x * (s[1] * (w[i] & 0xf0) + bias);
}
- }
- else if (bits == 8) {
+ } else if (bits == 8) {
for (int i = 0; i < values_per_thread; i++) {
result[i] += x * (scale * w[i] + bias);
}
@@ -102,7 +106,8 @@ inline U qdot(
const thread U* x_thread,
U scale,
U bias,
- U sum) {
+ U sum
+) {
static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");
U accum = 0;
@@ -128,7 +133,8 @@ inline U qdot_zero_point(
const device uint8_t* w,
const thread U* x_thread,
U scale,
- U zero_point) {
+ U zero_point
+) {
static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");
U accum = 0;
@@ -142,13 +148,21 @@ inline U qdot_zero_point(
for (int i = 0; i < (values_per_thread / 4); i++) {
uint16_t word = ws[i];
accum += x_thread[4 * i] *
- static_cast<U>(static_cast<int>(word & 0x000f) - static_cast<int>(zp0));
+ static_cast<U>(
+ static_cast<int>(word & 0x000f) - static_cast<int>(zp0)
+ );
accum += x_thread[4 * i + 1] *
- static_cast<U>(static_cast<int>(word & 0x00f0) - static_cast<int>(zp1));
+ static_cast<U>(
+ static_cast<int>(word & 0x00f0) - static_cast<int>(zp1)
+ );
accum += x_thread[4 * i + 2] *
- static_cast<U>(static_cast<int>(word & 0x0f00) - static_cast<int>(zp2));
+ static_cast<U>(
+ static_cast<int>(word & 0x0f00) - static_cast<int>(zp2)
+ );
accum += x_thread[4 * i + 3] *
- static_cast<U>(static_cast<int>(word & 0xf000) - static_cast<int>(zp3));
+ static_cast<U>(
+ static_cast<int>(word & 0xf000) - static_cast<int>(zp3)
+ );
}
} else if (bits == 8) {
for (int i = 0; i < values_per_thread; i++) {
@@ -165,7 +179,8 @@ inline U qdot_safe(
U scale,
U bias,
U sum,
- int N) {
+ int N
+) {
static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");
U accum = 0;
@@ -185,9 +200,12 @@ inline U qdot_safe(
if (rem > 0) {
uint16_t wv = ws[full];
int base = 4 * full;
- if (rem > 0) accum += x_thread[base] * (wv & 0x000f);
- if (rem > 1) accum += x_thread[base + 1] * (wv & 0x00f0);
- if (rem > 2) accum += x_thread[base + 2] * (wv & 0x0f00);
+ if (rem > 0)
+ accum += x_thread[base] * (wv & 0x000f);
+ if (rem > 1)
+ accum += x_thread[base + 1] * (wv & 0x00f0);
+ if (rem > 2)
+ accum += x_thread[base + 2] * (wv & 0x0f00);
}
} else if (bits == 8) {
for (int i = 0; i < N; i++) {
@@ -199,7 +217,12 @@ inline U qdot_safe(
}
template <typename U, int N, int bits>
-inline void dequantize(const device uint8_t* w, U scale, U bias, threadgroup U* w_local) {
+inline void dequantize(
+ const device uint8_t* w,
+ U scale,
+ U bias,
+ threadgroup U* w_local
+) {
static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");
if (bits == 4) {
@@ -216,30 +239,35 @@ inline void dequantize(const device uint8_t* w, U scale, U bias, threadgroup U*
}
template <>
-inline void dequantize<bfloat, 8, 4>(const device uint8_t* w, bfloat scale, bfloat bias, threadgroup bfloat* w_local) {
- const device uint32_t* w_ptr = (const device uint32_t*)w;
- uint32_t packed = *w_ptr;
-
- bfloat4 v0, v1;
-
- // Low 4 nibbles
- v0.x = static_cast<bfloat>(packed & 0xF);
- v0.y = static_cast<bfloat>((packed >> 4) & 0xF);
- v0.z = static_cast<bfloat>((packed >> 8) & 0xF);
- v0.w = static_cast<bfloat>((packed >> 12) & 0xF);
-
- // High 4 nibbles
- v1.x = static_cast<bfloat>((packed >> 16) & 0xF);
- v1.y = static_cast<bfloat>((packed >> 20) & 0xF);
- v1.z = static_cast<bfloat>((packed >> 24) & 0xF);
- v1.w = static_cast<bfloat>((packed >> 28) & 0xF);
-
- v0 = v0 * scale + bias;
- v1 = v1 * scale + bias;
-
- threadgroup bfloat4* out_ptr = (threadgroup bfloat4*)w_local;
- out_ptr[0] = v0;
- out_ptr[1] = v1;
+inline void dequantize<bfloat, 8, 4>(
+ const device uint8_t* w,
+ bfloat scale,
+ bfloat bias,
+ threadgroup bfloat* w_local
+) {
+ const device uint32_t* w_ptr = (const device uint32_t*)w;
+ uint32_t packed = *w_ptr;
+
+ bfloat4 v0, v1;
+
+ // Low 4 nibbles
+ v0.x = static_cast<bfloat>(packed & 0xF);
+ v0.y = static_cast<bfloat>((packed >> 4) & 0xF);
+ v0.z = static_cast<bfloat>((packed >> 8) & 0xF);
+ v0.w = static_cast<bfloat>((packed >> 12) & 0xF);
+
+ // High 4 nibbles
+ v1.x = static_cast<bfloat>((packed >> 16) & 0xF);
+ v1.y = static_cast<bfloat>((packed >> 20) & 0xF);
+ v1.z = static_cast<bfloat>((packed >> 24) & 0xF);
+ v1.w = static_cast<bfloat>((packed >> 28) & 0xF);
+
+ v0 = v0 * scale + bias;
+ v1 = v1 * scale + bias;
+
+ threadgroup bfloat4* out_ptr = (threadgroup bfloat4*)w_local;
+ out_ptr[0] = v0;
+ out_ptr[1] = v1;
}
template <
@@ -252,14 +280,21 @@ template <
short group_size,
short bits>
struct QuantizedBlockLoaderMlx {
- static_assert(BCOLS <= group_size, "Group size should be larger than columns");
- static_assert(group_size % BCOLS == 0, "Group size should be divisible by columns");
+ static_assert(
+ BCOLS <= group_size,
+ "Group size should be larger than columns"
+ );
+ static_assert(
+ group_size % BCOLS == 0,
+ "Group size should be divisible by columns"
+ );
static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");
UZU_MTL_CONST short pack_factor = get_pack_factor<bits, 8>();
UZU_MTL_CONST short bytes_per_pack = get_bytes_per_pack<bits>();
UZU_MTL_CONST short BCOLS_PACKED = BCOLS / pack_factor;
- UZU_MTL_CONST short n_reads = (BCOLS_PACKED * BROWS < tgp_size) ? 1 : (BCOLS_PACKED * BROWS) / tgp_size;
+ UZU_MTL_CONST short n_reads =
+ (BCOLS_PACKED * BROWS < tgp_size) ? 1 : (BCOLS_PACKED * BROWS) / tgp_size;
UZU_MTL_CONST short group_steps = group_size / BCOLS;
const int src_ld;
@@ -283,18 +318,20 @@ struct QuantizedBlockLoaderMlx {
const int src_ld_,
threadgroup T* dst_,
ushort simd_group_id [[simdgroup_index_in_threadgroup]],
- ushort simd_lane_id [[thread_index_in_simdgroup]])
+ ushort simd_lane_id [[thread_index_in_simdgroup]]
+ )
: src_ld(src_ld_),
tile_stride(
reduction_dim ? BCOLS_PACKED * bytes_per_pack
- : BROWS * src_ld * bytes_per_pack / pack_factor),
- group_step_cnt(0),
- group_stride(BROWS * src_ld / group_size),
+ : BROWS * src_ld * bytes_per_pack / pack_factor
+ ),
+ group_step_cnt(0), group_stride(BROWS * src_ld / group_size),
thread_idx(simd_group_id * 32 + simd_lane_id),
bi(n_reads * thread_idx / BCOLS_PACKED),
bj((n_reads * thread_idx) % BCOLS_PACKED),
dst(dst_ + bi * dst_ld + bj * pack_factor),
- src(src_ + bi * src_ld * bytes_per_pack / pack_factor + bj * bytes_per_pack),
+ src(src_ + bi * src_ld * bytes_per_pack / pack_factor +
+ bj * bytes_per_pack),
scales(scales_ + bi * src_ld / group_size),
biases(biases_ + bi * src_ld / group_size) {}
@@ -307,7 +344,11 @@ struct QuantizedBlockLoaderMlx {
T bias = *biases;
for (int i = 0; i < n_reads; i++) {
dequantize<T, pack_factor, bits>(
- src + i * bytes_per_pack, scale, bias, dst + i * pack_factor);
+ src + i * bytes_per_pack,
+ scale,
+ bias,
+ dst + i * pack_factor
+ );
}
}
@@ -337,7 +378,8 @@ struct QuantizedBlockLoaderMlx {
(device uint8_t*)(src + i * bytes_per_pack),
scale,
bias,
- dst + i * pack_factor);
+ dst + i * pack_factor
+ );
}
}
@@ -373,14 +415,21 @@ template <
short bits,
bool per_output_layout = false>
struct QuantizedBlockLoaderZp {
- static_assert(BCOLS <= group_size, "Group size should be larger than columns");
- static_assert(group_size % BCOLS == 0, "Group size should be divisible by columns");
+ static_assert(
+ BCOLS <= group_size,
+ "Group size should be larger than columns"
+ );
+ static_assert(
+ group_size % BCOLS == 0,
+ "Group size should be divisible by columns"
+ );
static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");
UZU_MTL_CONST short pack_factor = get_pack_factor<bits, 8>();
UZU_MTL_CONST short bytes_per_pack = get_bytes_per_pack<bits>();
UZU_MTL_CONST short BCOLS_PACKED = BCOLS / pack_factor;
- UZU_MTL_CONST short n_reads = (BCOLS_PACKED * BROWS < tgp_size) ? 1 : (BCOLS_PACKED * BROWS) / tgp_size;
+ UZU_MTL_CONST short n_reads =
+ (BCOLS_PACKED * BROWS < tgp_size) ? 1 : (BCOLS_PACKED * BROWS) / tgp_size;
UZU_MTL_CONST short group_steps = group_size / BCOLS;
const int src_ld;
@@ -414,30 +463,38 @@ struct QuantizedBlockLoaderZp {
ushort simd_lane_id [[thread_index_in_simdgroup]],
const int out_group_base_ = 0,
const int out_groups_total_ = 0,
- const int zp_stride_total_ = 0)
- : src_ld(src_ld_),
- groups_per_row(groups_per_row_),
+ const int zp_stride_total_ = 0
+ )
+ : src_ld(src_ld_), groups_per_row(groups_per_row_),
tile_stride(
reduction_dim ? BCOLS_PACKED * bytes_per_pack
- : BROWS * src_ld * bytes_per_pack / pack_factor),
- group_step_cnt(0),
- k_base(0),
- group_stride(BROWS * groups_per_row_),
+ : BROWS * src_ld * bytes_per_pack / pack_factor
+ ),
+ group_step_cnt(0), k_base(0), group_stride(BROWS * groups_per_row_),
thread_idx(simd_group_id * 32 + simd_lane_id),
bi(n_reads * thread_idx / BCOLS_PACKED),
bj((n_reads * thread_idx) % BCOLS_PACKED),
dst(dst_ + bi * dst_ld + bj * pack_factor),
- src(src_ + bi * src_ld * bytes_per_pack / pack_factor + bj * bytes_per_pack),
+ src(src_ + bi * src_ld * bytes_per_pack / pack_factor +
+ bj * bytes_per_pack),
scales(reduction_dim == 1 ? (scales_ + bi * groups_per_row_) : scales_),
- scales_row_start(reduction_dim == 1 ? (scales_ + bi * groups_per_row_) : scales_),
- zps_row_start(reduction_dim == 1
- ? (zero_points_row_start_ + bi * (bits == 4 ? ((groups_per_row_ + 1) / 2) : groups_per_row_))
- : zero_points_row_start_),
+ scales_row_start(
+ reduction_dim == 1 ? (scales_ + bi * groups_per_row_) : scales_
+ ),
+ zps_row_start(
+ reduction_dim == 1 ? (zero_points_row_start_ +
+ bi * (bits == 4 ? ((groups_per_row_ + 1) / 2)
+ : groups_per_row_))
+ : zero_points_row_start_
+ ),
out_group_base(per_output_layout ? out_group_base_ : 0),
out_groups_total(per_output_layout ? out_groups_total_ : 0),
zp_stride_total(per_output_layout ? zp_stride_total_ : 0) {}
- inline void current_scale_bias(thread T& out_scale, thread T& out_bias) const {
+ inline void current_scale_bias(
+ thread T& out_scale,
+ thread T& out_bias
+ ) const {
uint zp_n;
T scale_val;
if (per_output_layout) {
@@ -445,7 +502,8 @@ struct QuantizedBlockLoaderZp {
const int scale_index = row_idx * groups_per_row + out_group_base;
scale_val = scales_row_start[scale_index];
if (bits == 4) {
- const int byte_index = row_idx * zp_stride_total + (out_group_base >> 1);
+ const int byte_index =
+ row_idx * zp_stride_total + (out_group_base >> 1);
uint8_t zp_b = zps_row_start[byte_index];
zp_n = (out_group_base & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F);
} else {
@@ -457,11 +515,11 @@ struct QuantizedBlockLoaderZp {
: (int)(scales - scales_row_start);
scale_val = reduction_dim == 0 ? scales_row_start[g] : *scales;
if (bits == 4) {
- const device uint8_t* zp_ptr = zps_row_start + (g >> 1);
- uint8_t zp_b = *zp_ptr;
- zp_n = (g & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F);
+ const device uint8_t* zp_ptr = zps_row_start + (g >> 1);
+ uint8_t zp_b = *zp_ptr;
+ zp_n = (g & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F);
} else {
- zp_n = zps_row_start[g];
+ zp_n = zps_row_start[g];
}
}
out_scale = scale_val;
@@ -478,7 +536,11 @@ struct QuantizedBlockLoaderZp {
current_scale_bias(scale, bias);
for (int i = 0; i < n_reads; i++) {
dequantize<T, pack_factor, bits>(
- src + i * bytes_per_pack, scale, bias, dst + i * pack_factor);
+ src + i * bytes_per_pack,
+ scale,
+ bias,
+ dst + i * pack_factor
+ );
}
}
@@ -506,8 +568,12 @@ struct QuantizedBlockLoaderZp {
int pack_idx = bj + i; // global pack index across the BK packs
if (pack_idx < valid_packs) {
dequantize<T, pack_factor, bits>(
- src + i * bytes_per_pack, scale, bias, dst + i * pack_factor);
-
+ src + i * bytes_per_pack,
+ scale,
+ bias,
+ dst + i * pack_factor
+ );
+
// Mask the last pack if needed
if (pack_idx == valid_packs - 1) {
int rem = valid_cols - pack_idx * pack_factor;
@@ -538,7 +604,11 @@ struct QuantizedBlockLoaderZp {
current_scale_bias(scale, bias);
for (int i = 0; i < n_reads; i++) {
dequantize<T, pack_factor, bits>(
- src + i * bytes_per_pack, scale, bias, dst + i * pack_factor);
+ src + i * bytes_per_pack,
+ scale,
+ bias,
+ dst + i * pack_factor
+ );
}
}
@@ -580,7 +650,8 @@ inline void qmm_core(
const constant int& N,
threadgroup T* Xs,
threadgroup T* Ws,
- uint lid [[thread_index_in_threadgroup]]) {
+ uint lid [[thread_index_in_threadgroup]]
+) {
if (num_els < BM) {
if ((K % BK) != 0) {
const int k_blocks = K / BK;
@@ -673,7 +744,8 @@ void qmm_impl(
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]) {
+ uint simd_lid [[thread_index_in_simdgroup]]
+) {
static_assert(BK >= 32, "BK should be larger than SIMD_SIZE");
static_assert(BK % 32 == 0, "BK should be divisible by SIMD_SIZE");
@@ -687,8 +759,10 @@ void qmm_impl(
constexpr int BK_padded = (BK + 16 / sizeof(T));
constexpr int BN_padded = (BN + 16 / sizeof(T));
- using mma_t = matmul_utils::BlockMMA<T, T, BM, BN, BK, WM, WN, false, false, BK_padded, BN_padded>;
- using loader_x_t = matmul_utils::BlockLoader<T, BM, BK, BK_padded, 1, WM * WN * 32, 1, 4>;
+ using mma_t = matmul_utils::
+ BlockMMA<T, T, BM, BN, BK, WM, WN, false, false, BK_padded, BN_padded>;
+ using loader_x_t =
+ matmul_utils::BlockLoader<T, BM, BK, BK_padded, 1, WM * WN * 32, 1, 4>;
auto wl = (const device uint8_t*)w;
@@ -707,19 +781,47 @@ void qmm_impl(
mma_t mma_op(simd_gid, simd_lid);
if (kUseMlxQuant) {
- using loader_w_t = QuantizedBlockLoaderMlx<T, BK, BN, BN_padded, 0, WM * WN * 32, group_size, bits>;
+ using loader_w_t = QuantizedBlockLoaderMlx<
+ T,
+ BK,
+ BN,
+ BN_padded,
+ 0,
+ WM * WN * 32,
+ group_size,
+ bits>;
const device T* scales_mlx = scales_base + (y_col / group_size);
const device T* biases_mlx = biases_base + (y_col / group_size);
loader_w_t loader_w(wl, scales_mlx, biases_mlx, N, Ws, simd_gid, simd_lid);
qmm_core<loader_w_t, loader_x_t, mma_t, T, aligned_K, BM, BK, BN>(
- loader_x, loader_w, mma_op, num_els, num_outs, K, y, N, Xs, Ws, lid);
+ loader_x,
+ loader_w,
+ mma_op,
+ num_els,
+ num_outs,
+ K,
+ y,
+ N,
+ Xs,
+ Ws,
+ lid
+ );
} else {
const int out_groups_total = (N + group_size - 1) / group_size;
const int groups_per_row = out_groups_total;
const int out_group = y_col / group_size;
const int zp_stride_out =
(bits == 4) ? ((out_groups_total + 1) / 2) : out_groups_total;
- using loader_w_t = QuantizedBlockLoaderZp<T, BK, BN, BN_padded, 0, WM * WN * 32, group_size, bits, true>;
+ using loader_w_t = QuantizedBlockLoaderZp<
+ T,
+ BK,
+ BN,
+ BN_padded,
+ 0,
+ WM * WN * 32,
+ group_size,
+ bits,
+ true>;
loader_w_t loader_w(
wl,
scales_base,
@@ -731,9 +833,21 @@ void qmm_impl(
simd_lid,
out_group,
out_groups_total,
- zp_stride_out);
+ zp_stride_out
+ );
qmm_core<loader_w_t, loader_x_t, mma_t, T, aligned_K, BM, BK, BN>(
- loader_x, loader_w, mma_op, num_els, num_outs, K, y, N, Xs, Ws, lid);
+ loader_x,
+ loader_w,
+ mma_op,
+ num_els,
+ num_outs,
+ K,
+ y,
+ N,
+ Xs,
+ Ws,
+ lid
+ );
}
}
@@ -757,7 +871,8 @@ inline void qmm_transposed_core(
const constant int& N,
threadgroup T* Xs,
threadgroup T* Ws,
- uint lid [[thread_index_in_threadgroup]]) {
+ uint lid [[thread_index_in_threadgroup]]
+) {
if (num_els < BM) {
if (!aligned_N && num_outs < BN) {
for (int k = 0; k < K; k += BK) {
@@ -837,7 +952,8 @@ void qmm_transposed_impl(
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]) {
+ uint simd_lid [[thread_index_in_simdgroup]]
+) {
static_assert(BK >= 32, "BK should be larger than SIMD_SIZE");
static_assert(BK % 32 == 0, "BK should be divisible by SIMD_SIZE");
@@ -847,8 +963,10 @@ void qmm_transposed_impl(
constexpr int bytes_per_pack = get_bytes_per_pack<bits>();
constexpr int BK_padded = (BK + 16 / sizeof(T));
- using mma_t = matmul_utils::BlockMMA<T, T, BM, BN, BK, WM, WN, false, true, BK_padded, BK_padded>;
- using loader_x_t = matmul_utils::BlockLoader<T, BM, BK, BK_padded, 1, WM * WN * 32>;
+ using mma_t = matmul_utils::
+ BlockMMA<T, T, BM, BN, BK, WM, WN, false, true, BK_padded, BK_padded>;
+ using loader_x_t =
+ matmul_utils::BlockLoader<T, BM, BK, BK_padded, 1, WM * WN * 32>;
const int K_w = K * bytes_per_pack / pack_factor;
const int K_g = (K + group_size - 1) / group_size;
@@ -869,16 +987,80 @@ void qmm_transposed_impl(
mma_t mma_op(simd_gid, simd_lid);
if (kUseMlxQuant) {
- using loader_w_t = QuantizedBlockLoaderMlx< T, BN, BK, BK_padded, 1, WM * WN * 32, group_size, bits>;
+ using loader_w_t = QuantizedBlockLoaderMlx<
+ T,
+ BN,
+ BK,
+ BK_padded,
+ 1,
+ WM * WN * 32,
+ group_size,
+ bits>;
loader_w_t loader_w(w_block, scales, biases, K, Ws, simd_gid, simd_lid);
- qmm_transposed_core<loader_w_t, loader_x_t, mma_t, T, aligned_N, BM, BK, BN>(
- loader_x, loader_w, mma_op, num_els, num_outs, K, y_block, N, Xs, Ws, lid);
+ qmm_transposed_core<
+ loader_w_t,
+ loader_x_t,
+ mma_t,
+ T,
+ aligned_N,
+ BM,
+ BK,
+ BN>(
+ loader_x,
+ loader_w,
+ mma_op,
+ num_els,
+ num_outs,
+ K,
+ y_block,
+ N,
+ Xs,
+ Ws,
+ lid
+ );
} else {
- using loader_w_t = QuantizedBlockLoaderZp<T, BN, BK, BK_padded, 1, WM * WN * 32, group_size, bits>;
- const device uint8_t* zero_points_row = zero_points + y_col * (bits == 4 ? ((K_g + 1) / 2) : K_g);
- loader_w_t loader_w(w_block, scales, zero_points_row, K, K_g, Ws, simd_gid, simd_lid);
- qmm_transposed_core<loader_w_t, loader_x_t, mma_t, T, aligned_N, BM, BK, BN>(
- loader_x, loader_w, mma_op, num_els, num_outs, K, y_block, N, Xs, Ws, lid);
+ using loader_w_t = QuantizedBlockLoaderZp<
+ T,
+ BN,
+ BK,
+ BK_padded,
+ 1,
+ WM * WN * 32,
+ group_size,
+ bits>;
+ const device uint8_t* zero_points_row =
+ zero_points + y_col * (bits == 4 ? ((K_g + 1) / 2) : K_g);
+ loader_w_t loader_w(
+ w_block,
+ scales,
+ zero_points_row,
+ K,
+ K_g,
+ Ws,
+ simd_gid,
+ simd_lid
+ );
+ qmm_transposed_core<
+ loader_w_t,
+ loader_x_t,
+ mma_t,
+ T,
+ aligned_N,
+ BM,
+ BK,
+ BN>(
+ loader_x,
+ loader_w,
+ mma_op,
+ num_els,
+ num_outs,
+ K,
+ y_block,
+ N,
+ Xs,
+ Ws,
+ lid
+ );
}
}
@@ -894,7 +1076,8 @@ void qmv_impl_dispatch(
const constant int& N, // out_vec_size
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]) {
+ uint simd_lid [[thread_index_in_simdgroup]]
+) {
constexpr int num_simdgroups = 2;
constexpr int results_per_simdgroup = 4;
constexpr int packs_per_thread = 1;
@@ -911,10 +1094,12 @@ void qmv_impl_dispatch(
thread U result[results_per_simdgroup] = {0};
const int in_vec_size_w = K * bytes_per_pack / pack_factor;
- const int in_vec_size_g = (K + group_size - 1) / group_size; // ceil(K / group_size)
- const device T* scales_base = scales;
+ const int in_vec_size_g =
+ (K + group_size - 1) / group_size; // ceil(K / group_size)
+ const device T* scales_base = scales;
const device uint8_t* zero_points_base = zero_points;
- const int out_row = tid.y * (num_simdgroups * results_per_simdgroup) + simd_gid * results_per_simdgroup;
+ const int out_row = tid.y * (num_simdgroups * results_per_simdgroup) +
+ simd_gid * results_per_simdgroup;
const int used_out_row = min(N - results_per_simdgroup, out_row);
if (out_row >= N) {
@@ -922,9 +1107,10 @@ void qmv_impl_dispatch(
}
if (N < (num_simdgroups * results_per_simdgroup)) {
- ws += out_row * in_vec_size_w + simd_lid * packs_per_thread * bytes_per_pack;
+ ws +=
+ out_row * in_vec_size_w + simd_lid * packs_per_thread * bytes_per_pack;
scales += out_row * in_vec_size_g + simd_lid / scale_step_per_thread;
-
+
const int zp_stride = bits == 4 ? ((in_vec_size_g + 1) / 2) : in_vec_size_g;
const device uint8_t* zps_row_base = nullptr;
const device T* biases_row_base = nullptr;
@@ -942,7 +1128,8 @@ void qmv_impl_dispatch(
U sum = load_vector<T, U, values_per_thread, bits>(x, x_thread);
for (int row = 0; out_row + row < N; row++) {
- if (row >= results_per_simdgroup) break;
+ if (row >= results_per_simdgroup)
+ break;
auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);
const int row_idx = out_row + row;
const device T* sr = scales_base + row_idx * in_vec_size_g;
@@ -952,7 +1139,8 @@ void qmv_impl_dispatch(
if (UseMlx) {
const device T* bl = biases_row_base + row * in_vec_size_g;
U b = static_cast<U>(bl[g]);
- result[row] += qdot<U, values_per_thread, bits>(wl, x_thread, s, b, sum);
+ result[row] +=
+ qdot<U, values_per_thread, bits>(wl, x_thread, s, b, sum);
} else {
const device uint8_t* zl = zps_row_base + row * zp_stride;
U zp;
@@ -974,50 +1162,57 @@ void qmv_impl_dispatch(
const int remaining = clamp(
static_cast<int>(K - k - simd_lid * values_per_thread),
0,
- values_per_thread);
+ values_per_thread
+ );
if (remaining > 0) {
- U sum = load_vector_safe<T, U, values_per_thread, bits>(
- x, x_thread, remaining);
-
- for (int row = 0; out_row + row < N; row++) {
- if (row >= results_per_simdgroup) break;
- auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);
- const int row_idx = out_row + row;
- const device T* sr = scales_base + row_idx * in_vec_size_g;
-
- int g = (k + simd_lid * values_per_thread) / group_size;
- U s = static_cast<U>(sr[g]);
- if (UseMlx) {
- const device T* bl = biases_row_base + row * in_vec_size_g;
- U b = static_cast<U>(bl[g]);
- result[row] += qdot<U, values_per_thread, bits>(wl, x_thread, s, b, sum);
- } else {
- const device uint8_t* zl = zps_row_base + row * zp_stride;
- U zp;
- if (bits == 4) {
- uint8_t zp_b = zl[g >> 1];
- zp = static_cast<U>((g & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F));
- } else {
- zp = static_cast<U>(zl[g]);
- }
- result[row] += qdot_zero_point<U, values_per_thread, bits>(
- wl, x_thread, s, zp);
- }
+ U sum = load_vector_safe<T, U, values_per_thread, bits>(
+ x,
+ x_thread,
+ remaining
+ );
+
+ for (int row = 0; out_row + row < N; row++) {
+ if (row >= results_per_simdgroup)
+ break;
+ auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);
+ const int row_idx = out_row + row;
+ const device T* sr = scales_base + row_idx * in_vec_size_g;
+
+ int g = (k + simd_lid * values_per_thread) / group_size;
+ U s = static_cast<U>(sr[g]);
+ if (UseMlx) {
+ const device T* bl = biases_row_base + row * in_vec_size_g;
+ U b = static_cast<U>(bl[g]);
+ result[row] +=
+ qdot<U, values_per_thread, bits>(wl, x_thread, s, b, sum);
+ } else {
+ const device uint8_t* zl = zps_row_base + row * zp_stride;
+ U zp;
+ if (bits == 4) {
+ uint8_t zp_b = zl[g >> 1];
+ zp = static_cast<U>((g & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F));
+ } else {
+ zp = static_cast<U>(zl[g]);
+ }
+ result[row] +=
+ qdot_zero_point<U, values_per_thread, bits>(wl, x_thread, s, zp);
}
+ }
}
for (int row = 0; out_row + row < N; row++) {
- if (row >= results_per_simdgroup) break;
+ if (row >= results_per_simdgroup)
+ break;
result[row] = simd_sum(result[row]);
if (simd_lid == 0) {
y[row] = static_cast<T>(result[row]);
}
}
- }
- else {
- ws += used_out_row * in_vec_size_w + simd_lid * packs_per_thread * bytes_per_pack;
+ } else {
+ ws += used_out_row * in_vec_size_w +
+ simd_lid * packs_per_thread * bytes_per_pack;
scales += used_out_row * in_vec_size_g + simd_lid / scale_step_per_thread;
-
+
const int zp_stride = bits == 4 ? ((in_vec_size_g + 1) / 2) : in_vec_size_g;
const device uint8_t* zps_row_base = nullptr;
const device T* biases_row_base = nullptr;
@@ -1044,7 +1239,8 @@ void qmv_impl_dispatch(
if (UseMlx) {
const device T* bl = biases_row_base + row * in_vec_size_g;
U b = static_cast<U>(bl[g]);
- result[row] += qdot<U, values_per_thread, bits>(wl, x_thread, s, b, sum);
+ result[row] +=
+ qdot<U, values_per_thread, bits>(wl, x_thread, s, b, sum);
} else {
const device uint8_t* zl = zps_row_base + row * zp_stride;
U zp;
@@ -1066,37 +1262,47 @@ void qmv_impl_dispatch(
const int remaining = clamp(
static_cast<int>(K - k - simd_lid * values_per_thread),
0,
- values_per_thread);
+ values_per_thread
+ );
if (remaining > 0) {
- U sum = load_vector_safe<T, U, values_per_thread, bits>(
- x, x_thread, remaining);
-
- for (int row = 0; row < results_per_simdgroup; row++) {
- auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);
- const int row_idx = used_out_row + row;
- const device T* sr = scales_base + row_idx * in_vec_size_g;
-
- int g = (k + simd_lid * values_per_thread) / group_size;
- U s = static_cast<U>(sr[g]);
- if (UseMlx) {
- const device T* bl = biases_row_base + row * in_vec_size_g;
- U b = static_cast<U>(bl[g]);
- result[row] += qdot_safe<U, values_per_thread, bits>(
- wl, x_thread, s, b, sum, remaining);
- } else {
- const device uint8_t* zl = zps_row_base + row * zp_stride;
- U zp;
- if (bits == 4) {
- uint8_t zp_b = zl[g >> 1];
- zp = static_cast<U>((g & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F));
- } else {
- zp = static_cast<U>(zl[g]);
- }
- result[row] += qdot_zero_point<U, values_per_thread, bits>(
- wl, x_thread, s, zp);
- }
+ U sum = load_vector_safe<T, U, values_per_thread, bits>(
+ x,
+ x_thread,
+ remaining
+ );
+
+ for (int row = 0; row < results_per_simdgroup; row++) {
+ auto wl = (const device uint8_t*)(ws + row * in_vec_size_w);
+ const int row_idx = used_out_row + row;
+ const device T* sr = scales_base + row_idx * in_vec_size_g;
+
+ int g = (k + simd_lid * values_per_thread) / group_size;
+ U s = static_cast<U>(sr[g]);
+ if (UseMlx) {
+ const device T* bl = biases_row_base + row * in_vec_size_g;
+ U b = static_cast<U>(bl[g]);
+ result[row] += qdot_safe<U, values_per_thread, bits>(
+ wl,
+ x_thread,
+ s,
+ b,
+ sum,
+ remaining
+ );
+ } else {
+ const device uint8_t* zl = zps_row_base + row * zp_stride;
+ U zp;
+ if (bits == 4) {
+ uint8_t zp_b = zl[g >> 1];
+ zp = static_cast<U>((g & 1) ? ((zp_b >> 4) & 0x0F) : (zp_b & 0x0F));
+ } else {
+ zp = static_cast<U>(zl[g]);
+ }
+ result[row] +=
+ qdot_zero_point<U, values_per_thread, bits>(wl, x_thread, s, zp);
}
+ }
}
for (int row = 0; row < results_per_simdgroup; row++) {
@@ -1120,13 +1326,36 @@ void qmv_impl(
const constant int& N,
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]) {
+ uint simd_lid [[thread_index_in_simdgroup]]
+) {
if (kUseMlxQuant) {
qmv_impl_dispatch<T, group_size, bits, true>(
- w, scales, zero_points, biases, x, y, K, N, tid, simd_gid, simd_lid);
+ w,
+ scales,
+ zero_points,
+ biases,
+ x,
+ y,
+ K,
+ N,
+ tid,
+ simd_gid,
+ simd_lid
+ );
} else {
qmv_impl_dispatch<T, group_size, bits, false>(
- w, scales, zero_points, biases, x, y, K, N, tid, simd_gid, simd_lid);
+ w,
+ scales,
+ zero_points,
+ biases,
+ x,
+ y,
+ K,
+ N,
+ tid,
+ simd_gid,
+ simd_lid
+ );
}
}
@@ -1142,7 +1371,8 @@ void qmv_fast_impl(
const constant int& out_vec_size,
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]) {
+ uint simd_lid [[thread_index_in_simdgroup]]
+) {
constexpr int packs_per_thread = bits == 2 ? 1 : 2;
constexpr int num_simdgroups = 2;
constexpr int results_per_simdgroup = 4;
@@ -1159,10 +1389,10 @@ void qmv_fast_impl(
const int in_vec_size_w = in_vec_size * bytes_per_pack / pack_factor;
const int in_vec_size_g = in_vec_size / group_size;
const int out_row = tid.y * (num_simdgroups * results_per_simdgroup) +
- simd_gid * results_per_simdgroup;
+ simd_gid * results_per_simdgroup;
ws += out_row * in_vec_size_w + simd_lid * packs_per_thread * bytes_per_pack;
scales += out_row * in_vec_size_g + simd_lid / scale_step_per_thread;
-
+
int zp_stride = 0;
const device uint8_t* zps = nullptr;
bool high_nibble = false;
@@ -1171,15 +1401,15 @@ void qmv_fast_impl(
biases += out_row * in_vec_size_g + simd_lid / scale_step_per_thread;
} else {
if (bits == 4) {
- zp_stride = (in_vec_size_g + 1) / 2;
- zps = zero_points + out_row * zp_stride;
- int g_offset = simd_lid / scale_step_per_thread;
- zps += g_offset / 2;
- high_nibble = (g_offset & 1);
+ zp_stride = (in_vec_size_g + 1) / 2;
+ zps = zero_points + out_row * zp_stride;
+ int g_offset = simd_lid / scale_step_per_thread;
+ zps += g_offset / 2;
+ high_nibble = (g_offset & 1);
} else {
- zp_stride = in_vec_size_g;
- zps = zero_points + out_row * zp_stride;
- zps += simd_lid / scale_step_per_thread;
+ zp_stride = in_vec_size_g;
+ zps = zero_points + out_row * zp_stride;
+ zps += simd_lid / scale_step_per_thread;
}
}
@@ -1205,29 +1435,45 @@ void qmv_fast_impl(
U b1 = static_cast<U>(biases[in_vec_size_g]);
U b2 = static_cast<U>(biases[2 * in_vec_size_g]);
U b3 = static_cast<U>(biases[3 * in_vec_size_g]);
- result[0] += qdot<U, values_per_thread, bits>(wl0, x_thread, s0, b0, sum);
- result[1] += qdot<U, values_per_thread, bits>(wl1, x_thread, s1, b1, sum);
- result[2] += qdot<U, values_per_thread, bits>(wl2, x_thread, s2, b2, sum);
- result[3] += qdot<U, values_per_thread, bits>(wl3, x_thread, s3, b3, sum);
+ result[0] +=
+ qdot<U, values_per_thread, bits>(wl0, x_thread, s0, b0, sum);
+ result[1] +=
+ qdot<U, values_per_thread, bits>(wl1, x_thread, s1, b1, sum);
+ result[2] +=
+ qdot<U, values_per_thread, bits>(wl2, x_thread, s2, b2, sum);
+ result[3] +=
+ qdot<U, values_per_thread, bits>(wl3, x_thread, s3, b3, sum);
} else {
uint8_t zp_byte0 = zps[0];
uint8_t zp_byte1 = zps[zp_stride];
uint8_t zp_byte2 = zps[2 * zp_stride];
uint8_t zp_byte3 = zps[3 * zp_stride];
- U zp0 = static_cast<U>((bits == 4 && high_nibble) ? (zp_byte0 >> 4) : (zp_byte0 & 0x0F));
- U zp1 = static_cast<U>((bits == 4 && high_nibble) ? (zp_byte1 >> 4) : (zp_byte1 & 0x0F));
- U zp2 = static_cast<U>((bits == 4 && high_nibble) ? (zp_byte2 >> 4) : (zp_byte2 & 0x0F));
- U zp3 = static_cast<U>((bits == 4 && high_nibble) ? (zp_byte3 >> 4) : (zp_byte3 & 0x0F));
+ U zp0 = static_cast<U>(
+ (bits == 4 && high_nibble) ? (zp_byte0 >> 4) : (zp_byte0 & 0x0F)
+ );
+ U zp1 = static_cast<U>(
+ (bits == 4 && high_nibble) ? (zp_byte1 >> 4) : (zp_byte1 & 0x0F)
+ );
+ U zp2 = static_cast<U>(
+ (bits == 4 && high_nibble) ? (zp_byte2 >> 4) : (zp_byte2 & 0x0F)
+ );
+ U zp3 = static_cast<U>(
+ (bits == 4 && high_nibble) ? (zp_byte3 >> 4) : (zp_byte3 & 0x0F)
+ );
if (bits == 8) {
zp0 = static_cast<U>(zp_byte0);
zp1 = static_cast<U>(zp_byte1);
zp2 = static_cast<U>(zp_byte2);
zp3 = static_cast<U>(zp_byte3);
}
- result[0] += qdot_zero_point<U, values_per_thread, bits>(wl0, x_thread, s0, zp0);
- result[1] += qdot_zero_point<U, values_per_thread, bits>(wl1, x_thread, s1, zp1);
- result[2] += qdot_zero_point<U, values_per_thread, bits>(wl2, x_thread, s2, zp2);
- result[3] += qdot_zero_point<U, values_per_thread, bits>(wl3, x_thread, s3, zp3);
+ result[0] +=
+ qdot_zero_point<U, values_per_thread, bits>(wl0, x_thread, s0, zp0);
+ result[1] +=
+ qdot_zero_point<U, values_per_thread, bits>(wl1, x_thread, s1, zp1);
+ result[2] +=
+ qdot_zero_point<U, values_per_thread, bits>(wl2, x_thread, s2, zp2);
+ result[3] +=
+ qdot_zero_point<U, values_per_thread, bits>(wl3, x_thread, s3, zp3);
}
}
@@ -1237,9 +1483,9 @@ void qmv_fast_impl(
biases += block_size / group_size;
} else {
if (bits == 4) {
- zps += (block_size / group_size) / 2;
+ zps += (block_size / group_size) / 2;
} else {
- zps += block_size / group_size;
+ zps += block_size / group_size;
}
}
x += block_size;
@@ -1265,7 +1511,8 @@ void qvm_impl_core(
const int out_vec_size,
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]) {
+ uint simd_lid [[thread_index_in_simdgroup]]
+) {
constexpr int num_simdgroups = 2;
constexpr int pack_factor = get_pack_factor<bits, 32>();
constexpr int bytes_per_pack = get_bytes_per_pack<bits, 32>();
@@ -1283,18 +1530,20 @@ void qvm_impl_core(
const int out_vec_size_w = out_vec_size * bytes_per_pack / pack_factor;
const int out_vec_size_g = out_vec_size / group_size;
const int out_col = pack_factor * tn * (tid.y * num_simdgroups + simd_gid);
-
+
if (out_col >= out_vec_size) {
return;
}
const int out_group = out_col / group_size;
- const int zp_row_stride = use_zero_points ? ((bits == 4) ? ((out_vec_size_g + 1) / 2) : out_vec_size_g) : 0;
-
- const device uint8_t* ws_ptr =
- (const device uint8_t*)ws +
- out_col * bytes_per_pack / pack_factor +
- simd_lid * out_vec_size_w;
+ const int zp_row_stride =
+ use_zero_points
+ ? ((bits == 4) ? ((out_vec_size_g + 1) / 2) : out_vec_size_g)
+ : 0;
+
+ const device uint8_t* ws_ptr = (const device uint8_t*)ws +
+ out_col * bytes_per_pack / pack_factor +
+ simd_lid * out_vec_size_w;
const device T* scales_base = scales;
const device T* biases_base = biases;
x += tid.x * in_vec_size + simd_lid;
@@ -1303,11 +1552,14 @@ void qvm_impl_core(
for (int k_base = 0; k_base < in_vec_size; k_base += block_size) {
const int k_index = k_base + simd_lid;
const bool active = k_index < in_vec_size;
-
+
U x_local = active ? static_cast<U>(*x) : U(0);
- U scale = active ? static_cast<U>(scales_base[k_index * out_vec_size_g + out_group]) : U(0);
+ U scale =
+ active
+ ? static_cast<U>(scales_base[k_index * out_vec_size_g + out_group])
+ : U(0);
U bias;
-
+
if (use_zero_points && active) {
U zp;
if (bits == 4) {
@@ -1315,7 +1567,9 @@ void qvm_impl_core(
const device uint8_t* zp_ptr = row_ptr + (out_group >> 1);
uint8_t zp_byte = *zp_ptr;
bool high_nibble = (out_group & 1) != 0;
- zp = static_cast<U>(high_nibble ? ((zp_byte >> 4) & 0x0F) : (zp_byte & 0x0F));
+ zp = static_cast<U>(
+ high_nibble ? ((zp_byte >> 4) & 0x0F) : (zp_byte & 0x0F)
+ );
} else {
const device uint8_t* row_ptr = zero_points + k_index * zp_row_stride;
const device uint8_t* zp_ptr = row_ptr + out_group;
@@ -1324,19 +1578,24 @@ void qvm_impl_core(
bias = -scale * zp;
} else if (active) {
bias = static_cast<U>(biases_base[k_index * out_vec_size_g + out_group]);
- } else {
+ } else {
bias = U(0);
}
-
+
if (active) {
w_local = *((device vec_w*)ws_ptr);
}
- qouter<U, tn * pack_factor, bits>(
- (thread uint8_t*)&w_local, x_local, scale, bias, result);
+ qouter<U, tn * pack_factor, bits>(
+ (thread uint8_t*)&w_local,
+ x_local,
+ scale,
+ bias,
+ result
+ );
- x += block_size;
- scales += block_size * out_vec_size_g;
+ x += block_size;
+ scales += block_size * out_vec_size_g;
if (!use_zero_points) {
biases += block_size * out_vec_size_g;
}
@@ -1367,9 +1626,21 @@ void qvm_impl_mlx(
const int out_vec_size,
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]) {
+ uint simd_lid [[thread_index_in_simdgroup]]
+) {
qvm_impl_core<T, group_size, bits, false>(
- ws, scales, biases, nullptr, x, y, in_vec_size, out_vec_size, tid, simd_gid, simd_lid);
+ ws,
+ scales,
+ biases,
+ nullptr,
+ x,
+ y,
+ in_vec_size,
+ out_vec_size,
+ tid,
+ simd_gid,
+ simd_lid
+ );
}
template <typename T, int group_size, int bits>
@@ -1383,9 +1654,21 @@ void qvm_impl_zeropoint(
const int out_vec_size,
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]) {
+ uint simd_lid [[thread_index_in_simdgroup]]
+) {
qvm_impl_core<T, group_size, bits, true>(
- w, scales, nullptr, zero_points, x, y, in_vec_size, out_vec_size, tid, simd_gid, simd_lid);
+ w,
+ scales,
+ nullptr,
+ zero_points,
+ x,
+ y,
+ in_vec_size,
+ out_vec_size,
+ tid,
+ simd_gid,
+ simd_lid
+ );
}
template <typename T, int group_size, int bits>
@@ -1400,17 +1683,37 @@ void qvm_impl(
const constant int& N, // out_vec_size
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]) {
+ uint simd_lid [[thread_index_in_simdgroup]]
+) {
if (kUseMlxQuant) {
qvm_impl_mlx<T, group_size, bits>(
- w, scales, biases, x, y, K, N, tid, simd_gid, simd_lid);
+ w,
+ scales,
+ biases,
+ x,
+ y,
+ K,
+ N,
+ tid,
+ simd_gid,
+ simd_lid
+ );
} else {
qvm_impl_zeropoint<T, group_size, bits>(
- w, scales, zero_points, x, y, K, N, tid, simd_gid, simd_lid);
+ w,
+ scales,
+ zero_points,
+ x,
+ y,
+ K,
+ N,
+ tid,
+ simd_gid,
+ simd_lid
+ );
}
}
-
template <
typename T,
const int group_size,
@@ -1422,7 +1725,8 @@ template <
[[kernel]] void qmm(
const device uint32_t* w [[buffer(0)]],
const device T* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],
+ const device uint8_t* zero_points
+ [[buffer(2), function_constant(kUseZeroPoints)]],
const device T* biases [[buffer(2), function_constant(kUseMlxQuant)]],
const device T* x [[buffer(3)]],
device T* y [[buffer(4)]],
@@ -1432,7 +1736,8 @@ template <
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]) {
+ uint simd_lid [[thread_index_in_simdgroup]]
+) {
(void)lid;
@@ -1443,7 +1748,22 @@ template <
threadgroup T Ws[BK * BN_padded];
qmm_impl<T, group_size, bits, aligned_K, BM, BK, BN>(
- w, scales, zero_points, biases, x, y, Xs, Ws, K, N, M, tid, lid, simd_gid, simd_lid);
+ w,
+ scales,
+ zero_points,
+ biases,
+ x,
+ y,
+ Xs,
+ Ws,
+ K,
+ N,
+ M,
+ tid,
+ lid,
+ simd_gid,
+ simd_lid
+ );
}
template <
@@ -1457,7 +1777,8 @@ template <
[[kernel]] void qmm_transposed(
const device uint32_t* w [[buffer(0)]],
const device T* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],
+ const device uint8_t* zero_points
+ [[buffer(2), function_constant(kUseZeroPoints)]],
const device T* biases [[buffer(2), function_constant(kUseMlxQuant)]],
const device T* x [[buffer(3)]],
device T* y [[buffer(4)]],
@@ -1467,7 +1788,8 @@ template <
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]) {
+ uint simd_lid [[thread_index_in_simdgroup]]
+) {
(void)lid;
@@ -1477,14 +1799,30 @@ template <
threadgroup T Ws[BN * BK_padded];
qmm_transposed_impl<T, group_size, bits, aligned_N, BM, BK, BN>(
- w, scales, zero_points, biases, x, y, Xs, Ws, K, N, M, tid, lid, simd_gid, simd_lid);
+ w,
+ scales,
+ zero_points,
+ biases,
+ x,
+ y,
+ Xs,
+ Ws,
+ K,
+ N,
+ M,
+ tid,
+ lid,
+ simd_gid,
+ simd_lid
+ );
}
template <typename T, int group_size, int bits>
[[kernel]] void qmv(
const device uint32_t* w [[buffer(0)]],
const device T* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],
+ const device uint8_t* zero_points
+ [[buffer(2), function_constant(kUseZeroPoints)]],
const device T* biases [[buffer(2), function_constant(kUseMlxQuant)]],
const device T* x [[buffer(3)]],
device T* y [[buffer(4)]],
@@ -1492,16 +1830,30 @@ template <typename T, int group_size, int bits>
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]) {
-
- qmv_impl<T, group_size, bits>(w, scales, zero_points, biases, x, y, K, N, tid, simd_gid, simd_lid);
+ uint simd_lid [[thread_index_in_simdgroup]]
+) {
+
+ qmv_impl<T, group_size, bits>(
+ w,
+ scales,
+ zero_points,
+ biases,
+ x,
+ y,
+ K,
+ N,
+ tid,
+ simd_gid,
+ simd_lid
+ );
}
template <typename T, int group_size, int bits>
[[kernel]] void qvm(
const device uint32_t* w [[buffer(0)]],
const device T* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],
+ const device uint8_t* zero_points
+ [[buffer(2), function_constant(kUseZeroPoints)]],
const device T* biases [[buffer(2), function_constant(kUseMlxQuant)]],
const device T* x [[buffer(3)]],
device T* y [[buffer(4)]],
@@ -1509,17 +1861,30 @@ template <typename T, int group_size, int bits>
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]) {
+ uint simd_lid [[thread_index_in_simdgroup]]
+) {
qvm_impl<T, group_size, bits>(
- w, scales, zero_points, biases, x, y, K, N, tid, simd_gid, simd_lid);
+ w,
+ scales,
+ zero_points,
+ biases,
+ x,
+ y,
+ K,
+ N,
+ tid,
+ simd_gid,
+ simd_lid
+ );
}
template <typename T, int group_size, int bits>
[[kernel]] void qmv_fast(
const device uint32_t* w [[buffer(0)]],
const device T* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],
+ const device uint8_t* zero_points
+ [[buffer(2), function_constant(kUseZeroPoints)]],
const device T* biases [[buffer(2), function_constant(kUseMlxQuant)]],
const device T* x [[buffer(3)]],
device T* y [[buffer(4)]],
@@ -1527,12 +1892,27 @@ template <typename T, int group_size, int bits>
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]) {
- qmv_fast_impl<T, group_size, bits>(w, scales, zero_points, biases, x, y, K, N, tid, simd_gid, simd_lid);
+ uint simd_lid [[thread_index_in_simdgroup]]
+) {
+ qmv_fast_impl<T, group_size, bits>(
+ w,
+ scales,
+ zero_points,
+ biases,
+ x,
+ y,
+ K,
+ N,
+ tid,
+ simd_gid,
+ simd_lid
+ );
}
-template [[host_name("qmv_f16_g32_b4_fast")]]
-[[kernel]] void qmv_fast<half, 32, 4>(
+template [[host_name("qmv_f16_g32_b4_fast")]] [[kernel]] void qmv_fast<
+ half,
+ 32,
+ 4>(
const device uint32_t* w [[buffer(0)]],
const device half* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -1543,10 +1923,13 @@ template [[host_name("qmv_f16_g32_b4_fast")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmv_f16_g64_b4_fast")]]
-[[kernel]] void qmv_fast<half, 64, 4>(
+template [[host_name("qmv_f16_g64_b4_fast")]] [[kernel]] void qmv_fast<
+ half,
+ 64,
+ 4>(
const device uint32_t* w [[buffer(0)]],
const device half* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -1557,10 +1940,13 @@ template [[host_name("qmv_f16_g64_b4_fast")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmv_f16_g128_b4_fast")]]
-[[kernel]] void qmv_fast<half, 128, 4>(
+template [[host_name("qmv_f16_g128_b4_fast")]] [[kernel]] void qmv_fast<
+ half,
+ 128,
+ 4>(
const device uint32_t* w [[buffer(0)]],
const device half* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -1571,10 +1957,13 @@ template [[host_name("qmv_f16_g128_b4_fast")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmv_bf16_g32_b4_fast")]]
-[[kernel]] void qmv_fast<bfloat, 32, 4>(
+template [[host_name("qmv_bf16_g32_b4_fast")]] [[kernel]] void qmv_fast<
+ bfloat,
+ 32,
+ 4>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -1585,10 +1974,13 @@ template [[host_name("qmv_bf16_g32_b4_fast")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmv_bf16_g64_b4_fast")]]
-[[kernel]] void qmv_fast<bfloat, 64, 4>(
+template [[host_name("qmv_bf16_g64_b4_fast")]] [[kernel]] void qmv_fast<
+ bfloat,
+ 64,
+ 4>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -1599,10 +1991,13 @@ template [[host_name("qmv_bf16_g64_b4_fast")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmv_bf16_g128_b4_fast")]]
-[[kernel]] void qmv_fast<bfloat, 128, 4>(
+template [[host_name("qmv_bf16_g128_b4_fast")]] [[kernel]] void qmv_fast<
+ bfloat,
+ 128,
+ 4>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -1613,10 +2008,13 @@ template [[host_name("qmv_bf16_g128_b4_fast")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmv_f32_g32_b4_fast")]]
-[[kernel]] void qmv_fast<float, 32, 4>(
+template [[host_name("qmv_f32_g32_b4_fast")]] [[kernel]] void qmv_fast<
+ float,
+ 32,
+ 4>(
const device uint32_t* w [[buffer(0)]],
const device float* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -1627,10 +2025,13 @@ template [[host_name("qmv_f32_g32_b4_fast")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmv_f32_g64_b4_fast")]]
-[[kernel]] void qmv_fast<float, 64, 4>(
+template [[host_name("qmv_f32_g64_b4_fast")]] [[kernel]] void qmv_fast<
+ float,
+ 64,
+ 4>(
const device uint32_t* w [[buffer(0)]],
const device float* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -1641,10 +2042,13 @@ template [[host_name("qmv_f32_g64_b4_fast")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmv_f32_g128_b4_fast")]]
-[[kernel]] void qmv_fast<float, 128, 4>(
+template [[host_name("qmv_f32_g128_b4_fast")]] [[kernel]] void qmv_fast<
+ float,
+ 128,
+ 4>(
const device uint32_t* w [[buffer(0)]],
const device float* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -1655,10 +2059,13 @@ template [[host_name("qmv_f32_g128_b4_fast")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmv_f16_g32_b8_fast")]]
-[[kernel]] void qmv_fast<half, 32, 8>(
+template [[host_name("qmv_f16_g32_b8_fast")]] [[kernel]] void qmv_fast<
+ half,
+ 32,
+ 8>(
const device uint32_t* w [[buffer(0)]],
const device half* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -1669,10 +2076,13 @@ template [[host_name("qmv_f16_g32_b8_fast")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmv_f16_g64_b8_fast")]]
-[[kernel]] void qmv_fast<half, 64, 8>(
+template [[host_name("qmv_f16_g64_b8_fast")]] [[kernel]] void qmv_fast<
+ half,
+ 64,
+ 8>(
const device uint32_t* w [[buffer(0)]],
const device half* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -1683,10 +2093,13 @@ template [[host_name("qmv_f16_g64_b8_fast")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmv_f16_g128_b8_fast")]]
-[[kernel]] void qmv_fast<half, 128, 8>(
+template [[host_name("qmv_f16_g128_b8_fast")]] [[kernel]] void qmv_fast<
+ half,
+ 128,
+ 8>(
const device uint32_t* w [[buffer(0)]],
const device half* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -1697,10 +2110,13 @@ template [[host_name("qmv_f16_g128_b8_fast")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmv_bf16_g32_b8_fast")]]
-[[kernel]] void qmv_fast<bfloat, 32, 8>(
+template [[host_name("qmv_bf16_g32_b8_fast")]] [[kernel]] void qmv_fast<
+ bfloat,
+ 32,
+ 8>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -1711,10 +2127,13 @@ template [[host_name("qmv_bf16_g32_b8_fast")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmv_bf16_g64_b8_fast")]]
-[[kernel]] void qmv_fast<bfloat, 64, 8>(
+template [[host_name("qmv_bf16_g64_b8_fast")]] [[kernel]] void qmv_fast<
+ bfloat,
+ 64,
+ 8>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -1725,10 +2144,13 @@ template [[host_name("qmv_bf16_g64_b8_fast")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmv_bf16_g128_b8_fast")]]
-[[kernel]] void qmv_fast<bfloat, 128, 8>(
+template [[host_name("qmv_bf16_g128_b8_fast")]] [[kernel]] void qmv_fast<
+ bfloat,
+ 128,
+ 8>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -1739,10 +2161,13 @@ template [[host_name("qmv_bf16_g128_b8_fast")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmv_f32_g32_b8_fast")]]
-[[kernel]] void qmv_fast<float, 32, 8>(
+template [[host_name("qmv_f32_g32_b8_fast")]] [[kernel]] void qmv_fast<
+ float,
+ 32,
+ 8>(
const device uint32_t* w [[buffer(0)]],
const device float* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -1753,10 +2178,13 @@ template [[host_name("qmv_f32_g32_b8_fast")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmv_f32_g64_b8_fast")]]
-[[kernel]] void qmv_fast<float, 64, 8>(
+template [[host_name("qmv_f32_g64_b8_fast")]] [[kernel]] void qmv_fast<
+ float,
+ 64,
+ 8>(
const device uint32_t* w [[buffer(0)]],
const device float* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -1767,10 +2195,13 @@ template [[host_name("qmv_f32_g64_b8_fast")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmv_f32_g128_b8_fast")]]
-[[kernel]] void qmv_fast<float, 128, 8>(
+template [[host_name("qmv_f32_g128_b8_fast")]] [[kernel]] void qmv_fast<
+ float,
+ 128,
+ 8>(
const device uint32_t* w [[buffer(0)]],
const device float* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -1781,11 +2212,15 @@ template [[host_name("qmv_f32_g128_b8_fast")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
// Group size 32 (F16)
-template [[host_name("qmm_f16_g32_b4")]]
-[[kernel]] void qmm<half, 32, 4, false>(
+template [[host_name("qmm_f16_g32_b4")]] [[kernel]] void qmm<
+ half,
+ 32,
+ 4,
+ false>(
const device uint32_t* w [[buffer(0)]],
const device half* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -1798,10 +2233,11 @@ template [[host_name("qmm_f16_g32_b4")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_transposed_f16_g32_b4")]]
-[[kernel]] void qmm_transposed<half, 32, 4, true>(
+template [[host_name("qmm_transposed_f16_g32_b4")]] [[kernel]] void
+qmm_transposed<half, 32, 4, true>(
const device uint32_t* w [[buffer(0)]],
const device half* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -1814,10 +2250,11 @@ template [[host_name("qmm_transposed_f16_g32_b4")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_transposed_f16_g32_b4_unaligned")]]
-[[kernel]] void qmm_transposed<half, 32, 4, false>(
+template [[host_name("qmm_transposed_f16_g32_b4_unaligned")]] [[kernel]] void
+qmm_transposed<half, 32, 4, false>(
const device uint32_t* w [[buffer(0)]],
const device half* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -1830,10 +2267,14 @@ template [[host_name("qmm_transposed_f16_g32_b4_unaligned")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_f16_g32_b4_alignedk")]]
-[[kernel]] void qmm<half, 32, 4, true>(
+template [[host_name("qmm_f16_g32_b4_alignedk")]] [[kernel]] void qmm<
+ half,
+ 32,
+ 4,
+ true>(
const device uint32_t* w [[buffer(0)]],
const device half* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -1846,10 +2287,10 @@ template [[host_name("qmm_f16_g32_b4_alignedk")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmv_f16_g32_b4")]]
-[[kernel]] void qmv<half, 32, 4>(
+template [[host_name("qmv_f16_g32_b4")]] [[kernel]] void qmv<half, 32, 4>(
const device uint32_t* w [[buffer(0)]],
const device half* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -1860,13 +2301,14 @@ template [[host_name("qmv_f16_g32_b4")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qvm_f16_g32_b4")]]
-[[kernel]] void qvm<half, 32, 4>(
+template [[host_name("qvm_f16_g32_b4")]] [[kernel]] void qvm<half, 32, 4>(
const device uint32_t* w [[buffer(0)]],
const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],
+ const device uint8_t* zero_points
+ [[buffer(2), function_constant(kUseZeroPoints)]],
const device half* biases [[buffer(2), function_constant(kUseMlxQuant)]],
const device half* x [[buffer(3)]],
device half* y [[buffer(4)]],
@@ -1874,11 +2316,15 @@ template [[host_name("qvm_f16_g32_b4")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
// Group size 32 (F16) - 8 bit
-template [[host_name("qmm_f16_g32_b8")]]
-[[kernel]] void qmm<half, 32, 8, false>(
+template [[host_name("qmm_f16_g32_b8")]] [[kernel]] void qmm<
+ half,
+ 32,
+ 8,
+ false>(
const device uint32_t* w [[buffer(0)]],
const device half* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -1891,10 +2337,11 @@ template [[host_name("qmm_f16_g32_b8")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_transposed_f16_g32_b8")]]
-[[kernel]] void qmm_transposed<half, 32, 8, true>(
+template [[host_name("qmm_transposed_f16_g32_b8")]] [[kernel]] void
+qmm_transposed<half, 32, 8, true>(
const device uint32_t* w [[buffer(0)]],
const device half* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -1907,10 +2354,11 @@ template [[host_name("qmm_transposed_f16_g32_b8")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_transposed_f16_g32_b8_unaligned")]]
-[[kernel]] void qmm_transposed<half, 32, 8, false>(
+template [[host_name("qmm_transposed_f16_g32_b8_unaligned")]] [[kernel]] void
+qmm_transposed<half, 32, 8, false>(
const device uint32_t* w [[buffer(0)]],
const device half* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -1923,10 +2371,14 @@ template [[host_name("qmm_transposed_f16_g32_b8_unaligned")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_f16_g32_b8_alignedk")]]
-[[kernel]] void qmm<half, 32, 8, true>(
+template [[host_name("qmm_f16_g32_b8_alignedk")]] [[kernel]] void qmm<
+ half,
+ 32,
+ 8,
+ true>(
const device uint32_t* w [[buffer(0)]],
const device half* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -1939,10 +2391,10 @@ template [[host_name("qmm_f16_g32_b8_alignedk")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmv_f16_g32_b8")]]
-[[kernel]] void qmv<half, 32, 8>(
+template [[host_name("qmv_f16_g32_b8")]] [[kernel]] void qmv<half, 32, 8>(
const device uint32_t* w [[buffer(0)]],
const device half* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -1953,13 +2405,14 @@ template [[host_name("qmv_f16_g32_b8")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qvm_f16_g32_b8")]]
-[[kernel]] void qvm<half, 32, 8>(
+template [[host_name("qvm_f16_g32_b8")]] [[kernel]] void qvm<half, 32, 8>(
const device uint32_t* w [[buffer(0)]],
const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],
+ const device uint8_t* zero_points
+ [[buffer(2), function_constant(kUseZeroPoints)]],
const device half* biases [[buffer(2), function_constant(kUseMlxQuant)]],
const device half* x [[buffer(3)]],
device half* y [[buffer(4)]],
@@ -1967,11 +2420,15 @@ template [[host_name("qvm_f16_g32_b8")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
// Group size 64 (F16)
-template [[host_name("qmm_f16_g64_b4")]]
-[[kernel]] void qmm<half, 64, 4, false>(
+template [[host_name("qmm_f16_g64_b4")]] [[kernel]] void qmm<
+ half,
+ 64,
+ 4,
+ false>(
const device uint32_t* w [[buffer(0)]],
const device half* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -1984,10 +2441,14 @@ template [[host_name("qmm_f16_g64_b4")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_f16_g64_b4_alignedk")]]
-[[kernel]] void qmm<half, 64, 4, true>(
+template [[host_name("qmm_f16_g64_b4_alignedk")]] [[kernel]] void qmm<
+ half,
+ 64,
+ 4,
+ true>(
const device uint32_t* w [[buffer(0)]],
const device half* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2000,10 +2461,11 @@ template [[host_name("qmm_f16_g64_b4_alignedk")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_transposed_f16_g64_b4")]]
-[[kernel]] void qmm_transposed<half, 64, 4, true>(
+template [[host_name("qmm_transposed_f16_g64_b4")]] [[kernel]] void
+qmm_transposed<half, 64, 4, true>(
const device uint32_t* w [[buffer(0)]],
const device half* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2016,10 +2478,11 @@ template [[host_name("qmm_transposed_f16_g64_b4")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_transposed_f16_g64_b4_unaligned")]]
-[[kernel]] void qmm_transposed<half, 64, 4, false>(
+template [[host_name("qmm_transposed_f16_g64_b4_unaligned")]] [[kernel]] void
+qmm_transposed<half, 64, 4, false>(
const device uint32_t* w [[buffer(0)]],
const device half* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2032,10 +2495,10 @@ template [[host_name("qmm_transposed_f16_g64_b4_unaligned")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmv_f16_g64_b4")]]
-[[kernel]] void qmv<half, 64, 4>(
+template [[host_name("qmv_f16_g64_b4")]] [[kernel]] void qmv<half, 64, 4>(
const device uint32_t* w [[buffer(0)]],
const device half* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2046,13 +2509,14 @@ template [[host_name("qmv_f16_g64_b4")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qvm_f16_g64_b4")]]
-[[kernel]] void qvm<half, 64, 4>(
+template [[host_name("qvm_f16_g64_b4")]] [[kernel]] void qvm<half, 64, 4>(
const device uint32_t* w [[buffer(0)]],
const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],
+ const device uint8_t* zero_points
+ [[buffer(2), function_constant(kUseZeroPoints)]],
const device half* biases [[buffer(2), function_constant(kUseMlxQuant)]],
const device half* x [[buffer(3)]],
device half* y [[buffer(4)]],
@@ -2060,11 +2524,15 @@ template [[host_name("qvm_f16_g64_b4")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
// Group size 64 (F16) - 8 bit
-template [[host_name("qmm_f16_g64_b8")]]
-[[kernel]] void qmm<half, 64, 8, false>(
+template [[host_name("qmm_f16_g64_b8")]] [[kernel]] void qmm<
+ half,
+ 64,
+ 8,
+ false>(
const device uint32_t* w [[buffer(0)]],
const device half* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2077,10 +2545,14 @@ template [[host_name("qmm_f16_g64_b8")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_f16_g64_b8_alignedk")]]
-[[kernel]] void qmm<half, 64, 8, true>(
+template [[host_name("qmm_f16_g64_b8_alignedk")]] [[kernel]] void qmm<
+ half,
+ 64,
+ 8,
+ true>(
const device uint32_t* w [[buffer(0)]],
const device half* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2093,10 +2565,11 @@ template [[host_name("qmm_f16_g64_b8_alignedk")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_transposed_f16_g64_b8")]]
-[[kernel]] void qmm_transposed<half, 64, 8, true>(
+template [[host_name("qmm_transposed_f16_g64_b8")]] [[kernel]] void
+qmm_transposed<half, 64, 8, true>(
const device uint32_t* w [[buffer(0)]],
const device half* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2109,10 +2582,11 @@ template [[host_name("qmm_transposed_f16_g64_b8")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_transposed_f16_g64_b8_unaligned")]]
-[[kernel]] void qmm_transposed<half, 64, 8, false>(
+template [[host_name("qmm_transposed_f16_g64_b8_unaligned")]] [[kernel]] void
+qmm_transposed<half, 64, 8, false>(
const device uint32_t* w [[buffer(0)]],
const device half* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2125,10 +2599,10 @@ template [[host_name("qmm_transposed_f16_g64_b8_unaligned")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmv_f16_g64_b8")]]
-[[kernel]] void qmv<half, 64, 8>(
+template [[host_name("qmv_f16_g64_b8")]] [[kernel]] void qmv<half, 64, 8>(
const device uint32_t* w [[buffer(0)]],
const device half* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2139,13 +2613,14 @@ template [[host_name("qmv_f16_g64_b8")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qvm_f16_g64_b8")]]
-[[kernel]] void qvm<half, 64, 8>(
+template [[host_name("qvm_f16_g64_b8")]] [[kernel]] void qvm<half, 64, 8>(
const device uint32_t* w [[buffer(0)]],
const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],
+ const device uint8_t* zero_points
+ [[buffer(2), function_constant(kUseZeroPoints)]],
const device half* biases [[buffer(2), function_constant(kUseMlxQuant)]],
const device half* x [[buffer(3)]],
device half* y [[buffer(4)]],
@@ -2153,11 +2628,15 @@ template [[host_name("qvm_f16_g64_b8")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
// Group size 128 (F16)
-template [[host_name("qmm_f16_g128_b4")]]
-[[kernel]] void qmm<half, 128, 4, false>(
+template [[host_name("qmm_f16_g128_b4")]] [[kernel]] void qmm<
+ half,
+ 128,
+ 4,
+ false>(
const device uint32_t* w [[buffer(0)]],
const device half* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2170,10 +2649,14 @@ template [[host_name("qmm_f16_g128_b4")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_f16_g128_b4_alignedk")]]
-[[kernel]] void qmm<half, 128, 4, true>(
+template [[host_name("qmm_f16_g128_b4_alignedk")]] [[kernel]] void qmm<
+ half,
+ 128,
+ 4,
+ true>(
const device uint32_t* w [[buffer(0)]],
const device half* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2186,10 +2669,11 @@ template [[host_name("qmm_f16_g128_b4_alignedk")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_transposed_f16_g128_b4")]]
-[[kernel]] void qmm_transposed<half, 128, 4, true>(
+template [[host_name("qmm_transposed_f16_g128_b4")]] [[kernel]] void
+qmm_transposed<half, 128, 4, true>(
const device uint32_t* w [[buffer(0)]],
const device half* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2202,10 +2686,11 @@ template [[host_name("qmm_transposed_f16_g128_b4")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_transposed_f16_g128_b4_unaligned")]]
-[[kernel]] void qmm_transposed<half, 128, 4, false>(
+template [[host_name("qmm_transposed_f16_g128_b4_unaligned")]] [[kernel]] void
+qmm_transposed<half, 128, 4, false>(
const device uint32_t* w [[buffer(0)]],
const device half* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2218,10 +2703,10 @@ template [[host_name("qmm_transposed_f16_g128_b4_unaligned")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmv_f16_g128_b4")]]
-[[kernel]] void qmv<half, 128, 4>(
+template [[host_name("qmv_f16_g128_b4")]] [[kernel]] void qmv<half, 128, 4>(
const device uint32_t* w [[buffer(0)]],
const device half* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2232,13 +2717,14 @@ template [[host_name("qmv_f16_g128_b4")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qvm_f16_g128_b4")]]
-[[kernel]] void qvm<half, 128, 4>(
+template [[host_name("qvm_f16_g128_b4")]] [[kernel]] void qvm<half, 128, 4>(
const device uint32_t* w [[buffer(0)]],
const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],
+ const device uint8_t* zero_points
+ [[buffer(2), function_constant(kUseZeroPoints)]],
const device half* biases [[buffer(2), function_constant(kUseMlxQuant)]],
const device half* x [[buffer(3)]],
device half* y [[buffer(4)]],
@@ -2246,11 +2732,15 @@ template [[host_name("qvm_f16_g128_b4")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
// Group size 128 (F16) - 8 bit
-template [[host_name("qmm_f16_g128_b8")]]
-[[kernel]] void qmm<half, 128, 8, false>(
+template [[host_name("qmm_f16_g128_b8")]] [[kernel]] void qmm<
+ half,
+ 128,
+ 8,
+ false>(
const device uint32_t* w [[buffer(0)]],
const device half* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2263,10 +2753,14 @@ template [[host_name("qmm_f16_g128_b8")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_f16_g128_b8_alignedk")]]
-[[kernel]] void qmm<half, 128, 8, true>(
+template [[host_name("qmm_f16_g128_b8_alignedk")]] [[kernel]] void qmm<
+ half,
+ 128,
+ 8,
+ true>(
const device uint32_t* w [[buffer(0)]],
const device half* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2279,10 +2773,11 @@ template [[host_name("qmm_f16_g128_b8_alignedk")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_transposed_f16_g128_b8")]]
-[[kernel]] void qmm_transposed<half, 128, 8, true>(
+template [[host_name("qmm_transposed_f16_g128_b8")]] [[kernel]] void
+qmm_transposed<half, 128, 8, true>(
const device uint32_t* w [[buffer(0)]],
const device half* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2295,10 +2790,11 @@ template [[host_name("qmm_transposed_f16_g128_b8")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_transposed_f16_g128_b8_unaligned")]]
-[[kernel]] void qmm_transposed<half, 128, 8, false>(
+template [[host_name("qmm_transposed_f16_g128_b8_unaligned")]] [[kernel]] void
+qmm_transposed<half, 128, 8, false>(
const device uint32_t* w [[buffer(0)]],
const device half* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2311,10 +2807,10 @@ template [[host_name("qmm_transposed_f16_g128_b8_unaligned")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmv_f16_g128_b8")]]
-[[kernel]] void qmv<half, 128, 8>(
+template [[host_name("qmv_f16_g128_b8")]] [[kernel]] void qmv<half, 128, 8>(
const device uint32_t* w [[buffer(0)]],
const device half* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2325,13 +2821,14 @@ template [[host_name("qmv_f16_g128_b8")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qvm_f16_g128_b8")]]
-[[kernel]] void qvm<half, 128, 8>(
+template [[host_name("qvm_f16_g128_b8")]] [[kernel]] void qvm<half, 128, 8>(
const device uint32_t* w [[buffer(0)]],
const device half* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],
+ const device uint8_t* zero_points
+ [[buffer(2), function_constant(kUseZeroPoints)]],
const device half* biases [[buffer(2), function_constant(kUseMlxQuant)]],
const device half* x [[buffer(3)]],
device half* y [[buffer(4)]],
@@ -2339,11 +2836,15 @@ template [[host_name("qvm_f16_g128_b8")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
// Group size 32 (F32)
-template [[host_name("qmm_f32_g32_b4")]]
-[[kernel]] void qmm<float, 32, 4, false>(
+template [[host_name("qmm_f32_g32_b4")]] [[kernel]] void qmm<
+ float,
+ 32,
+ 4,
+ false>(
const device uint32_t* w [[buffer(0)]],
const device float* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2356,10 +2857,11 @@ template [[host_name("qmm_f32_g32_b4")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_transposed_f32_g32_b4")]]
-[[kernel]] void qmm_transposed<float, 32, 4, true>(
+template [[host_name("qmm_transposed_f32_g32_b4")]] [[kernel]] void
+qmm_transposed<float, 32, 4, true>(
const device uint32_t* w [[buffer(0)]],
const device float* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2372,10 +2874,11 @@ template [[host_name("qmm_transposed_f32_g32_b4")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_transposed_f32_g32_b4_unaligned")]]
-[[kernel]] void qmm_transposed<float, 32, 4, false>(
+template [[host_name("qmm_transposed_f32_g32_b4_unaligned")]] [[kernel]] void
+qmm_transposed<float, 32, 4, false>(
const device uint32_t* w [[buffer(0)]],
const device float* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2388,10 +2891,14 @@ template [[host_name("qmm_transposed_f32_g32_b4_unaligned")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_f32_g32_b4_alignedk")]]
-[[kernel]] void qmm<float, 32, 4, true>(
+template [[host_name("qmm_f32_g32_b4_alignedk")]] [[kernel]] void qmm<
+ float,
+ 32,
+ 4,
+ true>(
const device uint32_t* w [[buffer(0)]],
const device float* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2404,10 +2911,10 @@ template [[host_name("qmm_f32_g32_b4_alignedk")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmv_f32_g32_b4")]]
-[[kernel]] void qmv<float, 32, 4>(
+template [[host_name("qmv_f32_g32_b4")]] [[kernel]] void qmv<float, 32, 4>(
const device uint32_t* w [[buffer(0)]],
const device float* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2418,13 +2925,14 @@ template [[host_name("qmv_f32_g32_b4")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qvm_f32_g32_b4")]]
-[[kernel]] void qvm<float, 32, 4>(
+template [[host_name("qvm_f32_g32_b4")]] [[kernel]] void qvm<float, 32, 4>(
const device uint32_t* w [[buffer(0)]],
const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],
+ const device uint8_t* zero_points
+ [[buffer(2), function_constant(kUseZeroPoints)]],
const device float* biases [[buffer(2), function_constant(kUseMlxQuant)]],
const device float* x [[buffer(3)]],
device float* y [[buffer(4)]],
@@ -2432,11 +2940,15 @@ template [[host_name("qvm_f32_g32_b4")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
// Group size 32 (F32) - 8 bit
-template [[host_name("qmm_f32_g32_b8")]]
-[[kernel]] void qmm<float, 32, 8, false>(
+template [[host_name("qmm_f32_g32_b8")]] [[kernel]] void qmm<
+ float,
+ 32,
+ 8,
+ false>(
const device uint32_t* w [[buffer(0)]],
const device float* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2449,10 +2961,11 @@ template [[host_name("qmm_f32_g32_b8")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_transposed_f32_g32_b8")]]
-[[kernel]] void qmm_transposed<float, 32, 8, true>(
+template [[host_name("qmm_transposed_f32_g32_b8")]] [[kernel]] void
+qmm_transposed<float, 32, 8, true>(
const device uint32_t* w [[buffer(0)]],
const device float* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2465,10 +2978,11 @@ template [[host_name("qmm_transposed_f32_g32_b8")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_transposed_f32_g32_b8_unaligned")]]
-[[kernel]] void qmm_transposed<float, 32, 8, false>(
+template [[host_name("qmm_transposed_f32_g32_b8_unaligned")]] [[kernel]] void
+qmm_transposed<float, 32, 8, false>(
const device uint32_t* w [[buffer(0)]],
const device float* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2481,10 +2995,14 @@ template [[host_name("qmm_transposed_f32_g32_b8_unaligned")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_f32_g32_b8_alignedk")]]
-[[kernel]] void qmm<float, 32, 8, true>(
+template [[host_name("qmm_f32_g32_b8_alignedk")]] [[kernel]] void qmm<
+ float,
+ 32,
+ 8,
+ true>(
const device uint32_t* w [[buffer(0)]],
const device float* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2497,10 +3015,10 @@ template [[host_name("qmm_f32_g32_b8_alignedk")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmv_f32_g32_b8")]]
-[[kernel]] void qmv<float, 32, 8>(
+template [[host_name("qmv_f32_g32_b8")]] [[kernel]] void qmv<float, 32, 8>(
const device uint32_t* w [[buffer(0)]],
const device float* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2511,13 +3029,14 @@ template [[host_name("qmv_f32_g32_b8")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qvm_f32_g32_b8")]]
-[[kernel]] void qvm<float, 32, 8>(
+template [[host_name("qvm_f32_g32_b8")]] [[kernel]] void qvm<float, 32, 8>(
const device uint32_t* w [[buffer(0)]],
const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],
+ const device uint8_t* zero_points
+ [[buffer(2), function_constant(kUseZeroPoints)]],
const device float* biases [[buffer(2), function_constant(kUseMlxQuant)]],
const device float* x [[buffer(3)]],
device float* y [[buffer(4)]],
@@ -2525,11 +3044,15 @@ template [[host_name("qvm_f32_g32_b8")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
// Group size 64 (F32)
-template [[host_name("qmm_f32_g64_b4")]]
-[[kernel]] void qmm<float, 64, 4, false>(
+template [[host_name("qmm_f32_g64_b4")]] [[kernel]] void qmm<
+ float,
+ 64,
+ 4,
+ false>(
const device uint32_t* w [[buffer(0)]],
const device float* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2542,10 +3065,14 @@ template [[host_name("qmm_f32_g64_b4")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_f32_g64_b4_alignedk")]]
-[[kernel]] void qmm<float, 64, 4, true>(
+template [[host_name("qmm_f32_g64_b4_alignedk")]] [[kernel]] void qmm<
+ float,
+ 64,
+ 4,
+ true>(
const device uint32_t* w [[buffer(0)]],
const device float* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2558,10 +3085,11 @@ template [[host_name("qmm_f32_g64_b4_alignedk")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_transposed_f32_g64_b4")]]
-[[kernel]] void qmm_transposed<float, 64, 4, true>(
+template [[host_name("qmm_transposed_f32_g64_b4")]] [[kernel]] void
+qmm_transposed<float, 64, 4, true>(
const device uint32_t* w [[buffer(0)]],
const device float* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2574,10 +3102,11 @@ template [[host_name("qmm_transposed_f32_g64_b4")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_transposed_f32_g64_b4_unaligned")]]
-[[kernel]] void qmm_transposed<float, 64, 4, false>(
+template [[host_name("qmm_transposed_f32_g64_b4_unaligned")]] [[kernel]] void
+qmm_transposed<float, 64, 4, false>(
const device uint32_t* w [[buffer(0)]],
const device float* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2590,10 +3119,10 @@ template [[host_name("qmm_transposed_f32_g64_b4_unaligned")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmv_f32_g64_b4")]]
-[[kernel]] void qmv<float, 64, 4>(
+template [[host_name("qmv_f32_g64_b4")]] [[kernel]] void qmv<float, 64, 4>(
const device uint32_t* w [[buffer(0)]],
const device float* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2604,13 +3133,14 @@ template [[host_name("qmv_f32_g64_b4")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qvm_f32_g64_b4")]]
-[[kernel]] void qvm<float, 64, 4>(
+template [[host_name("qvm_f32_g64_b4")]] [[kernel]] void qvm<float, 64, 4>(
const device uint32_t* w [[buffer(0)]],
const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],
+ const device uint8_t* zero_points
+ [[buffer(2), function_constant(kUseZeroPoints)]],
const device float* biases [[buffer(2), function_constant(kUseMlxQuant)]],
const device float* x [[buffer(3)]],
device float* y [[buffer(4)]],
@@ -2618,11 +3148,15 @@ template [[host_name("qvm_f32_g64_b4")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
// Group size 64 (F32) - 8 bit
-template [[host_name("qmm_f32_g64_b8")]]
-[[kernel]] void qmm<float, 64, 8, false>(
+template [[host_name("qmm_f32_g64_b8")]] [[kernel]] void qmm<
+ float,
+ 64,
+ 8,
+ false>(
const device uint32_t* w [[buffer(0)]],
const device float* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2635,10 +3169,14 @@ template [[host_name("qmm_f32_g64_b8")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_f32_g64_b8_alignedk")]]
-[[kernel]] void qmm<float, 64, 8, true>(
+template [[host_name("qmm_f32_g64_b8_alignedk")]] [[kernel]] void qmm<
+ float,
+ 64,
+ 8,
+ true>(
const device uint32_t* w [[buffer(0)]],
const device float* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2651,10 +3189,11 @@ template [[host_name("qmm_f32_g64_b8_alignedk")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_transposed_f32_g64_b8")]]
-[[kernel]] void qmm_transposed<float, 64, 8, true>(
+template [[host_name("qmm_transposed_f32_g64_b8")]] [[kernel]] void
+qmm_transposed<float, 64, 8, true>(
const device uint32_t* w [[buffer(0)]],
const device float* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2667,10 +3206,11 @@ template [[host_name("qmm_transposed_f32_g64_b8")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_transposed_f32_g64_b8_unaligned")]]
-[[kernel]] void qmm_transposed<float, 64, 8, false>(
+template [[host_name("qmm_transposed_f32_g64_b8_unaligned")]] [[kernel]] void
+qmm_transposed<float, 64, 8, false>(
const device uint32_t* w [[buffer(0)]],
const device float* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2683,10 +3223,10 @@ template [[host_name("qmm_transposed_f32_g64_b8_unaligned")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmv_f32_g64_b8")]]
-[[kernel]] void qmv<float, 64, 8>(
+template [[host_name("qmv_f32_g64_b8")]] [[kernel]] void qmv<float, 64, 8>(
const device uint32_t* w [[buffer(0)]],
const device float* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2697,13 +3237,14 @@ template [[host_name("qmv_f32_g64_b8")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qvm_f32_g64_b8")]]
-[[kernel]] void qvm<float, 64, 8>(
+template [[host_name("qvm_f32_g64_b8")]] [[kernel]] void qvm<float, 64, 8>(
const device uint32_t* w [[buffer(0)]],
const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],
+ const device uint8_t* zero_points
+ [[buffer(2), function_constant(kUseZeroPoints)]],
const device float* biases [[buffer(2), function_constant(kUseMlxQuant)]],
const device float* x [[buffer(3)]],
device float* y [[buffer(4)]],
@@ -2711,11 +3252,15 @@ template [[host_name("qvm_f32_g64_b8")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
// Group size 128 (F32)
-template [[host_name("qmm_f32_g128_b4")]]
-[[kernel]] void qmm<float, 128, 4, false>(
+template [[host_name("qmm_f32_g128_b4")]] [[kernel]] void qmm<
+ float,
+ 128,
+ 4,
+ false>(
const device uint32_t* w [[buffer(0)]],
const device float* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2728,10 +3273,14 @@ template [[host_name("qmm_f32_g128_b4")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_f32_g128_b4_alignedk")]]
-[[kernel]] void qmm<float, 128, 4, true>(
+template [[host_name("qmm_f32_g128_b4_alignedk")]] [[kernel]] void qmm<
+ float,
+ 128,
+ 4,
+ true>(
const device uint32_t* w [[buffer(0)]],
const device float* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2744,10 +3293,11 @@ template [[host_name("qmm_f32_g128_b4_alignedk")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_transposed_f32_g128_b4")]]
-[[kernel]] void qmm_transposed<float, 128, 4, true>(
+template [[host_name("qmm_transposed_f32_g128_b4")]] [[kernel]] void
+qmm_transposed<float, 128, 4, true>(
const device uint32_t* w [[buffer(0)]],
const device float* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2760,10 +3310,11 @@ template [[host_name("qmm_transposed_f32_g128_b4")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_transposed_f32_g128_b4_unaligned")]]
-[[kernel]] void qmm_transposed<float, 128, 4, false>(
+template [[host_name("qmm_transposed_f32_g128_b4_unaligned")]] [[kernel]] void
+qmm_transposed<float, 128, 4, false>(
const device uint32_t* w [[buffer(0)]],
const device float* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2776,10 +3327,10 @@ template [[host_name("qmm_transposed_f32_g128_b4_unaligned")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmv_f32_g128_b4")]]
-[[kernel]] void qmv<float, 128, 4>(
+template [[host_name("qmv_f32_g128_b4")]] [[kernel]] void qmv<float, 128, 4>(
const device uint32_t* w [[buffer(0)]],
const device float* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2790,13 +3341,14 @@ template [[host_name("qmv_f32_g128_b4")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qvm_f32_g128_b4")]]
-[[kernel]] void qvm<float, 128, 4>(
+template [[host_name("qvm_f32_g128_b4")]] [[kernel]] void qvm<float, 128, 4>(
const device uint32_t* w [[buffer(0)]],
const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],
+ const device uint8_t* zero_points
+ [[buffer(2), function_constant(kUseZeroPoints)]],
const device float* biases [[buffer(2), function_constant(kUseMlxQuant)]],
const device float* x [[buffer(3)]],
device float* y [[buffer(4)]],
@@ -2804,11 +3356,15 @@ template [[host_name("qvm_f32_g128_b4")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
// Group size 128 (F32) - 8 bit
-template [[host_name("qmm_f32_g128_b8")]]
-[[kernel]] void qmm<float, 128, 8, false>(
+template [[host_name("qmm_f32_g128_b8")]] [[kernel]] void qmm<
+ float,
+ 128,
+ 8,
+ false>(
const device uint32_t* w [[buffer(0)]],
const device float* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2821,10 +3377,14 @@ template [[host_name("qmm_f32_g128_b8")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_f32_g128_b8_alignedk")]]
-[[kernel]] void qmm<float, 128, 8, true>(
+template [[host_name("qmm_f32_g128_b8_alignedk")]] [[kernel]] void qmm<
+ float,
+ 128,
+ 8,
+ true>(
const device uint32_t* w [[buffer(0)]],
const device float* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2837,10 +3397,11 @@ template [[host_name("qmm_f32_g128_b8_alignedk")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_transposed_f32_g128_b8")]]
-[[kernel]] void qmm_transposed<float, 128, 8, true>(
+template [[host_name("qmm_transposed_f32_g128_b8")]] [[kernel]] void
+qmm_transposed<float, 128, 8, true>(
const device uint32_t* w [[buffer(0)]],
const device float* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2853,10 +3414,11 @@ template [[host_name("qmm_transposed_f32_g128_b8")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_transposed_f32_g128_b8_unaligned")]]
-[[kernel]] void qmm_transposed<float, 128, 8, false>(
+template [[host_name("qmm_transposed_f32_g128_b8_unaligned")]] [[kernel]] void
+qmm_transposed<float, 128, 8, false>(
const device uint32_t* w [[buffer(0)]],
const device float* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2869,10 +3431,10 @@ template [[host_name("qmm_transposed_f32_g128_b8_unaligned")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmv_f32_g128_b8")]]
-[[kernel]] void qmv<float, 128, 8>(
+template [[host_name("qmv_f32_g128_b8")]] [[kernel]] void qmv<float, 128, 8>(
const device uint32_t* w [[buffer(0)]],
const device float* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2883,13 +3445,14 @@ template [[host_name("qmv_f32_g128_b8")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qvm_f32_g128_b8")]]
-[[kernel]] void qvm<float, 128, 8>(
+template [[host_name("qvm_f32_g128_b8")]] [[kernel]] void qvm<float, 128, 8>(
const device uint32_t* w [[buffer(0)]],
const device float* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],
+ const device uint8_t* zero_points
+ [[buffer(2), function_constant(kUseZeroPoints)]],
const device float* biases [[buffer(2), function_constant(kUseMlxQuant)]],
const device float* x [[buffer(3)]],
device float* y [[buffer(4)]],
@@ -2897,11 +3460,15 @@ template [[host_name("qvm_f32_g128_b8")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
// Group size 32 (BF16)
-template [[host_name("qmm_bf16_g32_b4")]]
-[[kernel]] void qmm<bfloat, 32, 4, false>(
+template [[host_name("qmm_bf16_g32_b4")]] [[kernel]] void qmm<
+ bfloat,
+ 32,
+ 4,
+ false>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2914,10 +3481,14 @@ template [[host_name("qmm_bf16_g32_b4")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_bf16_g32_b4_alignedk")]]
-[[kernel]] void qmm<bfloat, 32, 4, true>(
+template [[host_name("qmm_bf16_g32_b4_alignedk")]] [[kernel]] void qmm<
+ bfloat,
+ 32,
+ 4,
+ true>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2930,10 +3501,11 @@ template [[host_name("qmm_bf16_g32_b4_alignedk")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_transposed_bf16_g32_b4")]]
-[[kernel]] void qmm_transposed<bfloat, 32, 4, true>(
+template [[host_name("qmm_transposed_bf16_g32_b4")]] [[kernel]] void
+qmm_transposed<bfloat, 32, 4, true>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2946,10 +3518,11 @@ template [[host_name("qmm_transposed_bf16_g32_b4")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_transposed_bf16_g32_b4_unaligned")]]
-[[kernel]] void qmm_transposed<bfloat, 32, 4, false>(
+template [[host_name("qmm_transposed_bf16_g32_b4_unaligned")]] [[kernel]] void
+qmm_transposed<bfloat, 32, 4, false>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2962,10 +3535,10 @@ template [[host_name("qmm_transposed_bf16_g32_b4_unaligned")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmv_bf16_g32_b4")]]
-[[kernel]] void qmv<bfloat, 32, 4>(
+template [[host_name("qmv_bf16_g32_b4")]] [[kernel]] void qmv<bfloat, 32, 4>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -2976,13 +3549,14 @@ template [[host_name("qmv_bf16_g32_b4")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qvm_bf16_g32_b4")]]
-[[kernel]] void qvm<bfloat, 32, 4>(
+template [[host_name("qvm_bf16_g32_b4")]] [[kernel]] void qvm<bfloat, 32, 4>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],
+ const device uint8_t* zero_points
+ [[buffer(2), function_constant(kUseZeroPoints)]],
const device bfloat* biases [[buffer(2), function_constant(kUseMlxQuant)]],
const device bfloat* x [[buffer(3)]],
device bfloat* y [[buffer(4)]],
@@ -2990,11 +3564,15 @@ template [[host_name("qvm_bf16_g32_b4")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
// Group size 32 (BF16) - 8 bit
-template [[host_name("qmm_bf16_g32_b8")]]
-[[kernel]] void qmm<bfloat, 32, 8, false>(
+template [[host_name("qmm_bf16_g32_b8")]] [[kernel]] void qmm<
+ bfloat,
+ 32,
+ 8,
+ false>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -3007,10 +3585,14 @@ template [[host_name("qmm_bf16_g32_b8")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_bf16_g32_b8_alignedk")]]
-[[kernel]] void qmm<bfloat, 32, 8, true>(
+template [[host_name("qmm_bf16_g32_b8_alignedk")]] [[kernel]] void qmm<
+ bfloat,
+ 32,
+ 8,
+ true>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -3023,10 +3605,11 @@ template [[host_name("qmm_bf16_g32_b8_alignedk")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_transposed_bf16_g32_b8")]]
-[[kernel]] void qmm_transposed<bfloat, 32, 8, true>(
+template [[host_name("qmm_transposed_bf16_g32_b8")]] [[kernel]] void
+qmm_transposed<bfloat, 32, 8, true>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -3039,10 +3622,11 @@ template [[host_name("qmm_transposed_bf16_g32_b8")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_transposed_bf16_g32_b8_unaligned")]]
-[[kernel]] void qmm_transposed<bfloat, 32, 8, false>(
+template [[host_name("qmm_transposed_bf16_g32_b8_unaligned")]] [[kernel]] void
+qmm_transposed<bfloat, 32, 8, false>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -3055,10 +3639,10 @@ template [[host_name("qmm_transposed_bf16_g32_b8_unaligned")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmv_bf16_g32_b8")]]
-[[kernel]] void qmv<bfloat, 32, 8>(
+template [[host_name("qmv_bf16_g32_b8")]] [[kernel]] void qmv<bfloat, 32, 8>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -3069,13 +3653,14 @@ template [[host_name("qmv_bf16_g32_b8")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qvm_bf16_g32_b8")]]
-[[kernel]] void qvm<bfloat, 32, 8>(
+template [[host_name("qvm_bf16_g32_b8")]] [[kernel]] void qvm<bfloat, 32, 8>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],
+ const device uint8_t* zero_points
+ [[buffer(2), function_constant(kUseZeroPoints)]],
const device bfloat* biases [[buffer(2), function_constant(kUseMlxQuant)]],
const device bfloat* x [[buffer(3)]],
device bfloat* y [[buffer(4)]],
@@ -3083,11 +3668,15 @@ template [[host_name("qvm_bf16_g32_b8")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
// Group size 64 (BF16)
-template [[host_name("qmm_bf16_g64_b4")]]
-[[kernel]] void qmm<bfloat, 64, 4, false>(
+template [[host_name("qmm_bf16_g64_b4")]] [[kernel]] void qmm<
+ bfloat,
+ 64,
+ 4,
+ false>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -3100,10 +3689,14 @@ template [[host_name("qmm_bf16_g64_b4")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_bf16_g64_b4_alignedk")]]
-[[kernel]] void qmm<bfloat, 64, 4, true>(
+template [[host_name("qmm_bf16_g64_b4_alignedk")]] [[kernel]] void qmm<
+ bfloat,
+ 64,
+ 4,
+ true>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -3116,10 +3709,11 @@ template [[host_name("qmm_bf16_g64_b4_alignedk")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_transposed_bf16_g64_b4")]]
-[[kernel]] void qmm_transposed<bfloat, 64, 4, true>(
+template [[host_name("qmm_transposed_bf16_g64_b4")]] [[kernel]] void
+qmm_transposed<bfloat, 64, 4, true>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -3132,10 +3726,11 @@ template [[host_name("qmm_transposed_bf16_g64_b4")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_transposed_bf16_g64_b4_unaligned")]]
-[[kernel]] void qmm_transposed<bfloat, 64, 4, false>(
+template [[host_name("qmm_transposed_bf16_g64_b4_unaligned")]] [[kernel]] void
+qmm_transposed<bfloat, 64, 4, false>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -3148,10 +3743,10 @@ template [[host_name("qmm_transposed_bf16_g64_b4_unaligned")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmv_bf16_g64_b4")]]
-[[kernel]] void qmv<bfloat, 64, 4>(
+template [[host_name("qmv_bf16_g64_b4")]] [[kernel]] void qmv<bfloat, 64, 4>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -3162,13 +3757,14 @@ template [[host_name("qmv_bf16_g64_b4")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qvm_bf16_g64_b4")]]
-[[kernel]] void qvm<bfloat, 64, 4>(
+template [[host_name("qvm_bf16_g64_b4")]] [[kernel]] void qvm<bfloat, 64, 4>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],
+ const device uint8_t* zero_points
+ [[buffer(2), function_constant(kUseZeroPoints)]],
const device bfloat* biases [[buffer(2), function_constant(kUseMlxQuant)]],
const device bfloat* x [[buffer(3)]],
device bfloat* y [[buffer(4)]],
@@ -3176,11 +3772,15 @@ template [[host_name("qvm_bf16_g64_b4")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
// Group size 64 (BF16) - 8 bit
-template [[host_name("qmm_bf16_g64_b8")]]
-[[kernel]] void qmm<bfloat, 64, 8, false>(
+template [[host_name("qmm_bf16_g64_b8")]] [[kernel]] void qmm<
+ bfloat,
+ 64,
+ 8,
+ false>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -3193,10 +3793,14 @@ template [[host_name("qmm_bf16_g64_b8")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_bf16_g64_b8_alignedk")]]
-[[kernel]] void qmm<bfloat, 64, 8, true>(
+template [[host_name("qmm_bf16_g64_b8_alignedk")]] [[kernel]] void qmm<
+ bfloat,
+ 64,
+ 8,
+ true>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -3209,10 +3813,11 @@ template [[host_name("qmm_bf16_g64_b8_alignedk")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_transposed_bf16_g64_b8")]]
-[[kernel]] void qmm_transposed<bfloat, 64, 8, true>(
+template [[host_name("qmm_transposed_bf16_g64_b8")]] [[kernel]] void
+qmm_transposed<bfloat, 64, 8, true>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -3225,10 +3830,11 @@ template [[host_name("qmm_transposed_bf16_g64_b8")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_transposed_bf16_g64_b8_unaligned")]]
-[[kernel]] void qmm_transposed<bfloat, 64, 8, false>(
+template [[host_name("qmm_transposed_bf16_g64_b8_unaligned")]] [[kernel]] void
+qmm_transposed<bfloat, 64, 8, false>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -3241,10 +3847,10 @@ template [[host_name("qmm_transposed_bf16_g64_b8_unaligned")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmv_bf16_g64_b8")]]
-[[kernel]] void qmv<bfloat, 64, 8>(
+template [[host_name("qmv_bf16_g64_b8")]] [[kernel]] void qmv<bfloat, 64, 8>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -3255,13 +3861,14 @@ template [[host_name("qmv_bf16_g64_b8")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qvm_bf16_g64_b8")]]
-[[kernel]] void qvm<bfloat, 64, 8>(
+template [[host_name("qvm_bf16_g64_b8")]] [[kernel]] void qvm<bfloat, 64, 8>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],
+ const device uint8_t* zero_points
+ [[buffer(2), function_constant(kUseZeroPoints)]],
const device bfloat* biases [[buffer(2), function_constant(kUseMlxQuant)]],
const device bfloat* x [[buffer(3)]],
device bfloat* y [[buffer(4)]],
@@ -3269,11 +3876,15 @@ template [[host_name("qvm_bf16_g64_b8")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
// Group size 128 (BF16)
-template [[host_name("qmm_bf16_g128_b4")]]
-[[kernel]] void qmm<bfloat, 128, 4, false>(
+template [[host_name("qmm_bf16_g128_b4")]] [[kernel]] void qmm<
+ bfloat,
+ 128,
+ 4,
+ false>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -3286,10 +3897,14 @@ template [[host_name("qmm_bf16_g128_b4")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_bf16_g128_b4_alignedk")]]
-[[kernel]] void qmm<bfloat, 128, 4, true>(
+template [[host_name("qmm_bf16_g128_b4_alignedk")]] [[kernel]] void qmm<
+ bfloat,
+ 128,
+ 4,
+ true>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -3302,10 +3917,11 @@ template [[host_name("qmm_bf16_g128_b4_alignedk")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_transposed_bf16_g128_b4")]]
-[[kernel]] void qmm_transposed<bfloat, 128, 4, true>(
+template [[host_name("qmm_transposed_bf16_g128_b4")]] [[kernel]] void
+qmm_transposed<bfloat, 128, 4, true>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -3318,10 +3934,11 @@ template [[host_name("qmm_transposed_bf16_g128_b4")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_transposed_bf16_g128_b4_unaligned")]]
-[[kernel]] void qmm_transposed<bfloat, 128, 4, false>(
+template [[host_name("qmm_transposed_bf16_g128_b4_unaligned")]] [[kernel]] void
+qmm_transposed<bfloat, 128, 4, false>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -3334,10 +3951,10 @@ template [[host_name("qmm_transposed_bf16_g128_b4_unaligned")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmv_bf16_g128_b4")]]
-[[kernel]] void qmv<bfloat, 128, 4>(
+template [[host_name("qmv_bf16_g128_b4")]] [[kernel]] void qmv<bfloat, 128, 4>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -3348,13 +3965,14 @@ template [[host_name("qmv_bf16_g128_b4")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qvm_bf16_g128_b4")]]
-[[kernel]] void qvm<bfloat, 128, 4>(
+template [[host_name("qvm_bf16_g128_b4")]] [[kernel]] void qvm<bfloat, 128, 4>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],
+ const device uint8_t* zero_points
+ [[buffer(2), function_constant(kUseZeroPoints)]],
const device bfloat* biases [[buffer(2), function_constant(kUseMlxQuant)]],
const device bfloat* x [[buffer(3)]],
device bfloat* y [[buffer(4)]],
@@ -3362,11 +3980,15 @@ template [[host_name("qvm_bf16_g128_b4")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
// Group size 128 (BF16) - 8 bit
-template [[host_name("qmm_bf16_g128_b8")]]
-[[kernel]] void qmm<bfloat, 128, 8, false>(
+template [[host_name("qmm_bf16_g128_b8")]] [[kernel]] void qmm<
+ bfloat,
+ 128,
+ 8,
+ false>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -3379,10 +4001,14 @@ template [[host_name("qmm_bf16_g128_b8")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_bf16_g128_b8_alignedk")]]
-[[kernel]] void qmm<bfloat, 128, 8, true>(
+template [[host_name("qmm_bf16_g128_b8_alignedk")]] [[kernel]] void qmm<
+ bfloat,
+ 128,
+ 8,
+ true>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -3395,10 +4021,11 @@ template [[host_name("qmm_bf16_g128_b8_alignedk")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_transposed_bf16_g128_b8")]]
-[[kernel]] void qmm_transposed<bfloat, 128, 8, true>(
+template [[host_name("qmm_transposed_bf16_g128_b8")]] [[kernel]] void
+qmm_transposed<bfloat, 128, 8, true>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -3411,10 +4038,11 @@ template [[host_name("qmm_transposed_bf16_g128_b8")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_transposed_bf16_g128_b8_unaligned")]]
-[[kernel]] void qmm_transposed<bfloat, 128, 8, false>(
+template [[host_name("qmm_transposed_bf16_g128_b8_unaligned")]] [[kernel]] void
+qmm_transposed<bfloat, 128, 8, false>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -3427,10 +4055,10 @@ template [[host_name("qmm_transposed_bf16_g128_b8_unaligned")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmv_bf16_g128_b8")]]
-[[kernel]] void qmv<bfloat, 128, 8>(
+template [[host_name("qmv_bf16_g128_b8")]] [[kernel]] void qmv<bfloat, 128, 8>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -3441,13 +4069,14 @@ template [[host_name("qmv_bf16_g128_b8")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qvm_bf16_g128_b8")]]
-[[kernel]] void qvm<bfloat, 128, 8>(
+template [[host_name("qvm_bf16_g128_b8")]] [[kernel]] void qvm<bfloat, 128, 8>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
- const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],
+ const device uint8_t* zero_points
+ [[buffer(2), function_constant(kUseZeroPoints)]],
const device bfloat* biases [[buffer(2), function_constant(kUseMlxQuant)]],
const device bfloat* x [[buffer(3)]],
device bfloat* y [[buffer(4)]],
@@ -3455,10 +4084,11 @@ template [[host_name("qvm_bf16_g128_b8")]]
const constant int& N [[buffer(6)]],
uint3 tid [[threadgroup_position_in_grid]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_transposed_bf16_g128_b4_64x64")]]
-[[kernel]] void qmm_transposed<bfloat, 128, 4, true, 64, 32, 64>(
+template [[host_name("qmm_transposed_bf16_g128_b4_64x64")]] [[kernel]] void
+qmm_transposed<bfloat, 128, 4, true, 64, 32, 64>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -3471,10 +4101,11 @@ template [[host_name("qmm_transposed_bf16_g128_b4_64x64")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_transposed_bf16_g128_b4_64x128")]]
-[[kernel]] void qmm_transposed<bfloat, 128, 4, true, 64, 32, 128>(
+template [[host_name("qmm_transposed_bf16_g128_b4_64x128")]] [[kernel]] void
+qmm_transposed<bfloat, 128, 4, true, 64, 32, 128>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -3487,11 +4118,11 @@ template [[host_name("qmm_transposed_bf16_g128_b4_64x128")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
-
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
-template [[host_name("qmm_transposed_bf16_g128_b4_128x64")]]
-[[kernel]] void qmm_transposed<bfloat, 128, 4, true, 128, 32, 64>(
+template [[host_name("qmm_transposed_bf16_g128_b4_128x64")]] [[kernel]] void
+qmm_transposed<bfloat, 128, 4, true, 128, 32, 64>(
const device uint32_t* w [[buffer(0)]],
const device bfloat* scales [[buffer(1)]],
const device uint8_t* zero_points [[buffer(2)]],
@@ -3504,4 +4135,5 @@ template [[host_name("qmm_transposed_bf16_g128_b4_128x64")]]
uint3 tid [[threadgroup_position_in_grid]],
uint lid [[thread_index_in_threadgroup]],
uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]);
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
diff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
index 30e0b7573..060fb09d8 100644
--- a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
+++ b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
@@ -8,7 +8,7 @@ using namespace metal;
#define GRAIN_SIZE 4
// Main template kernel - single implementation for all types
-template<typename InputT, typename ScaleT, typename OutputT, typename AccumT>
+template <typename InputT, typename ScaleT, typename OutputT, typename AccumT>
void rms_norm_core(
const device InputT* input_data,
const device ScaleT* scales_data,
@@ -20,279 +20,299 @@ void rms_norm_core(
uint thread_in_row,
bool full_layer
) {
- AccumT partial_sum = static_cast<AccumT>(0.0f);
+ AccumT partial_sum = static_cast<AccumT>(0.0f);
- // Compute thread local partial sum
- for (uint base_i = thread_in_row * GRAIN_SIZE; base_i < element_count; base_i += BLOCK_SIZE * GRAIN_SIZE) {
- AccumT vals[GRAIN_SIZE];
-
- for (uint j = 0; j < GRAIN_SIZE; ++j) {
- uint i = base_i + j;
- vals[j] = (i < element_count) ? static_cast<AccumT>(input_data[i]) : 0.0f;
- }
-
- for (uint j = 0; j < GRAIN_SIZE; ++j) {
- partial_sum += vals[j] * vals[j];
- }
+ // Compute thread local partial sum
+ for (uint base_i = thread_in_row * GRAIN_SIZE; base_i < element_count;
+ base_i += BLOCK_SIZE * GRAIN_SIZE) {
+ AccumT vals[GRAIN_SIZE];
+
+ for (uint j = 0; j < GRAIN_SIZE; ++j) {
+ uint i = base_i + j;
+ vals[j] = (i < element_count) ? static_cast<AccumT>(input_data[i]) : 0.0f;
+ }
+
+ for (uint j = 0; j < GRAIN_SIZE; ++j) {
+ partial_sum += vals[j] * vals[j];
}
+ }
+
+ // Compute total sum across threadgroup
+ AccumT total_sum = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(
+ partial_sum,
+ shared_sum,
+ thread_in_row
+ );
+
+ // Compute RMS norm factor
+ AccumT mean_square =
+ static_cast<AccumT>(total_sum) / static_cast<AccumT>(element_count);
+ AccumT rms_norm = rsqrt(mean_square + static_cast<AccumT>(epsilon));
- // Compute total sum across threadgroup
- AccumT total_sum = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(partial_sum, shared_sum, thread_in_row);
+ // Apply normalization and scaling using the same vectorized pattern
+ for (uint base_i = thread_in_row * GRAIN_SIZE; base_i < element_count;
+ base_i += BLOCK_SIZE * GRAIN_SIZE) {
+ AccumT vals[GRAIN_SIZE];
+ AccumT scaled_vals[GRAIN_SIZE];
- // Compute RMS norm factor
- AccumT mean_square = static_cast<AccumT>(total_sum) / static_cast<AccumT>(element_count);
- AccumT rms_norm = rsqrt(mean_square + static_cast<AccumT>(epsilon));
+ // Load GRAIN_SIZE input elements
+ for (uint j = 0; j < GRAIN_SIZE; ++j) {
+ uint i = base_i + j;
+ vals[j] = (i < element_count) ? static_cast<AccumT>(input_data[i]) : 0.0f;
+ }
+
+ // Process GRAIN_SIZE elements: normalize and scale
+ for (uint j = 0; j < GRAIN_SIZE; ++j) {
+ uint i = base_i + j;
+ if (i >= element_count) {
+ continue;
+ }
- // Apply normalization and scaling using the same vectorized pattern
- for (uint base_i = thread_in_row * GRAIN_SIZE; base_i < element_count; base_i += BLOCK_SIZE * GRAIN_SIZE) {
- AccumT vals[GRAIN_SIZE];
- AccumT scaled_vals[GRAIN_SIZE];
-
- // Load GRAIN_SIZE input elements
- for (uint j = 0; j < GRAIN_SIZE; ++j) {
- uint i = base_i + j;
- vals[j] = (i < element_count) ? static_cast<AccumT>(input_data[i]) : 0.0f;
- }
-
- // Process GRAIN_SIZE elements: normalize and scale
- for (uint j = 0; j < GRAIN_SIZE; ++j) {
- uint i = base_i + j;
- if (i >= element_count) { continue; }
+ AccumT normalized_high = vals[j] * rms_norm;
- AccumT normalized_high = vals[j] * rms_norm;
+ if (full_layer) {
+ // Full-layer: keep everything in accumulation precision
+ AccumT scale_value_high = static_cast<AccumT>(scales_data[i]) +
+ static_cast<AccumT>(scale_offset);
+ scaled_vals[j] = normalized_high * scale_value_high;
+ } else {
+ // Only-normalization: cast down for the scale multiply
+ OutputT normalized_low = static_cast<OutputT>(normalized_high);
+ OutputT scale_value_low = static_cast<OutputT>(
+ static_cast<AccumT>(scales_data[i]) +
+ static_cast<AccumT>(scale_offset)
+ );
+ OutputT product_low = normalized_low * scale_value_low;
+ scaled_vals[j] = static_cast<AccumT>(product_low);
+ }
+ }
- if (full_layer) {
- // Full-layer: keep everything in accumulation precision
- AccumT scale_value_high = static_cast<AccumT>(scales_data[i]) + static_cast<AccumT>(scale_offset);
- scaled_vals[j] = normalized_high * scale_value_high;
- } else {
- // Only-normalization: cast down for the scale multiply
- OutputT normalized_low = static_cast<OutputT>(normalized_high);
- OutputT scale_value_low = static_cast<OutputT>(static_cast<AccumT>(scales_data[i]) + static_cast<AccumT>(scale_offset));
- OutputT product_low = normalized_low * scale_value_low;
- scaled_vals[j] = static_cast<AccumT>(product_low);
- }
- }
-
- // Store GRAIN_SIZE output elements
- for (uint j = 0; j < GRAIN_SIZE; ++j) {
- uint i = base_i + j;
- if (i < element_count) {
- output_data[i] = static_cast<OutputT>(scaled_vals[j]);
- }
- }
+ // Store GRAIN_SIZE output elements
+ for (uint j = 0; j < GRAIN_SIZE; ++j) {
+ uint i = base_i + j;
+ if (i < element_count) {
+ output_data[i] = static_cast<OutputT>(scaled_vals[j]);
+ }
}
+ }
}
// X-macro table defining all supported type combinations
-// Format: (InputType, ScaleType, OutputType, AccumType, full_layer_mode, suffix)
-#define FOREACH_RMS_COMBO(_) \
- /* float combinations */ \
- _(float, float, float, float, false, f32_f32_f32_f32_norm) \
- _(float, float, float, float, true, f32_f32_f32_f32_full) \
- _(float, float, half, float, false, f32_f32_f16_f32_norm) \
- _(float, float, half, float, true, f32_f32_f16_f32_full) \
- _(float, float, bfloat, float, false, f32_f32_bf16_f32_norm) \
- _(float, float, bfloat, float, true, f32_f32_bf16_f32_full) \
- _(float, half, float, float, false, f32_f16_f32_f32_norm) \
- _(float, half, float, float, true, f32_f16_f32_f32_full) \
- _(float, half, half, float, false, f32_f16_f16_f32_norm) \
- _(float, half, half, float, true, f32_f16_f16_f32_full) \
- _(float, half, bfloat, float, false, f32_f16_bf16_f32_norm) \
- _(float, half, bfloat, float, true, f32_f16_bf16_f32_full) \
- _(float, bfloat, float, float, false, f32_bf16_f32_f32_norm) \
- _(float, bfloat, float, float, true, f32_bf16_f32_f32_full) \
- _(float, bfloat, half, float, false, f32_bf16_f16_f32_norm) \
- _(float, bfloat, half, float, true, f32_bf16_f16_f32_full) \
- _(float, bfloat, bfloat, float, false, f32_bf16_bf16_f32_norm) \
- _(float, bfloat, bfloat, float, true, f32_bf16_bf16_f32_full) \
- \
- /* half combinations */ \
- _(half, float, float, float, false, f16_f32_f32_f32_norm) \
- _(half, float, float, float, true, f16_f32_f32_f32_full) \
- _(half, float, half, float, false, f16_f32_f16_f32_norm) \
- _(half, float, half, float, true, f16_f32_f16_f32_full) \
- _(half, float, bfloat, float, false, f16_f32_bf16_f32_norm) \
- _(half, float, bfloat, float, true, f16_f32_bf16_f32_full) \
- _(half, half, float, float, false, f16_f16_f32_f32_norm) \
- _(half, half, float, float, true, f16_f16_f32_f32_full) \
- _(half, half, half, float, false, f16_f16_f16_f32_norm) \
- _(half, half, half, float, true, f16_f16_f16_f32_full) \
- _(half, half, bfloat, float, false, f16_f16_bf16_f32_norm) \
- _(half, half, bfloat, float, true, f16_f16_bf16_f32_full) \
- _(half, bfloat, float, float, false, f16_bf16_f32_f32_norm) \
- _(half, bfloat, float, float, true, f16_bf16_f32_f32_full) \
- _(half, bfloat, half, float, false, f16_bf16_f16_f32_norm) \
- _(half, bfloat, half, float, true, f16_bf16_f16_f32_full) \
- _(half, bfloat, bfloat, float, false, f16_bf16_bf16_f32_norm) \
- _(half, bfloat, bfloat, float, true, f16_bf16_bf16_f32_full) \
- \
- /* bfloat combinations */ \
- _(bfloat, float, float, float, false, bf16_f32_f32_f32_norm) \
- _(bfloat, float, float, float, true, bf16_f32_f32_f32_full) \
- _(bfloat, float, half, float, false, bf16_f32_f16_f32_norm) \
- _(bfloat, float, half, float, true, bf16_f32_f16_f32_full) \
- _(bfloat, float, bfloat, float, false, bf16_f32_bf16_f32_norm) \
- _(bfloat, float, bfloat, float, true, bf16_f32_bf16_f32_full) \
- _(bfloat, half, float, float, false, bf16_f16_f32_f32_norm) \
- _(bfloat, half, float, float, true, bf16_f16_f32_f32_full) \
- _(bfloat, half, half, float, false, bf16_f16_f16_f32_norm) \
- _(bfloat, half, half, float, true, bf16_f16_f16_f32_full) \
- _(bfloat, half, bfloat, float, false, bf16_f16_bf16_f32_norm) \
- _(bfloat, half, bfloat, float, true, bf16_f16_bf16_f32_full) \
- _(bfloat, bfloat, float, float, false, bf16_bf16_f32_f32_norm) \
- _(bfloat, bfloat, float, float, true, bf16_bf16_f32_f32_full) \
- _(bfloat, bfloat, half, float, false, bf16_bf16_f16_f32_norm) \
- _(bfloat, bfloat, half, float, true, bf16_bf16_f16_f32_full) \
- _(bfloat, bfloat, bfloat, float, false, bf16_bf16_bf16_f32_norm) \
- _(bfloat, bfloat, bfloat, float, true, bf16_bf16_bf16_f32_full) \
- \
- /* All combinations with half accumulation */ \
- _(float, float, float, half, false, f32_f32_f32_f16_norm) \
- _(float, float, float, half, true, f32_f32_f32_f16_full) \
- _(float, float, half, half, false, f32_f32_f16_f16_norm) \
- _(float, float, half, half, true, f32_f32_f16_f16_full) \
- _(float, float, bfloat, half, false, f32_f32_bf16_f16_norm) \
- _(float, float, bfloat, half, true, f32_f32_bf16_f16_full) \
- _(float, half, float, half, false, f32_f16_f32_f16_norm) \
- _(float, half, float, half, true, f32_f16_f32_f16_full) \
- _(float, half, half, half, false, f32_f16_f16_f16_norm) \
- _(float, half, half, half, true, f32_f16_f16_f16_full) \
- _(float, half, bfloat, half, false, f32_f16_bf16_f16_norm) \
- _(float, half, bfloat, half, true, f32_f16_bf16_f16_full) \
- _(float, bfloat, float, half, false, f32_bf16_f32_f16_norm) \
- _(float, bfloat, float, half, true, f32_bf16_f32_f16_full) \
- _(float, bfloat, half, half, false, f32_bf16_f16_f16_norm) \
- _(float, bfloat, half, half, true, f32_bf16_f16_f16_full) \
- _(float, bfloat, bfloat, half, false, f32_bf16_bf16_f16_norm) \
- _(float, bfloat, bfloat, half, true, f32_bf16_bf16_f16_full) \
- \
- _(half, float, float, half, false, f16_f32_f32_f16_norm) \
- _(half, float, float, half, true, f16_f32_f32_f16_full) \
- _(half, float, half, half, false, f16_f32_f16_f16_norm) \
- _(half, float, half, half, true, f16_f32_f16_f16_full) \
- _(half, float, bfloat, half, false, f16_f32_bf16_f16_norm) \
- _(half, float, bfloat, half, true, f16_f32_bf16_f16_full) \
- _(half, half, float, half, false, f16_f16_f32_f16_norm) \
- _(half, half, float, half, true, f16_f16_f32_f16_full) \
- _(half, half, half, half, false, f16_f16_f16_f16_norm) \
- _(half, half, half, half, true, f16_f16_f16_f16_full) \
- _(half, half, bfloat, half, false, f16_f16_bf16_f16_norm) \
- _(half, half, bfloat, half, true, f16_f16_bf16_f16_full) \
- _(half, bfloat, float, half, false, f16_bf16_f32_f16_norm) \
- _(half, bfloat, float, half, true, f16_bf16_f32_f16_full) \
- _(half, bfloat, half, half, false, f16_bf16_f16_f16_norm) \
- _(half, bfloat, half, half, true, f16_bf16_f16_f16_full) \
- _(half, bfloat, bfloat, half, false, f16_bf16_bf16_f16_norm) \
- _(half, bfloat, bfloat, half, true, f16_bf16_bf16_f16_full) \
- \
- _(bfloat, float, float, half, false, bf16_f32_f32_f16_norm) \
- _(bfloat, float, float, half, true, bf16_f32_f32_f16_full) \
- _(bfloat, float, half, half, false, bf16_f32_f16_f16_norm) \
- _(bfloat, float, half, half, true, bf16_f32_f16_f16_full) \
- _(bfloat, float, bfloat, half, false, bf16_f32_bf16_f16_norm) \
- _(bfloat, float, bfloat, half, true, bf16_f32_bf16_f16_full) \
- _(bfloat, half, float, half, false, bf16_f16_f32_f16_norm) \
- _(bfloat, half, float, half, true, bf16_f16_f32_f16_full) \
- _(bfloat, half, half, half, false, bf16_f16_f16_f16_norm) \
- _(bfloat, half, half, half, true, bf16_f16_f16_f16_full) \
- _(bfloat, half, bfloat, half, false, bf16_f16_bf16_f16_norm) \
- _(bfloat, half, bfloat, half, true, bf16_f16_bf16_f16_full) \
- _(bfloat, bfloat, float, half, false, bf16_bf16_f32_f16_norm) \
- _(bfloat, bfloat, float, half, true, bf16_bf16_f32_f16_full) \
- _(bfloat, bfloat, half, half, false, bf16_bf16_f16_f16_norm) \
- _(bfloat, bfloat, half, half, true, bf16_bf16_f16_f16_full) \
- _(bfloat, bfloat, bfloat, half, false, bf16_bf16_bf16_f16_norm) \
- _(bfloat, bfloat, bfloat, half, true, bf16_bf16_bf16_f16_full)
+// Format: (InputType, ScaleType, OutputType, AccumType, full_layer_mode,
+// suffix)
+#define FOREACH_RMS_COMBO(_) \
+ /* float combinations */ \
+ _(float, float, float, float, false, f32_f32_f32_f32_norm) \
+ _(float, float, float, float, true, f32_f32_f32_f32_full) \
+ _(float, float, half, float, false, f32_f32_f16_f32_norm) \
+ _(float, float, half, float, true, f32_f32_f16_f32_full) \
+ _(float, float, bfloat, float, false, f32_f32_bf16_f32_norm) \
+ _(float, float, bfloat, float, true, f32_f32_bf16_f32_full) \
+ _(float, half, float, float, false, f32_f16_f32_f32_norm) \
+ _(float, half, float, float, true, f32_f16_f32_f32_full) \
+ _(float, half, half, float, false, f32_f16_f16_f32_norm) \
+ _(float, half, half, float, true, f32_f16_f16_f32_full) \
+ _(float, half, bfloat, float, false, f32_f16_bf16_f32_norm) \
+ _(float, half, bfloat, float, true, f32_f16_bf16_f32_full) \
+ _(float, bfloat, float, float, false, f32_bf16_f32_f32_norm) \
+ _(float, bfloat, float, float, true, f32_bf16_f32_f32_full) \
+ _(float, bfloat, half, float, false, f32_bf16_f16_f32_norm) \
+ _(float, bfloat, half, float, true, f32_bf16_f16_f32_full) \
+ _(float, bfloat, bfloat, float, false, f32_bf16_bf16_f32_norm) \
+ _(float, bfloat, bfloat, float, true, f32_bf16_bf16_f32_full) \
+ \
+ /* half combinations */ \
+ _(half, float, float, float, false, f16_f32_f32_f32_norm) \
+ _(half, float, float, float, true, f16_f32_f32_f32_full) \
+ _(half, float, half, float, false, f16_f32_f16_f32_norm) \
+ _(half, float, half, float, true, f16_f32_f16_f32_full) \
+ _(half, float, bfloat, float, false, f16_f32_bf16_f32_norm) \
+ _(half, float, bfloat, float, true, f16_f32_bf16_f32_full) \
+ _(half, half, float, float, false, f16_f16_f32_f32_norm) \
+ _(half, half, float, float, true, f16_f16_f32_f32_full) \
+ _(half, half, half, float, false, f16_f16_f16_f32_norm) \
+ _(half, half, half, float, true, f16_f16_f16_f32_full) \
+ _(half, half, bfloat, float, false, f16_f16_bf16_f32_norm) \
+ _(half, half, bfloat, float, true, f16_f16_bf16_f32_full) \
+ _(half, bfloat, float, float, false, f16_bf16_f32_f32_norm) \
+ _(half, bfloat, float, float, true, f16_bf16_f32_f32_full) \
+ _(half, bfloat, half, float, false, f16_bf16_f16_f32_norm) \
+ _(half, bfloat, half, float, true, f16_bf16_f16_f32_full) \
+ _(half, bfloat, bfloat, float, false, f16_bf16_bf16_f32_norm) \
+ _(half, bfloat, bfloat, float, true, f16_bf16_bf16_f32_full) \
+ \
+ /* bfloat combinations */ \
+ _(bfloat, float, float, float, false, bf16_f32_f32_f32_norm) \
+ _(bfloat, float, float, float, true, bf16_f32_f32_f32_full) \
+ _(bfloat, float, half, float, false, bf16_f32_f16_f32_norm) \
+ _(bfloat, float, half, float, true, bf16_f32_f16_f32_full) \
+ _(bfloat, float, bfloat, float, false, bf16_f32_bf16_f32_norm) \
+ _(bfloat, float, bfloat, float, true, bf16_f32_bf16_f32_full) \
+ _(bfloat, half, float, float, false, bf16_f16_f32_f32_norm) \
+ _(bfloat, half, float, float, true, bf16_f16_f32_f32_full) \
+ _(bfloat, half, half, float, false, bf16_f16_f16_f32_norm) \
+ _(bfloat, half, half, float, true, bf16_f16_f16_f32_full) \
+ _(bfloat, half, bfloat, float, false, bf16_f16_bf16_f32_norm) \
+ _(bfloat, half, bfloat, float, true, bf16_f16_bf16_f32_full) \
+ _(bfloat, bfloat, float, float, false, bf16_bf16_f32_f32_norm) \
+ _(bfloat, bfloat, float, float, true, bf16_bf16_f32_f32_full) \
+ _(bfloat, bfloat, half, float, false, bf16_bf16_f16_f32_norm) \
+ _(bfloat, bfloat, half, float, true, bf16_bf16_f16_f32_full) \
+ _(bfloat, bfloat, bfloat, float, false, bf16_bf16_bf16_f32_norm) \
+ _(bfloat, bfloat, bfloat, float, true, bf16_bf16_bf16_f32_full) \
+ \
+ /* All combinations with half accumulation */ \
+ _(float, float, float, half, false, f32_f32_f32_f16_norm) \
+ _(float, float, float, half, true, f32_f32_f32_f16_full) \
+ _(float, float, half, half, false, f32_f32_f16_f16_norm) \
+ _(float, float, half, half, true, f32_f32_f16_f16_full) \
+ _(float, float, bfloat, half, false, f32_f32_bf16_f16_norm) \
+ _(float, float, bfloat, half, true, f32_f32_bf16_f16_full) \
+ _(float, half, float, half, false, f32_f16_f32_f16_norm) \
+ _(float, half, float, half, true, f32_f16_f32_f16_full) \
+ _(float, half, half, half, false, f32_f16_f16_f16_norm) \
+ _(float, half, half, half, true, f32_f16_f16_f16_full) \
+ _(float, half, bfloat, half, false, f32_f16_bf16_f16_norm) \
+ _(float, half, bfloat, half, true, f32_f16_bf16_f16_full) \
+ _(float, bfloat, float, half, false, f32_bf16_f32_f16_norm) \
+ _(float, bfloat, float, half, true, f32_bf16_f32_f16_full) \
+ _(float, bfloat, half, half, false, f32_bf16_f16_f16_norm) \
+ _(float, bfloat, half, half, true, f32_bf16_f16_f16_full) \
+ _(float, bfloat, bfloat, half, false, f32_bf16_bf16_f16_norm) \
+ _(float, bfloat, bfloat, half, true, f32_bf16_bf16_f16_full) \
+ \
+ _(half, float, float, half, false, f16_f32_f32_f16_norm) \
+ _(half, float, float, half, true, f16_f32_f32_f16_full) \
+ _(half, float, half, half, false, f16_f32_f16_f16_norm) \
+ _(half, float, half, half, true, f16_f32_f16_f16_full) \
+ _(half, float, bfloat, half, false, f16_f32_bf16_f16_norm) \
+ _(half, float, bfloat, half, true, f16_f32_bf16_f16_full) \
+ _(half, half, float, half, false, f16_f16_f32_f16_norm) \
+ _(half, half, float, half, true, f16_f16_f32_f16_full) \
+ _(half, half, half, half, false, f16_f16_f16_f16_norm) \
+ _(half, half, half, half, true, f16_f16_f16_f16_full) \
+ _(half, half, bfloat, half, false, f16_f16_bf16_f16_norm) \
+ _(half, half, bfloat, half, true, f16_f16_bf16_f16_full) \
+ _(half, bfloat, float, half, false, f16_bf16_f32_f16_norm) \
+ _(half, bfloat, float, half, true, f16_bf16_f32_f16_full) \
+ _(half, bfloat, half, half, false, f16_bf16_f16_f16_norm) \
+ _(half, bfloat, half, half, true, f16_bf16_f16_f16_full) \
+ _(half, bfloat, bfloat, half, false, f16_bf16_bf16_f16_norm) \
+ _(half, bfloat, bfloat, half, true, f16_bf16_bf16_f16_full) \
+ \
+ _(bfloat, float, float, half, false, bf16_f32_f32_f16_norm) \
+ _(bfloat, float, float, half, true, bf16_f32_f32_f16_full) \
+ _(bfloat, float, half, half, false, bf16_f32_f16_f16_norm) \
+ _(bfloat, float, half, half, true, bf16_f32_f16_f16_full) \
+ _(bfloat, float, bfloat, half, false, bf16_f32_bf16_f16_norm) \
+ _(bfloat, float, bfloat, half, true, bf16_f32_bf16_f16_full) \
+ _(bfloat, half, float, half, false, bf16_f16_f32_f16_norm) \
+ _(bfloat, half, float, half, true, bf16_f16_f32_f16_full) \
+ _(bfloat, half, half, half, false, bf16_f16_f16_f16_norm) \
+ _(bfloat, half, half, half, true, bf16_f16_f16_f16_full) \
+ _(bfloat, half, bfloat, half, false, bf16_f16_bf16_f16_norm) \
+ _(bfloat, half, bfloat, half, true, bf16_f16_bf16_f16_full) \
+ _(bfloat, bfloat, float, half, false, bf16_bf16_f32_f16_norm) \
+ _(bfloat, bfloat, float, half, true, bf16_bf16_f32_f16_full) \
+ _(bfloat, bfloat, half, half, false, bf16_bf16_f16_f16_norm) \
+ _(bfloat, bfloat, half, half, true, bf16_bf16_f16_f16_full) \
+ _(bfloat, bfloat, bfloat, half, false, bf16_bf16_bf16_f16_norm) \
+ _(bfloat, bfloat, bfloat, half, true, bf16_bf16_bf16_f16_full)
// Generate RMS norm kernels
-#define DEFINE_RMS_KERNEL(IN, SC, OUT, ACC, FULL_LAYER, SUF) \
-[[max_total_threads_per_threadgroup(1024)]] kernel void rms_norm_##SUF( \
- const device IN* input [[buffer(0)]], \
- const device SC* scales [[buffer(1)]], \
- device OUT* output [[buffer(2)]], \
- constant uint& batch_size [[buffer(3)]], \
- constant uint& model_dim [[buffer(4)]], \
- constant float& epsilon [[buffer(5)]], \
- constant float& scale_offset [[buffer(6)]], \
- uint batch_idx [[threadgroup_position_in_grid]], \
- uint thread_in_row [[thread_position_in_threadgroup]] \
-) { \
- if (batch_idx >= batch_size) return; \
- \
- threadgroup ACC shared_sum[SIMD_SIZE]; \
- const uint input_offset = batch_idx * model_dim; \
- \
- rms_norm_core<IN, SC, OUT, ACC>( \
- input + input_offset, \
- scales, \
- output + input_offset, \
- model_dim, \
- epsilon, \
- scale_offset, \
- shared_sum, \
- thread_in_row, \
- FULL_LAYER \
- ); \
-}
+#define DEFINE_RMS_KERNEL(IN, SC, OUT, ACC, FULL_LAYER, SUF) \
+ [[max_total_threads_per_threadgroup(1024)]] kernel void rms_norm_##SUF( \
+ const device IN* input [[buffer(0)]], \
+ const device SC* scales [[buffer(1)]], \
+ device OUT* output [[buffer(2)]], \
+ constant uint& batch_size [[buffer(3)]], \
+ constant uint& model_dim [[buffer(4)]], \
+ constant float& epsilon [[buffer(5)]], \
+ constant float& scale_offset [[buffer(6)]], \
+ uint batch_idx [[threadgroup_position_in_grid]], \
+ uint thread_in_row [[thread_position_in_threadgroup]] \
+ ) { \
+ if (batch_idx >= batch_size) \
+ return; \
+ \
+ threadgroup ACC shared_sum[SIMD_SIZE]; \
+ const uint input_offset = batch_idx * model_dim; \
+ \
+ rms_norm_core<IN, SC, OUT, ACC>( \
+ input + input_offset, \
+ scales, \
+ output + input_offset, \
+ model_dim, \
+ epsilon, \
+ scale_offset, \
+ shared_sum, \
+ thread_in_row, \
+ FULL_LAYER \
+ ); \
+ }
FOREACH_RMS_COMBO(DEFINE_RMS_KERNEL)
#undef DEFINE_RMS_KERNEL
// Generate QK norm kernels
-#define DEFINE_QK_KERNEL(IN, SC, OUT, ACC, FULL_LAYER, SUF) \
-[[max_total_threads_per_threadgroup(1024)]] kernel void qk_norm_##SUF( \
- const device IN* qkv_input [[buffer(0)]], \
- const device SC* scales [[buffer(1)]], \
- device OUT* qkv_output [[buffer(2)]], \
- constant uint& batch_size [[buffer(3)]], \
- constant uint& num_q_heads [[buffer(4)]], \
- constant uint& num_kv_heads [[buffer(5)]], \
- constant uint& head_dim [[buffer(6)]], \
- constant float& epsilon [[buffer(7)]], \
- constant float& scale_offset [[buffer(8)]], \
- constant uint& active_type_mask [[buffer(9)]], \
- uint3 threadgroup_position_in_grid [[threadgroup_position_in_grid]], \
- uint3 thread_position_in_threadgroup [[thread_position_in_threadgroup]] \
-) { \
- uint batch_idx = threadgroup_position_in_grid.x; \
- uint sequential_head_idx = threadgroup_position_in_grid.y; \
- uint thread_in_threadgroup = thread_position_in_threadgroup.x; \
- \
- if (batch_idx >= batch_size) return; \
- \
- uint qk_type, actual_head_idx, logical_head_idx; \
- if (sequential_head_idx < num_q_heads) { \
- qk_type = 0; \
- actual_head_idx = sequential_head_idx; \
- logical_head_idx = actual_head_idx; \
- } else { \
- qk_type = 1; \
- actual_head_idx = sequential_head_idx - num_q_heads; \
- logical_head_idx = num_q_heads + actual_head_idx; \
- } \
- \
- if ((active_type_mask & (1 << qk_type)) == 0) return; \
- if (qk_type == 0 && actual_head_idx >= num_q_heads) return; \
- if (qk_type == 1 && actual_head_idx >= num_kv_heads) return; \
- \
- const uint total_heads_in_buffer = num_q_heads + 2 * num_kv_heads; \
- const uint slice_offset = batch_idx * total_heads_in_buffer * head_dim + logical_head_idx * head_dim; \
- \
- threadgroup ACC shared_sum[SIMD_SIZE]; \
- \
- rms_norm_core<IN, SC, OUT, ACC>( \
- qkv_input + slice_offset, \
- scales, \
- qkv_output + slice_offset, \
- head_dim, \
- epsilon, \
- scale_offset, \
- shared_sum, \
- thread_in_threadgroup, \
- FULL_LAYER \
- ); \
-}
+#define DEFINE_QK_KERNEL(IN, SC, OUT, ACC, FULL_LAYER, SUF) \
+ [[max_total_threads_per_threadgroup(1024)]] kernel void qk_norm_##SUF( \
+ const device IN* qkv_input [[buffer(0)]], \
+ const device SC* scales [[buffer(1)]], \
+ device OUT* qkv_output [[buffer(2)]], \
+ constant uint& batch_size [[buffer(3)]], \
+ constant uint& num_q_heads [[buffer(4)]], \
+ constant uint& num_kv_heads [[buffer(5)]], \
+ constant uint& head_dim [[buffer(6)]], \
+ constant float& epsilon [[buffer(7)]], \
+ constant float& scale_offset [[buffer(8)]], \
+ constant uint& active_type_mask [[buffer(9)]], \
+ uint3 threadgroup_position_in_grid [[threadgroup_position_in_grid]], \
+ uint3 thread_position_in_threadgroup [[thread_position_in_threadgroup]] \
+ ) { \
+ uint batch_idx = threadgroup_position_in_grid.x; \
+ uint sequential_head_idx = threadgroup_position_in_grid.y; \
+ uint thread_in_threadgroup = thread_position_in_threadgroup.x; \
+ \
+ if (batch_idx >= batch_size) \
+ return; \
+ \
+ uint qk_type, actual_head_idx, logical_head_idx; \
+ if (sequential_head_idx < num_q_heads) { \
+ qk_type = 0; \
+ actual_head_idx = sequential_head_idx; \
+ logical_head_idx = actual_head_idx; \
+ } else { \
+ qk_type = 1; \
+ actual_head_idx = sequential_head_idx - num_q_heads; \
+ logical_head_idx = num_q_heads + actual_head_idx; \
+ } \
+ \
+ if ((active_type_mask & (1 << qk_type)) == 0) \
+ return; \
+ if (qk_type == 0 && actual_head_idx >= num_q_heads) \
+ return; \
+ if (qk_type == 1 && actual_head_idx >= num_kv_heads) \
+ return; \
+ \
+ const uint total_heads_in_buffer = num_q_heads + 2 * num_kv_heads; \
+ const uint slice_offset = batch_idx * total_heads_in_buffer * head_dim + \
+ logical_head_idx * head_dim; \
+ \
+ threadgroup ACC shared_sum[SIMD_SIZE]; \
+ \
+ rms_norm_core<IN, SC, OUT, ACC>( \
+ qkv_input + slice_offset, \
+ scales, \
+ qkv_output + slice_offset, \
+ head_dim, \
+ epsilon, \
+ scale_offset, \
+ shared_sum, \
+ thread_in_threadgroup, \
+ FULL_LAYER \
+ ); \
+ }
FOREACH_RMS_COMBO(DEFINE_QK_KERNEL)
-#undef DEFINE_QK_KERNEL
+#undef DEFINE_QK_KERNEL
diff --git a/crates/uzu/src/backends/metal/kernel/rng.metal b/crates/uzu/src/backends/metal/kernel/rng.metal
index 565a901df..97979277f 100644
--- a/crates/uzu/src/backends/metal/kernel/rng.metal
+++ b/crates/uzu/src/backends/metal/kernel/rng.metal
@@ -6,105 +6,125 @@ using namespace metal;
/* ───────────── Philox4x32-10 RNG ───────────── */
-#define PHILOX_W32_0 0x9E3779B9u
-#define PHILOX_W32_1 0xBB67AE85u
+#define PHILOX_W32_0 0x9E3779B9u
+#define PHILOX_W32_1 0xBB67AE85u
#define PHILOX_M4x32_0 0xD2511F53u
#define PHILOX_M4x32_1 0xCD9E8D57u
struct PhiloxState {
- uint key[2];
- uint ctr[4];
- int state_idx;
- uint output[4];
+ uint key[2];
+ uint ctr[4];
+ int state_idx;
+ uint output[4];
};
inline uint mulhilo32(uint a, uint b, thread uint* hip) {
- uint64_t product = (uint64_t(a)) * (uint64_t(b));
- *hip = uint(product >> 32);
- return uint(product);
+ uint64_t product = (uint64_t(a)) * (uint64_t(b));
+ *hip = uint(product >> 32);
+ return uint(product);
}
inline void philox4x32round(thread uint* ctr, thread uint* key) {
- uint hi0, hi1;
- uint lo0 = mulhilo32(PHILOX_M4x32_0, ctr[0], &hi0);
- uint lo1 = mulhilo32(PHILOX_M4x32_1, ctr[2], &hi1);
-
- uint new_ctr[4];
- new_ctr[0] = hi1 ^ ctr[1] ^ key[0];
- new_ctr[1] = lo1;
- new_ctr[2] = hi0 ^ ctr[3] ^ key[1];
- new_ctr[3] = lo0;
-
- for(int i = 0; i < 4; i++) {
- ctr[i] = new_ctr[i];
- }
+ uint hi0, hi1;
+ uint lo0 = mulhilo32(PHILOX_M4x32_0, ctr[0], &hi0);
+ uint lo1 = mulhilo32(PHILOX_M4x32_1, ctr[2], &hi1);
+
+ uint new_ctr[4];
+ new_ctr[0] = hi1 ^ ctr[1] ^ key[0];
+ new_ctr[1] = lo1;
+ new_ctr[2] = hi0 ^ ctr[3] ^ key[1];
+ new_ctr[3] = lo0;
+
+ for (int i = 0; i < 4; i++) {
+ ctr[i] = new_ctr[i];
+ }
}
inline void philox4x32bumpkey(thread uint* key) {
- key[0] += PHILOX_W32_0;
- key[1] += PHILOX_W32_1;
+ key[0] += PHILOX_W32_0;
+ key[1] += PHILOX_W32_1;
}
-inline void curand_philox4x32_10(thread uint* ctr, thread uint* key, thread uint* out) {
- uint local_ctr[4] = {ctr[0], ctr[1], ctr[2], ctr[3]};
- uint local_key[2] = {key[0], key[1]};
-
- // 10 rounds
- philox4x32round(local_ctr, local_key); // 1
- philox4x32bumpkey(local_key); philox4x32round(local_ctr, local_key); // 2
- philox4x32bumpkey(local_key); philox4x32round(local_ctr, local_key); // 3
- philox4x32bumpkey(local_key); philox4x32round(local_ctr, local_key); // 4
- philox4x32bumpkey(local_key); philox4x32round(local_ctr, local_key); // 5
- philox4x32bumpkey(local_key); philox4x32round(local_ctr, local_key); // 6
- philox4x32bumpkey(local_key); philox4x32round(local_ctr, local_key); // 7
- philox4x32bumpkey(local_key); philox4x32round(local_ctr, local_key); // 8
- philox4x32bumpkey(local_key); philox4x32round(local_ctr, local_key); // 9
- philox4x32bumpkey(local_key); philox4x32round(local_ctr, local_key); // 10
-
- for(int i = 0; i < 4; i++) {
- out[i] = local_ctr[i];
- }
+inline void curand_philox4x32_10(
+ thread uint* ctr,
+ thread uint* key,
+ thread uint* out
+) {
+ uint local_ctr[4] = {ctr[0], ctr[1], ctr[2], ctr[3]};
+ uint local_key[2] = {key[0], key[1]};
+
+ // 10 rounds
+ philox4x32round(local_ctr, local_key); // 1
+ philox4x32bumpkey(local_key);
+ philox4x32round(local_ctr, local_key); // 2
+ philox4x32bumpkey(local_key);
+ philox4x32round(local_ctr, local_key); // 3
+ philox4x32bumpkey(local_key);
+ philox4x32round(local_ctr, local_key); // 4
+ philox4x32bumpkey(local_key);
+ philox4x32round(local_ctr, local_key); // 5
+ philox4x32bumpkey(local_key);
+ philox4x32round(local_ctr, local_key); // 6
+ philox4x32bumpkey(local_key);
+ philox4x32round(local_ctr, local_key); // 7
+ philox4x32bumpkey(local_key);
+ philox4x32round(local_ctr, local_key); // 8
+ philox4x32bumpkey(local_key);
+ philox4x32round(local_ctr, local_key); // 9
+ philox4x32bumpkey(local_key);
+ philox4x32round(local_ctr, local_key); // 10
+
+ for (int i = 0; i < 4; i++) {
+ out[i] = local_ctr[i];
+ }
}
inline void philox_state_incr(thread PhiloxState* s) {
- ++s->ctr[0];
- if(s->ctr[0] != 0) return;
- ++s->ctr[1];
- if(s->ctr[1] != 0) return;
- ++s->ctr[2];
- if(s->ctr[2] != 0) return;
- ++s->ctr[3];
+ ++s->ctr[0];
+ if (s->ctr[0] != 0)
+ return;
+ ++s->ctr[1];
+ if (s->ctr[1] != 0)
+ return;
+ ++s->ctr[2];
+ if (s->ctr[2] != 0)
+ return;
+ ++s->ctr[3];
}
-inline void philox_init(thread PhiloxState* state, uint64_t seed, uint64_t offset) {
- // Initialize key from seed
- state->key[0] = uint(seed);
- state->key[1] = uint(seed >> 32);
-
- // Initialize counter from offset
- state->ctr[0] = uint(offset);
- state->ctr[1] = uint(offset >> 32);
- state->ctr[2] = 0;
- state->ctr[3] = 0;
-
- state->state_idx = 0;
-
- // Generate first batch of random numbers
- curand_philox4x32_10(state->ctr, state->key, state->output);
- philox_state_incr(state);
+inline void philox_init(
+ thread PhiloxState* state,
+ uint64_t seed,
+ uint64_t offset
+) {
+ // Initialize key from seed
+ state->key[0] = uint(seed);
+ state->key[1] = uint(seed >> 32);
+
+ // Initialize counter from offset
+ state->ctr[0] = uint(offset);
+ state->ctr[1] = uint(offset >> 32);
+ state->ctr[2] = 0;
+ state->ctr[3] = 0;
+
+ state->state_idx = 0;
+
+ // Generate first batch of random numbers
+ curand_philox4x32_10(state->ctr, state->key, state->output);
+ philox_state_incr(state);
}
inline uint philox_next(thread PhiloxState* state) {
- if(state->state_idx >= 4) {
- curand_philox4x32_10(state->ctr, state->key, state->output);
- philox_state_incr(state);
- state->state_idx = 0;
- }
- return state->output[state->state_idx++];
+ if (state->state_idx >= 4) {
+ curand_philox4x32_10(state->ctr, state->key, state->output);
+ philox_state_incr(state);
+ state->state_idx = 0;
+ }
+ return state->output[state->state_idx++];
}
inline float uniform_float(thread PhiloxState* state) {
- return float(philox_next(state)) * (1.0f/4294967296.0f); /* (0,1) */
+ return float(philox_next(state)) * (1.0f / 4294967296.0f); /* (0,1) */
}
#endif /* rng_metal */
diff --git a/crates/uzu/src/backends/metal/kernel/rope/rope.metal b/crates/uzu/src/backends/metal/kernel/rope/rope.metal
index 5d03ebb00..1f61c9b92 100644
--- a/crates/uzu/src/backends/metal/kernel/rope/rope.metal
+++ b/crates/uzu/src/backends/metal/kernel/rope/rope.metal
@@ -11,89 +11,127 @@ inline T applyRopeTransform(
T cosVal,
T sinVal
) {
- T inputVal = qkvTensorView(tokenIdx, headIdx, dimIdx);
- T pairedVal = (dimIdx < halfDim) ?
- -qkvTensorView(tokenIdx, headIdx, dimIdx + halfDim) :
- qkvTensorView(tokenIdx, headIdx, dimIdx - halfDim);
- return inputVal * cosVal + pairedVal * sinVal;
+ T inputVal = qkvTensorView(tokenIdx, headIdx, dimIdx);
+ T pairedVal = (dimIdx < halfDim)
+ ? -qkvTensorView(tokenIdx, headIdx, dimIdx + halfDim)
+ : qkvTensorView(tokenIdx, headIdx, dimIdx - halfDim);
+ return inputVal * cosVal + pairedVal * sinVal;
}
template <typename T>
void applyRope(
- device const T* qkv, // [suffix_len, (num_heads + 2*num_groups) * head_dim]
- device const T* cosines, // [max_seq_len, head_dim]
- device const T* sines, // [max_seq_len, head_dim]
- device const int* tokenPositions, // [suffix_len] - actual token positions
- device T* rotatedQueries, // [num_heads, suffix_len, head_dim]
- device T* rotatedKeys, // [num_groups, suffix_len, head_dim]
+ device const T* qkv, // [suffix_len, (num_heads + 2*num_groups) * head_dim]
+ device const T* cosines, // [max_seq_len, head_dim]
+ device const T* sines, // [max_seq_len, head_dim]
+ device const int* tokenPositions, // [suffix_len] - actual token positions
+ device T* rotatedQueries, // [num_heads, suffix_len, head_dim]
+ device T* rotatedKeys, // [num_groups, suffix_len, head_dim]
constant int& headDim,
constant int& numHeads,
constant int& numGroups,
constant int& suffixLength,
constant int& maxSequenceLength,
- const uint3 position // x: headIdx, y: tokenIdx, z: dimIdx
+ const uint3 position // x: headIdx, y: tokenIdx, z: dimIdx
) {
- const uint headIndex = position.x;
- const uint tokenIndex = position.y;
- const uint dimensionIndex = position.z;
+ const uint headIndex = position.x;
+ const uint tokenIndex = position.y;
+ const uint dimensionIndex = position.z;
- if (headIndex >= numHeads || tokenIndex >= suffixLength || dimensionIndex >= headDim) return;
- if (headDim & 1) return; // head_dim must be even
- if (numHeads % numGroups != 0) return;
+ if (headIndex >= numHeads || tokenIndex >= suffixLength ||
+ dimensionIndex >= headDim)
+ return;
+ if (headDim & 1)
+ return; // head_dim must be even
+ if (numHeads % numGroups != 0)
+ return;
- const uint groupIndex = headIndex / (numHeads / numGroups); // which KV group this head belongs to
- const uint totalHeads = numHeads + 2 * numGroups;
+ const uint groupIndex =
+ headIndex / (numHeads / numGroups); // which KV group this head belongs to
+ const uint totalHeads = numHeads + 2 * numGroups;
- // Use actual token position from buffer
- const uint rawPosition = tokenPositions[tokenIndex];
- const uint absolutePosition = rawPosition > maxSequenceLength ? 0 : rawPosition;
+ // Use actual token position from buffer
+ const uint rawPosition = tokenPositions[tokenIndex];
+ const uint absolutePosition =
+ rawPosition > maxSequenceLength ? 0 : rawPosition;
- const uint halfDimension = headDim / 2;
+ const uint halfDimension = headDim / 2;
- TensorView3D<const T> qkvTensorView = TensorView3D<const T>(qkv).shaped(suffixLength, totalHeads, headDim);
- TensorView2D<const T> cosinesTensorView = TensorView2D<const T>(cosines).shaped(maxSequenceLength, headDim);
- TensorView2D<const T> sinesTensorView = TensorView2D<const T>(sines).shaped(maxSequenceLength, headDim);
- TensorView3D<T> rotatedQueriesTensorView = TensorView3D<T>(rotatedQueries).shaped(numHeads, suffixLength, headDim);
- TensorView3D<T> rotatedKeysTensorView = TensorView3D<T>(rotatedKeys).shaped(numGroups, suffixLength, headDim);
-
- const T cosVal = cosinesTensorView(absolutePosition, dimensionIndex);
- const T sinVal = sinesTensorView(absolutePosition, dimensionIndex);
+ TensorView3D<const T> qkvTensorView =
+ TensorView3D<const T>(qkv).shaped(suffixLength, totalHeads, headDim);
+ TensorView2D<const T> cosinesTensorView =
+ TensorView2D<const T>(cosines).shaped(maxSequenceLength, headDim);
+ TensorView2D<const T> sinesTensorView =
+ TensorView2D<const T>(sines).shaped(maxSequenceLength, headDim);
+ TensorView3D<T> rotatedQueriesTensorView =
+ TensorView3D<T>(rotatedQueries).shaped(numHeads, suffixLength, headDim);
+ TensorView3D<T> rotatedKeysTensorView =
+ TensorView3D<T>(rotatedKeys).shaped(numGroups, suffixLength, headDim);
- /* ---------- QUERIES ---------- */
- T queryResult = applyRopeTransform(qkvTensorView, tokenIndex, headIndex, dimensionIndex, halfDimension, cosVal, sinVal);
- rotatedQueriesTensorView(headIndex, tokenIndex, dimensionIndex) = queryResult;
+ const T cosVal = cosinesTensorView(absolutePosition, dimensionIndex);
+ const T sinVal = sinesTensorView(absolutePosition, dimensionIndex);
- /* ---------- KEYS & VALUES (only first head of each group processes) ---------- */
- uint firstHeadInGroup = groupIndex * (numHeads / numGroups);
- if (headIndex == firstHeadInGroup) {
- /* ---------- keys ---------- */
- uint keyHeadIndex = numHeads + groupIndex; // Keys start after all query heads
- T keyResult = applyRopeTransform(qkvTensorView, tokenIndex, keyHeadIndex, dimensionIndex, halfDimension, cosVal, sinVal);
- rotatedKeysTensorView(groupIndex, tokenIndex, dimensionIndex) = keyResult;
- }
+ /* ---------- QUERIES ---------- */
+ T queryResult = applyRopeTransform(
+ qkvTensorView,
+ tokenIndex,
+ headIndex,
+ dimensionIndex,
+ halfDimension,
+ cosVal,
+ sinVal
+ );
+ rotatedQueriesTensorView(headIndex, tokenIndex, dimensionIndex) = queryResult;
+
+ /* ---------- KEYS & VALUES (only first head of each group processes)
+ * ---------- */
+ uint firstHeadInGroup = groupIndex * (numHeads / numGroups);
+ if (headIndex == firstHeadInGroup) {
+ /* ---------- keys ---------- */
+ uint keyHeadIndex =
+ numHeads + groupIndex; // Keys start after all query heads
+ T keyResult = applyRopeTransform(
+ qkvTensorView,
+ tokenIndex,
+ keyHeadIndex,
+ dimensionIndex,
+ halfDimension,
+ cosVal,
+ sinVal
+ );
+ rotatedKeysTensorView(groupIndex, tokenIndex, dimensionIndex) = keyResult;
+ }
}
/* ---------- boiler-plate kernel wrappers ---------- */
-#define outerArguments(T) \
-(device const T* qkv [[ buffer(0) ]], \
- device const T* cosines [[ buffer(1) ]], \
- device const T* sines [[ buffer(2) ]], \
- device const int* tokenPositions [[ buffer(3) ]], \
- device T* rotatedQueries [[ buffer(4) ]], \
- device T* rotatedKeys [[ buffer(5) ]], \
- constant int& headDim [[ buffer(6) ]], \
- constant int& numHeads [[ buffer(7) ]], \
- constant int& numGroups [[ buffer(8) ]], \
- constant int& suffixLength [[ buffer(9) ]], \
- constant int& maxSequenceLength [[ buffer(10) ]], \
- const uint3 position [[ thread_position_in_grid ]]) \
+#define outerArguments(T) \
+ (device const T* qkv [[buffer(0)]], \
+ device const T* cosines [[buffer(1)]], \
+ device const T* sines [[buffer(2)]], \
+ device const int* tokenPositions [[buffer(3)]], \
+ device T* rotatedQueries [[buffer(4)]], \
+ device T* rotatedKeys [[buffer(5)]], \
+ constant int& headDim [[buffer(6)]], \
+ constant int& numHeads [[buffer(7)]], \
+ constant int& numGroups [[buffer(8)]], \
+ constant int& suffixLength [[buffer(9)]], \
+ constant int& maxSequenceLength [[buffer(10)]], \
+ const uint3 position [[thread_position_in_grid]])
-#define innerArguments \
-(qkv, cosines, sines, tokenPositions, rotatedQueries, rotatedKeys, \
- headDim, numHeads, numGroups, suffixLength, maxSequenceLength, position) \
+#define innerArguments \
+ (qkv, \
+ cosines, \
+ sines, \
+ tokenPositions, \
+ rotatedQueries, \
+ rotatedKeys, \
+ headDim, \
+ numHeads, \
+ numGroups, \
+ suffixLength, \
+ maxSequenceLength, \
+ position)
generateKernels(applyRope)
#undef outerArguments
#undef innerArguments
-
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal b/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal
index c3966c0c3..7cb44b155 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal
@@ -6,302 +6,379 @@
#define GRAIN_SIZE 4
struct ArgmaxPair {
- float value;
- uint index;
+ float value;
+ uint index;
};
constant ArgmaxPair ARGMAX_INIT = {-INFINITY, UINT_MAX};
bool argmax_is_better(ArgmaxPair a, ArgmaxPair b) {
- return a.value > b.value || (a.value == b.value && a.index < b.index);
+ return a.value > b.value || (a.value == b.value && a.index < b.index);
}
ArgmaxPair argmax_combine(ArgmaxPair a, ArgmaxPair b) {
- return argmax_is_better(a, b) ? a : b;
+ return argmax_is_better(a, b) ? a : b;
}
//------------------------------------------------------------------------------------------------//
// Cooperative threadgroup argmax reduction (2-level hierarchical)
-template<ushort BLOCK_SIZE_PARAM>
-static ArgmaxPair threadgroup_cooperative_argmax(ArgmaxPair value, threadgroup ArgmaxPair* shared, const ushort lid) {
- const ushort simd_group_id = lid / 32;
- const ushort simd_lane_id = lid % 32;
-
- // Reduce within simdgroup using manual shuffle operations
- ArgmaxPair local_result = value;
- for (ushort offset = 16; offset > 0; offset /= 2) {
- ArgmaxPair other = {
- simd_shuffle_down(local_result.value, offset),
- simd_shuffle_down(local_result.index, offset)
- };
- local_result = argmax_combine(local_result, other);
- }
-
- // First thread in each simdgroup writes to shared memory
- if (simd_lane_id == 0) {
- shared[simd_group_id] = local_result;
- }
-
- // Synchronize across the threadgroup
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // Reduce across simdgroups
- const ushort num_simd_groups = (BLOCK_SIZE_PARAM + 31) / 32;
- ArgmaxPair total_result = ARGMAX_INIT;
- if (lid < num_simd_groups) {
- total_result = shared[lid];
- }
-
- // Final simdgroup reduction
- for (ushort offset = 16; offset > 0; offset /= 2) {
- ArgmaxPair other = {
- simd_shuffle_down(total_result.value, offset),
- simd_shuffle_down(total_result.index, offset)
- };
- total_result = argmax_combine(total_result, other);
- }
-
- // Broadcast the result to all threads
- if (lid == 0) {
- shared[0] = total_result;
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- return shared[0];
+template <ushort BLOCK_SIZE_PARAM>
+static ArgmaxPair threadgroup_cooperative_argmax(
+ ArgmaxPair value,
+ threadgroup ArgmaxPair* shared,
+ const ushort lid
+) {
+ const ushort simd_group_id = lid / 32;
+ const ushort simd_lane_id = lid % 32;
+
+ // Reduce within simdgroup using manual shuffle operations
+ ArgmaxPair local_result = value;
+ for (ushort offset = 16; offset > 0; offset /= 2) {
+ ArgmaxPair other = {
+ simd_shuffle_down(local_result.value, offset),
+ simd_shuffle_down(local_result.index, offset)
+ };
+ local_result = argmax_combine(local_result, other);
+ }
+
+ // First thread in each simdgroup writes to shared memory
+ if (simd_lane_id == 0) {
+ shared[simd_group_id] = local_result;
+ }
+
+ // Synchronize across the threadgroup
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // Reduce across simdgroups
+ const ushort num_simd_groups = (BLOCK_SIZE_PARAM + 31) / 32;
+ ArgmaxPair total_result = ARGMAX_INIT;
+ if (lid < num_simd_groups) {
+ total_result = shared[lid];
+ }
+
+ // Final simdgroup reduction
+ for (ushort offset = 16; offset > 0; offset /= 2) {
+ ArgmaxPair other = {
+ simd_shuffle_down(total_result.value, offset),
+ simd_shuffle_down(total_result.index, offset)
+ };
+ total_result = argmax_combine(total_result, other);
+ }
+
+ // Broadcast the result to all threads
+ if (lid == 0) {
+ shared[0] = total_result;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ return shared[0];
}
-template<ushort GRAIN_SIZE_PARAM>
-static inline ArgmaxPair thread_argmax_reduce(thread ArgmaxPair (&values)[GRAIN_SIZE_PARAM]) {
- ArgmaxPair result = values[0];
- for (ushort i = 1; i < GRAIN_SIZE_PARAM; i++) {
- result = argmax_combine(result, values[i]);
- }
- return result;
+template <ushort GRAIN_SIZE_PARAM>
+static inline ArgmaxPair thread_argmax_reduce(
+ thread ArgmaxPair (&values)[GRAIN_SIZE_PARAM]
+) {
+ ArgmaxPair result = values[0];
+ for (ushort i = 1; i < GRAIN_SIZE_PARAM; i++) {
+ result = argmax_combine(result, values[i]);
+ }
+ return result;
}
-template<ushort GRAIN_SIZE_PARAM, typename T>
-static void load_blocked_argmax_batched(thread ArgmaxPair (&values)[GRAIN_SIZE_PARAM],
- const device T* logits_data,
- const ushort local_id,
- const uint batch_idx,
- const uint vocab_offset,
- const uint vocab_size) {
- for (ushort i = 0; i < GRAIN_SIZE_PARAM; i++) {
- uint vocab_idx = vocab_offset + local_id * GRAIN_SIZE_PARAM + i;
- if (vocab_idx < vocab_size) {
- uint global_idx = batch_idx * vocab_size + vocab_idx;
- float value;
- value = static_cast<float>(logits_data[global_idx]);
- values[i] = {value, vocab_idx};
- } else {
- values[i] = ARGMAX_INIT;
- }
+template <ushort GRAIN_SIZE_PARAM, typename T>
+static void load_blocked_argmax_batched(
+ thread ArgmaxPair (&values)[GRAIN_SIZE_PARAM],
+ const device T* logits_data,
+ const ushort local_id,
+ const uint batch_idx,
+ const uint vocab_offset,
+ const uint vocab_size
+) {
+ for (ushort i = 0; i < GRAIN_SIZE_PARAM; i++) {
+ uint vocab_idx = vocab_offset + local_id * GRAIN_SIZE_PARAM + i;
+ if (vocab_idx < vocab_size) {
+ uint global_idx = batch_idx * vocab_size + vocab_idx;
+ float value;
+ value = static_cast<float>(logits_data[global_idx]);
+ values[i] = {value, vocab_idx};
+ } else {
+ values[i] = ARGMAX_INIT;
}
+ }
}
-template<ushort BLOCK_SIZE_PARAM>
-static ArgmaxPair threadgroup_raking_argmax(ArgmaxPair value, threadgroup ArgmaxPair* shared, const ushort lid) {
- shared[lid] = value;
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- if (lid < 32) {
- const ushort values_per_thread = BLOCK_SIZE_PARAM / 32;
- const ushort first_index = lid * values_per_thread;
-
- ArgmaxPair thread_result = shared[first_index];
- for (ushort i = first_index + 1; i < first_index + values_per_thread; i++) {
- thread_result = argmax_combine(thread_result, shared[i]);
- }
-
- ArgmaxPair simd_result = thread_result;
-
- for (ushort offset = 16; offset > 0; offset /= 2) {
- ArgmaxPair other = {
- simd_shuffle_down(simd_result.value, offset),
- simd_shuffle_down(simd_result.index, offset)
- };
- simd_result = argmax_combine(simd_result, other);
- }
-
- ArgmaxPair final_result = {
- simd_broadcast(simd_result.value, 0),
- simd_broadcast(simd_result.index, 0)
- };
-
- for (ushort i = first_index; i < first_index + values_per_thread; i++) {
- shared[i] = final_result;
- }
+template <ushort BLOCK_SIZE_PARAM>
+static ArgmaxPair threadgroup_raking_argmax(
+ ArgmaxPair value,
+ threadgroup ArgmaxPair* shared,
+ const ushort lid
+) {
+ shared[lid] = value;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ if (lid < 32) {
+ const ushort values_per_thread = BLOCK_SIZE_PARAM / 32;
+ const ushort first_index = lid * values_per_thread;
+
+ ArgmaxPair thread_result = shared[first_index];
+ for (ushort i = first_index + 1; i < first_index + values_per_thread; i++) {
+ thread_result = argmax_combine(thread_result, shared[i]);
}
- threadgroup_barrier(mem_flags::mem_threadgroup);
- return shared[lid];
-}
-
-template<typename T>
-void batched_argmax_main_impl(const device T* logits_data,
- device ArgmaxPair* partial_results,
- constant uint& batch_size,
- constant uint& vocab_size,
- threadgroup ArgmaxPair* shared,
- uint batch_idx,
- uint vocab_group_idx,
- ushort local_id) {
-
- if (batch_idx >= batch_size) return;
-
- const uint elements_per_group = BLOCK_SIZE * GRAIN_SIZE;
- const uint vocab_offset = vocab_group_idx * elements_per_group;
+ ArgmaxPair simd_result = thread_result;
- ArgmaxPair values[GRAIN_SIZE];
- load_blocked_argmax_batched<GRAIN_SIZE>(values, logits_data, local_id, batch_idx, vocab_offset, vocab_size);
-
- ArgmaxPair thread_result = thread_argmax_reduce<GRAIN_SIZE>(values);
- ArgmaxPair group_result = threadgroup_cooperative_argmax<BLOCK_SIZE>(thread_result, shared, local_id);
-
- if (local_id == 0) {
- uint vocab_groups_per_batch = (vocab_size + elements_per_group - 1) / elements_per_group;
- uint partial_idx = batch_idx * vocab_groups_per_batch + vocab_group_idx;
- partial_results[partial_idx] = group_result;
+ for (ushort offset = 16; offset > 0; offset /= 2) {
+ ArgmaxPair other = {
+ simd_shuffle_down(simd_result.value, offset),
+ simd_shuffle_down(simd_result.index, offset)
+ };
+ simd_result = argmax_combine(simd_result, other);
}
-}
-
-void batched_argmax_final_impl(const device ArgmaxPair* partial_results,
- device uint* final_tokens,
- constant uint& batch_size,
- constant uint& vocab_size,
- threadgroup ArgmaxPair* shared,
- uint batch_idx,
- ushort local_id) {
-
- if (batch_idx >= batch_size) return;
- const uint elements_per_group = BLOCK_SIZE * GRAIN_SIZE;
- const uint vocab_groups_per_batch = (vocab_size + elements_per_group - 1) / elements_per_group;
- const uint partial_offset = batch_idx * vocab_groups_per_batch;
+ ArgmaxPair final_result = {
+ simd_broadcast(simd_result.value, 0),
+ simd_broadcast(simd_result.index, 0)
+ };
- ArgmaxPair thread_result = ARGMAX_INIT;
-
- for (uint i = local_id; i < vocab_groups_per_batch; i += BLOCK_SIZE) {
- thread_result = argmax_combine(thread_result, partial_results[partial_offset + i]);
+ for (ushort i = first_index; i < first_index + values_per_thread; i++) {
+ shared[i] = final_result;
}
+ }
- ArgmaxPair result = threadgroup_cooperative_argmax<BLOCK_SIZE>(thread_result, shared, local_id);
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ return shared[lid];
+}
- if (local_id == 0) {
- final_tokens[batch_idx] = result.index;
- }
+template <typename T>
+void batched_argmax_main_impl(
+ const device T* logits_data,
+ device ArgmaxPair* partial_results,
+ constant uint& batch_size,
+ constant uint& vocab_size,
+ threadgroup ArgmaxPair* shared,
+ uint batch_idx,
+ uint vocab_group_idx,
+ ushort local_id
+) {
+
+ if (batch_idx >= batch_size)
+ return;
+
+ const uint elements_per_group = BLOCK_SIZE * GRAIN_SIZE;
+ const uint vocab_offset = vocab_group_idx * elements_per_group;
+
+ ArgmaxPair values[GRAIN_SIZE];
+ load_blocked_argmax_batched<GRAIN_SIZE>(
+ values,
+ logits_data,
+ local_id,
+ batch_idx,
+ vocab_offset,
+ vocab_size
+ );
+
+ ArgmaxPair thread_result = thread_argmax_reduce<GRAIN_SIZE>(values);
+ ArgmaxPair group_result = threadgroup_cooperative_argmax<BLOCK_SIZE>(
+ thread_result,
+ shared,
+ local_id
+ );
+
+ if (local_id == 0) {
+ uint vocab_groups_per_batch =
+ (vocab_size + elements_per_group - 1) / elements_per_group;
+ uint partial_idx = batch_idx * vocab_groups_per_batch + vocab_group_idx;
+ partial_results[partial_idx] = group_result;
+ }
}
-// Generate argmax main kernels using macro approach
-#define outerArguments(T) \
-(device const T* logits_data [[ buffer(0) ]], \
-device ArgmaxPair* partial_results [[ buffer(1) ]], \
-constant uint& batch_size [[ buffer(2) ]], \
-constant uint& vocab_size [[ buffer(3) ]], \
-uint2 group_position [[ threadgroup_position_in_grid ]], \
-uint2 local_position [[ thread_position_in_threadgroup ]])
-
-#define innerArguments \
-(logits_data, partial_results, batch_size, vocab_size, shared, group_position.x, group_position.y, (ushort)local_position.x)
-
-#define generateArgmaxMainKernel(functionName, T, outerArgs, innerArgs) \
-[[max_total_threads_per_threadgroup(1024)]] kernel void functionName##_##T outerArgs { \
- threadgroup ArgmaxPair shared[BLOCK_SIZE]; \
- batched_argmax_main_impl<T> innerArgs; \
+void batched_argmax_final_impl(
+ const device ArgmaxPair* partial_results,
+ device uint* final_tokens,
+ constant uint& batch_size,
+ constant uint& vocab_size,
+ threadgroup ArgmaxPair* shared,
+ uint batch_idx,
+ ushort local_id
+) {
+
+ if (batch_idx >= batch_size)
+ return;
+
+ const uint elements_per_group = BLOCK_SIZE * GRAIN_SIZE;
+ const uint vocab_groups_per_batch =
+ (vocab_size + elements_per_group - 1) / elements_per_group;
+ const uint partial_offset = batch_idx * vocab_groups_per_batch;
+
+ ArgmaxPair thread_result = ARGMAX_INIT;
+
+ for (uint i = local_id; i < vocab_groups_per_batch; i += BLOCK_SIZE) {
+ thread_result =
+ argmax_combine(thread_result, partial_results[partial_offset + i]);
+ }
+
+ ArgmaxPair result = threadgroup_cooperative_argmax<BLOCK_SIZE>(
+ thread_result,
+ shared,
+ local_id
+ );
+
+ if (local_id == 0) {
+ final_tokens[batch_idx] = result.index;
+ }
}
-generateArgmaxMainKernel(batched_argmax_main, float, outerArguments(float), innerArguments)
-generateArgmaxMainKernel(batched_argmax_main, half, outerArguments(half), innerArguments)
-generateArgmaxMainKernel(batched_argmax_main, bfloat, outerArguments(bfloat), innerArguments)
+// Generate argmax main kernels using macro approach
+#define outerArguments(T) \
+ (device const T* logits_data [[buffer(0)]], \
+ device ArgmaxPair* partial_results [[buffer(1)]], \
+ constant uint& batch_size [[buffer(2)]], \
+ constant uint& vocab_size [[buffer(3)]], \
+ uint2 group_position [[threadgroup_position_in_grid]], \
+ uint2 local_position [[thread_position_in_threadgroup]])
+
+#define innerArguments \
+ (logits_data, \
+ partial_results, \
+ batch_size, \
+ vocab_size, \
+ shared, \
+ group_position.x, \
+ group_position.y, \
+ (ushort)local_position.x)
+
+#define generateArgmaxMainKernel(functionName, T, outerArgs, innerArgs) \
+ [[max_total_threads_per_threadgroup( \
+ 1024 \
+ )]] kernel void functionName##_##T outerArgs { \
+ threadgroup ArgmaxPair shared[BLOCK_SIZE]; \
+ batched_argmax_main_impl<T> innerArgs; \
+ }
+
+generateArgmaxMainKernel(batched_argmax_main, float, outerArguments(float), innerArguments) generateArgmaxMainKernel(
+ batched_argmax_main,
+ half,
+ outerArguments(half),
+ innerArguments
+) generateArgmaxMainKernel(batched_argmax_main, bfloat, outerArguments(bfloat), innerArguments)
#undef outerArguments
#undef innerArguments
#undef generateArgmaxMainKernel
-// Generate argmax final kernels using macro approach - note: no T parameter needed for final stage
-#define outerArguments(T) \
-(device const ArgmaxPair* partial_results [[ buffer(0) ]], \
-device uint* final_tokens [[ buffer(1) ]], \
-constant uint& batch_size [[ buffer(2) ]], \
-constant uint& vocab_size [[ buffer(3) ]], \
-uint batch_idx [[ threadgroup_position_in_grid ]], \
-uint local_id [[ thread_position_in_threadgroup ]])
-
-#define innerArguments \
-(partial_results, final_tokens, batch_size, vocab_size, shared, batch_idx, (ushort)local_id)
-
-#define generateArgmaxFinalKernel(functionName, T, outerArgs, innerArgs) \
-[[max_total_threads_per_threadgroup(1024)]] kernel void functionName##_##T outerArgs { \
- threadgroup ArgmaxPair shared[BLOCK_SIZE]; \
- batched_argmax_final_impl innerArgs; \
-}
-
-generateArgmaxFinalKernel(batched_argmax_final, float, outerArguments(float), innerArguments)
-generateArgmaxFinalKernel(batched_argmax_final, half, outerArguments(half), innerArguments)
-generateArgmaxFinalKernel(batched_argmax_final, bfloat, outerArguments(bfloat), innerArguments)
+// Generate argmax final kernels using macro approach - note: no T parameter
+// needed for final stage
+#define outerArguments(T) \
+ (device const ArgmaxPair* partial_results [[buffer(0)]], \
+ device uint* final_tokens [[buffer(1)]], \
+ constant uint& batch_size [[buffer(2)]], \
+ constant uint& vocab_size [[buffer(3)]], \
+ uint batch_idx [[threadgroup_position_in_grid]], \
+ uint local_id [[thread_position_in_threadgroup]])
+
+#define innerArguments \
+ (partial_results, \
+ final_tokens, \
+ batch_size, \
+ vocab_size, \
+ shared, \
+ batch_idx, \
+ (ushort)local_id)
+
+#define generateArgmaxFinalKernel(functionName, T, outerArgs, innerArgs) \
+ [[max_total_threads_per_threadgroup( \
+ 1024 \
+ )]] kernel void functionName##_##T outerArgs { \
+ threadgroup ArgmaxPair shared[BLOCK_SIZE]; \
+ batched_argmax_final_impl innerArgs; \
+ }
+
+ generateArgmaxFinalKernel(batched_argmax_final, float, outerArguments(float), innerArguments) generateArgmaxFinalKernel(
+ batched_argmax_final,
+ half,
+ outerArguments(half),
+ innerArguments
+ ) generateArgmaxFinalKernel(batched_argmax_final, bfloat, outerArguments(bfloat), innerArguments)
#undef outerArguments
#undef innerArguments
#undef generateArgmaxFinalKernel
-template<typename T>
-void batched_argmax_single_impl(const device T* logits_data,
- device uint* final_tokens,
- constant uint& batch_size,
- constant uint& vocab_size,
- threadgroup ArgmaxPair* shared,
- uint batch_idx,
- ushort local_id)
-{
- if (batch_idx >= batch_size) return;
-
- const uint elements_per_tile = BLOCK_SIZE * GRAIN_SIZE;
- ArgmaxPair best = ARGMAX_INIT;
-
- for (uint vocab_offset = 0; vocab_offset < vocab_size; vocab_offset += elements_per_tile) {
- ArgmaxPair buf[GRAIN_SIZE];
- load_blocked_argmax_batched<GRAIN_SIZE>(buf,
- logits_data,
- local_id,
- batch_idx,
- vocab_offset,
- vocab_size);
- ArgmaxPair tile_best = thread_argmax_reduce<GRAIN_SIZE>(buf);
- best = argmax_combine(best, tile_best);
- }
-
- ArgmaxPair group_best = threadgroup_raking_argmax<BLOCK_SIZE>(best, shared, local_id);
-
- if (local_id == 0) {
- final_tokens[batch_idx] = group_best.index;
- }
+ template <typename T>
+ void batched_argmax_single_impl(
+ const device T* logits_data,
+ device uint* final_tokens,
+ constant uint& batch_size,
+ constant uint& vocab_size,
+ threadgroup ArgmaxPair* shared,
+ uint batch_idx,
+ ushort local_id
+ ) {
+ if (batch_idx >= batch_size)
+ return;
+
+ const uint elements_per_tile = BLOCK_SIZE * GRAIN_SIZE;
+ ArgmaxPair best = ARGMAX_INIT;
+
+ for (uint vocab_offset = 0; vocab_offset < vocab_size;
+ vocab_offset += elements_per_tile) {
+ ArgmaxPair buf[GRAIN_SIZE];
+ load_blocked_argmax_batched<GRAIN_SIZE>(
+ buf,
+ logits_data,
+ local_id,
+ batch_idx,
+ vocab_offset,
+ vocab_size
+ );
+ ArgmaxPair tile_best = thread_argmax_reduce<GRAIN_SIZE>(buf);
+ best = argmax_combine(best, tile_best);
+ }
+
+ ArgmaxPair group_best =
+ threadgroup_raking_argmax<BLOCK_SIZE>(best, shared, local_id);
+
+ if (local_id == 0) {
+ final_tokens[batch_idx] = group_best.index;
+ }
}
// Generate argmax single pass kernels using macro approach
-#define outerArguments(T) \
-(device const T* logits_data [[ buffer(0) ]], \
-device uint* final_tokens [[ buffer(1) ]], \
-constant uint& batch_size [[ buffer(2) ]], \
-constant uint& vocab_size [[ buffer(3) ]], \
-uint batch_idx [[ threadgroup_position_in_grid ]], \
-uint local_id [[ thread_position_in_threadgroup ]])
-
-#define innerArguments \
-(logits_data, final_tokens, batch_size, vocab_size, shared, batch_idx, (ushort)local_id)
-
-#define generateArgmaxSingleKernel(functionName, T, outerArgs, innerArgs) \
-[[max_total_threads_per_threadgroup(1024)]] kernel void functionName##_##T outerArgs { \
- threadgroup ArgmaxPair shared[BLOCK_SIZE]; \
- batched_argmax_single_impl<T> innerArgs; \
-}
-
-generateArgmaxSingleKernel(batched_argmax_single, float, outerArguments(float), innerArguments)
-generateArgmaxSingleKernel(batched_argmax_single, half, outerArguments(half), innerArguments)
-generateArgmaxSingleKernel(batched_argmax_single, bfloat, outerArguments(bfloat), innerArguments)
+#define outerArguments(T) \
+ (device const T* logits_data [[buffer(0)]], \
+ device uint* final_tokens [[buffer(1)]], \
+ constant uint& batch_size [[buffer(2)]], \
+ constant uint& vocab_size [[buffer(3)]], \
+ uint batch_idx [[threadgroup_position_in_grid]], \
+ uint local_id [[thread_position_in_threadgroup]])
+
+#define innerArguments \
+ (logits_data, \
+ final_tokens, \
+ batch_size, \
+ vocab_size, \
+ shared, \
+ batch_idx, \
+ (ushort)local_id)
+
+#define generateArgmaxSingleKernel(functionName, T, outerArgs, innerArgs) \
+ [[max_total_threads_per_threadgroup( \
+ 1024 \
+ )]] kernel void functionName##_##T outerArgs { \
+ threadgroup ArgmaxPair shared[BLOCK_SIZE]; \
+ batched_argmax_single_impl<T> innerArgs; \
+ }
+
+generateArgmaxSingleKernel(batched_argmax_single, float, outerArguments(float), innerArguments) generateArgmaxSingleKernel(
+ batched_argmax_single,
+ half,
+ outerArguments(half),
+ innerArguments
+) generateArgmaxSingleKernel(batched_argmax_single, bfloat, outerArguments(bfloat), innerArguments)
#undef outerArguments
#undef innerArguments
#undef generateArgmaxSingleKernel
#undef BLOCK_SIZE
-#undef GRAIN_SIZE
+#undef GRAIN_SIZE
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/bitmask.metal b/crates/uzu/src/backends/metal/kernel/sampling/bitmask.metal
index 999f7b150..27e7c0780 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/bitmask.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/bitmask.metal
@@ -16,38 +16,66 @@ void batched_bitmask(
uint batch_idx,
uint thread_idx
) {
- uint base_idx = batch_idx * vocab_size + group_idx * BLOCK_SIZE * GRAIN_SIZE + thread_idx;
- uint batch_end = batch_idx * vocab_size + vocab_size;
-
- #pragma unroll(4)
- for (uint i = 0; i < GRAIN_SIZE; i++) {
- uint global_idx = base_idx + i * BLOCK_SIZE;
- if (global_idx < batch_end) {
- bool mask = (bitmask_data[global_idx / BITS_IN_U32] >> (global_idx % BITS_IN_U32)) & 0b1;
- processed_logits[global_idx] = select(T(-INFINITY), logits_data[global_idx], mask);
- }
+ uint base_idx =
+ batch_idx * vocab_size + group_idx * BLOCK_SIZE * GRAIN_SIZE + thread_idx;
+ uint batch_end = batch_idx * vocab_size + vocab_size;
+
+#pragma unroll(4)
+ for (uint i = 0; i < GRAIN_SIZE; i++) {
+ uint global_idx = base_idx + i * BLOCK_SIZE;
+ if (global_idx < batch_end) {
+ bool mask = (bitmask_data[global_idx / BITS_IN_U32] >>
+ (global_idx % BITS_IN_U32)) &
+ 0b1;
+ processed_logits[global_idx] =
+ select(T(-INFINITY), logits_data[global_idx], mask);
}
+ }
}
-#define outerArguments(T) \
-(device const T* logits_data [[ buffer(0) ]], \
-device const uint32_t* bitmask_data [[ buffer(1) ]], \
-device T* processed_logits [[ buffer(2) ]], \
-constant uint& vocab_size [[ buffer(3) ]], \
-uint3 threadgroup_idx [[ threadgroup_position_in_grid ]], \
-uint3 thread_idx [[ thread_position_in_threadgroup ]])
+#define outerArguments(T) \
+ (device const T* logits_data [[buffer(0)]], \
+ device const uint32_t* bitmask_data [[buffer(1)]], \
+ device T* processed_logits [[buffer(2)]], \
+ constant uint& vocab_size [[buffer(3)]], \
+ uint3 threadgroup_idx [[threadgroup_position_in_grid]], \
+ uint3 thread_idx [[thread_position_in_threadgroup]])
-#define innerArguments (logits_data, bitmask_data, processed_logits, vocab_size, threadgroup_idx.x, threadgroup_idx.y, thread_idx.x)
+#define innerArguments \
+ (logits_data, \
+ bitmask_data, \
+ processed_logits, \
+ vocab_size, \
+ threadgroup_idx.x, \
+ threadgroup_idx.y, \
+ thread_idx.x)
-#define generateBitmaskKernel(functionName, scalarType, outerArgs, innerArgs) \
-[[max_total_threads_per_threadgroup(1024)]] kernel void functionName##_##scalarType outerArgs { \
- functionName innerArgs; \
-}
+#define generateBitmaskKernel(functionName, scalarType, outerArgs, innerArgs) \
+ [[max_total_threads_per_threadgroup( \
+ 1024 \
+ )]] kernel void functionName##_##scalarType outerArgs { \
+ functionName innerArgs; \
+ }
-#define generateBitmaskKernels(functionName) \
-generateBitmaskKernel(functionName, float, outerArguments(float), innerArguments); \
-generateBitmaskKernel(functionName, bfloat, outerArguments(bfloat), innerArguments); \
-generateBitmaskKernel(functionName, half, outerArguments(half), innerArguments);
+#define generateBitmaskKernels(functionName) \
+ generateBitmaskKernel( \
+ functionName, \
+ float, \
+ outerArguments(float), \
+ innerArguments \
+ ); \
+ generateBitmaskKernel( \
+ functionName, \
+ bfloat, \
+ outerArguments(bfloat), \
+ innerArguments \
+ ); \
+ generateBitmaskKernel( \
+ functionName, \
+ half, \
+ outerArguments(half), \
+ innerArguments \
+ );
generateBitmaskKernels(batched_bitmask)
@@ -55,4 +83,3 @@ generateBitmaskKernels(batched_bitmask)
#undef innerArguments
#undef generatebitmaskKernel
#undef generatebitmaskKernels
-
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metal b/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metal
index 47b804e33..b51b9bb94 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metal
@@ -17,46 +17,73 @@ void batched_gumbel(
uint batch_idx,
uint thread_idx
) {
- uint batch_start = vocab_size * batch_idx;
- uint batch_end = batch_start + vocab_size;
+ uint batch_start = vocab_size * batch_idx;
+ uint batch_end = batch_start + vocab_size;
- uint grain_offset_in_batch = group_idx * BLOCK_SIZE * GRAIN_SIZE + thread_idx;
+ uint grain_offset_in_batch = group_idx * BLOCK_SIZE * GRAIN_SIZE + thread_idx;
- uint grain_offset = batch_start + grain_offset_in_batch;
+ uint grain_offset = batch_start + grain_offset_in_batch;
- uint64_t rng_seed = batch_seeds[batch_idx];
- uint64_t rng_offset = (group_idx * BLOCK_SIZE + thread_idx) * (GRAIN_SIZE + WORDS_PER_OFFSET - 1) / WORDS_PER_OFFSET;
- PhiloxState rng;
- philox_init(&rng, rng_seed, rng_offset);
+ uint64_t rng_seed = batch_seeds[batch_idx];
+ uint64_t rng_offset = (group_idx * BLOCK_SIZE + thread_idx) *
+ (GRAIN_SIZE + WORDS_PER_OFFSET - 1) / WORDS_PER_OFFSET;
+ PhiloxState rng;
+ philox_init(&rng, rng_seed, rng_offset);
- #pragma unroll(4)
- for (uint i = 0; i < GRAIN_SIZE; i++) {
- uint global_idx = grain_offset + i * BLOCK_SIZE;
- if (global_idx < batch_end) {
- processed_logits[global_idx] = logits_data[global_idx] + T(-fast::log(-fast::log(uniform_float(&rng))));
- }
+#pragma unroll(4)
+ for (uint i = 0; i < GRAIN_SIZE; i++) {
+ uint global_idx = grain_offset + i * BLOCK_SIZE;
+ if (global_idx < batch_end) {
+ processed_logits[global_idx] =
+ logits_data[global_idx] +
+ T(-fast::log(-fast::log(uniform_float(&rng))));
}
+ }
}
-#define outerArguments(T) \
-(device const T* logits_data [[ buffer(0) ]], \
-device const uint64_t* batch_seeds [[ buffer(1) ]], \
-device T* processed_logits [[ buffer(2) ]], \
-constant uint& vocab_size [[ buffer(3) ]], \
-uint3 threadgroup_idx [[ threadgroup_position_in_grid ]], \
-uint3 thread_idx [[ thread_position_in_threadgroup ]])
+#define outerArguments(T) \
+ (device const T* logits_data [[buffer(0)]], \
+ device const uint64_t* batch_seeds [[buffer(1)]], \
+ device T* processed_logits [[buffer(2)]], \
+ constant uint& vocab_size [[buffer(3)]], \
+ uint3 threadgroup_idx [[threadgroup_position_in_grid]], \
+ uint3 thread_idx [[thread_position_in_threadgroup]])
-#define innerArguments (logits_data, batch_seeds, processed_logits, vocab_size, threadgroup_idx.x, threadgroup_idx.y, thread_idx.x)
+#define innerArguments \
+ (logits_data, \
+ batch_seeds, \
+ processed_logits, \
+ vocab_size, \
+ threadgroup_idx.x, \
+ threadgroup_idx.y, \
+ thread_idx.x)
-#define generateGumbelKernel(functionName, scalarType, outerArgs, innerArgs) \
-[[max_total_threads_per_threadgroup(1024)]] kernel void functionName##_##scalarType outerArgs { \
- functionName innerArgs; \
-}
+#define generateGumbelKernel(functionName, scalarType, outerArgs, innerArgs) \
+ [[max_total_threads_per_threadgroup( \
+ 1024 \
+ )]] kernel void functionName##_##scalarType outerArgs { \
+ functionName innerArgs; \
+ }
-#define generateGumbelKernels(functionName) \
-generateGumbelKernel(functionName, float, outerArguments(float), innerArguments); \
-generateGumbelKernel(functionName, bfloat, outerArguments(bfloat), innerArguments); \
-generateGumbelKernel(functionName, half, outerArguments(half), innerArguments);
+#define generateGumbelKernels(functionName) \
+ generateGumbelKernel( \
+ functionName, \
+ float, \
+ outerArguments(float), \
+ innerArguments \
+ ); \
+ generateGumbelKernel( \
+ functionName, \
+ bfloat, \
+ outerArguments(bfloat), \
+ innerArguments \
+ ); \
+ generateGumbelKernel( \
+ functionName, \
+ half, \
+ outerArguments(half), \
+ innerArguments \
+ );
generateGumbelKernels(batched_gumbel)
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/temperature.metal b/crates/uzu/src/backends/metal/kernel/sampling/temperature.metal
index 50b2e83e7..027813031 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/temperature.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/temperature.metal
@@ -14,37 +14,68 @@ void batched_temperature(
uint batch_idx,
uint thread_idx
) {
- uint base_idx = batch_idx * vocab_size + group_idx * BLOCK_SIZE * GRAIN_SIZE + thread_idx;
- uint batch_end = batch_idx * vocab_size + vocab_size;
-
- #pragma unroll(4)
- for (uint i = 0; i < GRAIN_SIZE; i++) {
- uint global_idx = base_idx + i * BLOCK_SIZE;
- if (global_idx < batch_end) {
- processed_logits[global_idx] = T(float(logits_data[global_idx]) / temperature);
- }
+ uint base_idx =
+ batch_idx * vocab_size + group_idx * BLOCK_SIZE * GRAIN_SIZE + thread_idx;
+ uint batch_end = batch_idx * vocab_size + vocab_size;
+
+#pragma unroll(4)
+ for (uint i = 0; i < GRAIN_SIZE; i++) {
+ uint global_idx = base_idx + i * BLOCK_SIZE;
+ if (global_idx < batch_end) {
+ processed_logits[global_idx] =
+ T(float(logits_data[global_idx]) / temperature);
}
+ }
}
-#define outerArguments(T) \
-(device const T* logits_data [[ buffer(0) ]], \
-device T* processed_logits [[ buffer(1) ]], \
-constant uint& vocab_size [[ buffer(2) ]], \
-constant float& temperature [[ buffer(3) ]], \
-uint3 threadgroup_idx [[ threadgroup_position_in_grid ]], \
-uint3 thread_idx [[ thread_position_in_threadgroup ]])
+#define outerArguments(T) \
+ (device const T* logits_data [[buffer(0)]], \
+ device T* processed_logits [[buffer(1)]], \
+ constant uint& vocab_size [[buffer(2)]], \
+ constant float& temperature [[buffer(3)]], \
+ uint3 threadgroup_idx [[threadgroup_position_in_grid]], \
+ uint3 thread_idx [[thread_position_in_threadgroup]])
-#define innerArguments (logits_data, processed_logits, vocab_size, temperature, threadgroup_idx.x, threadgroup_idx.y, thread_idx.x)
+#define innerArguments \
+ (logits_data, \
+ processed_logits, \
+ vocab_size, \
+ temperature, \
+ threadgroup_idx.x, \
+ threadgroup_idx.y, \
+ thread_idx.x)
-#define generateTemperatureKernel(functionName, scalarType, outerArgs, innerArgs) \
-[[max_total_threads_per_threadgroup(1024)]] kernel void functionName##_##scalarType outerArgs { \
- functionName innerArgs; \
-}
+#define generateTemperatureKernel( \
+ functionName, \
+ scalarType, \
+ outerArgs, \
+ innerArgs \
+) \
+ [[max_total_threads_per_threadgroup( \
+ 1024 \
+ )]] kernel void functionName##_##scalarType outerArgs { \
+ functionName innerArgs; \
+ }
-#define generateTemperatureKernels(functionName) \
-generateTemperatureKernel(functionName, float, outerArguments(float), innerArguments); \
-generateTemperatureKernel(functionName, bfloat, outerArguments(bfloat), innerArguments); \
-generateTemperatureKernel(functionName, half, outerArguments(half), innerArguments);
+#define generateTemperatureKernels(functionName) \
+ generateTemperatureKernel( \
+ functionName, \
+ float, \
+ outerArguments(float), \
+ innerArguments \
+ ); \
+ generateTemperatureKernel( \
+ functionName, \
+ bfloat, \
+ outerArguments(bfloat), \
+ innerArguments \
+ ); \
+ generateTemperatureKernel( \
+ functionName, \
+ half, \
+ outerArguments(half), \
+ innerArguments \
+ );
generateTemperatureKernels(batched_temperature)
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal b/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal
index 69f7f70c1..51d7ef616 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal
@@ -14,77 +14,112 @@ void batched_topk(
uint batch_idx,
uint thread_idx
) {
- uint batch_start = batch_idx * vocab_size;
+ uint batch_start = batch_idx * vocab_size;
- // Find min and max logit for binary search
- float local_max = -INFINITY;
- float local_min = INFINITY;
- #pragma unroll(4)
- for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {
- float logit_value = float(logits_data[batch_start + i]);
- local_max = fmax(local_max, logit_value);
- local_min = select(local_min, fmin(local_min, logit_value), logit_value > -INFINITY);
- }
- float max_logit = threadgroup_cooperative_reduce_max<BLOCK_SIZE>(local_max, shared_reduce_buffer, thread_idx);
- float min_logit = threadgroup_cooperative_reduce_min<BLOCK_SIZE>(local_min, shared_reduce_buffer, thread_idx);
+ // Find min and max logit for binary search
+ float local_max = -INFINITY;
+ float local_min = INFINITY;
+#pragma unroll(4)
+ for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {
+ float logit_value = float(logits_data[batch_start + i]);
+ local_max = fmax(local_max, logit_value);
+ local_min = select(
+ local_min,
+ fmin(local_min, logit_value),
+ logit_value > -INFINITY
+ );
+ }
+ float max_logit = threadgroup_cooperative_reduce_max<BLOCK_SIZE>(
+ local_max,
+ shared_reduce_buffer,
+ thread_idx
+ );
+ float min_logit = threadgroup_cooperative_reduce_min<BLOCK_SIZE>(
+ local_min,
+ shared_reduce_buffer,
+ thread_idx
+ );
+ // Do the binary search on the threshold
+ float low = min_logit;
+ float high = max_logit;
+ float threshold = (min_logit + max_logit) / 2.0;
- // Do the binary search on the threshold
- float low = min_logit;
- float high = max_logit;
- float threshold = (min_logit + max_logit) / 2.0;
-
- for (uint iter = 0; iter < MAX_ITERS; iter++) {
- // Find the mass above threshold
- uint local_num_above_threshold = 0;
- #pragma unroll(4)
- for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {
- float logit_value = float(logits_data[batch_start + i]);
- local_num_above_threshold += select(0, 1, logit_value >= threshold);
- }
- uint num_above_threshold = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(local_num_above_threshold, (threadgroup uint*)shared_reduce_buffer, thread_idx);
+ for (uint iter = 0; iter < MAX_ITERS; iter++) {
+ // Find the mass above threshold
+ uint local_num_above_threshold = 0;
+#pragma unroll(4)
+ for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {
+ float logit_value = float(logits_data[batch_start + i]);
+ local_num_above_threshold += select(0, 1, logit_value >= threshold);
+ }
+ uint num_above_threshold = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(
+ local_num_above_threshold,
+ (threadgroup uint*)shared_reduce_buffer,
+ thread_idx
+ );
- // Update binary search
- if (num_above_threshold == top_k) {
- break;
- } else if (num_above_threshold > top_k) {
- low = threshold;
- } else {
- high = threshold;
- }
- threshold = (low + high) / 2.0;
+ // Update binary search
+ if (num_above_threshold == top_k) {
+ break;
+ } else if (num_above_threshold > top_k) {
+ low = threshold;
+ } else {
+ high = threshold;
}
+ threshold = (low + high) / 2.0;
+ }
- T t_threshold = T(threshold);
+ T t_threshold = T(threshold);
- // We know the threshold, just mask everything below it
- #pragma unroll(4)
- for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {
- T logit_value = logits_data[batch_start + i];
- processed_logits[batch_start + i] = select(T(-INFINITY), logit_value, logit_value >= t_threshold);
- }
+// We know the threshold, just mask everything below it
+#pragma unroll(4)
+ for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {
+ T logit_value = logits_data[batch_start + i];
+ processed_logits[batch_start + i] =
+ select(T(-INFINITY), logit_value, logit_value >= t_threshold);
+ }
}
-#define outerArguments(T) \
-(device const T* logits_data [[ buffer(0) ]], \
-device T* processed_logits [[ buffer(1) ]], \
-constant uint& vocab_size [[ buffer(2) ]], \
-constant uint& top_k [[ buffer(3) ]], \
-uint3 threadgroup_idx [[ threadgroup_position_in_grid ]], \
-uint3 thread_idx [[ thread_position_in_threadgroup ]])
+#define outerArguments(T) \
+ (device const T* logits_data [[buffer(0)]], \
+ device T* processed_logits [[buffer(1)]], \
+ constant uint& vocab_size [[buffer(2)]], \
+ constant uint& top_k [[buffer(3)]], \
+ uint3 threadgroup_idx [[threadgroup_position_in_grid]], \
+ uint3 thread_idx [[thread_position_in_threadgroup]])
-#define innerArguments (logits_data, processed_logits, shared_reduce_buffer, vocab_size, top_k, threadgroup_idx.x, thread_idx.x)
+#define innerArguments \
+ (logits_data, \
+ processed_logits, \
+ shared_reduce_buffer, \
+ vocab_size, \
+ top_k, \
+ threadgroup_idx.x, \
+ thread_idx.x)
-#define generateTopkKernel(functionName, scalarType, outerArgs, innerArgs) \
-[[max_total_threads_per_threadgroup(1024)]] kernel void functionName##_##scalarType outerArgs { \
- threadgroup float shared_reduce_buffer[BLOCK_SIZE]; \
- functionName innerArgs; \
-}
+#define generateTopkKernel(functionName, scalarType, outerArgs, innerArgs) \
+ [[max_total_threads_per_threadgroup( \
+ 1024 \
+ )]] kernel void functionName##_##scalarType outerArgs { \
+ threadgroup float shared_reduce_buffer[BLOCK_SIZE]; \
+ functionName innerArgs; \
+ }
-#define generateTopkKernels(functionName) \
-generateTopkKernel(functionName, float, outerArguments(float), innerArguments); \
-generateTopkKernel(functionName, bfloat, outerArguments(bfloat), innerArguments); \
-generateTopkKernel(functionName, half, outerArguments(half), innerArguments);
+#define generateTopkKernels(functionName) \
+ generateTopkKernel( \
+ functionName, \
+ float, \
+ outerArguments(float), \
+ innerArguments \
+ ); \
+ generateTopkKernel( \
+ functionName, \
+ bfloat, \
+ outerArguments(bfloat), \
+ innerArguments \
+ ); \
+ generateTopkKernel(functionName, half, outerArguments(half), innerArguments);
generateTopkKernels(batched_topk)
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal b/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal
index 4bbcc8478..f16125107 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal
@@ -16,94 +16,146 @@ void batched_topp(
uint batch_idx,
uint thread_idx
) {
- uint batch_start = batch_idx * vocab_size;
-
- // Find min (for binary search) and max (for binary search and softmax)
- float local_max = -INFINITY;
- float local_min = INFINITY;
- #pragma unroll(4)
+ uint batch_start = batch_idx * vocab_size;
+
+ // Find min (for binary search) and max (for binary search and softmax)
+ float local_max = -INFINITY;
+ float local_min = INFINITY;
+#pragma unroll(4)
+ for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {
+ float logit_value = float(logits_data[batch_start + i]);
+ local_max = fmax(local_max, logit_value);
+ local_min = select(
+ local_min,
+ fmin(local_min, logit_value),
+ logit_value > -INFINITY
+ );
+ }
+ float max_logit = threadgroup_cooperative_reduce_max<BLOCK_SIZE>(
+ local_max,
+ shared_reduce_buffer,
+ thread_idx
+ );
+ float min_logit = threadgroup_cooperative_reduce_min<BLOCK_SIZE>(
+ local_min,
+ shared_reduce_buffer,
+ thread_idx
+ );
+
+ // Find denominator for softmax
+ float local_sum = 0.0f;
+#pragma unroll(4)
+ for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {
+ float logit_value = float(logits_data[batch_start + i]);
+ local_sum += select(
+ 0.0,
+ fast::exp(logit_value - max_logit),
+ logit_value > -INFINITY
+ );
+ }
+ float total_sum = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(
+ local_sum,
+ shared_reduce_buffer,
+ thread_idx
+ );
+
+ // Do the binary search on the threshold
+ float target_mass = top_p * total_sum;
+
+ float low = min_logit;
+ float high = max_logit;
+ float threshold = (min_logit + max_logit) / 2.0;
+
+ for (uint iter = 0; iter < MAX_ITERS; iter++) {
+ // Find the mass above threshold
+ float local_sum_above_threshold = 0.0f;
+ float local_min_above_threshold = INFINITY;
+#pragma unroll(4)
for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {
- float logit_value = float(logits_data[batch_start + i]);
- local_max = fmax(local_max, logit_value);
- local_min = select(local_min, fmin(local_min, logit_value), logit_value > -INFINITY);
+ float logit_value = float(logits_data[batch_start + i]);
+ float logit_mass = fast::exp(logit_value - max_logit);
+ local_sum_above_threshold +=
+ select(0.0, logit_mass, logit_value >= threshold);
+ local_min_above_threshold = fmin(
+ local_min_above_threshold,
+ select(INFINITY, logit_mass, logit_value >= threshold)
+ );
}
- float max_logit = threadgroup_cooperative_reduce_max<BLOCK_SIZE>(local_max, shared_reduce_buffer, thread_idx);
- float min_logit = threadgroup_cooperative_reduce_min<BLOCK_SIZE>(local_min, shared_reduce_buffer, thread_idx);
-
- // Find denominator for softmax
- float local_sum = 0.0f;
- #pragma unroll(4)
- for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {
- float logit_value = float(logits_data[batch_start + i]);
- local_sum += select(0.0, fast::exp(logit_value - max_logit), logit_value > -INFINITY);
+ float sum_above_threshold = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(
+ local_sum_above_threshold,
+ shared_reduce_buffer,
+ thread_idx
+ );
+ float min_above_threshold = threadgroup_cooperative_reduce_min<BLOCK_SIZE>(
+ local_min_above_threshold,
+ shared_reduce_buffer,
+ thread_idx
+ );
+
+ // Early exit
+ if (sum_above_threshold >= target_mass &&
+ sum_above_threshold - min_above_threshold < target_mass) {
+ break;
}
- float total_sum = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(local_sum, shared_reduce_buffer, thread_idx);
-
-
- // Do the binary search on the threshold
- float target_mass = top_p * total_sum;
-
- float low = min_logit;
- float high = max_logit;
- float threshold = (min_logit + max_logit) / 2.0;
-
- for (uint iter = 0; iter < MAX_ITERS; iter++) {
- // Find the mass above threshold
- float local_sum_above_threshold = 0.0f;
- float local_min_above_threshold = INFINITY;
- #pragma unroll(4)
- for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {
- float logit_value = float(logits_data[batch_start + i]);
- float logit_mass = fast::exp(logit_value - max_logit);
- local_sum_above_threshold += select(0.0, logit_mass, logit_value >= threshold);
- local_min_above_threshold = fmin(local_min_above_threshold, select(INFINITY, logit_mass, logit_value >= threshold));
- }
- float sum_above_threshold = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(local_sum_above_threshold, shared_reduce_buffer, thread_idx);
- float min_above_threshold = threadgroup_cooperative_reduce_min<BLOCK_SIZE>(local_min_above_threshold, shared_reduce_buffer, thread_idx);
-
- // Early exit
- if (sum_above_threshold >= target_mass && sum_above_threshold - min_above_threshold < target_mass) {
- break;
- }
- // Update binary search
- if (sum_above_threshold >= target_mass) {
- low = threshold;
- } else {
- high = threshold;
- }
- threshold = (low + high) / 2.0;
+ // Update binary search
+ if (sum_above_threshold >= target_mass) {
+ low = threshold;
+ } else {
+ high = threshold;
}
-
- T t_threshold = T(threshold);
-
- // We know the threshold, just mask everything below it
- #pragma unroll(4)
- for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {
- T logit_value = logits_data[batch_start + i];
- processed_logits[batch_start + i] = select(T(-INFINITY), logit_value, logit_value >= t_threshold);
- }
-}
-
-#define outerArguments(T) \
-(device const T* logits_data [[ buffer(0) ]], \
-device T* processed_logits [[ buffer(1) ]], \
-constant uint& vocab_size [[ buffer(2) ]], \
-constant float& top_p [[ buffer(3) ]], \
-uint3 threadgroup_idx [[ threadgroup_position_in_grid ]], \
-uint3 thread_idx [[ thread_position_in_threadgroup ]])
-
-#define innerArguments (logits_data, processed_logits, shared_reduce_buffer, vocab_size, top_p, threadgroup_idx.x, thread_idx.x)
-
-#define generateToppKernel(functionName, scalarType, outerArgs, innerArgs) \
-[[max_total_threads_per_threadgroup(1024)]] kernel void functionName##_##scalarType outerArgs { \
- threadgroup float shared_reduce_buffer[BLOCK_SIZE_IN_SIMDS]; \
- functionName innerArgs; \
+ threshold = (low + high) / 2.0;
+ }
+
+ T t_threshold = T(threshold);
+
+// We know the threshold, just mask everything below it
+#pragma unroll(4)
+ for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {
+ T logit_value = logits_data[batch_start + i];
+ processed_logits[batch_start + i] =
+ select(T(-INFINITY), logit_value, logit_value >= t_threshold);
+ }
}
-#define generateToppKernels(functionName) \
-generateToppKernel(functionName, float, outerArguments(float), innerArguments); \
-generateToppKernel(functionName, bfloat, outerArguments(bfloat), innerArguments); \
-generateToppKernel(functionName, half, outerArguments(half), innerArguments);
+#define outerArguments(T) \
+ (device const T* logits_data [[buffer(0)]], \
+ device T* processed_logits [[buffer(1)]], \
+ constant uint& vocab_size [[buffer(2)]], \
+ constant float& top_p [[buffer(3)]], \
+ uint3 threadgroup_idx [[threadgroup_position_in_grid]], \
+ uint3 thread_idx [[thread_position_in_threadgroup]])
+
+#define innerArguments \
+ (logits_data, \
+ processed_logits, \
+ shared_reduce_buffer, \
+ vocab_size, \
+ top_p, \
+ threadgroup_idx.x, \
+ thread_idx.x)
+
+#define generateToppKernel(functionName, scalarType, outerArgs, innerArgs) \
+ [[max_total_threads_per_threadgroup( \
+ 1024 \
+ )]] kernel void functionName##_##scalarType outerArgs { \
+ threadgroup float shared_reduce_buffer[BLOCK_SIZE_IN_SIMDS]; \
+ functionName innerArgs; \
+ }
+
+#define generateToppKernels(functionName) \
+ generateToppKernel( \
+ functionName, \
+ float, \
+ outerArguments(float), \
+ innerArguments \
+ ); \
+ generateToppKernel( \
+ functionName, \
+ bfloat, \
+ outerArguments(bfloat), \
+ innerArguments \
+ ); \
+ generateToppKernel(functionName, half, outerArguments(half), innerArguments);
generateToppKernels(batched_topp)
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.metal b/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.metal
index dc477a170..369dc79ea 100644
--- a/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.metal
+++ b/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.metal
@@ -13,240 +13,241 @@ constant uint CONV_SCAN_THREADS = 32u;
template <typename T>
inline T apply_silu(T x) {
- float xf = float(x);
- float y = 1.0f / (1.0f + fast::exp(-fabs(xf)));
- float out = (xf < 0.0f) ? (1.0f - y) * xf : y * xf;
- return static_cast<T>(out);
+ float xf = float(x);
+ float y = 1.0f / (1.0f + fast::exp(-fabs(xf)));
+ float out = (xf < 0.0f) ? (1.0f - y) * xf : y * xf;
+ return static_cast<T>(out);
}
template <typename T>
inline T apply_gelu(T x) {
- float xf = float(x);
- return static_cast<T>(0.5f * xf * (1.0f + fast::tanh(0.797885f * (xf + 0.044715f * xf * xf * xf))));
+ float xf = float(x);
+ return static_cast<T>(
+ 0.5f * xf *
+ (1.0f + fast::tanh(0.797885f * (xf + 0.044715f * xf * xf * xf)))
+ );
}
template <typename T>
inline T apply_activation_fn(T x, int activation_type) {
- if (activation_type == ACTIVATION_SILU) {
- return apply_silu(x);
- } else if (activation_type == ACTIVATION_GELU) {
- return apply_gelu(x);
- } else {
- return x; // Identity
- }
+ if (activation_type == ACTIVATION_SILU) {
+ return apply_silu(x);
+ } else if (activation_type == ACTIVATION_GELU) {
+ return apply_gelu(x);
+ } else {
+ return x; // Identity
+ }
}
template <typename T>
kernel void conv1d_pack_prefix_kernel(
- device const T* state_in [[ buffer(0) ]],
- device const T* x [[ buffer(1) ]],
- device T* padded [[ buffer(2) ]],
- constant const size_t& state_stride [[ buffer(3) ]],
- constant const size_t& row_stride [[ buffer(4) ]],
- constant const size_t& suffix_len [[ buffer(5) ]],
- constant const uint& num_channels [[ buffer(6) ]],
- uint2 grid_idx [[ thread_position_in_grid ]]
+ device const T* state_in [[buffer(0)]],
+ device const T* x [[buffer(1)]],
+ device T* padded [[buffer(2)]],
+ constant const size_t& state_stride [[buffer(3)]],
+ constant const size_t& row_stride [[buffer(4)]],
+ constant const size_t& suffix_len [[buffer(5)]],
+ constant const uint& num_channels [[buffer(6)]],
+ uint2 grid_idx [[thread_position_in_grid]]
) {
- const uint channel_idx = grid_idx.x;
- const uint row_idx = grid_idx.y;
- const size_t padded_rows = state_stride + suffix_len;
- if (channel_idx >= num_channels || row_idx >= padded_rows) {
- return;
- }
- const size_t padded_index =
- size_t(row_idx) * row_stride + size_t(channel_idx);
- if (row_idx < state_stride) {
- const size_t state_index =
- size_t(channel_idx) * state_stride + size_t(row_idx);
- padded[padded_index] = state_in[state_index];
- } else {
- const size_t token = row_idx - state_stride;
- const size_t x_index =
- size_t(token) * row_stride + size_t(channel_idx);
- padded[padded_index] = x[x_index];
- }
+ const uint channel_idx = grid_idx.x;
+ const uint row_idx = grid_idx.y;
+ const size_t padded_rows = state_stride + suffix_len;
+ if (channel_idx >= num_channels || row_idx >= padded_rows) {
+ return;
+ }
+ const size_t padded_index =
+ size_t(row_idx) * row_stride + size_t(channel_idx);
+ if (row_idx < state_stride) {
+ const size_t state_index =
+ size_t(channel_idx) * state_stride + size_t(row_idx);
+ padded[padded_index] = state_in[state_index];
+ } else {
+ const size_t token = row_idx - state_stride;
+ const size_t x_index = size_t(token) * row_stride + size_t(channel_idx);
+ padded[padded_index] = x[x_index];
+ }
}
template <typename T>
kernel void conv1d_scan_kernel(
- device const T* padded [[ buffer(0) ]], // (prefix+suffix, channels)
- device const T* w [[ buffer(1) ]], // (channels, kernel)
- device const T* b [[ buffer(2) ]], // optional (channels)
- device T* x_out [[ buffer(3) ]], // (suffix, inner_dim)
- device T* b_out [[ buffer(4) ]], // (suffix, proj_dim)
- device T* c_out [[ buffer(5) ]], // (suffix, proj_dim)
- device T* state_out [[ buffer(6) ]], // (channels, kernel-1)
- constant const size_t& suffix_len [[ buffer(7) ]],
- constant const int& kernel_size [[ buffer(8) ]],
- constant const size_t& row_stride [[ buffer(9) ]],
- constant const size_t& state_stride [[ buffer(10) ]],
- constant const uint& num_channels [[ buffer(11) ]],
- constant const uint& inner_dim [[ buffer(12) ]],
- constant const uint& proj_dim [[ buffer(13) ]],
- uint3 grid_idx [[ thread_position_in_grid ]]
+ device const T* padded [[buffer(0)]], // (prefix+suffix, channels)
+ device const T* w [[buffer(1)]], // (channels, kernel)
+ device const T* b [[buffer(2)]], // optional (channels)
+ device T* x_out [[buffer(3)]], // (suffix, inner_dim)
+ device T* b_out [[buffer(4)]], // (suffix, proj_dim)
+ device T* c_out [[buffer(5)]], // (suffix, proj_dim)
+ device T* state_out [[buffer(6)]], // (channels, kernel-1)
+ constant const size_t& suffix_len [[buffer(7)]],
+ constant const int& kernel_size [[buffer(8)]],
+ constant const size_t& row_stride [[buffer(9)]],
+ constant const size_t& state_stride [[buffer(10)]],
+ constant const uint& num_channels [[buffer(11)]],
+ constant const uint& inner_dim [[buffer(12)]],
+ constant const uint& proj_dim [[buffer(13)]],
+ uint3 grid_idx [[thread_position_in_grid]]
) {
- const int kernel_value = kernel_size;
- if (kernel_value <= 0) {
- return;
- }
+ const int kernel_value = kernel_size;
+ if (kernel_value <= 0) {
+ return;
+ }
- const int tap_count = max(kernel_value - 1, 0);
- const size_t work_len = suffix_len + size_t(tap_count);
+ const int tap_count = max(kernel_value - 1, 0);
+ const size_t work_len = suffix_len + size_t(tap_count);
- const uint token_idx = grid_idx.x;
- const uint channel_idx = grid_idx.y;
- if (channel_idx >= num_channels || token_idx >= work_len) {
- return;
- }
+ const uint token_idx = grid_idx.x;
+ const uint channel_idx = grid_idx.y;
+ if (channel_idx >= num_channels || token_idx >= work_len) {
+ return;
+ }
- const size_t state_offset = size_t(channel_idx) * state_stride;
- const size_t weight_offset = size_t(channel_idx) * size_t(kernel_value);
- const device T* w_row = w + weight_offset;
- const bool has_bias = b != nullptr;
+ const size_t state_offset = size_t(channel_idx) * state_stride;
+ const size_t weight_offset = size_t(channel_idx) * size_t(kernel_value);
+ const device T* w_row = w + weight_offset;
+ const bool has_bias = b != nullptr;
- if (token_idx < suffix_len) {
- float acc = has_bias ? float(b[channel_idx]) : 0.0f;
- for (int tap = 0; tap < kernel_value; ++tap) {
- const size_t padded_idx = size_t(token_idx) + size_t(tap);
- float sample = 0.0f;
- const size_t padded_index =
- size_t(padded_idx) * row_stride + channel_idx;
- sample = float(padded[padded_index]);
- acc += float(w_row[tap]) * sample;
- }
+ if (token_idx < suffix_len) {
+ float acc = has_bias ? float(b[channel_idx]) : 0.0f;
+ for (int tap = 0; tap < kernel_value; ++tap) {
+ const size_t padded_idx = size_t(token_idx) + size_t(tap);
+ float sample = 0.0f;
+ const size_t padded_index = size_t(padded_idx) * row_stride + channel_idx;
+ sample = float(padded[padded_index]);
+ acc += float(w_row[tap]) * sample;
+ }
- const T activated = apply_activation_fn(static_cast<T>(acc), activation_type);
- if (channel_idx < inner_dim) {
- const size_t dst = size_t(token_idx) * inner_dim + channel_idx;
- x_out[dst] = activated;
- } else if (channel_idx < inner_dim + proj_dim) {
- const size_t dst = size_t(token_idx) * proj_dim + (channel_idx - inner_dim);
- b_out[dst] = activated;
- } else if (channel_idx < inner_dim + 2 * proj_dim) {
- const size_t dst = size_t(token_idx) * proj_dim + (channel_idx - inner_dim - proj_dim);
- c_out[dst] = activated;
- }
- } else if (tap_count > 0) {
- const size_t tap = size_t(token_idx - suffix_len);
- if (tap >= size_t(tap_count)) {
- return;
- }
- const size_t padded_index =
- size_t(token_idx) * row_stride + channel_idx;
- const float sample = float(padded[padded_index]);
- state_out[state_offset + tap] = static_cast<T>(sample);
+ const T activated =
+ apply_activation_fn(static_cast<T>(acc), activation_type);
+ if (channel_idx < inner_dim) {
+ const size_t dst = size_t(token_idx) * inner_dim + channel_idx;
+ x_out[dst] = activated;
+ } else if (channel_idx < inner_dim + proj_dim) {
+ const size_t dst =
+ size_t(token_idx) * proj_dim + (channel_idx - inner_dim);
+ b_out[dst] = activated;
+ } else if (channel_idx < inner_dim + 2 * proj_dim) {
+ const size_t dst =
+ size_t(token_idx) * proj_dim + (channel_idx - inner_dim - proj_dim);
+ c_out[dst] = activated;
+ }
+ } else if (tap_count > 0) {
+ const size_t tap = size_t(token_idx - suffix_len);
+ if (tap >= size_t(tap_count)) {
+ return;
}
+ const size_t padded_index = size_t(token_idx) * row_stride + channel_idx;
+ const float sample = float(padded[padded_index]);
+ state_out[state_offset + tap] = static_cast<T>(sample);
+ }
}
template <typename T>
kernel void conv1d_decode_kernel(
- device const T* x [[ buffer(0) ]],
- device const T* w [[ buffer(1) ]],
- device const T* b [[ buffer(2) ]],
- device const T* state [[ buffer(3) ]],
- device T* x_out [[ buffer(4) ]],
- device T* b_out [[ buffer(5) ]],
- device T* c_out [[ buffer(6) ]],
- device T* next_state [[ buffer(7) ]],
- constant const int& kernel_size [[ buffer(8) ]],
- constant const size_t& row_stride [[ buffer(9) ]],
- constant const size_t& state_stride [[ buffer(10) ]],
- constant const uint& num_channels [[ buffer(11) ]],
- constant const size_t& suffix_len [[ buffer(12) ]],
- constant const uint& inner_dim [[ buffer(13) ]],
- constant const uint& proj_dim [[ buffer(14) ]],
- uint2 grid_idx [[ thread_position_in_grid ]]
+ device const T* x [[buffer(0)]],
+ device const T* w [[buffer(1)]],
+ device const T* b [[buffer(2)]],
+ device const T* state [[buffer(3)]],
+ device T* x_out [[buffer(4)]],
+ device T* b_out [[buffer(5)]],
+ device T* c_out [[buffer(6)]],
+ device T* next_state [[buffer(7)]],
+ constant const int& kernel_size [[buffer(8)]],
+ constant const size_t& row_stride [[buffer(9)]],
+ constant const size_t& state_stride [[buffer(10)]],
+ constant const uint& num_channels [[buffer(11)]],
+ constant const size_t& suffix_len [[buffer(12)]],
+ constant const uint& inner_dim [[buffer(13)]],
+ constant const uint& proj_dim [[buffer(14)]],
+ uint2 grid_idx [[thread_position_in_grid]]
) {
- const int kernel_value = kernel_size;
- if (kernel_value <= 0) {
- return;
- }
+ const int kernel_value = kernel_size;
+ if (kernel_value <= 0) {
+ return;
+ }
- const uint token_idx = grid_idx.x;
- const uint channel_idx = grid_idx.y;
- if (channel_idx >= num_channels || token_idx >= suffix_len) {
- return;
- }
+ const uint token_idx = grid_idx.x;
+ const uint channel_idx = grid_idx.y;
+ if (channel_idx >= num_channels || token_idx >= suffix_len) {
+ return;
+ }
- const size_t x_idx =
- size_t(token_idx) * row_stride + size_t(channel_idx);
- const size_t state_offset =
- size_t(channel_idx) * state_stride;
- const device T* w_row =
- w + size_t(channel_idx) * size_t(kernel_value);
- const bool has_bias = b != nullptr;
+ const size_t x_idx = size_t(token_idx) * row_stride + size_t(channel_idx);
+ const size_t state_offset = size_t(channel_idx) * state_stride;
+ const device T* w_row = w + size_t(channel_idx) * size_t(kernel_value);
+ const bool has_bias = b != nullptr;
- float acc = has_bias ? float(b[channel_idx]) : 0.0f;
- const int state_taps = max(kernel_value - 1, 0);
- for (int tap = 0; tap < state_taps; ++tap) {
- const size_t state_index = state_offset + size_t(tap);
- const float sample = float(state[state_index]);
- acc += float(w_row[tap]) * sample;
- }
+ float acc = has_bias ? float(b[channel_idx]) : 0.0f;
+ const int state_taps = max(kernel_value - 1, 0);
+ for (int tap = 0; tap < state_taps; ++tap) {
+ const size_t state_index = state_offset + size_t(tap);
+ const float sample = float(state[state_index]);
+ acc += float(w_row[tap]) * sample;
+ }
- const float current = float(x[x_idx]);
- const int current_tap_index = state_taps;
- if (current_tap_index < kernel_value) {
- acc += float(w_row[current_tap_index]) * current;
- }
+ const float current = float(x[x_idx]);
+ const int current_tap_index = state_taps;
+ if (current_tap_index < kernel_value) {
+ acc += float(w_row[current_tap_index]) * current;
+ }
- const T activated = apply_activation_fn(static_cast<T>(acc), activation_type);
- if (channel_idx < inner_dim) {
- const size_t dst = size_t(token_idx) * inner_dim + channel_idx;
- x_out[dst] = activated;
- } else if (channel_idx < inner_dim + proj_dim) {
- const size_t dst = size_t(token_idx) * proj_dim + (channel_idx - inner_dim);
- b_out[dst] = activated;
- } else if (channel_idx < inner_dim + 2 * proj_dim) {
- const size_t dst = size_t(token_idx) * proj_dim + (channel_idx - inner_dim - proj_dim);
- c_out[dst] = activated;
- }
+ const T activated = apply_activation_fn(static_cast<T>(acc), activation_type);
+ if (channel_idx < inner_dim) {
+ const size_t dst = size_t(token_idx) * inner_dim + channel_idx;
+ x_out[dst] = activated;
+ } else if (channel_idx < inner_dim + proj_dim) {
+ const size_t dst = size_t(token_idx) * proj_dim + (channel_idx - inner_dim);
+ b_out[dst] = activated;
+ } else if (channel_idx < inner_dim + 2 * proj_dim) {
+ const size_t dst =
+ size_t(token_idx) * proj_dim + (channel_idx - inner_dim - proj_dim);
+ c_out[dst] = activated;
+ }
- if (state_taps == 0) {
- return;
- }
+ if (state_taps == 0) {
+ return;
+ }
- for (int tap = 0; tap < state_taps - 1; ++tap) {
- const size_t dst_index = state_offset + size_t(tap);
- const size_t src_index = state_offset + size_t(tap + 1);
- next_state[dst_index] = state[src_index];
- }
+ for (int tap = 0; tap < state_taps - 1; ++tap) {
+ const size_t dst_index = state_offset + size_t(tap);
+ const size_t src_index = state_offset + size_t(tap + 1);
+ next_state[dst_index] = state[src_index];
+ }
- const size_t tail_index = state_offset + size_t(state_taps - 1);
- next_state[tail_index] = static_cast<T>(current);
+ const size_t tail_index = state_offset + size_t(state_taps - 1);
+ next_state[tail_index] = static_cast<T>(current);
}
-#define instantiate_conv1d_pack_prefix_kernel(type_name, type) \
- template [[host_name("conv1d_pack_prefix_kernel_" #type_name)]] \
- kernel void conv1d_pack_prefix_kernel<type>( \
- device const type* state_in [[ buffer(0) ]], \
- device const type* x [[ buffer(1) ]], \
- device type* padded [[ buffer(2) ]], \
- constant const size_t& state_stride [[ buffer(3) ]], \
- constant const size_t& row_stride [[ buffer(4) ]], \
- constant const size_t& suffix_len [[ buffer(5) ]], \
- constant const uint& num_channels [[ buffer(6) ]], \
- uint2 grid_idx [[ thread_position_in_grid ]] \
+#define instantiate_conv1d_pack_prefix_kernel(type_name, type) \
+ template [[host_name("conv1d_pack_prefix_kernel_" #type_name)]] \
+ kernel void conv1d_pack_prefix_kernel<type>( \
+ device const type* state_in [[buffer(0)]], \
+ device const type* x [[buffer(1)]], \
+ device type* padded [[buffer(2)]], \
+ constant const size_t& state_stride [[buffer(3)]], \
+ constant const size_t& row_stride [[buffer(4)]], \
+ constant const size_t& suffix_len [[buffer(5)]], \
+ constant const uint& num_channels [[buffer(6)]], \
+ uint2 grid_idx [[thread_position_in_grid]] \
);
-#define instantiate_conv1d_scan_kernel(type_name, type) \
- template [[host_name("conv1d_scan_kernel_" #type_name)]] \
- kernel void conv1d_scan_kernel<type>( \
- device const type* padded [[ buffer(0) ]], \
- device const type* w [[ buffer(1) ]], \
- device const type* b [[ buffer(2) ]], \
- device type* x_out [[ buffer(3) ]], \
- device type* b_out [[ buffer(4) ]], \
- device type* c_out [[ buffer(5) ]], \
- device type* state_out [[ buffer(6) ]], \
- constant const size_t& suffix_len [[ buffer(7) ]], \
- constant const int& kernel_size [[ buffer(8) ]], \
- constant const size_t& row_stride [[ buffer(9) ]], \
- constant const size_t& state_stride [[ buffer(10) ]], \
- constant const uint& num_channels [[ buffer(11) ]], \
- constant const uint& inner_dim [[ buffer(12) ]], \
- constant const uint& proj_dim [[ buffer(13) ]], \
- uint3 grid_idx [[ thread_position_in_grid ]] \
+#define instantiate_conv1d_scan_kernel(type_name, type) \
+ template [[host_name("conv1d_scan_kernel_" #type_name)]] \
+ kernel void conv1d_scan_kernel<type>( \
+ device const type* padded [[buffer(0)]], \
+ device const type* w [[buffer(1)]], \
+ device const type* b [[buffer(2)]], \
+ device type* x_out [[buffer(3)]], \
+ device type* b_out [[buffer(4)]], \
+ device type* c_out [[buffer(5)]], \
+ device type* state_out [[buffer(6)]], \
+ constant const size_t& suffix_len [[buffer(7)]], \
+ constant const int& kernel_size [[buffer(8)]], \
+ constant const size_t& row_stride [[buffer(9)]], \
+ constant const size_t& state_stride [[buffer(10)]], \
+ constant const uint& num_channels [[buffer(11)]], \
+ constant const uint& inner_dim [[buffer(12)]], \
+ constant const uint& proj_dim [[buffer(13)]], \
+ uint3 grid_idx [[thread_position_in_grid]] \
);
instantiate_conv1d_pack_prefix_kernel(float, float);
@@ -257,25 +258,25 @@ instantiate_conv1d_scan_kernel(float, float);
instantiate_conv1d_scan_kernel(bfloat, bfloat);
instantiate_conv1d_scan_kernel(half, half);
-#define instantiate_conv1d_decode_kernel(type_name, type) \
- template [[host_name("conv1d_decode_kernel_" #type_name)]] \
- kernel void conv1d_decode_kernel<type>( \
- device const type* x [[ buffer(0) ]], \
- device const type* w [[ buffer(1) ]], \
- device const type* b [[ buffer(2) ]], \
- device const type* state [[ buffer(3) ]], \
- device type* x_out [[ buffer(4) ]], \
- device type* b_out [[ buffer(5) ]], \
- device type* c_out [[ buffer(6) ]], \
- device type* next_state [[ buffer(7) ]], \
- constant const int& kernel_size [[ buffer(8) ]], \
- constant const size_t& row_stride [[ buffer(9) ]], \
- constant const size_t& state_stride [[ buffer(10) ]], \
- constant const uint& num_channels [[ buffer(11) ]], \
- constant const size_t& suffix_len [[ buffer(12) ]], \
- constant const uint& inner_dim [[ buffer(13) ]], \
- constant const uint& proj_dim [[ buffer(14) ]], \
- uint2 grid_idx [[ thread_position_in_grid ]] \
+#define instantiate_conv1d_decode_kernel(type_name, type) \
+ template [[host_name("conv1d_decode_kernel_" #type_name)]] \
+ kernel void conv1d_decode_kernel<type>( \
+ device const type* x [[buffer(0)]], \
+ device const type* w [[buffer(1)]], \
+ device const type* b [[buffer(2)]], \
+ device const type* state [[buffer(3)]], \
+ device type* x_out [[buffer(4)]], \
+ device type* b_out [[buffer(5)]], \
+ device type* c_out [[buffer(6)]], \
+ device type* next_state [[buffer(7)]], \
+ constant const int& kernel_size [[buffer(8)]], \
+ constant const size_t& row_stride [[buffer(9)]], \
+ constant const size_t& state_stride [[buffer(10)]], \
+ constant const uint& num_channels [[buffer(11)]], \
+ constant const size_t& suffix_len [[buffer(12)]], \
+ constant const uint& inner_dim [[buffer(13)]], \
+ constant const uint& proj_dim [[buffer(14)]], \
+ uint2 grid_idx [[thread_position_in_grid]] \
);
#undef instantiate_conv1d_pack_prefix_kernel
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/split_inproj.metal b/crates/uzu/src/backends/metal/kernel/ssm/split_inproj.metal
index 9f6565fd2..ae0066ead 100644
--- a/crates/uzu/src/backends/metal/kernel/ssm/split_inproj.metal
+++ b/crates/uzu/src/backends/metal/kernel/ssm/split_inproj.metal
@@ -5,53 +5,53 @@ using namespace metal;
template <typename T>
kernel void ssm_split_inproj_kernel(
- device const T* input [[ buffer(0) ]],
- device T* conv_out [[ buffer(1) ]],
- device T* z_out [[ buffer(2) ]],
- device T* dt_out [[ buffer(3) ]],
- device const T* z_bias [[ buffer(4) ]],
- constant const int& total_dim [[ buffer(5) ]],
- constant const int& conv_dim [[ buffer(6) ]],
- constant const int& inner_dim [[ buffer(7) ]],
- constant const int& num_heads [[ buffer(8) ]],
- uint2 grid_idx [[ thread_position_in_grid ]],
- uint2 grid_size [[ threads_per_grid ]]
+ device const T* input [[buffer(0)]],
+ device T* conv_out [[buffer(1)]],
+ device T* z_out [[buffer(2)]],
+ device T* dt_out [[buffer(3)]],
+ device const T* z_bias [[buffer(4)]],
+ constant const int& total_dim [[buffer(5)]],
+ constant const int& conv_dim [[buffer(6)]],
+ constant const int& inner_dim [[buffer(7)]],
+ constant const int& num_heads [[buffer(8)]],
+ uint2 grid_idx [[thread_position_in_grid]],
+ uint2 grid_size [[threads_per_grid]]
) {
- const int row = static_cast<int>(grid_idx.x);
- const int col = static_cast<int>(grid_idx.y);
+ const int row = static_cast<int>(grid_idx.x);
+ const int col = static_cast<int>(grid_idx.y);
- if (row >= grid_size.x || col >= grid_size.y || col >= total_dim) {
- return;
- }
+ if (row >= grid_size.x || col >= grid_size.y || col >= total_dim) {
+ return;
+ }
- const int input_idx = row * total_dim + col;
- if (col < conv_dim) {
- const int dst = row * conv_dim + col;
- conv_out[dst] = input[input_idx];
- } else if (col < conv_dim + inner_dim) {
- const int dst = row * inner_dim + (col - conv_dim);
- const int bias_idx = col - conv_dim;
- z_out[dst] = input[input_idx] + z_bias[bias_idx];
- } else if (col < conv_dim + inner_dim + num_heads) {
- const int dst = row * num_heads + (col - conv_dim - inner_dim);
- dt_out[dst] = input[input_idx];
- }
+ const int input_idx = row * total_dim + col;
+ if (col < conv_dim) {
+ const int dst = row * conv_dim + col;
+ conv_out[dst] = input[input_idx];
+ } else if (col < conv_dim + inner_dim) {
+ const int dst = row * inner_dim + (col - conv_dim);
+ const int bias_idx = col - conv_dim;
+ z_out[dst] = input[input_idx] + z_bias[bias_idx];
+ } else if (col < conv_dim + inner_dim + num_heads) {
+ const int dst = row * num_heads + (col - conv_dim - inner_dim);
+ dt_out[dst] = input[input_idx];
+ }
}
-#define instantiate_ssm_split_inproj_kernel(type_name, type) \
- template [[host_name("ssm_split_inproj_kernel_" #type_name)]] \
- kernel void ssm_split_inproj_kernel<type>( \
- device const type* input [[ buffer(0) ]], \
- device type* conv_out [[ buffer(1) ]], \
- device type* z_out [[ buffer(2) ]], \
- device type* dt_out [[ buffer(3) ]], \
- device const type* z_bias [[ buffer(4) ]], \
- constant const int& total_dim [[ buffer(5) ]], \
- constant const int& conv_dim [[ buffer(6) ]], \
- constant const int& inner_dim [[ buffer(7) ]], \
- constant const int& num_heads [[ buffer(8) ]], \
- uint2 grid_idx [[ thread_position_in_grid ]], \
- uint2 grid_size [[ threads_per_grid ]] \
+#define instantiate_ssm_split_inproj_kernel(type_name, type) \
+ template [[host_name("ssm_split_inproj_kernel_" #type_name)]] \
+ kernel void ssm_split_inproj_kernel<type>( \
+ device const type* input [[buffer(0)]], \
+ device type* conv_out [[buffer(1)]], \
+ device type* z_out [[buffer(2)]], \
+ device type* dt_out [[buffer(3)]], \
+ device const type* z_bias [[buffer(4)]], \
+ constant const int& total_dim [[buffer(5)]], \
+ constant const int& conv_dim [[buffer(6)]], \
+ constant const int& inner_dim [[buffer(7)]], \
+ constant const int& num_heads [[buffer(8)]], \
+ uint2 grid_idx [[thread_position_in_grid]], \
+ uint2 grid_size [[threads_per_grid]] \
);
instantiate_ssm_split_inproj_kernel(float, float);
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal b/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal
index c39c50ec8..72c0bebbe 100644
--- a/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal
+++ b/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal
@@ -18,141 +18,143 @@ struct SILU {
template <typename T>
inline T softplus(T x) {
- float xf = float(x);
- if (xf > 20.0f) {
- return x;
- }
- return static_cast<T>(log(1.0f + fast::exp(xf)));
+ float xf = float(x);
+ if (xf > 20.0f) {
+ return x;
+ }
+ return static_cast<T>(log(1.0f + fast::exp(xf)));
}
template <typename T>
kernel void ssd_prefill_kernel_64(
- device const T* x [[ buffer(0) ]], // (suffix, h, dh)
- device const T* dt_raw [[ buffer(1) ]], // (suffix, h) - raw dt values
- device const T* B [[ buffer(2) ]], // (suffix, g, n)
- device const T* C [[ buffer(3) ]], // (suffix, g, n)
- device const T* D [[ buffer(4) ]], // (h)
- device const T* z [[ buffer(5) ]], // (suffix, h, dh)
- device T* state [[ buffer(6) ]], // (h, dh, n)
- device T* y [[ buffer(7) ]], // (suffix, h, dh)
- constant const size_t& suffix_len [[ buffer(8) ]],
- constant const int& group_size [[ buffer(9) ]],
- constant const int& state_size [[ buffer(10) ]],
- constant const size_t* x_strides [[ buffer(11) ]],
- constant const size_t* dt_strides [[ buffer(12) ]],
- constant const size_t* cb_strides [[ buffer(13) ]],
- constant const size_t* state_strides [[ buffer(14) ]],
- constant const uint& num_heads [[ buffer(15) ]],
- constant const uint& head_dim [[ buffer(16) ]],
- uint3 tg_pos [[ threadgroup_position_in_grid ]],
- ushort lane [[ thread_index_in_threadgroup ]]
+ device const T* x [[buffer(0)]], // (suffix, h, dh)
+ device const T* dt_raw [[buffer(1)]], // (suffix, h) - raw dt values
+ device const T* B [[buffer(2)]], // (suffix, g, n)
+ device const T* C [[buffer(3)]], // (suffix, g, n)
+ device const T* D [[buffer(4)]], // (h)
+ device const T* z [[buffer(5)]], // (suffix, h, dh)
+ device T* state [[buffer(6)]], // (h, dh, n)
+ device T* y [[buffer(7)]], // (suffix, h, dh)
+ constant const size_t& suffix_len [[buffer(8)]],
+ constant const int& group_size [[buffer(9)]],
+ constant const int& state_size [[buffer(10)]],
+ constant const size_t* x_strides [[buffer(11)]],
+ constant const size_t* dt_strides [[buffer(12)]],
+ constant const size_t* cb_strides [[buffer(13)]],
+ constant const size_t* state_strides [[buffer(14)]],
+ constant const uint& num_heads [[buffer(15)]],
+ constant const uint& head_dim [[buffer(16)]],
+ uint3 tg_pos [[threadgroup_position_in_grid]],
+ ushort lane [[thread_index_in_threadgroup]]
) {
- const uint simd_width = 32;
- const int state_dim = state_size;
+ const uint simd_width = 32;
+ const int state_dim = state_size;
- const uint total_pairs = num_heads * head_dim;
- const uint pair_idx = tg_pos.x;
- if (pair_idx >= total_pairs) {
- return;
- }
+ const uint total_pairs = num_heads * head_dim;
+ const uint pair_idx = tg_pos.x;
+ if (pair_idx >= total_pairs) {
+ return;
+ }
- const uint lane_idx = uint(lane);
- const uint h_idx = pair_idx / head_dim;
- const uint dh_idx = pair_idx % head_dim;
- const uint safe_group = uint(max(group_size, 1));
- const uint group_idx = h_idx / safe_group;
-
- const size_t x_token_stride = x_strides[0];
- const size_t x_head_stride = x_strides[1];
- const size_t x_dim_stride = x_strides[2];
- const size_t dt_token_stride = dt_strides[0];
- const size_t dt_head_stride = dt_strides[1];
- const size_t cb_token_stride = cb_strides[0];
- const size_t cb_group_stride = cb_strides[1];
- const size_t cb_state_stride = cb_strides[2];
- const size_t state_head_stride = state_strides[0];
- const size_t state_dim_stride = state_strides[1];
- const size_t state_inner_stride = state_strides[2];
-
- const size_t x_base = size_t(h_idx) * x_head_stride
- + size_t(dh_idx) * x_dim_stride;
- const size_t dt_base = size_t(h_idx) * dt_head_stride;
- const size_t state_base =
- size_t(h_idx) * state_head_stride + size_t(dh_idx) * state_dim_stride;
- const size_t cb_group_base = size_t(group_idx) * cb_group_stride;
- const float d_scalar = float(D[h_idx]);
-
- const uint idx0 = lane_idx;
- const uint idx1 = lane_idx + simd_width;
-
- float state0 = 0.0f;
- float state1 = 0.0f;
- size_t state_idx0 = 0;
- size_t state_idx1 = 0;
- state_idx0 = state_base + size_t(idx0) * state_inner_stride;
- state0 = float(state[state_idx0]);
-
- state_idx1 = state_base + size_t(idx1) * state_inner_stride;
- state1 = float(state[state_idx1]);
-
- size_t cb_idx0 = 0;
- size_t cb_idx1 = 0;
- cb_idx0 = cb_group_base + size_t(idx0) * cb_state_stride;
- cb_idx1 = cb_group_base + size_t(idx1) * cb_state_stride;
-
- for (size_t token = 0; token < suffix_len; ++token) {
- const size_t x_idx = token * x_token_stride + x_base;
- const size_t dt_idx = token * dt_token_stride + dt_base;
-
- const float x_val = float(x[x_idx]);
- const float decay_val = fast::exp(-float(softplus(float(dt_raw[dt_idx]))));
- const float gate = float(SILU{}(z[x_idx]));
- const float skip = d_scalar * x_val;
- const float dt_scaled_input = x_val;
-
- float contrib = 0.0f;
- const float new_state0 = decay_val * state0 + dt_scaled_input * float(B[cb_idx0]);
- state0 = new_state0;
- contrib += new_state0 * float(C[cb_idx0]);
- cb_idx0 += cb_token_stride;
-
- const float new_state1 = decay_val * state1 + dt_scaled_input * float(B[cb_idx1]);
- state1 = new_state1;
- contrib += new_state1 * float(C[cb_idx1]);
- cb_idx1 += cb_token_stride;
-
- float dot = simd_sum(contrib);
- if (lane_idx == 0) {
- y[x_idx] = static_cast<T>((skip + dot) * gate);
- }
+ const uint lane_idx = uint(lane);
+ const uint h_idx = pair_idx / head_dim;
+ const uint dh_idx = pair_idx % head_dim;
+ const uint safe_group = uint(max(group_size, 1));
+ const uint group_idx = h_idx / safe_group;
+
+ const size_t x_token_stride = x_strides[0];
+ const size_t x_head_stride = x_strides[1];
+ const size_t x_dim_stride = x_strides[2];
+ const size_t dt_token_stride = dt_strides[0];
+ const size_t dt_head_stride = dt_strides[1];
+ const size_t cb_token_stride = cb_strides[0];
+ const size_t cb_group_stride = cb_strides[1];
+ const size_t cb_state_stride = cb_strides[2];
+ const size_t state_head_stride = state_strides[0];
+ const size_t state_dim_stride = state_strides[1];
+ const size_t state_inner_stride = state_strides[2];
+
+ const size_t x_base =
+ size_t(h_idx) * x_head_stride + size_t(dh_idx) * x_dim_stride;
+ const size_t dt_base = size_t(h_idx) * dt_head_stride;
+ const size_t state_base =
+ size_t(h_idx) * state_head_stride + size_t(dh_idx) * state_dim_stride;
+ const size_t cb_group_base = size_t(group_idx) * cb_group_stride;
+ const float d_scalar = float(D[h_idx]);
+
+ const uint idx0 = lane_idx;
+ const uint idx1 = lane_idx + simd_width;
+
+ float state0 = 0.0f;
+ float state1 = 0.0f;
+ size_t state_idx0 = 0;
+ size_t state_idx1 = 0;
+ state_idx0 = state_base + size_t(idx0) * state_inner_stride;
+ state0 = float(state[state_idx0]);
+
+ state_idx1 = state_base + size_t(idx1) * state_inner_stride;
+ state1 = float(state[state_idx1]);
+
+ size_t cb_idx0 = 0;
+ size_t cb_idx1 = 0;
+ cb_idx0 = cb_group_base + size_t(idx0) * cb_state_stride;
+ cb_idx1 = cb_group_base + size_t(idx1) * cb_state_stride;
+
+ for (size_t token = 0; token < suffix_len; ++token) {
+ const size_t x_idx = token * x_token_stride + x_base;
+ const size_t dt_idx = token * dt_token_stride + dt_base;
+
+ const float x_val = float(x[x_idx]);
+ const float decay_val = fast::exp(-float(softplus(float(dt_raw[dt_idx]))));
+ const float gate = float(SILU{}(z[x_idx]));
+ const float skip = d_scalar * x_val;
+ const float dt_scaled_input = x_val;
+
+ float contrib = 0.0f;
+ const float new_state0 =
+ decay_val * state0 + dt_scaled_input * float(B[cb_idx0]);
+ state0 = new_state0;
+ contrib += new_state0 * float(C[cb_idx0]);
+ cb_idx0 += cb_token_stride;
+
+ const float new_state1 =
+ decay_val * state1 + dt_scaled_input * float(B[cb_idx1]);
+ state1 = new_state1;
+ contrib += new_state1 * float(C[cb_idx1]);
+ cb_idx1 += cb_token_stride;
+
+ float dot = simd_sum(contrib);
+ if (lane_idx == 0) {
+ y[x_idx] = static_cast<T>((skip + dot) * gate);
}
+ }
- state[state_idx0] = static_cast<T>(state0);
- state[state_idx1] = static_cast<T>(state1);
+ state[state_idx0] = static_cast<T>(state0);
+ state[state_idx1] = static_cast<T>(state1);
}
-#define instantiate_ssd_prefill_kernel_64(type_name, type) \
- template [[host_name("ssd_prefill_kernel_64_" #type_name)]] \
- kernel void ssd_prefill_kernel_64<type>( \
- device const type* x [[ buffer(0) ]], \
- device const type* dt_raw [[ buffer(1) ]], \
- device const type* B [[ buffer(2) ]], \
- device const type* C [[ buffer(3) ]], \
- device const type* D [[ buffer(4) ]], \
- device const type* z [[ buffer(5) ]], \
- device type* state [[ buffer(6) ]], \
- device type* y [[ buffer(7) ]], \
- constant const size_t& suffix_len [[ buffer(8) ]], \
- constant const int& group_size [[ buffer(9) ]], \
- constant const int& state_size [[ buffer(10) ]], \
- constant const size_t* x_strides [[ buffer(11) ]], \
- constant const size_t* dt_strides [[ buffer(12) ]], \
- constant const size_t* cb_strides [[ buffer(13) ]], \
- constant const size_t* state_strides [[ buffer(14) ]], \
- constant const uint& num_heads [[ buffer(15) ]], \
- constant const uint& head_dim [[ buffer(16) ]], \
- uint3 tg_pos [[ threadgroup_position_in_grid ]], \
- ushort lane [[ thread_index_in_threadgroup ]] \
+#define instantiate_ssd_prefill_kernel_64(type_name, type) \
+ template [[host_name("ssd_prefill_kernel_64_" #type_name)]] \
+ kernel void ssd_prefill_kernel_64<type>( \
+ device const type* x [[buffer(0)]], \
+ device const type* dt_raw [[buffer(1)]], \
+ device const type* B [[buffer(2)]], \
+ device const type* C [[buffer(3)]], \
+ device const type* D [[buffer(4)]], \
+ device const type* z [[buffer(5)]], \
+ device type* state [[buffer(6)]], \
+ device type* y [[buffer(7)]], \
+ constant const size_t& suffix_len [[buffer(8)]], \
+ constant const int& group_size [[buffer(9)]], \
+ constant const int& state_size [[buffer(10)]], \
+ constant const size_t* x_strides [[buffer(11)]], \
+ constant const size_t* dt_strides [[buffer(12)]], \
+ constant const size_t* cb_strides [[buffer(13)]], \
+ constant const size_t* state_strides [[buffer(14)]], \
+ constant const uint& num_heads [[buffer(15)]], \
+ constant const uint& head_dim [[buffer(16)]], \
+ uint3 tg_pos [[threadgroup_position_in_grid]], \
+ ushort lane [[thread_index_in_threadgroup]] \
);
instantiate_ssd_prefill_kernel_64(float, float);
@@ -163,133 +165,135 @@ instantiate_ssd_prefill_kernel_64(half, half);
template <typename T>
kernel void ssd_prefill_kernel(
- device const T* x [[ buffer(0) ]], // (suffix, h, dh)
- device const T* dt_raw [[ buffer(1) ]], // (suffix, h) - raw dt values
- device const T* B [[ buffer(2) ]], // (suffix, g, n)
- device const T* C [[ buffer(3) ]], // (suffix, g, n)
- device const T* D [[ buffer(4) ]], // (h)
- device const T* z [[ buffer(5) ]], // (suffix, h, dh)
- device T* state [[ buffer(6) ]], // (h, dh, n)
- device T* y [[ buffer(7) ]], // (suffix, h, dh)
- constant const size_t& suffix_len [[ buffer(8) ]],
- constant const int& group_size [[ buffer(9) ]],
- constant const int& state_size [[ buffer(10) ]],
- constant const size_t* x_strides [[ buffer(11) ]],
- constant const size_t* dt_strides [[ buffer(12) ]],
- constant const size_t* cb_strides [[ buffer(13) ]],
- constant const size_t* state_strides [[ buffer(14) ]],
- constant const uint& num_heads [[ buffer(15) ]],
- constant const uint& head_dim [[ buffer(16) ]],
- uint3 tg_pos [[ threadgroup_position_in_grid ]],
- ushort lane [[ thread_index_in_threadgroup ]]
+ device const T* x [[buffer(0)]], // (suffix, h, dh)
+ device const T* dt_raw [[buffer(1)]], // (suffix, h) - raw dt values
+ device const T* B [[buffer(2)]], // (suffix, g, n)
+ device const T* C [[buffer(3)]], // (suffix, g, n)
+ device const T* D [[buffer(4)]], // (h)
+ device const T* z [[buffer(5)]], // (suffix, h, dh)
+ device T* state [[buffer(6)]], // (h, dh, n)
+ device T* y [[buffer(7)]], // (suffix, h, dh)
+ constant const size_t& suffix_len [[buffer(8)]],
+ constant const int& group_size [[buffer(9)]],
+ constant const int& state_size [[buffer(10)]],
+ constant const size_t* x_strides [[buffer(11)]],
+ constant const size_t* dt_strides [[buffer(12)]],
+ constant const size_t* cb_strides [[buffer(13)]],
+ constant const size_t* state_strides [[buffer(14)]],
+ constant const uint& num_heads [[buffer(15)]],
+ constant const uint& head_dim [[buffer(16)]],
+ uint3 tg_pos [[threadgroup_position_in_grid]],
+ ushort lane [[thread_index_in_threadgroup]]
) {
- const uint simd_width = 32;
- const int state_dim = state_size;
+ const uint simd_width = 32;
+ const int state_dim = state_size;
- const uint total_pairs = num_heads * head_dim;
- const uint pair_idx = tg_pos.x;
- if (pair_idx >= total_pairs) {
- return;
- }
+ const uint total_pairs = num_heads * head_dim;
+ const uint pair_idx = tg_pos.x;
+ if (pair_idx >= total_pairs) {
+ return;
+ }
- const uint lane_idx = uint(lane);
- const uint h_idx = pair_idx / head_dim;
- const uint dh_idx = pair_idx % head_dim;
- const uint safe_group = uint(max(group_size, 1));
- const uint group_idx = h_idx / safe_group;
-
- const size_t x_token_stride = x_strides[0];
- const size_t x_head_stride = x_strides[1];
- const size_t x_dim_stride = x_strides[2];
- const size_t dt_token_stride = dt_strides[0];
- const size_t dt_head_stride = dt_strides[1];
- const size_t cb_token_stride = cb_strides[0];
- const size_t cb_group_stride = cb_strides[1];
- const size_t cb_state_stride = cb_strides[2];
- const size_t state_head_stride = state_strides[0];
- const size_t state_dim_stride = state_strides[1];
- const size_t state_inner_stride = state_strides[2];
-
- const size_t x_base = size_t(h_idx) * x_head_stride + size_t(dh_idx) * x_dim_stride;
- const size_t dt_base = size_t(h_idx) * dt_head_stride;
- const size_t state_base = size_t(h_idx) * state_head_stride + size_t(dh_idx) * state_dim_stride;
- const size_t cb_group_base = size_t(group_idx) * cb_group_stride;
- const float d_scalar = float(D[h_idx]);
-
- const int max_chunks = SSM_PREFILL_MAX_STATE / simd_width;
- const int chunk_count =
- (state_dim + int(simd_width) - 1) / int(simd_width);
- if (state_dim <= 0 || chunk_count > max_chunks) {
- return;
- }
+ const uint lane_idx = uint(lane);
+ const uint h_idx = pair_idx / head_dim;
+ const uint dh_idx = pair_idx % head_dim;
+ const uint safe_group = uint(max(group_size, 1));
+ const uint group_idx = h_idx / safe_group;
+
+ const size_t x_token_stride = x_strides[0];
+ const size_t x_head_stride = x_strides[1];
+ const size_t x_dim_stride = x_strides[2];
+ const size_t dt_token_stride = dt_strides[0];
+ const size_t dt_head_stride = dt_strides[1];
+ const size_t cb_token_stride = cb_strides[0];
+ const size_t cb_group_stride = cb_strides[1];
+ const size_t cb_state_stride = cb_strides[2];
+ const size_t state_head_stride = state_strides[0];
+ const size_t state_dim_stride = state_strides[1];
+ const size_t state_inner_stride = state_strides[2];
+
+ const size_t x_base =
+ size_t(h_idx) * x_head_stride + size_t(dh_idx) * x_dim_stride;
+ const size_t dt_base = size_t(h_idx) * dt_head_stride;
+ const size_t state_base =
+ size_t(h_idx) * state_head_stride + size_t(dh_idx) * state_dim_stride;
+ const size_t cb_group_base = size_t(group_idx) * cb_group_stride;
+ const float d_scalar = float(D[h_idx]);
+
+ const int max_chunks = SSM_PREFILL_MAX_STATE / simd_width;
+ const int chunk_count = (state_dim + int(simd_width) - 1) / int(simd_width);
+ if (state_dim <= 0 || chunk_count > max_chunks) {
+ return;
+ }
+
+ thread float lane_states[SSM_PREFILL_MAX_STATE / 32];
- thread float lane_states[SSM_PREFILL_MAX_STATE / 32];
+#pragma unroll
+ for (int chunk = 0; chunk < chunk_count; ++chunk) {
+ const int idx = chunk * int(simd_width) + int(lane_idx);
+ const size_t state_idx = state_base + size_t(idx) * state_inner_stride;
+ lane_states[chunk] = float(state[state_idx]);
+ }
- #pragma unroll
+ for (size_t token = 0; token < suffix_len; ++token) {
+ const size_t x_idx = token * x_token_stride + x_base;
+ const size_t dt_idx = token * dt_token_stride + dt_base;
+
+ const float x_val = float(x[x_idx]);
+ const float decay_val = fast::exp(-float(softplus(float(dt_raw[dt_idx]))));
+ const float gate = float(SILU{}(z[x_idx]));
+ const float skip = d_scalar * x_val;
+ const float dt_scaled_input = x_val;
+
+ float contrib_sum = 0.0f;
+#pragma unroll
for (int chunk = 0; chunk < chunk_count; ++chunk) {
- const int idx = chunk * int(simd_width) + int(lane_idx);
- const size_t state_idx =
- state_base + size_t(idx) * state_inner_stride;
- lane_states[chunk] = float(state[state_idx]);
+ const int idx = chunk * int(simd_width) + int(lane_idx);
+ const size_t cb_idx = cb_group_base + size_t(idx) * cb_state_stride +
+ token * cb_token_stride;
+
+ const float new_state =
+ decay_val * lane_states[chunk] + dt_scaled_input * float(B[cb_idx]);
+ lane_states[chunk] = new_state;
+ contrib_sum += new_state * float(C[cb_idx]);
}
- for (size_t token = 0; token < suffix_len; ++token) {
- const size_t x_idx = token * x_token_stride + x_base;
- const size_t dt_idx = token * dt_token_stride + dt_base;
-
- const float x_val = float(x[x_idx]);
- const float decay_val = fast::exp(-float(softplus(float(dt_raw[dt_idx]))));
- const float gate = float(SILU{}(z[x_idx]));
- const float skip = d_scalar * x_val;
- const float dt_scaled_input = x_val;
-
- float contrib_sum = 0.0f;
- #pragma unroll
- for (int chunk = 0; chunk < chunk_count; ++chunk) {
- const int idx = chunk * int(simd_width) + int(lane_idx);
- const size_t cb_idx = cb_group_base + size_t(idx) * cb_state_stride + token * cb_token_stride;
-
- const float new_state = decay_val * lane_states[chunk] + dt_scaled_input * float(B[cb_idx]);
- lane_states[chunk] = new_state;
- contrib_sum += new_state * float(C[cb_idx]);
- }
-
- float dot = simd_sum(contrib_sum);
- if (lane_idx == 0) {
- y[x_idx] = static_cast<T>((skip + dot) * gate);
- }
+ float dot = simd_sum(contrib_sum);
+ if (lane_idx == 0) {
+ y[x_idx] = static_cast<T>((skip + dot) * gate);
}
+ }
- #pragma unroll
- for (int chunk = 0; chunk < chunk_count; ++chunk) {
- const int idx = chunk * int(simd_width) + int(lane_idx);
- const size_t state_idx = state_base + size_t(idx) * state_inner_stride;
- state[state_idx] = static_cast<T>(lane_states[chunk]);
- }
+#pragma unroll
+ for (int chunk = 0; chunk < chunk_count; ++chunk) {
+ const int idx = chunk * int(simd_width) + int(lane_idx);
+ const size_t state_idx = state_base + size_t(idx) * state_inner_stride;
+ state[state_idx] = static_cast<T>(lane_states[chunk]);
+ }
}
-#define instantiate_ssd_prefill_kernel(type_name, type) \
- template [[host_name("ssd_prefill_kernel_" #type_name)]] \
- kernel void ssd_prefill_kernel<type>( \
- device const type* x [[ buffer(0) ]], \
- device const type* dt_raw [[ buffer(1) ]], \
- device const type* B [[ buffer(2) ]], \
- device const type* C [[ buffer(3) ]], \
- device const type* D [[ buffer(4) ]], \
- device const type* z [[ buffer(5) ]], \
- device type* state [[ buffer(6) ]], \
- device type* y [[ buffer(7) ]], \
- constant const size_t& suffix_len [[ buffer(8) ]], \
- constant const int& group_size [[ buffer(9) ]], \
- constant const int& state_size [[ buffer(10) ]], \
- constant const size_t* x_strides [[ buffer(11) ]], \
- constant const size_t* dt_strides [[ buffer(12) ]], \
- constant const size_t* cb_strides [[ buffer(13) ]], \
- constant const size_t* state_strides [[ buffer(14) ]], \
- constant const uint& num_heads [[ buffer(15) ]], \
- constant const uint& head_dim [[ buffer(16) ]], \
- uint3 tg_pos [[ threadgroup_position_in_grid ]], \
- ushort lane [[ thread_index_in_threadgroup ]] \
+#define instantiate_ssd_prefill_kernel(type_name, type) \
+ template [[host_name("ssd_prefill_kernel_" #type_name)]] \
+ kernel void ssd_prefill_kernel<type>( \
+ device const type* x [[buffer(0)]], \
+ device const type* dt_raw [[buffer(1)]], \
+ device const type* B [[buffer(2)]], \
+ device const type* C [[buffer(3)]], \
+ device const type* D [[buffer(4)]], \
+ device const type* z [[buffer(5)]], \
+ device type* state [[buffer(6)]], \
+ device type* y [[buffer(7)]], \
+ constant const size_t& suffix_len [[buffer(8)]], \
+ constant const int& group_size [[buffer(9)]], \
+ constant const int& state_size [[buffer(10)]], \
+ constant const size_t* x_strides [[buffer(11)]], \
+ constant const size_t* dt_strides [[buffer(12)]], \
+ constant const size_t* cb_strides [[buffer(13)]], \
+ constant const size_t* state_strides [[buffer(14)]], \
+ constant const uint& num_heads [[buffer(15)]], \
+ constant const uint& head_dim [[buffer(16)]], \
+ uint3 tg_pos [[threadgroup_position_in_grid]], \
+ ushort lane [[thread_index_in_threadgroup]] \
);
instantiate_ssd_prefill_kernel(float, float);
@@ -300,101 +304,99 @@ instantiate_ssd_prefill_kernel(half, half);
template <typename T>
kernel void ssd_prefill_kernel_sequential(
- device const T* x [[ buffer(0) ]], // (suffix, h, dh)
- device const T* dt_raw [[ buffer(1) ]], // (suffix, h) - raw dt values
- device const T* B [[ buffer(2) ]], // (suffix, g, n)
- device const T* C [[ buffer(3) ]], // (suffix, g, n)
- device const T* D [[ buffer(4) ]], // (h)
- device const T* z [[ buffer(5) ]], // (suffix, h, dh)
- device T* state [[ buffer(6) ]], // (h, dh, n)
- device T* y [[ buffer(7) ]], // (suffix, h, dh)
- constant const size_t& suffix_len [[ buffer(8) ]],
- constant const int& group_size [[ buffer(9) ]],
- constant const int& state_size [[ buffer(10) ]],
- constant const size_t* x_strides [[ buffer(11) ]],
- constant const size_t* dt_strides [[ buffer(12) ]],
- constant const size_t* cb_strides [[ buffer(13) ]],
- constant const size_t* state_strides [[ buffer(14) ]],
- uint3 tid [[ thread_position_in_grid ]],
- uint3 grid_dim [[ threads_per_grid ]]
+ device const T* x [[buffer(0)]], // (suffix, h, dh)
+ device const T* dt_raw [[buffer(1)]], // (suffix, h) - raw dt values
+ device const T* B [[buffer(2)]], // (suffix, g, n)
+ device const T* C [[buffer(3)]], // (suffix, g, n)
+ device const T* D [[buffer(4)]], // (h)
+ device const T* z [[buffer(5)]], // (suffix, h, dh)
+ device T* state [[buffer(6)]], // (h, dh, n)
+ device T* y [[buffer(7)]], // (suffix, h, dh)
+ constant const size_t& suffix_len [[buffer(8)]],
+ constant const int& group_size [[buffer(9)]],
+ constant const int& state_size [[buffer(10)]],
+ constant const size_t* x_strides [[buffer(11)]],
+ constant const size_t* dt_strides [[buffer(12)]],
+ constant const size_t* cb_strides [[buffer(13)]],
+ constant const size_t* state_strides [[buffer(14)]],
+ uint3 tid [[thread_position_in_grid]],
+ uint3 grid_dim [[threads_per_grid]]
) {
- const uint h_idx = tid.x;
- const uint dh_idx = tid.y;
- if (h_idx >= grid_dim.x || dh_idx >= grid_dim.y) {
- return;
- }
+ const uint h_idx = tid.x;
+ const uint dh_idx = tid.y;
+ if (h_idx >= grid_dim.x || dh_idx >= grid_dim.y) {
+ return;
+ }
- const uint safe_group = uint(max(group_size, 1));
- const uint group_idx = h_idx / safe_group;
- device T* state_row =
- state + size_t(h_idx) * state_strides[0] + size_t(dh_idx) * state_strides[1];
-
- for (size_t token = 0; token < suffix_len; ++token) {
- const size_t x_idx = token * x_strides[0]
- + size_t(h_idx) * x_strides[1]
- + size_t(dh_idx) * x_strides[2];
- const size_t dt_idx =
- token * dt_strides[0] + size_t(h_idx) * dt_strides[1];
- const size_t cb_base =
- token * cb_strides[0] + group_idx * cb_strides[1];
-
- const T this_x = x[x_idx];
- const T dt_raw_val = dt_raw[dt_idx];
- const T this_dt = softplus(dt_raw_val);
- const T this_decay = static_cast<T>(fast::exp(-float(this_dt)));
- const T this_D = D[h_idx];
- const T this_z = SILU{}(z[x_idx]);
- const T dt_scaled_input = this_x;
-
- T acc = T(0);
- int s = 0;
- const int vec_bound = (state_size / 4) * 4;
- for (; s < vec_bound; s += 4) {
- const size_t state_idx = s * state_strides[2];
- const size_t cb_idx = cb_base + s * cb_strides[2];
- auto prev_state = *reinterpret_cast<device vec<T, 4>*>(state_row + state_idx);
- auto b_vec = *reinterpret_cast<device const vec<T, 4>*>(B + cb_idx);
- auto c_vec = *reinterpret_cast<device const vec<T, 4>*>(C + cb_idx);
- vec<T, 4> new_state = prev_state * this_decay + b_vec * dt_scaled_input;
- *reinterpret_cast<device vec<T, 4>*>(state_row + state_idx) = new_state;
- vec<T, 4> prod = new_state * c_vec;
- acc += prod.x + prod.y + prod.z + prod.w;
- }
- for (; s < state_size; ++s) {
- const size_t state_idx = s * state_strides[2];
- const T prev_state = state_row[state_idx];
- const size_t cb_idx = cb_base + s * cb_strides[2];
- const T new_state = prev_state * this_decay + B[cb_idx] * dt_scaled_input;
- state_row[state_idx] = new_state;
- acc += new_state * C[cb_idx];
- }
-
- acc += this_D * this_x;
- acc *= this_z;
- y[x_idx] = acc;
+ const uint safe_group = uint(max(group_size, 1));
+ const uint group_idx = h_idx / safe_group;
+ device T* state_row = state + size_t(h_idx) * state_strides[0] +
+ size_t(dh_idx) * state_strides[1];
+
+ for (size_t token = 0; token < suffix_len; ++token) {
+ const size_t x_idx = token * x_strides[0] + size_t(h_idx) * x_strides[1] +
+ size_t(dh_idx) * x_strides[2];
+ const size_t dt_idx = token * dt_strides[0] + size_t(h_idx) * dt_strides[1];
+ const size_t cb_base = token * cb_strides[0] + group_idx * cb_strides[1];
+
+ const T this_x = x[x_idx];
+ const T dt_raw_val = dt_raw[dt_idx];
+ const T this_dt = softplus(dt_raw_val);
+ const T this_decay = static_cast<T>(fast::exp(-float(this_dt)));
+ const T this_D = D[h_idx];
+ const T this_z = SILU{}(z[x_idx]);
+ const T dt_scaled_input = this_x;
+
+ T acc = T(0);
+ int s = 0;
+ const int vec_bound = (state_size / 4) * 4;
+ for (; s < vec_bound; s += 4) {
+ const size_t state_idx = s * state_strides[2];
+ const size_t cb_idx = cb_base + s * cb_strides[2];
+ auto prev_state =
+ *reinterpret_cast<device vec<T, 4>*>(state_row + state_idx);
+ auto b_vec = *reinterpret_cast<device const vec<T, 4>*>(B + cb_idx);
+ auto c_vec = *reinterpret_cast<device const vec<T, 4>*>(C + cb_idx);
+ vec<T, 4> new_state = prev_state * this_decay + b_vec * dt_scaled_input;
+ *reinterpret_cast<device vec<T, 4>*>(state_row + state_idx) = new_state;
+ vec<T, 4> prod = new_state * c_vec;
+ acc += prod.x + prod.y + prod.z + prod.w;
+ }
+ for (; s < state_size; ++s) {
+ const size_t state_idx = s * state_strides[2];
+ const T prev_state = state_row[state_idx];
+ const size_t cb_idx = cb_base + s * cb_strides[2];
+ const T new_state = prev_state * this_decay + B[cb_idx] * dt_scaled_input;
+ state_row[state_idx] = new_state;
+ acc += new_state * C[cb_idx];
}
+
+ acc += this_D * this_x;
+ acc *= this_z;
+ y[x_idx] = acc;
+ }
}
-#define instantiate_ssd_prefill_kernel_sequential(type_name, type) \
- template [[host_name("ssd_prefill_kernel_sequential_" #type_name)]] \
- kernel void ssd_prefill_kernel_sequential<type>( \
- device const type* x [[ buffer(0) ]], \
- device const type* dt_raw [[ buffer(1) ]], \
- device const type* B [[ buffer(2) ]], \
- device const type* C [[ buffer(3) ]], \
- device const type* D [[ buffer(4) ]], \
- device const type* z [[ buffer(5) ]], \
- device type* state [[ buffer(6) ]], \
- device type* y [[ buffer(7) ]], \
- constant const size_t& suffix_len [[ buffer(8) ]], \
- constant const int& group_size [[ buffer(9) ]], \
- constant const int& state_size [[ buffer(10) ]], \
- constant const size_t* x_strides [[ buffer(11) ]], \
- constant const size_t* dt_strides [[ buffer(12) ]], \
- constant const size_t* cb_strides [[ buffer(13) ]], \
- constant const size_t* state_strides [[ buffer(14) ]], \
- uint3 tid [[ thread_position_in_grid ]], \
- uint3 grid_dim [[ threads_per_grid ]] \
+#define instantiate_ssd_prefill_kernel_sequential(type_name, type) \
+ template [[host_name("ssd_prefill_kernel_sequential_" #type_name)]] \
+ kernel void ssd_prefill_kernel_sequential<type>( \
+ device const type* x [[buffer(0)]], \
+ device const type* dt_raw [[buffer(1)]], \
+ device const type* B [[buffer(2)]], \
+ device const type* C [[buffer(3)]], \
+ device const type* D [[buffer(4)]], \
+ device const type* z [[buffer(5)]], \
+ device type* state [[buffer(6)]], \
+ device type* y [[buffer(7)]], \
+ constant const size_t& suffix_len [[buffer(8)]], \
+ constant const int& group_size [[buffer(9)]], \
+ constant const int& state_size [[buffer(10)]], \
+ constant const size_t* x_strides [[buffer(11)]], \
+ constant const size_t* dt_strides [[buffer(12)]], \
+ constant const size_t* cb_strides [[buffer(13)]], \
+ constant const size_t* state_strides [[buffer(14)]], \
+ uint3 tid [[thread_position_in_grid]], \
+ uint3 grid_dim [[threads_per_grid]] \
);
instantiate_ssd_prefill_kernel_sequential(float, float);
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal b/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal
index 048023aca..085fc63eb 100644
--- a/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal
+++ b/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal
@@ -15,90 +15,100 @@ struct SILU {
template <typename T>
inline T softplus(T x) {
- float xf = float(x);
- if (xf > 20.0f) {
- return x;
- }
- return static_cast<T>(log(1.0f + fast::exp(xf)));
+ float xf = float(x);
+ if (xf > 20.0f) {
+ return x;
+ }
+ return static_cast<T>(log(1.0f + fast::exp(xf)));
}
template <typename T>
kernel void ssd_update_kernel(
- // Input
- device const T* x [[ buffer(0) ]], // (b, h, dh)
- device const T* dt_raw [[ buffer(1) ]], // (b, h)
- device const T* B [[ buffer(2) ]], // (b, g, n)
- device const T* C [[ buffer(3) ]], // (b, g, n)
- device const T* D [[ buffer(4) ]], // (h)
- device const T* z [[ buffer(5) ]], // (b, d)
- device const T* state [[ buffer(6) ]], // (b, h, dh, n)
- device T* y [[ buffer(7) ]],
- device T* next_state [[ buffer(8) ]],
+ // Input
+ device const T* x [[buffer(0)]], // (b, h, dh)
+ device const T* dt_raw [[buffer(1)]], // (b, h)
+ device const T* B [[buffer(2)]], // (b, g, n)
+ device const T* C [[buffer(3)]], // (b, g, n)
+ device const T* D [[buffer(4)]], // (h)
+ device const T* z [[buffer(5)]], // (b, d)
+ device const T* state [[buffer(6)]], // (b, h, dh, n)
+ device T* y [[buffer(7)]],
+ device T* next_state [[buffer(8)]],
// Parameters
- constant const int& group_size [[ buffer(9) ]], // h = group_size * g
- constant const int& state_size [[ buffer(10) ]],
+ constant const int& group_size [[buffer(9)]], // h = group_size * g
+ constant const int& state_size [[buffer(10)]],
// Strides
- constant const size_t* x_strides [[ buffer(11) ]],
- constant const size_t* dt_strides [[ buffer(12) ]],
- constant const size_t* CB_strides [[ buffer(13) ]],
- constant const size_t* state_strides [[ buffer(14) ]],
+ constant const size_t* x_strides [[buffer(11)]],
+ constant const size_t* dt_strides [[buffer(12)]],
+ constant const size_t* CB_strides [[buffer(13)]],
+ constant const size_t* state_strides [[buffer(14)]],
// Grid
- uint3 grid_idx [[ thread_position_in_grid ]],
- uint3 grid_size [[ threads_per_grid ]]
+ uint3 grid_idx [[thread_position_in_grid]],
+ uint3 grid_size [[threads_per_grid]]
) {
- const int b_idx = grid_idx.x;
- const int h_idx = grid_idx.y;
- const int dh_idx = grid_idx.z;
+ const int b_idx = grid_idx.x;
+ const int h_idx = grid_idx.y;
+ const int dh_idx = grid_idx.z;
- const int CB_start_idx = static_cast<int>(b_idx * CB_strides[0] + (h_idx / group_size) * CB_strides[1]);
- const int x_idx = static_cast<int>(b_idx * x_strides[0] + h_idx * x_strides[1] + dh_idx * x_strides[2]);
- const int dt_idx = static_cast<int>(b_idx * dt_strides[0] + h_idx * dt_strides[1]);
- const int state_start_idx = static_cast<int>(b_idx * state_strides[0] + h_idx * state_strides[1] + dh_idx * state_strides[2]);
+ const int CB_start_idx = static_cast<int>(
+ b_idx * CB_strides[0] + (h_idx / group_size) * CB_strides[1]
+ );
+ const int x_idx = static_cast<int>(
+ b_idx * x_strides[0] + h_idx * x_strides[1] + dh_idx * x_strides[2]
+ );
+ const int dt_idx =
+ static_cast<int>(b_idx * dt_strides[0] + h_idx * dt_strides[1]);
+ const int state_start_idx = static_cast<int>(
+ b_idx * state_strides[0] + h_idx * state_strides[1] +
+ dh_idx * state_strides[2]
+ );
- // load data
- T this_x = x[x_idx];
- T dt_raw_val = dt_raw[dt_idx];
- T this_dt = softplus(dt_raw_val);
- T this_decay = static_cast<T>(fast::exp(-float(this_dt)));
- T this_D = D[h_idx];
- T this_z = z[x_idx];
- this_z = SILU{}(this_z);
- T dt_scaled_input = this_x;
+ // load data
+ T this_x = x[x_idx];
+ T dt_raw_val = dt_raw[dt_idx];
+ T this_dt = softplus(dt_raw_val);
+ T this_decay = static_cast<T>(fast::exp(-float(this_dt)));
+ T this_D = D[h_idx];
+ T this_z = z[x_idx];
+ this_z = SILU{}(this_z);
+ T dt_scaled_input = this_x;
- T temp = T(0);
- #pragma unroll
- for (int i = 0; i < state_size; ++i) {
- int CB_idx = CB_start_idx + i;
- int state_idx = state_start_idx + i;
- T this_new_state = state[state_idx] * this_decay + B[CB_idx] * dt_scaled_input;
- next_state[state_idx] = this_new_state;
- temp = temp + this_new_state * C[CB_idx];
- }
- temp = temp + this_D * this_x;
- temp = temp * this_z;
- y[x_idx] = temp;
+ T temp = T(0);
+#pragma unroll
+ for (int i = 0; i < state_size; ++i) {
+ int CB_idx = CB_start_idx + i;
+ int state_idx = state_start_idx + i;
+ T this_new_state =
+ state[state_idx] * this_decay + B[CB_idx] * dt_scaled_input;
+ next_state[state_idx] = this_new_state;
+ temp = temp + this_new_state * C[CB_idx];
+ }
+ temp = temp + this_D * this_x;
+ temp = temp * this_z;
+ y[x_idx] = temp;
}
-#define instantiate_ssd_update_kernel(type_name, type) \
- template [[host_name("ssd_update_kernel_" #type_name)]] \
- kernel void ssd_update_kernel<type>( \
- device const type* x [[buffer(0)]], \
- device const type* dt_raw [[buffer(1)]], \
- device const type* B [[buffer(2)]], \
- device const type* C [[buffer(3)]], \
- device const type* D [[buffer(4)]], \
- device const type* z [[buffer(5)]], \
- device const type* state [[buffer(6)]], \
- device type* y [[buffer(7)]], \
- device type* next_state [[buffer(8)]], \
- constant const int& group_size [[buffer(9)]], \
- constant const int& state_size [[buffer(10)]], \
- constant const size_t* x_strides [[buffer(11)]], \
- constant const size_t* dt_strides [[buffer(12)]], \
- constant const size_t* CB_strides [[buffer(13)]], \
- constant const size_t* state_strides [[buffer(14)]], \
- uint3 grid_idx [[thread_position_in_grid]], \
- uint3 grid_size [[threads_per_grid]]);
+#define instantiate_ssd_update_kernel(type_name, type) \
+ template [[host_name("ssd_update_kernel_" #type_name)]] \
+ kernel void ssd_update_kernel<type>( \
+ device const type* x [[buffer(0)]], \
+ device const type* dt_raw [[buffer(1)]], \
+ device const type* B [[buffer(2)]], \
+ device const type* C [[buffer(3)]], \
+ device const type* D [[buffer(4)]], \
+ device const type* z [[buffer(5)]], \
+ device const type* state [[buffer(6)]], \
+ device type* y [[buffer(7)]], \
+ device type* next_state [[buffer(8)]], \
+ constant const int& group_size [[buffer(9)]], \
+ constant const int& state_size [[buffer(10)]], \
+ constant const size_t* x_strides [[buffer(11)]], \
+ constant const size_t* dt_strides [[buffer(12)]], \
+ constant const size_t* CB_strides [[buffer(13)]], \
+ constant const size_t* state_strides [[buffer(14)]], \
+ uint3 grid_idx [[thread_position_in_grid]], \
+ uint3 grid_size [[threads_per_grid]] \
+ );
instantiate_ssd_update_kernel(float, float);
instantiate_ssd_update_kernel(bfloat, bfloat);
diff --git a/crates/uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal b/crates/uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal
index 45b57d9ed..ca865bfb4 100644
--- a/crates/uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal
+++ b/crates/uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal
@@ -10,31 +10,23 @@ void tensorAddBias(
constant int& length,
const uint position
) {
- if (position < length) {
- int col = position % numCols;
- output[position] = input[position] + bias[col];
- }
+ if (position < length) {
+ int col = position % numCols;
+ output[position] = input[position] + bias[col];
+ }
}
-#define outerArguments(T) \
-(const device T* input [[ buffer(0) ]], \
-const device T* bias [[ buffer(1) ]], \
-device T* output [[ buffer(2) ]], \
-constant int& numCols [[ buffer(3) ]], \
-constant int& length [[ buffer(4) ]], \
-const uint position [[ thread_position_in_grid ]])
+#define outerArguments(T) \
+ (const device T* input [[buffer(0)]], \
+ const device T* bias [[buffer(1)]], \
+ device T* output [[buffer(2)]], \
+ constant int& numCols [[buffer(3)]], \
+ constant int& length [[buffer(4)]], \
+ const uint position [[thread_position_in_grid]])
-#define innerArguments \
-(input, \
-bias, \
-output, \
-numCols, \
-length, \
-position)
+#define innerArguments (input, bias, output, numCols, length, position)
generateKernels(tensorAddBias)
#undef outerArguments
#undef innerArguments
-
-
diff --git a/crates/uzu/src/backends/metal/kernel/tensor_add_swap/tensor_add_swap.metal b/crates/uzu/src/backends/metal/kernel/tensor_add_swap/tensor_add_swap.metal
index 42a5e235b..2b7d32ab5 100644
--- a/crates/uzu/src/backends/metal/kernel/tensor_add_swap/tensor_add_swap.metal
+++ b/crates/uzu/src/backends/metal/kernel/tensor_add_swap/tensor_add_swap.metal
@@ -8,27 +8,23 @@ void tensorAddSwap(
constant int& length,
const uint position
) {
- if (position < length) {
- T skipValue = skipBuffer[position];
- T mainValue = mainBuffer[position];
- T result = mainValue + skipValue;
-
- mainBuffer[position] = result;
- skipBuffer[position] = result;
- }
+ if (position < length) {
+ T skipValue = skipBuffer[position];
+ T mainValue = mainBuffer[position];
+ T result = mainValue + skipValue;
+
+ mainBuffer[position] = result;
+ skipBuffer[position] = result;
+ }
}
-#define outerArguments(T) \
-(device T* skipBuffer [[ buffer(0) ]], \
-device T* mainBuffer [[ buffer(1) ]], \
-constant int& length [[ buffer(2) ]], \
-const uint position [[ thread_position_in_grid ]])\
+#define outerArguments(T) \
+ (device T * skipBuffer [[buffer(0)]], \
+ device T * mainBuffer [[buffer(1)]], \
+ constant int& length [[buffer(2)]], \
+ const uint position [[thread_position_in_grid]])
-#define innerArguments \
-(skipBuffer, \
-mainBuffer, \
-length, \
-position) \
+#define innerArguments (skipBuffer, mainBuffer, length, position)
generateKernels(tensorAddSwap)
diff --git a/crates/uzu/src/backends/metal/kernel/tensor_copy/tensor_copy.metal b/crates/uzu/src/backends/metal/kernel/tensor_copy/tensor_copy.metal
index 21733a303..6ba08dfd8 100644
--- a/crates/uzu/src/backends/metal/kernel/tensor_copy/tensor_copy.metal
+++ b/crates/uzu/src/backends/metal/kernel/tensor_copy/tensor_copy.metal
@@ -8,24 +8,20 @@ void tensorCopy(
constant int& length,
const uint position
) {
- if (position < length) {
- destinationBuffer[position] = sourceBuffer[position];
- }
+ if (position < length) {
+ destinationBuffer[position] = sourceBuffer[position];
+ }
}
-#define outerArguments(T) \
-(const device T* sourceBuffer [[ buffer(0) ]], \
-device T* destinationBuffer [[ buffer(1) ]], \
-constant int& length [[ buffer(2) ]], \
-const uint position [[ thread_position_in_grid ]])\
+#define outerArguments(T) \
+ (const device T* sourceBuffer [[buffer(0)]], \
+ device T* destinationBuffer [[buffer(1)]], \
+ constant int& length [[buffer(2)]], \
+ const uint position [[thread_position_in_grid]])
-#define innerArguments \
-(sourceBuffer, \
-destinationBuffer, \
-length, \
-position) \
+#define innerArguments (sourceBuffer, destinationBuffer, length, position)
generateKernels(tensorCopy)
#undef outerArguments
-#undef innerArguments
\ No newline at end of file
+#undef innerArguments
\ No newline at end of file
diff --git a/crates/uzu/src/backends/metal/kernel/token_copy/token_copy.metal b/crates/uzu/src/backends/metal/kernel/token_copy/token_copy.metal
index b9f458c16..cba725880 100644
--- a/crates/uzu/src/backends/metal/kernel/token_copy/token_copy.metal
+++ b/crates/uzu/src/backends/metal/kernel/token_copy/token_copy.metal
@@ -8,9 +8,9 @@ kernel void copy_sampled_token(
device uint64_t* dst [[buffer(1)]],
uint idx [[thread_position_in_grid]]
) {
- if (idx == 0) {
- dst[0] = static_cast<uint64_t>(src[0]);
- }
+ if (idx == 0) {
+ dst[0] = static_cast<uint64_t>(src[0]);
+ }
}
/// Copies a single sampled token (u32) to an indexed results buffer (u32).
@@ -20,7 +20,7 @@ kernel void copy_token_to_results(
device uint32_t* dst [[buffer(1)]],
uint idx [[thread_position_in_grid]]
) {
- if (idx == 0) {
- dst[0] = src[0];
- }
+ if (idx == 0) {
+ dst[0] = src[0];
+ }
}
diff --git a/scripts/clang-format.sh b/scripts/clang-format.sh
new file mode 100755
index 000000000..1295a4fcb
--- /dev/null
+++ b/scripts/clang-format.sh
@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Format C/C++/Metal headers and sources according to .clang-format at repo root.
+# Usage:
+# ./scripts/clang-format.sh
+# ./scripts/clang-format.sh --check # only check (no changes), non-zero exit on diffs
+#
+# Excludes:
+# - target/ (Rust build)
+# - external/ (third-party deps)
+# - .git/
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)"
+cd "$ROOT_DIR"
+
+# Ensure clang-format is available
+if ! command -v clang-format >/dev/null 2>&1; then
+ echo "Error: clang-format not found. Install it with: brew install clang-format" >&2
+ exit 127
+fi
+
+MODE="fix"
+if [[ "${1:-}" == "--check" ]]; then
+ MODE="check"
+fi
+
+# Files to consider (zsh/bash compatible)
+FILES=()
+while IFS= read -r -d '' file; do
+ FILES+=("$file")
+done < <(find . \
+ -path './target' -prune -o \
+ -path './external' -prune -o \
+ -path './.git' -prune -o \
+ -type f \( \
+ -name '*.h' -o -name '*.hpp' -o -name '*.hh' -o \
+ -name '*.c' -o -name '*.cc' -o -name '*.cpp' -o \
+ -name '*.metal' \
+ \) -print0)
+
+if [[ ${#FILES[@]} -eq 0 ]]; then
+ echo "No files to format."
+ exit 0
+fi
+
+if [[ "$MODE" == "check" ]]; then
+ # Show diffs without writing
+ FAILED=0
+ for f in "${FILES[@]}"; do
+ if ! diff -u "$f" <(clang-format "$f"); then
+ FAILED=1
+ fi
+ done
+ if [[ $FAILED -ne 0 ]]; then
+ echo "\nFormatting check failed. Run ./scripts/clang-format.sh to fix." >&2
+ exit 1
+ fi
+else
+ # In-place rewrite
+ printf '%s\0' "${FILES[@]}" | xargs -0 -n 50 clang-format -i
+ echo "Formatted ${#FILES[@]} file(s)."
+fi