Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

require [[max_total_threads_per_threadgroup(N)]] in legacy kernels (#88)

uuuvn3cde3efd255a
diff --git a/crates/uzu/build/metal/ast.rs b/crates/uzu/build/metal/ast.rs
index c860d56fb..933d16881 100644
--- a/crates/uzu/build/metal/ast.rs
+++ b/crates/uzu/build/metal/ast.rs
@@ -33,6 +33,8 @@ pub enum MetalAstKind {
StringLiteral {
value: Box<str>,
},
+ MetalKernelAttr,
+ MetalMaxTotalThreadsPerThreadGroupAttr,
Other,
}
@@ -385,3 +387,69 @@ impl MetalKernelInfo {
}))
}
}
+
+pub fn validate_raw_kernel(node: &MetalAstNode) -> anyhow::Result<()> {
+ let (node, is_template) =
+ if matches!(node.kind, MetalAstKind::FunctionTemplateDecl) {
+ let inner_fn = node.inner.iter().find(|c| {
+ matches!(
+ c.kind,
+ MetalAstKind::FunctionDecl {
+ name: _
+ }
+ )
+ });
+ match inner_fn {
+ Some(n) => (n, true),
+ None => return Ok(()),
+ }
+ } else if matches!(node.kind, MetalAstKind::FunctionDecl { .. }) {
+ (node, false)
+ } else {
+ return Ok(());
+ };
+
+ let MetalAstKind::FunctionDecl {
+ name,
+ } = &node.kind
+ else {
+ return Ok(());
+ };
+
+ let mut has_metal_kernel_attr = false;
+ let mut has_max_threads_attr = false;
+ let mut has_dsl_kernel = false;
+
+ for child in &node.inner {
+ match &child.kind {
+ MetalAstKind::MetalKernelAttr => has_metal_kernel_attr = true,
+ MetalAstKind::MetalMaxTotalThreadsPerThreadGroupAttr => {
+ has_max_threads_attr = true
+ },
+ MetalAstKind::AnnotateAttr => {
+ if let Ok(annotation) = annotation_from_ast_node(child.clone())
+ {
+ if annotation.first().map(|s| s.as_ref())
+ == Some("dsl.kernel")
+ {
+ has_dsl_kernel = true;
+ }
+ }
+ },
+ _ => (),
+ }
+ }
+
+ if has_metal_kernel_attr && !has_dsl_kernel && !has_max_threads_attr {
+ let kind = if is_template {
+ "template"
+ } else {
+ "function"
+ };
+ bail!(
+ "kernel {kind} '{name}' is missing `[[max_total_threads_per_threadgroup(N)]]` attribute! it **MUST** be present and correct!"
+ );
+ }
+
+ Ok(())
+}
diff --git a/crates/uzu/build/metal/toolchain.rs b/crates/uzu/build/metal/toolchain.rs
index dc28f26cb..a0682119a 100644
--- a/crates/uzu/build/metal/toolchain.rs
+++ b/crates/uzu/build/metal/toolchain.rs
@@ -10,7 +10,9 @@ use anyhow::{Context, bail};
use tempfile::NamedTempFile;
use tokio::{io::AsyncWriteExt, process::Command};
-use super::ast::{MetalAstKind, MetalAstNode, MetalKernelInfo};
+use super::ast::{
+ MetalAstKind, MetalAstNode, MetalKernelInfo, validate_raw_kernel,
+};
#[derive(Debug)]
pub enum MetalSdk {
@@ -172,6 +174,7 @@ impl MetalToolchain {
.args(["-x", "metal"])
.arg(format!("-std={}", self.std.to_str()))
.args(self.extra_options.as_ref())
+ .arg("-DDSL_ANALYZE")
.arg(path)
.arg("-fsyntax-only")
.args(["-MMD", "-MF"])
@@ -204,6 +207,11 @@ impl MetalToolchain {
let source_contents =
fs::read_to_string(path).context("cannot read source file")?;
+ for node in &ast_root.inner {
+ validate_raw_kernel(node)
+ .context("validation of legacy (non-dsl) kernel failed")?;
+ }
+
let kernel_infos = ast_root
.inner
.into_iter()
diff --git a/crates/uzu/src/backends/metal/kernel/activation/activation.metal b/crates/uzu/src/backends/metal/kernel/activation/activation.metal
index 283f342ee..b8ff92950 100644
--- a/crates/uzu/src/backends/metal/kernel/activation/activation.metal
+++ b/crates/uzu/src/backends/metal/kernel/activation/activation.metal
@@ -30,7 +30,8 @@ inline T activate(T x, ushort act) {
}
template <typename T>
-[[kernel]] void activation(
+[[kernel, max_total_threads_per_threadgroup(256)]]
+void activation(
const device T* input [[buffer(0)]],
device T* output [[buffer(1)]],
const constant int& N [[buffer(2)]],
diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention.metal b/crates/uzu/src/backends/metal/kernel/attention/attention.metal
index e813e54f3..115926208 100644
--- a/crates/uzu/src/backends/metal/kernel/attention/attention.metal
+++ b/crates/uzu/src/backends/metal/kernel/attention/attention.metal
@@ -866,7 +866,7 @@ void update_kv_cache(
max_sequence_length, \
position)
-generateKernels(update_kv_cache)
+generateKernels(64, update_kv_cache)
#undef outerArguments
#undef innerArguments
diff --git a/crates/uzu/src/backends/metal/kernel/definitions.metal b/crates/uzu/src/backends/metal/kernel/definitions.metal
index af5f31887..5ee29bb0d 100644
--- a/crates/uzu/src/backends/metal/kernel/definitions.metal
+++ b/crates/uzu/src/backends/metal/kernel/definitions.metal
@@ -462,7 +462,12 @@ static T threadgroup_cooperative_reduce_min(
// MARK: - DSL Annotation Helpers
+#ifdef DSL_ANALYZE
#define DSL_META(...) [[clang::annotate("", __VA_ARGS__)]]
+#else
+#define DSL_META(...)
+#endif
+
#define DSL_STR(X) #X
#define DSL_XSTR(X) DSL_STR(X)
@@ -477,17 +482,19 @@ static T threadgroup_cooperative_reduce_min(
// MARK: - Generate Template Kernels
-#define generateKernel(functionName, scalarType, outerArgs, innerArgs) \
+#define generateKernel(max_threads, functionName, scalarType, outerArgs, innerArgs) \
+ [[max_total_threads_per_threadgroup(max_threads)]] \
kernel void functionName##_##scalarType outerArgs { functionName innerArgs; }
-#define generateKernels(functionName) \
- generateKernel(functionName, float, outerArguments(float), innerArguments); \
+#define generateKernels(max_threads, functionName) \
+ generateKernel(max_threads, functionName, float, outerArguments(float), innerArguments); \
generateKernel( \
+ max_threads, \
functionName, \
bfloat, \
outerArguments(bfloat), \
innerArguments \
); \
- generateKernel(functionName, half, outerArguments(half), innerArguments);
+ generateKernel(max_threads, functionName, half, outerArguments(half), innerArguments);
#endif /* definitions_metal */
diff --git a/crates/uzu/src/backends/metal/kernel/embedding/full_precision_embedding.metal b/crates/uzu/src/backends/metal/kernel/embedding/full_precision_embedding.metal
index f1602f8ca..ee7667836 100644
--- a/crates/uzu/src/backends/metal/kernel/embedding/full_precision_embedding.metal
+++ b/crates/uzu/src/backends/metal/kernel/embedding/full_precision_embedding.metal
@@ -2,7 +2,8 @@
using namespace metal;
template <typename T>
-[[kernel]] void full_precision_embedding_lookup(
+[[kernel, max_total_threads_per_threadgroup(256)]]
+void full_precision_embedding_lookup(
const device uint64_t* token_ids [[buffer(0)]], // [batch_size]
const device T* weights [[buffer(1)]], // [vocab_size, model_dim]
device T* output [[buffer(2)]], // [batch_size, model_dim]
diff --git a/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal b/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal
index 071e540e9..354bc6809 100644
--- a/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal
+++ b/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal
@@ -2,7 +2,8 @@
using namespace metal;
template <typename T, uint PACKING_DIVISOR, bool SIGNED_STORAGE>
-[[kernel]] void quantized_embedding_lookup(
+[[kernel, max_total_threads_per_threadgroup(256)]]
+void quantized_embedding_lookup(
const device uint64_t* token_ids [[buffer(0)]], // [batch_size]
const device uint8_t* weights
[[buffer(1)]], // [vocab_size, model_dim/packing_divisor] packed
diff --git a/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal b/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal
index c4b0b52f6..b149770e7 100644
--- a/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal
+++ b/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal
@@ -52,7 +52,7 @@ void updateKVCache(
headDim, \
position)
-generateKernels(updateKVCache)
+generateKernels(1024, updateKVCache)
#undef outerArguments
#undef innerArguments
diff --git a/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal b/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal
index 37321f887..593d76575 100644
--- a/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal
+++ b/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal
@@ -128,6 +128,7 @@ void layer_norm_core(
// Generate LayerNorm kernels
#define DEFINE_LAYER_NORM_KERNEL(IN, SC, OUT, ACC, FULL_LAYER, SUF) \
+ [[max_total_threads_per_threadgroup(1024)]] \
kernel void layer_norm_##SUF( \
const device IN* input [[buffer(0)]], \
const device SC* scales [[buffer(1)]], \
diff --git a/crates/uzu/src/backends/metal/kernel/mask_update/mask_update.metal b/crates/uzu/src/backends/metal/kernel/mask_update/mask_update.metal
index 7898db785..c00b250a7 100644
--- a/crates/uzu/src/backends/metal/kernel/mask_update/mask_update.metal
+++ b/crates/uzu/src/backends/metal/kernel/mask_update/mask_update.metal
@@ -2,6 +2,7 @@
using namespace metal;
template <typename T>
+[[max_total_threads_per_threadgroup(1)]]
kernel void update_attention_mask(
device T* mask [[buffer(0)]],
constant int& unmask_col [[buffer(1)]],
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/split_k/shaders/steel_gemm_splitk.h b/crates/uzu/src/backends/metal/kernel/matmul/split_k/shaders/steel_gemm_splitk.h
index 24302129b..60d238d98 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/split_k/shaders/steel_gemm_splitk.h
+++ b/crates/uzu/src/backends/metal/kernel/matmul/split_k/shaders/steel_gemm_splitk.h
@@ -175,7 +175,7 @@ template <
typename AccT,
typename OutT,
typename Epilogue = TransformNone<OutT, AccT>>
-[[kernel]] void gemm_splitk_accum(
+[[kernel, max_total_threads_per_threadgroup(256)]] void gemm_splitk_accum(
const device AccT* C_split [[buffer(0)]],
device OutT* D [[buffer(1)]],
const constant int& k_partitions [[buffer(2)]],
@@ -203,7 +203,7 @@ template <
typename AccT,
typename OutT,
typename Epilogue = TransformAxpby<OutT, AccT>>
-[[kernel]] void gemm_splitk_accum_axpby(
+[[kernel, max_total_threads_per_threadgroup(256)]] void gemm_splitk_accum_axpby(
const device AccT* C_split [[buffer(0)]],
device OutT* D [[buffer(1)]],
const constant int& k_partitions [[buffer(2)]],
diff --git a/crates/uzu/src/backends/metal/kernel/media_kernels/image_ops.metal b/crates/uzu/src/backends/metal/kernel/media_kernels/image_ops.metal
index 1ef155c1c..687acf318 100644
--- a/crates/uzu/src/backends/metal/kernel/media_kernels/image_ops.metal
+++ b/crates/uzu/src/backends/metal/kernel/media_kernels/image_ops.metal
@@ -17,6 +17,7 @@ struct PatchParameters {
uint temporalSlices;
};
+[[max_total_threads_per_threadgroup(1024)]]
kernel void scalePadNormalizeImage(
texture2d<float, access::sample> inputTexture [[texture(0)]],
texture2d<float, access::write> outputTexture [[texture(1)]],
@@ -81,6 +82,7 @@ kernel void scalePadNormalizeImage(
outputTexture.write(float4(normalizedRgb, 1.0f), gid);
}
+[[max_total_threads_per_threadgroup(1024)]]
kernel void extractImagePatches(
texture2d<float, access::read> paddedNormalizedTexture [[texture(0)]],
device float* outputBuffer [[buffer(0)]],
diff --git a/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal b/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal
index 9a8c3a95c..81c5b1a25 100644
--- a/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal
+++ b/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal
@@ -32,7 +32,7 @@ inline T activate(T x, ushort act) {
}
template <typename T>
-[[kernel]] void mlp_activation_mul(
+[[kernel, max_total_threads_per_threadgroup(64)]] void mlp_activation_mul(
const device T* fused_up [[buffer(0)]],
device T* hidden [[buffer(1)]],
const constant int& H [[buffer(2)]],
diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/steel_gemm_splitk_mlp_fused.h b/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/steel_gemm_splitk_mlp_fused.h
index 3d36c9e9e..d37245d71 100644
--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/steel_gemm_splitk_mlp_fused.h
+++ b/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/steel_gemm_splitk_mlp_fused.h
@@ -201,7 +201,7 @@ gemm_splitk_mlp_fused(
///////////////////////////////////////////////////////////////////////////////
template <typename AccT, typename OutT>
-[[kernel]] void gemm_splitk_mlp_fused_accum(
+[[kernel, max_total_threads_per_threadgroup(256)]] void gemm_splitk_mlp_fused_accum(
const device AccT* C_split [[buffer(0)]],
device OutT* D [[buffer(1)]],
const constant int& k_partitions [[buffer(2)]],
diff --git a/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal b/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal
index 21ec0fccc..a6e14eb6a 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal
@@ -9,6 +9,7 @@ using namespace metal;
// Single-kernel fused: count all experts + scan to offsets
// This kernel is launched with SINGLE threadgroup
+[[max_total_threads_per_threadgroup(128)]]
kernel void moe_counts_offsets_fused(
device const int* topk_ids [[buffer(0)]],
device uint* offsets [[buffer(1)]], // output: exclusive scan [E+1]
diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal
index fe957960f..5f5d2ec28 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal
@@ -131,6 +131,7 @@ inline void moe_experts_decode_single_pass_a_impl(
}
#define MOE_DECODE_SINGLE_PASS_A_KERNEL(DTYPE, DTYPE4, SUFFIX) \
+ [[max_total_threads_per_threadgroup(128)]] \
kernel void moe_experts_decode_single_pass_a_##SUFFIX( \
device const DTYPE* x [[buffer(0)]], \
device const int* topk_ids [[buffer(1)]], \
@@ -249,6 +250,7 @@ inline void moe_experts_decode_single_pass_b_impl(
}
#define MOE_DECODE_SINGLE_PASS_B_KERNEL(DTYPE, DTYPE4, SUFFIX) \
+ [[max_total_threads_per_threadgroup(256)]] \
kernel void moe_experts_decode_single_pass_b_##SUFFIX( \
device const float* hidden [[buffer(0)]], \
device const int* topk_ids [[buffer(1)]], \
diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal
index ab55ce6c1..8e78049db 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal
@@ -132,54 +132,10 @@ void moe_experts_decode_pass_a_impl(
}
}
-#define MOE_PASS_A_KERNEL(DTYPE, DTYPE4, SUFFIX) \
- kernel void moe_experts_decode_pass_a_##SUFFIX( \
- device const DTYPE* X_perm [[buffer(0)]], \
- device const uint* expert_offsets [[buffer(1)]], \
- device const DTYPE* W13_all [[buffer(2)]], \
- device float* hidden_out [[buffer(3)]], \
- device const DTYPE* up_biases [[buffer(4)]], \
- constant uint& d_model [[buffer(5)]], \
- constant uint& d_ff [[buffer(6)]], \
- constant uint& E [[buffer(7)]], \
- constant float& gate_clip_min [[buffer(8)]], \
- constant float& gate_clip_max [[buffer(9)]], \
- constant float& up_clip_min [[buffer(10)]], \
- constant float& up_clip_max [[buffer(11)]], \
- constant float& silu_alpha [[buffer(12)]], \
- uint3 tgpig [[threadgroup_position_in_grid]], \
- uint simd_gid [[simdgroup_index_in_threadgroup]], \
- uint simd_lid [[thread_index_in_simdgroup]] \
- ) { \
- moe_experts_decode_pass_a_impl<DTYPE, DTYPE4>( \
- X_perm, \
- expert_offsets, \
- W13_all, \
- up_biases, \
- hidden_out, \
- d_model, \
- d_ff, \
- E, \
- gate_clip_min, \
- gate_clip_max, \
- up_clip_min, \
- up_clip_max, \
- silu_alpha, \
- tgpig.y, \
- tgpig.z, \
- tgpig.x, \
- simd_gid, \
- simd_lid \
- ); \
- }
-
-MOE_PASS_A_KERNEL(bfloat, bfloat4, bf16)
-MOE_PASS_A_KERNEL(half, half4, f16)
-MOE_PASS_A_KERNEL(float, float4, f32)
-
// === Helper kernels for indirect dispatch of Pass A ===
// Count tiles per expert: tiles = (num_rows > 0) ? num_rows * h_blocks : 0
+[[max_total_threads_per_threadgroup(256)]]
kernel void moe_pass_a_tile_counts(
device const uint* expert_offsets [[buffer(0)]], // [E+1]
device uint* tile_counts [[buffer(1)]], // [E]
@@ -196,6 +152,7 @@ kernel void moe_pass_a_tile_counts(
}
// Exclusive scan of tile_counts to get tile_offsets and total_tiles
+[[max_total_threads_per_threadgroup(1024)]]
kernel void moe_pass_a_tile_scan(
device const uint* tile_counts [[buffer(0)]], // [E]
device uint* tile_offsets [[buffer(1)]], // [E+1]
@@ -243,6 +200,7 @@ kernel void moe_pass_a_tile_scan(
}
// Build row→expert map: one thread per routed row
+[[max_total_threads_per_threadgroup(256)]]
kernel void moe_pass_a_build_row_map(
device const uint* expert_offsets [[buffer(0)]], // [E+1]
device uint* row_expert_map [[buffer(1)]], // [total_rows]
@@ -270,6 +228,7 @@ kernel void moe_pass_a_build_row_map(
}
// Build tile map entries from row→expert map
+[[max_total_threads_per_threadgroup(256)]]
kernel void moe_pass_a_build_tile_map(
device const uint* expert_offsets [[buffer(0)]], // [E+1]
device const uint* tile_offsets [[buffer(1)]], // [E+1]
@@ -301,6 +260,7 @@ kernel void moe_pass_a_build_tile_map(
}
// Write dispatch args for indirect dispatch (reusable from tiled version)
+[[max_total_threads_per_threadgroup(1)]]
kernel void moe_pass_a_write_dispatch_args(
device const uint* total_tiles [[buffer(0)]], // [1]
device uint* dispatch_args
@@ -365,6 +325,7 @@ void moe_experts_decode_pass_a_indirect_impl(
}
#define MOE_PASS_A_INDIRECT_KERNEL(DTYPE, DTYPE4, SUFFIX) \
+ [[max_total_threads_per_threadgroup(128)]] \
kernel void moe_experts_decode_pass_a_indirect_##SUFFIX( \
device const DTYPE* X_perm [[buffer(0)]], \
device const uint* expert_offsets [[buffer(1)]], \
@@ -535,6 +496,7 @@ void moe_experts_decode_down_fused_2d_impl(
}
#define MOE_PASS_B_FUSED_2D_KERNEL(DTYPE, SUFFIX) \
+ [[max_total_threads_per_threadgroup(256)]] \
kernel void moe_experts_decode_down_fused_2d_##SUFFIX( \
device const float* hidden [[buffer(0)]], \
device const uint* row_expert_map [[buffer(1)]], \
diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal
index 72076217f..42c13241e 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal
@@ -407,66 +407,6 @@ inline void pass_a_impl(
}
}
-// ----- Kernel entry points (Pass A) -----
-#define MOE_PASS_A_KERNEL(DTYPE, SUFFIX) \
- kernel void moe_two_pass_prefill_pass_a_##SUFFIX( \
- device const DTYPE* X_perm [[buffer(0)]], \
- device const uint* expert_offsets [[buffer(1)]], \
- device const DTYPE* W13_all [[buffer(2)]], \
- device const DTYPE* up_biases [[buffer(3)]], \
- device float* hidden_out [[buffer(4)]], \
- constant uint& d_model [[buffer(5)]], \
- constant uint& d_ff [[buffer(6)]], \
- constant uint& E [[buffer(7)]], \
- constant float& gate_clip_min [[buffer(8)]], \
- constant float& gate_clip_max [[buffer(9)]], \
- constant float& up_clip_min [[buffer(10)]], \
- constant float& up_clip_max [[buffer(11)]], \
- constant float& silu_alpha [[buffer(12)]], \
- uint sg_id [[simdgroup_index_in_threadgroup]], \
- uint3 threads_per_tg [[threads_per_threadgroup]], \
- uint3 tg_pos [[threadgroup_position_in_grid]], \
- uint3 local_tid [[thread_position_in_threadgroup]] \
- ) { \
- using StageT = typename StageStorage<DTYPE>::type; \
- threadgroup StageT Xs[PASSA_BM * (PASSA_BK + PASSA_TG_PAD)]; \
- threadgroup StageT Wk_up[PASSA_BN * (PASSA_BK + PASSA_TG_PAD)]; \
- threadgroup StageT Wk_gate[PASSA_BN * (PASSA_BK + PASSA_TG_PAD)]; \
- threadgroup float bias_up[PASSA_BN]; \
- threadgroup float bias_gate[PASSA_BN]; \
- pass_a_impl<DTYPE>( \
- X_perm, \
- expert_offsets, \
- W13_all, \
- up_biases, \
- hidden_out, \
- d_model, \
- d_ff, \
- E, \
- gate_clip_min, \
- gate_clip_max, \
- up_clip_min, \
- up_clip_max, \
- silu_alpha, \
- expert_offsets[tg_pos.z], \
- tg_pos.z, \
- tg_pos.y, \
- tg_pos.x, \
- sg_id, \
- threads_per_tg, \
- local_tid, \
- Xs, \
- Wk_up, \
- Wk_gate, \
- bias_up, \
- bias_gate \
- ); \
- }
-
-MOE_PASS_A_KERNEL(bfloat, bf16)
-MOE_PASS_A_KERNEL(half, f16)
-MOE_PASS_A_KERNEL(float, f32)
-
// Indirect variant (consumes [expert, seg_start, tile_row_offset] triples)
template <typename T>
inline void pass_a_indirect_impl(
@@ -534,6 +474,7 @@ inline void pass_a_indirect_impl(
}
#define MOE_PASS_A_INDIRECT_KERNEL(DTYPE, SUFFIX) \
+ [[max_total_threads_per_threadgroup(128)]] \
kernel void moe_two_pass_prefill_pass_a_indirect_##SUFFIX( \
device const DTYPE* X_perm [[buffer(0)]], \
device const uint* expert_offsets [[buffer(1)]], \
@@ -797,52 +738,6 @@ inline void pass_b_impl(
}
}
-// ----- Kernel entry points (Pass B) -----
-#define MOE_PASS_B_KERNEL(DTYPE, SUFFIX) \
- kernel void moe_two_pass_prefill_pass_b_##SUFFIX( \
- device const float* hidden [[buffer(0)]], \
- device const uint* expert_offsets [[buffer(1)]], \
- device const DTYPE* W2_all [[buffer(2)]], \
- device const DTYPE* down_biases [[buffer(3)]], \
- device DTYPE* output [[buffer(4)]], \
- constant uint& d_model [[buffer(5)]], \
- constant uint& d_ff [[buffer(6)]], \
- constant uint& E [[buffer(7)]], \
- uint sg_id [[simdgroup_index_in_threadgroup]], \
- uint simd_lid [[thread_index_in_simdgroup]], \
- uint3 tg_pos [[threadgroup_position_in_grid]], \
- uint3 local_tid [[thread_position_in_threadgroup]] \
- ) { \
- using StageT = typename StageStorage<DTYPE>::type; \
- threadgroup float Hs[PASSB_BM * (PASSB_BK + PASSB_TG_PAD)]; \
- threadgroup StageT Wk[PASSB_BN * (PASSB_BK + PASSB_TG_PAD)]; \
- threadgroup float bias[PASSB_BN]; \
- const uint lin = local_tid.x; \
- pass_b_impl<DTYPE>( \
- hidden, \
- expert_offsets, \
- W2_all, \
- down_biases, \
- output, \
- d_model, \
- d_ff, \
- E, \
- tg_pos.z, \
- tg_pos.y, \
- tg_pos.x, \
- sg_id, \
- simd_lid, \
- lin, \
- Hs, \
- Wk, \
- bias \
- ); \
- }
-
-MOE_PASS_B_KERNEL(bfloat, bf16)
-MOE_PASS_B_KERNEL(half, f16)
-MOE_PASS_B_KERNEL(float, f32)
-
// Indirect variant
template <typename T>
inline void pass_b_indirect_impl(
@@ -894,6 +789,7 @@ inline void pass_b_indirect_impl(
}
#define MOE_PASS_B_INDIRECT_KERNEL(DTYPE, SUFFIX) \
+ [[max_total_threads_per_threadgroup(128)]] \
kernel void moe_two_pass_prefill_pass_b_indirect_##SUFFIX( \
device const float* hidden [[buffer(0)]], \
device const uint* expert_offsets [[buffer(1)]], \
diff --git a/crates/uzu/src/backends/metal/kernel/moe/finalize.metal b/crates/uzu/src/backends/metal/kernel/moe/finalize.metal
index 168394b91..a64abac2c 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/finalize.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/finalize.metal
@@ -57,6 +57,7 @@ inline void moe_finalize_impl(
}
#define DEFINE_MOE_FINALIZE_KERNEL(SUFFIX, T) \
+ [[max_total_threads_per_threadgroup(128)]] \
kernel void moe_finalize_##SUFFIX( \
device const int* tok2row [[buffer(0)]], \
device const T* probs [[buffer(1)]], \
diff --git a/crates/uzu/src/backends/metal/kernel/moe/prepare.metal b/crates/uzu/src/backends/metal/kernel/moe/prepare.metal
index fb9589771..b32f844c6 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/prepare.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/prepare.metal
@@ -108,6 +108,7 @@ void moe_gather_x_perm_1d_impl(
// 2D Tiled kernels
#define GATHER_2D_KERNEL(DTYPE, SUFFIX) \
+ [[max_total_threads_per_threadgroup(256)]] \
kernel void moe_gather_x_perm_##SUFFIX##_2d( \
device const DTYPE* X [[buffer(0)]], \
device const int* bucketed_ids [[buffer(1)]], \
@@ -123,6 +124,7 @@ void moe_gather_x_perm_1d_impl(
// 1D kernels
#define GATHER_1D_KERNEL(DTYPE, SUFFIX) \
+ [[max_total_threads_per_threadgroup(256)]] \
kernel void moe_gather_x_perm_##SUFFIX( \
device const DTYPE* X [[buffer(0)]], \
device const int* bucketed_ids [[buffer(1)]], \
diff --git a/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal b/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal
index 7b19cf3ce..1cacbde62 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal
@@ -143,6 +143,7 @@ inline void moe_router_topk_impl(
}
#define DEFINE_ROUTER_TOPK_KERNEL(SUFFIX, VEC4, SCALAR, PROB) \
+ [[max_total_threads_per_threadgroup(256)]] \
kernel void moe_router_topk_##SUFFIX( \
const device VEC4* input [[buffer(0)]], \
const device VEC4* weight [[buffer(1)]], \
diff --git a/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal b/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal
index 3a2245045..cdb91e750 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal
@@ -9,6 +9,7 @@ using namespace metal;
// 4A: Compute per-block base offsets from partials
// partials layout: index = (block_id * num_tiles + tile_id) * TILE_E + te
+[[max_total_threads_per_threadgroup(256)]]
kernel void moe_block_bases_from_partials(
device const uint* partials [[buffer(0)]],
device uint* block_bases [[buffer(1)]],
@@ -188,6 +189,7 @@ inline void moe_scatter_buckets_impl(
}
#define DEFINE_MOE_SCATTER_BUCKETS_KERNEL(SUFFIX, ProbT) \
+ [[max_total_threads_per_threadgroup(256)]] \
kernel void moe_scatter_buckets_##SUFFIX( \
device const int* topk_ids [[buffer(0)]], \
device const ProbT* topk_probs [[buffer(1)]], \
@@ -235,6 +237,7 @@ DEFINE_MOE_SCATTER_BUCKETS_KERNEL(bf16, bfloat)
// Map variants
#define DEFINE_MOE_SCATTER_BUCKETS_MAP_KERNEL(SUFFIX, ProbT) \
+ [[max_total_threads_per_threadgroup(256)]] \
kernel void moe_scatter_buckets_map_##SUFFIX( \
device const int* topk_ids [[buffer(0)]], \
device const ProbT* topk_probs [[buffer(1)]], \
diff --git a/crates/uzu/src/backends/metal/kernel/moe/tiles.metal b/crates/uzu/src/backends/metal/kernel/moe/tiles.metal
index ccda7f661..fccd11b52 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/tiles.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/tiles.metal
@@ -5,6 +5,7 @@ using namespace metal;
#define BM 16
// Compute per-expert tile counts: tiles_e = ceil((seg_len)/BM)
+[[max_total_threads_per_threadgroup(256)]]
kernel void moe_tile_counts(
device const uint* offsets [[buffer(0)]], // [E+1]
device uint* tile_counts [[buffer(1)]], // [E]
@@ -22,6 +23,7 @@ kernel void moe_tile_counts(
// Single-threadgroup exclusive scan over tile_counts -> tile_row_offsets, also
// writes total_tiles
+[[max_total_threads_per_threadgroup(256)]]
kernel void moe_tile_scan(
device const uint* tile_counts [[buffer(0)]], // [E]
device uint* tile_row_offsets [[buffer(1)]], // [E+1]
@@ -75,6 +77,7 @@ kernel void moe_tile_scan(
// Build flattened tile_map of length total_tiles; each entry is 3 uints:
// [3*i+0]=expert_idx, [3*i+1]=seg_start, [3*i+2]=tile_m0
+[[max_total_threads_per_threadgroup(256)]]
kernel void moe_build_tile_map(
device const uint* offsets [[buffer(0)]], // [E+1]
device const uint* tile_row_offsets [[buffer(1)]], // [E+1]
@@ -98,6 +101,7 @@ kernel void moe_build_tile_map(
// Write MTLDispatchThreadgroupsIndirectArguments {x, y, z} where:
// x = num_tiles_n (computed on CPU and passed in), y = total_tiles, z = 1
+[[max_total_threads_per_threadgroup(1)]]
kernel void moe_write_dispatch_args(
device const uint* total_tiles_buf
[[buffer(0)]], // [>=1], total_tiles_buf[0] = total_rows
diff --git a/crates/uzu/src/backends/metal/kernel/moe/topk_select.metal b/crates/uzu/src/backends/metal/kernel/moe/topk_select.metal
deleted file mode 100644
index 68cf8b73c..000000000
--- a/crates/uzu/src/backends/metal/kernel/moe/topk_select.metal
+++ /dev/null
@@ -1,170 +0,0 @@
-#include <metal_stdlib>
-#include "../definitions.metal"
-using namespace metal;
-
-constant uint THREADS_PER_TG = 128;
-constant uint MAX_EXPERTS = 512;
-constant uint MAX_TOPK = 128;
-constant float NEG_INF = -INFINITY;
-
-template <typename LogitT, typename ProbT>
-inline void moe_topk_select_impl(
- device const LogitT* logits,
- device int* topk_ids,
- device ProbT* topk_probs,
- uint T,
- uint E,
- uint K,
- uint renorm,
- ushort lid,
- uint token,
- threadgroup float* tg_logits,
- threadgroup uint* tg_indices,
- threadgroup float* reduce_tmp,
- threadgroup uint* reduce_tmp_u,
- threadgroup float* selected_vals,
- threadgroup uint* selected_ids
-) {
- if (token >= T || K == 0) {
- return;
- }
-
- const uint effective_k = min(K, MAX_TOPK);
-
- for (uint i = lid; i < MAX_EXPERTS; i += THREADS_PER_TG) {
- if (i < E) {
- tg_logits[i] = float(logits[token * E + i]);
- } else {
- tg_logits[i] = NEG_INF;
- }
- tg_indices[i] = i;
- }
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- const uint selections = min(effective_k, E);
- for (uint sel = 0; sel < selections; ++sel) {
- float local_best_val = NEG_INF;
- uint local_best_idx = 0xFFFFFFFFu;
-
- for (uint i = lid; i < E; i += THREADS_PER_TG) {
- float candidate = tg_logits[i];
- if (candidate > local_best_val ||
- (candidate == local_best_val && i < local_best_idx)) {
- local_best_val = candidate;
- local_best_idx = i;
- }
- }
-
- float max_val = threadgroup_cooperative_reduce_max<THREADS_PER_TG>(
- local_best_val,
- reduce_tmp,
- lid
- );
-
- uint candidate_id =
- (local_best_val == max_val) ? local_best_idx : 0xFFFFFFFFu;
- uint best_idx = threadgroup_cooperative_reduce_min<THREADS_PER_TG>(
- candidate_id,
- reduce_tmp_u,
- lid
- );
-
- if (lid == 0) {
- selected_vals[sel] = max_val;
- selected_ids[sel] = best_idx;
- tg_logits[best_idx] = NEG_INF;
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
- }
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- if (lid == 0) {
- device int* out_ids = topk_ids + token * K;
- device ProbT* out_probs = topk_probs + token * K;
-
- const uint written = selections;
- for (uint i = 0; i < written; ++i) {
- out_ids[i] = int(selected_ids[i]);
- }
- for (uint i = written; i < K; ++i) {
- out_ids[i] = 0;
- }
-
- if (renorm != 0 && written > 0) {
- float max_logit = -INFINITY;
- for (uint i = 0; i < written; ++i) {
- max_logit = fmax(max_logit, selected_vals[i]);
- }
-
- float exp_values[MAX_TOPK];
- float sum_exp = 0.0f;
- for (uint i = 0; i < written; ++i) {
- float e_val = exp(selected_vals[i] - max_logit);
- exp_values[i] = e_val;
- sum_exp += e_val;
- }
-
- float default_prob = 1.0f / float(written);
- float inv_sum = (sum_exp > 0.0f) ? (1.0f / sum_exp) : default_prob;
-
- for (uint i = 0; i < written; ++i) {
- float prob =
- (sum_exp > 0.0f) ? (exp_values[i] * inv_sum) : default_prob;
- out_probs[i] = ProbT(prob);
- }
- for (uint i = written; i < K; ++i) {
- out_probs[i] = ProbT(0.0f);
- }
- } else {
- for (uint i = 0; i < written; ++i) {
- out_probs[i] = ProbT(selected_vals[i]);
- }
- for (uint i = written; i < K; ++i) {
- out_probs[i] = ProbT(0.0f);
- }
- }
- }
-}
-
-#define DEFINE_MOE_TOPK_KERNEL(LOGIT_T, PROB_T, SUFFIX) \
- kernel void moe_topk_select_##SUFFIX( \
- device const LOGIT_T* logits [[buffer(0)]], \
- device int* topk_ids [[buffer(1)]], \
- device PROB_T* topk_probs [[buffer(2)]], \
- constant uint& T [[buffer(3)]], \
- constant uint& E [[buffer(4)]], \
- constant uint& K [[buffer(5)]], \
- constant uint& renorm [[buffer(6)]], \
- ushort lid [[thread_index_in_threadgroup]], \
- uint3 tgpig [[threadgroup_position_in_grid]] \
- ) { \
- threadgroup float tg_logits[MAX_EXPERTS]; \
- threadgroup uint tg_indices[MAX_EXPERTS]; \
- threadgroup float reduce_tmp[THREADS_PER_TG]; \
- threadgroup uint reduce_tmp_u[THREADS_PER_TG]; \
- threadgroup float selected_vals[MAX_TOPK]; \
- threadgroup uint selected_ids[MAX_TOPK]; \
- moe_topk_select_impl<LOGIT_T, PROB_T>( \
- logits, \
- topk_ids, \
- topk_probs, \
- T, \
- E, \
- K, \
- renorm, \
- lid, \
- tgpig.x, \
- tg_logits, \
- tg_indices, \
- reduce_tmp, \
- reduce_tmp_u, \
- selected_vals, \
- selected_ids \
- ); \
- }
-
-DEFINE_MOE_TOPK_KERNEL(half, half, f16)
-DEFINE_MOE_TOPK_KERNEL(float, half, f32)
-DEFINE_MOE_TOPK_KERNEL(bfloat, bfloat, bf16)
diff --git a/crates/uzu/src/backends/metal/kernel/pooling/pooling.metal b/crates/uzu/src/backends/metal/kernel/pooling/pooling.metal
index e2c0eb7f2..2ec92926c 100644
--- a/crates/uzu/src/backends/metal/kernel/pooling/pooling.metal
+++ b/crates/uzu/src/backends/metal/kernel/pooling/pooling.metal
@@ -5,7 +5,8 @@ using namespace metal;
// CLS pooling: Extract first token [batch, seq_len, hidden_dim] → [batch,
// hidden_dim]
template <typename T>
-kernel void pool_cls(
+[[kernel, max_total_threads_per_threadgroup(256)]]
+void pool_cls(
const device T* input [[buffer(0)]],
device T* output [[buffer(1)]],
constant int& seq_len [[buffer(2)]],
@@ -25,7 +26,8 @@ kernel void pool_cls(
// Mean pooling: Average across sequence [batch, seq_len, hidden_dim] → [batch,
// hidden_dim]
template <typename T>
-kernel void pool_mean(
+[[kernel, max_total_threads_per_threadgroup(256)]]
+void pool_mean(
const device T* input [[buffer(0)]],
device T* output [[buffer(1)]],
constant int& seq_len [[buffer(2)]],
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal
index 89772b581..ce9591df8 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal
@@ -1725,7 +1725,7 @@ template <
const int BM = 32,
const int BK = 32,
const int BN = 32>
-[[kernel]] void qmm(
+[[kernel, max_total_threads_per_threadgroup(128)]] void qmm(
const device uint32_t* w [[buffer(0)]],
const device T* scales [[buffer(1)]],
const device uint8_t* zero_points
@@ -1777,7 +1777,7 @@ template <
const int BM = 32,
const int BK = 32,
const int BN = 32>
-[[kernel]] void qmm_transposed(
+[[kernel, max_total_threads_per_threadgroup(128)]] void qmm_transposed(
const device uint32_t* w [[buffer(0)]],
const device T* scales [[buffer(1)]],
const device uint8_t* zero_points
@@ -1821,7 +1821,7 @@ template <
}
template <typename T, int group_size, int bits>
-[[kernel]] void qmv(
+[[kernel, max_total_threads_per_threadgroup(64)]] void qmv(
const device uint32_t* w [[buffer(0)]],
const device T* scales [[buffer(1)]],
const device uint8_t* zero_points
@@ -1852,7 +1852,7 @@ template <typename T, int group_size, int bits>
}
template <typename T, int group_size, int bits>
-[[kernel]] void qvm(
+[[kernel, max_total_threads_per_threadgroup(64)]] void qvm(
const device uint32_t* w [[buffer(0)]],
const device T* scales [[buffer(1)]],
const device uint8_t* zero_points
@@ -1883,7 +1883,7 @@ template <typename T, int group_size, int bits>
}
template <typename T, int group_size, int bits>
-[[kernel]] void qmv_fast(
+[[kernel, max_total_threads_per_threadgroup(64)]] void qmv_fast(
const device uint32_t* w [[buffer(0)]],
const device T* scales [[buffer(1)]],
const device uint8_t* zero_points
@@ -4435,7 +4435,7 @@ void qmv_mlp_fused_impl(
}
template <typename T, int group_size, int bits>
-[[kernel]] void qmv_mlp_fused(
+[[kernel, max_total_threads_per_threadgroup(64)]] void qmv_mlp_fused(
const device uint32_t* w [[buffer(0)]],
const device T* scales [[buffer(1)]],
const device uint8_t* zero_points
@@ -4796,7 +4796,7 @@ void qmm_mlp_fused_impl(
}
template <typename T, int group_size, int bits>
-[[kernel]] void qmm_mlp_fused(
+[[kernel, max_total_threads_per_threadgroup(128)]] void qmm_mlp_fused(
const device uint32_t* w [[buffer(0)]],
const device T* scales [[buffer(1)]],
const device uint8_t* zero_points
diff --git a/crates/uzu/src/backends/metal/kernel/rope/rope.metal b/crates/uzu/src/backends/metal/kernel/rope/rope.metal
index 1f61c9b92..20f7f2244 100644
--- a/crates/uzu/src/backends/metal/kernel/rope/rope.metal
+++ b/crates/uzu/src/backends/metal/kernel/rope/rope.metal
@@ -131,7 +131,7 @@ void applyRope(
maxSequenceLength, \
position)
-generateKernels(applyRope)
+generateKernels(32, applyRope)
#undef outerArguments
#undef innerArguments
diff --git a/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal b/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal
index bdb7e0edb..12819218f 100644
--- a/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal
+++ b/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal
@@ -4,7 +4,8 @@
using namespace metal;
template <typename T>
-kernel void short_conv_pack_kernel(
+[[kernel, max_total_threads_per_threadgroup(32)]]
+void short_conv_pack_kernel(
device const T* state_in [[buffer(0)]],
device const T* in_proj [[buffer(1)]],
device T* padded [[buffer(2)]],
@@ -45,7 +46,8 @@ kernel void short_conv_pack_kernel(
}
template <typename T>
-kernel void short_conv_prefill_kernel(
+[[kernel, max_total_threads_per_threadgroup(32)]]
+void short_conv_prefill_kernel(
device const T* padded [[buffer(0)]],
device const T* in_proj [[buffer(1)]],
device const T* w [[buffer(2)]],
@@ -111,7 +113,8 @@ kernel void short_conv_prefill_kernel(
}
template <typename T>
-kernel void short_conv_decode_kernel(
+[[kernel, max_total_threads_per_threadgroup(32)]]
+void short_conv_decode_kernel(
device const T* in_proj [[buffer(0)]],
device const T* w [[buffer(1)]],
device const T* b [[buffer(2)]],
diff --git a/crates/uzu/src/backends/metal/kernel/sigmoid/sigmoid.metal b/crates/uzu/src/backends/metal/kernel/sigmoid/sigmoid.metal
index 8391a045e..a76de23c0 100644
--- a/crates/uzu/src/backends/metal/kernel/sigmoid/sigmoid.metal
+++ b/crates/uzu/src/backends/metal/kernel/sigmoid/sigmoid.metal
@@ -4,7 +4,8 @@ using namespace metal;
// Apply sigmoid element-wise to logits
template <typename T>
-kernel void apply_sigmoid(
+[[kernel, max_total_threads_per_threadgroup(256)]]
+void apply_sigmoid(
const device T* input [[buffer(0)]],
device T* output [[buffer(1)]],
constant int& total_elements [[buffer(2)]],
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.metal b/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.metal
index 369dc79ea..f0d8c0646 100644
--- a/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.metal
+++ b/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.metal
@@ -40,7 +40,8 @@ inline T apply_activation_fn(T x, int activation_type) {
}
template <typename T>
-kernel void conv1d_pack_prefix_kernel(
+[[kernel, max_total_threads_per_threadgroup(32)]]
+void conv1d_pack_prefix_kernel(
device const T* state_in [[buffer(0)]],
device const T* x [[buffer(1)]],
device T* padded [[buffer(2)]],
@@ -70,7 +71,8 @@ kernel void conv1d_pack_prefix_kernel(
}
template <typename T>
-kernel void conv1d_scan_kernel(
+[[kernel, max_total_threads_per_threadgroup(32)]]
+void conv1d_scan_kernel(
device const T* padded [[buffer(0)]], // (prefix+suffix, channels)
device const T* w [[buffer(1)]], // (channels, kernel)
device const T* b [[buffer(2)]], // optional (channels)
@@ -142,7 +144,8 @@ kernel void conv1d_scan_kernel(
}
template <typename T>
-kernel void conv1d_decode_kernel(
+[[kernel, max_total_threads_per_threadgroup(32)]]
+void conv1d_decode_kernel(
device const T* x [[buffer(0)]],
device const T* w [[buffer(1)]],
device const T* b [[buffer(2)]],
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/split_inproj.metal b/crates/uzu/src/backends/metal/kernel/ssm/split_inproj.metal
index ae0066ead..87562ff04 100644
--- a/crates/uzu/src/backends/metal/kernel/ssm/split_inproj.metal
+++ b/crates/uzu/src/backends/metal/kernel/ssm/split_inproj.metal
@@ -4,7 +4,8 @@
using namespace metal;
template <typename T>
-kernel void ssm_split_inproj_kernel(
+[[kernel, max_total_threads_per_threadgroup(256)]]
+void ssm_split_inproj_kernel(
device const T* input [[buffer(0)]],
device T* conv_out [[buffer(1)]],
device T* z_out [[buffer(2)]],
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal b/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal
index 72c0bebbe..a56e0e974 100644
--- a/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal
+++ b/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal
@@ -26,7 +26,8 @@ inline T softplus(T x) {
}
template <typename T>
-kernel void ssd_prefill_kernel_64(
+[[kernel, max_total_threads_per_threadgroup(32)]]
+void ssd_prefill_kernel_64(
device const T* x [[buffer(0)]], // (suffix, h, dh)
device const T* dt_raw [[buffer(1)]], // (suffix, h) - raw dt values
device const T* B [[buffer(2)]], // (suffix, g, n)
@@ -164,7 +165,8 @@ instantiate_ssd_prefill_kernel_64(half, half);
#undef instantiate_ssd_prefill_kernel_64
template <typename T>
-kernel void ssd_prefill_kernel(
+[[kernel, max_total_threads_per_threadgroup(32)]]
+void ssd_prefill_kernel(
device const T* x [[buffer(0)]], // (suffix, h, dh)
device const T* dt_raw [[buffer(1)]], // (suffix, h) - raw dt values
device const T* B [[buffer(2)]], // (suffix, g, n)
@@ -303,7 +305,8 @@ instantiate_ssd_prefill_kernel(half, half);
#undef instantiate_ssd_prefill_kernel
template <typename T>
-kernel void ssd_prefill_kernel_sequential(
+[[kernel, max_total_threads_per_threadgroup(1024)]]
+void ssd_prefill_kernel_sequential(
device const T* x [[buffer(0)]], // (suffix, h, dh)
device const T* dt_raw [[buffer(1)]], // (suffix, h) - raw dt values
device const T* B [[buffer(2)]], // (suffix, g, n)
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal b/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal
index 085fc63eb..10ce2519b 100644
--- a/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal
+++ b/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal
@@ -23,7 +23,8 @@ inline T softplus(T x) {
}
template <typename T>
-kernel void ssd_update_kernel(
+[[kernel, max_total_threads_per_threadgroup(1024)]]
+void ssd_update_kernel(
// Input
device const T* x [[buffer(0)]], // (b, h, dh)
device const T* dt_raw [[buffer(1)]], // (b, h)
diff --git a/crates/uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal b/crates/uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal
index ca865bfb4..b1b791801 100644
--- a/crates/uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal
+++ b/crates/uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal
@@ -26,7 +26,7 @@ void tensorAddBias(
#define innerArguments (input, bias, output, numCols, length, position)
-generateKernels(tensorAddBias)
+generateKernels(32, tensorAddBias)
#undef outerArguments
#undef innerArguments
diff --git a/crates/uzu/src/backends/metal/kernel/tensor_add_swap/tensor_add_swap.metal b/crates/uzu/src/backends/metal/kernel/tensor_add_swap/tensor_add_swap.metal
index 2b7d32ab5..cfc551b1f 100644
--- a/crates/uzu/src/backends/metal/kernel/tensor_add_swap/tensor_add_swap.metal
+++ b/crates/uzu/src/backends/metal/kernel/tensor_add_swap/tensor_add_swap.metal
@@ -26,7 +26,7 @@ void tensorAddSwap(
#define innerArguments (skipBuffer, mainBuffer, length, position)
-generateKernels(tensorAddSwap)
+generateKernels(32, tensorAddSwap)
#undef outerArguments
#undef innerArguments
\ No newline at end of file
diff --git a/crates/uzu/src/backends/metal/kernel/tensor_copy/tensor_copy.metal b/crates/uzu/src/backends/metal/kernel/tensor_copy/tensor_copy.metal
index 6ba08dfd8..a6e11cf6e 100644
--- a/crates/uzu/src/backends/metal/kernel/tensor_copy/tensor_copy.metal
+++ b/crates/uzu/src/backends/metal/kernel/tensor_copy/tensor_copy.metal
@@ -21,7 +21,7 @@ void tensorCopy(
#define innerArguments (sourceBuffer, destinationBuffer, length, position)
-generateKernels(tensorCopy)
+generateKernels(32, tensorCopy)
#undef outerArguments
#undef innerArguments
\ No newline at end of file
diff --git a/crates/uzu/src/backends/metal/kernel/token_copy/token_copy.metal b/crates/uzu/src/backends/metal/kernel/token_copy/token_copy.metal
index cba725880..9b2a1a8e8 100644
--- a/crates/uzu/src/backends/metal/kernel/token_copy/token_copy.metal
+++ b/crates/uzu/src/backends/metal/kernel/token_copy/token_copy.metal
@@ -3,6 +3,7 @@ using namespace metal;
/// Copies a single sampled token (u32) to the token_ids buffer (u64).
/// Used in async pipeline to pass sampled token to next forward pass.
+[[max_total_threads_per_threadgroup(1)]]
kernel void copy_sampled_token(
device const uint32_t* src [[buffer(0)]],
device uint64_t* dst [[buffer(1)]],
@@ -15,6 +16,7 @@ kernel void copy_sampled_token(
/// Copies a single sampled token (u32) to an indexed results buffer (u32).
/// Used in async pipeline to store result for callback access.
+[[max_total_threads_per_threadgroup(1)]]
kernel void copy_token_to_results(
device const uint32_t* src [[buffer(0)]],
device uint32_t* dst [[buffer(1)]],