A high-performance inference engine for AI models
require [[max_total_threads_per_threadgroup(N)]] in legacy kernels (#88)
uuuvn3cde3efd255a
diff --git a/crates/uzu/build/metal/ast.rs b/crates/uzu/build/metal/ast.rsindex c860d56fb..933d16881 100644--- a/crates/uzu/build/metal/ast.rs+++ b/crates/uzu/build/metal/ast.rs@@ -33,6 +33,8 @@ pub enum MetalAstKind {StringLiteral {value: Box<str>,},+ MetalKernelAttr,+ MetalMaxTotalThreadsPerThreadGroupAttr,Other,}@@ -385,3 +387,69 @@ impl MetalKernelInfo {}))}}++pub fn validate_raw_kernel(node: &MetalAstNode) -> anyhow::Result<()> {+ let (node, is_template) =+ if matches!(node.kind, MetalAstKind::FunctionTemplateDecl) {+ let inner_fn = node.inner.iter().find(|c| {+ matches!(+ c.kind,+ MetalAstKind::FunctionDecl {+ name: _+ }+ )+ });+ match inner_fn {+ Some(n) => (n, true),+ None => return Ok(()),+ }+ } else if matches!(node.kind, MetalAstKind::FunctionDecl { .. }) {+ (node, false)+ } else {+ return Ok(());+ };++ let MetalAstKind::FunctionDecl {+ name,+ } = &node.kind+ else {+ return Ok(());+ };++ let mut has_metal_kernel_attr = false;+ let mut has_max_threads_attr = false;+ let mut has_dsl_kernel = false;++ for child in &node.inner {+ match &child.kind {+ MetalAstKind::MetalKernelAttr => has_metal_kernel_attr = true,+ MetalAstKind::MetalMaxTotalThreadsPerThreadGroupAttr => {+ has_max_threads_attr = true+ },+ MetalAstKind::AnnotateAttr => {+ if let Ok(annotation) = annotation_from_ast_node(child.clone())+ {+ if annotation.first().map(|s| s.as_ref())+ == Some("dsl.kernel")+ {+ has_dsl_kernel = true;+ }+ }+ },+ _ => (),+ }+ }++ if has_metal_kernel_attr && !has_dsl_kernel && !has_max_threads_attr {+ let kind = if is_template {+ "template"+ } else {+ "function"+ };+ bail!(+ "kernel {kind} '{name}' is missing `[[max_total_threads_per_threadgroup(N)]]` attribute! it **MUST** be present and correct!"+ );+ }++ Ok(())+}diff --git a/crates/uzu/build/metal/toolchain.rs b/crates/uzu/build/metal/toolchain.rsindex dc28f26cb..a0682119a 100644--- a/crates/uzu/build/metal/toolchain.rs+++ b/crates/uzu/build/metal/toolchain.rs@@ -10,7 +10,9 @@ use anyhow::{Context, bail};use tempfile::NamedTempFile;use tokio::{io::AsyncWriteExt, process::Command};-use super::ast::{MetalAstKind, MetalAstNode, MetalKernelInfo};+use super::ast::{+ MetalAstKind, MetalAstNode, MetalKernelInfo, validate_raw_kernel,+};#[derive(Debug)]pub enum MetalSdk {@@ -172,6 +174,7 @@ impl MetalToolchain {.args(["-x", "metal"]).arg(format!("-std={}", self.std.to_str())).args(self.extra_options.as_ref())+ .arg("-DDSL_ANALYZE").arg(path).arg("-fsyntax-only").args(["-MMD", "-MF"])@@ -204,6 +207,11 @@ impl MetalToolchain {let source_contents =fs::read_to_string(path).context("cannot read source file")?;+ for node in &ast_root.inner {+ validate_raw_kernel(node)+ .context("validation of legacy (non-dsl) kernel failed")?;+ }+let kernel_infos = ast_root.inner.into_iter()diff --git a/crates/uzu/src/backends/metal/kernel/activation/activation.metal b/crates/uzu/src/backends/metal/kernel/activation/activation.metalindex 283f342ee..b8ff92950 100644--- a/crates/uzu/src/backends/metal/kernel/activation/activation.metal+++ b/crates/uzu/src/backends/metal/kernel/activation/activation.metal@@ -30,7 +30,8 @@ inline T activate(T x, ushort act) {}template <typename T>-[[kernel]] void activation(+[[kernel, max_total_threads_per_threadgroup(256)]]+void activation(const device T* input [[buffer(0)]],device T* output [[buffer(1)]],const constant int& N [[buffer(2)]],diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention.metal b/crates/uzu/src/backends/metal/kernel/attention/attention.metalindex e813e54f3..115926208 100644--- a/crates/uzu/src/backends/metal/kernel/attention/attention.metal+++ b/crates/uzu/src/backends/metal/kernel/attention/attention.metal@@ -866,7 +866,7 @@ void update_kv_cache(max_sequence_length, \position)-generateKernels(update_kv_cache)+generateKernels(64, update_kv_cache)#undef outerArguments#undef innerArgumentsdiff --git a/crates/uzu/src/backends/metal/kernel/definitions.metal b/crates/uzu/src/backends/metal/kernel/definitions.metalindex af5f31887..5ee29bb0d 100644--- a/crates/uzu/src/backends/metal/kernel/definitions.metal+++ b/crates/uzu/src/backends/metal/kernel/definitions.metal@@ -462,7 +462,12 @@ static T threadgroup_cooperative_reduce_min(// MARK: - DSL Annotation Helpers+#ifdef DSL_ANALYZE#define DSL_META(...) [[clang::annotate("", __VA_ARGS__)]]+#else+#define DSL_META(...)+#endif+#define DSL_STR(X) #X#define DSL_XSTR(X) DSL_STR(X)@@ -477,17 +482,19 @@ static T threadgroup_cooperative_reduce_min(// MARK: - Generate Template Kernels-#define generateKernel(functionName, scalarType, outerArgs, innerArgs) \+#define generateKernel(max_threads, functionName, scalarType, outerArgs, innerArgs) \+ [[max_total_threads_per_threadgroup(max_threads)]] \kernel void functionName##_##scalarType outerArgs { functionName innerArgs; }-#define generateKernels(functionName) \- generateKernel(functionName, float, outerArguments(float), innerArguments); \+#define generateKernels(max_threads, functionName) \+ generateKernel(max_threads, functionName, float, outerArguments(float), innerArguments); \generateKernel( \+ max_threads, \functionName, \bfloat, \outerArguments(bfloat), \innerArguments \); \- generateKernel(functionName, half, outerArguments(half), innerArguments);+ generateKernel(max_threads, functionName, half, outerArguments(half), innerArguments);#endif /* definitions_metal */diff --git a/crates/uzu/src/backends/metal/kernel/embedding/full_precision_embedding.metal b/crates/uzu/src/backends/metal/kernel/embedding/full_precision_embedding.metalindex f1602f8ca..ee7667836 100644--- a/crates/uzu/src/backends/metal/kernel/embedding/full_precision_embedding.metal+++ b/crates/uzu/src/backends/metal/kernel/embedding/full_precision_embedding.metal@@ -2,7 +2,8 @@using namespace metal;template <typename T>-[[kernel]] void full_precision_embedding_lookup(+[[kernel, max_total_threads_per_threadgroup(256)]]+void full_precision_embedding_lookup(const device uint64_t* token_ids [[buffer(0)]], // [batch_size]const device T* weights [[buffer(1)]], // [vocab_size, model_dim]device T* output [[buffer(2)]], // [batch_size, model_dim]diff --git a/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal b/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metalindex 071e540e9..354bc6809 100644--- a/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal+++ b/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal@@ -2,7 +2,8 @@using namespace metal;template <typename T, uint PACKING_DIVISOR, bool SIGNED_STORAGE>-[[kernel]] void quantized_embedding_lookup(+[[kernel, max_total_threads_per_threadgroup(256)]]+void quantized_embedding_lookup(const device uint64_t* token_ids [[buffer(0)]], // [batch_size]const device uint8_t* weights[[buffer(1)]], // [vocab_size, model_dim/packing_divisor] packeddiff --git a/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal b/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metalindex c4b0b52f6..b149770e7 100644--- a/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal+++ b/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal@@ -52,7 +52,7 @@ void updateKVCache(headDim, \position)-generateKernels(updateKVCache)+generateKernels(1024, updateKVCache)#undef outerArguments#undef innerArgumentsdiff --git a/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal b/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metalindex 37321f887..593d76575 100644--- a/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal+++ b/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal@@ -128,6 +128,7 @@ void layer_norm_core(// Generate LayerNorm kernels#define DEFINE_LAYER_NORM_KERNEL(IN, SC, OUT, ACC, FULL_LAYER, SUF) \+ [[max_total_threads_per_threadgroup(1024)]] \kernel void layer_norm_##SUF( \const device IN* input [[buffer(0)]], \const device SC* scales [[buffer(1)]], \diff --git a/crates/uzu/src/backends/metal/kernel/mask_update/mask_update.metal b/crates/uzu/src/backends/metal/kernel/mask_update/mask_update.metalindex 7898db785..c00b250a7 100644--- a/crates/uzu/src/backends/metal/kernel/mask_update/mask_update.metal+++ b/crates/uzu/src/backends/metal/kernel/mask_update/mask_update.metal@@ -2,6 +2,7 @@using namespace metal;template <typename T>+[[max_total_threads_per_threadgroup(1)]]kernel void update_attention_mask(device T* mask [[buffer(0)]],constant int& unmask_col [[buffer(1)]],diff --git a/crates/uzu/src/backends/metal/kernel/matmul/split_k/shaders/steel_gemm_splitk.h b/crates/uzu/src/backends/metal/kernel/matmul/split_k/shaders/steel_gemm_splitk.hindex 24302129b..60d238d98 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/split_k/shaders/steel_gemm_splitk.h+++ b/crates/uzu/src/backends/metal/kernel/matmul/split_k/shaders/steel_gemm_splitk.h@@ -175,7 +175,7 @@ template <typename AccT,typename OutT,typename Epilogue = TransformNone<OutT, AccT>>-[[kernel]] void gemm_splitk_accum(+[[kernel, max_total_threads_per_threadgroup(256)]] void gemm_splitk_accum(const device AccT* C_split [[buffer(0)]],device OutT* D [[buffer(1)]],const constant int& k_partitions [[buffer(2)]],@@ -203,7 +203,7 @@ template <typename AccT,typename OutT,typename Epilogue = TransformAxpby<OutT, AccT>>-[[kernel]] void gemm_splitk_accum_axpby(+[[kernel, max_total_threads_per_threadgroup(256)]] void gemm_splitk_accum_axpby(const device AccT* C_split [[buffer(0)]],device OutT* D [[buffer(1)]],const constant int& k_partitions [[buffer(2)]],diff --git a/crates/uzu/src/backends/metal/kernel/media_kernels/image_ops.metal b/crates/uzu/src/backends/metal/kernel/media_kernels/image_ops.metalindex 1ef155c1c..687acf318 100644--- a/crates/uzu/src/backends/metal/kernel/media_kernels/image_ops.metal+++ b/crates/uzu/src/backends/metal/kernel/media_kernels/image_ops.metal@@ -17,6 +17,7 @@ struct PatchParameters {uint temporalSlices;};+[[max_total_threads_per_threadgroup(1024)]]kernel void scalePadNormalizeImage(texture2d<float, access::sample> inputTexture [[texture(0)]],texture2d<float, access::write> outputTexture [[texture(1)]],@@ -81,6 +82,7 @@ kernel void scalePadNormalizeImage(outputTexture.write(float4(normalizedRgb, 1.0f), gid);}+[[max_total_threads_per_threadgroup(1024)]]kernel void extractImagePatches(texture2d<float, access::read> paddedNormalizedTexture [[texture(0)]],device float* outputBuffer [[buffer(0)]],diff --git a/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal b/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metalindex 9a8c3a95c..81c5b1a25 100644--- a/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal+++ b/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal@@ -32,7 +32,7 @@ inline T activate(T x, ushort act) {}template <typename T>-[[kernel]] void mlp_activation_mul(+[[kernel, max_total_threads_per_threadgroup(64)]] void mlp_activation_mul(const device T* fused_up [[buffer(0)]],device T* hidden [[buffer(1)]],const constant int& H [[buffer(2)]],diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/steel_gemm_splitk_mlp_fused.h b/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/steel_gemm_splitk_mlp_fused.hindex 3d36c9e9e..d37245d71 100644--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/steel_gemm_splitk_mlp_fused.h+++ b/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/steel_gemm_splitk_mlp_fused.h@@ -201,7 +201,7 @@ gemm_splitk_mlp_fused(///////////////////////////////////////////////////////////////////////////////template <typename AccT, typename OutT>-[[kernel]] void gemm_splitk_mlp_fused_accum(+[[kernel, max_total_threads_per_threadgroup(256)]] void gemm_splitk_mlp_fused_accum(const device AccT* C_split [[buffer(0)]],device OutT* D [[buffer(1)]],const constant int& k_partitions [[buffer(2)]],diff --git a/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal b/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metalindex 21ec0fccc..a6e14eb6a 100644--- a/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal@@ -9,6 +9,7 @@ using namespace metal;// Single-kernel fused: count all experts + scan to offsets// This kernel is launched with SINGLE threadgroup+[[max_total_threads_per_threadgroup(128)]]kernel void moe_counts_offsets_fused(device const int* topk_ids [[buffer(0)]],device uint* offsets [[buffer(1)]], // output: exclusive scan [E+1]diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_single.metalindex fe957960f..5f5d2ec28 100644--- a/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal@@ -131,6 +131,7 @@ inline void moe_experts_decode_single_pass_a_impl(}#define MOE_DECODE_SINGLE_PASS_A_KERNEL(DTYPE, DTYPE4, SUFFIX) \+ [[max_total_threads_per_threadgroup(128)]] \kernel void moe_experts_decode_single_pass_a_##SUFFIX( \device const DTYPE* x [[buffer(0)]], \device const int* topk_ids [[buffer(1)]], \@@ -249,6 +250,7 @@ inline void moe_experts_decode_single_pass_b_impl(}#define MOE_DECODE_SINGLE_PASS_B_KERNEL(DTYPE, DTYPE4, SUFFIX) \+ [[max_total_threads_per_threadgroup(256)]] \kernel void moe_experts_decode_single_pass_b_##SUFFIX( \device const float* hidden [[buffer(0)]], \device const int* topk_ids [[buffer(1)]], \diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metalindex ab55ce6c1..8e78049db 100644--- a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal@@ -132,54 +132,10 @@ void moe_experts_decode_pass_a_impl(}}-#define MOE_PASS_A_KERNEL(DTYPE, DTYPE4, SUFFIX) \- kernel void moe_experts_decode_pass_a_##SUFFIX( \- device const DTYPE* X_perm [[buffer(0)]], \- device const uint* expert_offsets [[buffer(1)]], \- device const DTYPE* W13_all [[buffer(2)]], \- device float* hidden_out [[buffer(3)]], \- device const DTYPE* up_biases [[buffer(4)]], \- constant uint& d_model [[buffer(5)]], \- constant uint& d_ff [[buffer(6)]], \- constant uint& E [[buffer(7)]], \- constant float& gate_clip_min [[buffer(8)]], \- constant float& gate_clip_max [[buffer(9)]], \- constant float& up_clip_min [[buffer(10)]], \- constant float& up_clip_max [[buffer(11)]], \- constant float& silu_alpha [[buffer(12)]], \- uint3 tgpig [[threadgroup_position_in_grid]], \- uint simd_gid [[simdgroup_index_in_threadgroup]], \- uint simd_lid [[thread_index_in_simdgroup]] \- ) { \- moe_experts_decode_pass_a_impl<DTYPE, DTYPE4>( \- X_perm, \- expert_offsets, \- W13_all, \- up_biases, \- hidden_out, \- d_model, \- d_ff, \- E, \- gate_clip_min, \- gate_clip_max, \- up_clip_min, \- up_clip_max, \- silu_alpha, \- tgpig.y, \- tgpig.z, \- tgpig.x, \- simd_gid, \- simd_lid \- ); \- }--MOE_PASS_A_KERNEL(bfloat, bfloat4, bf16)-MOE_PASS_A_KERNEL(half, half4, f16)-MOE_PASS_A_KERNEL(float, float4, f32)-// === Helper kernels for indirect dispatch of Pass A ===// Count tiles per expert: tiles = (num_rows > 0) ? num_rows * h_blocks : 0+[[max_total_threads_per_threadgroup(256)]]kernel void moe_pass_a_tile_counts(device const uint* expert_offsets [[buffer(0)]], // [E+1]device uint* tile_counts [[buffer(1)]], // [E]@@ -196,6 +152,7 @@ kernel void moe_pass_a_tile_counts(}// Exclusive scan of tile_counts to get tile_offsets and total_tiles+[[max_total_threads_per_threadgroup(1024)]]kernel void moe_pass_a_tile_scan(device const uint* tile_counts [[buffer(0)]], // [E]device uint* tile_offsets [[buffer(1)]], // [E+1]@@ -243,6 +200,7 @@ kernel void moe_pass_a_tile_scan(}// Build row→expert map: one thread per routed row+[[max_total_threads_per_threadgroup(256)]]kernel void moe_pass_a_build_row_map(device const uint* expert_offsets [[buffer(0)]], // [E+1]device uint* row_expert_map [[buffer(1)]], // [total_rows]@@ -270,6 +228,7 @@ kernel void moe_pass_a_build_row_map(}// Build tile map entries from row→expert map+[[max_total_threads_per_threadgroup(256)]]kernel void moe_pass_a_build_tile_map(device const uint* expert_offsets [[buffer(0)]], // [E+1]device const uint* tile_offsets [[buffer(1)]], // [E+1]@@ -301,6 +260,7 @@ kernel void moe_pass_a_build_tile_map(}// Write dispatch args for indirect dispatch (reusable from tiled version)+[[max_total_threads_per_threadgroup(1)]]kernel void moe_pass_a_write_dispatch_args(device const uint* total_tiles [[buffer(0)]], // [1]device uint* dispatch_args@@ -365,6 +325,7 @@ void moe_experts_decode_pass_a_indirect_impl(}#define MOE_PASS_A_INDIRECT_KERNEL(DTYPE, DTYPE4, SUFFIX) \+ [[max_total_threads_per_threadgroup(128)]] \kernel void moe_experts_decode_pass_a_indirect_##SUFFIX( \device const DTYPE* X_perm [[buffer(0)]], \device const uint* expert_offsets [[buffer(1)]], \@@ -535,6 +496,7 @@ void moe_experts_decode_down_fused_2d_impl(}#define MOE_PASS_B_FUSED_2D_KERNEL(DTYPE, SUFFIX) \+ [[max_total_threads_per_threadgroup(256)]] \kernel void moe_experts_decode_down_fused_2d_##SUFFIX( \device const float* hidden [[buffer(0)]], \device const uint* row_expert_map [[buffer(1)]], \diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metalindex 72076217f..42c13241e 100644--- a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal@@ -407,66 +407,6 @@ inline void pass_a_impl(}}-// ----- Kernel entry points (Pass A) ------#define MOE_PASS_A_KERNEL(DTYPE, SUFFIX) \- kernel void moe_two_pass_prefill_pass_a_##SUFFIX( \- device const DTYPE* X_perm [[buffer(0)]], \- device const uint* expert_offsets [[buffer(1)]], \- device const DTYPE* W13_all [[buffer(2)]], \- device const DTYPE* up_biases [[buffer(3)]], \- device float* hidden_out [[buffer(4)]], \- constant uint& d_model [[buffer(5)]], \- constant uint& d_ff [[buffer(6)]], \- constant uint& E [[buffer(7)]], \- constant float& gate_clip_min [[buffer(8)]], \- constant float& gate_clip_max [[buffer(9)]], \- constant float& up_clip_min [[buffer(10)]], \- constant float& up_clip_max [[buffer(11)]], \- constant float& silu_alpha [[buffer(12)]], \- uint sg_id [[simdgroup_index_in_threadgroup]], \- uint3 threads_per_tg [[threads_per_threadgroup]], \- uint3 tg_pos [[threadgroup_position_in_grid]], \- uint3 local_tid [[thread_position_in_threadgroup]] \- ) { \- using StageT = typename StageStorage<DTYPE>::type; \- threadgroup StageT Xs[PASSA_BM * (PASSA_BK + PASSA_TG_PAD)]; \- threadgroup StageT Wk_up[PASSA_BN * (PASSA_BK + PASSA_TG_PAD)]; \- threadgroup StageT Wk_gate[PASSA_BN * (PASSA_BK + PASSA_TG_PAD)]; \- threadgroup float bias_up[PASSA_BN]; \- threadgroup float bias_gate[PASSA_BN]; \- pass_a_impl<DTYPE>( \- X_perm, \- expert_offsets, \- W13_all, \- up_biases, \- hidden_out, \- d_model, \- d_ff, \- E, \- gate_clip_min, \- gate_clip_max, \- up_clip_min, \- up_clip_max, \- silu_alpha, \- expert_offsets[tg_pos.z], \- tg_pos.z, \- tg_pos.y, \- tg_pos.x, \- sg_id, \- threads_per_tg, \- local_tid, \- Xs, \- Wk_up, \- Wk_gate, \- bias_up, \- bias_gate \- ); \- }--MOE_PASS_A_KERNEL(bfloat, bf16)-MOE_PASS_A_KERNEL(half, f16)-MOE_PASS_A_KERNEL(float, f32)-// Indirect variant (consumes [expert, seg_start, tile_row_offset] triples)template <typename T>inline void pass_a_indirect_impl(@@ -534,6 +474,7 @@ inline void pass_a_indirect_impl(}#define MOE_PASS_A_INDIRECT_KERNEL(DTYPE, SUFFIX) \+ [[max_total_threads_per_threadgroup(128)]] \kernel void moe_two_pass_prefill_pass_a_indirect_##SUFFIX( \device const DTYPE* X_perm [[buffer(0)]], \device const uint* expert_offsets [[buffer(1)]], \@@ -797,52 +738,6 @@ inline void pass_b_impl(}}-// ----- Kernel entry points (Pass B) ------#define MOE_PASS_B_KERNEL(DTYPE, SUFFIX) \- kernel void moe_two_pass_prefill_pass_b_##SUFFIX( \- device const float* hidden [[buffer(0)]], \- device const uint* expert_offsets [[buffer(1)]], \- device const DTYPE* W2_all [[buffer(2)]], \- device const DTYPE* down_biases [[buffer(3)]], \- device DTYPE* output [[buffer(4)]], \- constant uint& d_model [[buffer(5)]], \- constant uint& d_ff [[buffer(6)]], \- constant uint& E [[buffer(7)]], \- uint sg_id [[simdgroup_index_in_threadgroup]], \- uint simd_lid [[thread_index_in_simdgroup]], \- uint3 tg_pos [[threadgroup_position_in_grid]], \- uint3 local_tid [[thread_position_in_threadgroup]] \- ) { \- using StageT = typename StageStorage<DTYPE>::type; \- threadgroup float Hs[PASSB_BM * (PASSB_BK + PASSB_TG_PAD)]; \- threadgroup StageT Wk[PASSB_BN * (PASSB_BK + PASSB_TG_PAD)]; \- threadgroup float bias[PASSB_BN]; \- const uint lin = local_tid.x; \- pass_b_impl<DTYPE>( \- hidden, \- expert_offsets, \- W2_all, \- down_biases, \- output, \- d_model, \- d_ff, \- E, \- tg_pos.z, \- tg_pos.y, \- tg_pos.x, \- sg_id, \- simd_lid, \- lin, \- Hs, \- Wk, \- bias \- ); \- }--MOE_PASS_B_KERNEL(bfloat, bf16)-MOE_PASS_B_KERNEL(half, f16)-MOE_PASS_B_KERNEL(float, f32)-// Indirect varianttemplate <typename T>inline void pass_b_indirect_impl(@@ -894,6 +789,7 @@ inline void pass_b_indirect_impl(}#define MOE_PASS_B_INDIRECT_KERNEL(DTYPE, SUFFIX) \+ [[max_total_threads_per_threadgroup(128)]] \kernel void moe_two_pass_prefill_pass_b_indirect_##SUFFIX( \device const float* hidden [[buffer(0)]], \device const uint* expert_offsets [[buffer(1)]], \diff --git a/crates/uzu/src/backends/metal/kernel/moe/finalize.metal b/crates/uzu/src/backends/metal/kernel/moe/finalize.metalindex 168394b91..a64abac2c 100644--- a/crates/uzu/src/backends/metal/kernel/moe/finalize.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/finalize.metal@@ -57,6 +57,7 @@ inline void moe_finalize_impl(}#define DEFINE_MOE_FINALIZE_KERNEL(SUFFIX, T) \+ [[max_total_threads_per_threadgroup(128)]] \kernel void moe_finalize_##SUFFIX( \device const int* tok2row [[buffer(0)]], \device const T* probs [[buffer(1)]], \diff --git a/crates/uzu/src/backends/metal/kernel/moe/prepare.metal b/crates/uzu/src/backends/metal/kernel/moe/prepare.metalindex fb9589771..b32f844c6 100644--- a/crates/uzu/src/backends/metal/kernel/moe/prepare.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/prepare.metal@@ -108,6 +108,7 @@ void moe_gather_x_perm_1d_impl(// 2D Tiled kernels#define GATHER_2D_KERNEL(DTYPE, SUFFIX) \+ [[max_total_threads_per_threadgroup(256)]] \kernel void moe_gather_x_perm_##SUFFIX##_2d( \device const DTYPE* X [[buffer(0)]], \device const int* bucketed_ids [[buffer(1)]], \@@ -123,6 +124,7 @@ void moe_gather_x_perm_1d_impl(// 1D kernels#define GATHER_1D_KERNEL(DTYPE, SUFFIX) \+ [[max_total_threads_per_threadgroup(256)]] \kernel void moe_gather_x_perm_##SUFFIX( \device const DTYPE* X [[buffer(0)]], \device const int* bucketed_ids [[buffer(1)]], \diff --git a/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal b/crates/uzu/src/backends/metal/kernel/moe/router_topk.metalindex 7b19cf3ce..1cacbde62 100644--- a/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal@@ -143,6 +143,7 @@ inline void moe_router_topk_impl(}#define DEFINE_ROUTER_TOPK_KERNEL(SUFFIX, VEC4, SCALAR, PROB) \+ [[max_total_threads_per_threadgroup(256)]] \kernel void moe_router_topk_##SUFFIX( \const device VEC4* input [[buffer(0)]], \const device VEC4* weight [[buffer(1)]], \diff --git a/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal b/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metalindex 3a2245045..cdb91e750 100644--- a/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal@@ -9,6 +9,7 @@ using namespace metal;// 4A: Compute per-block base offsets from partials// partials layout: index = (block_id * num_tiles + tile_id) * TILE_E + te+[[max_total_threads_per_threadgroup(256)]]kernel void moe_block_bases_from_partials(device const uint* partials [[buffer(0)]],device uint* block_bases [[buffer(1)]],@@ -188,6 +189,7 @@ inline void moe_scatter_buckets_impl(}#define DEFINE_MOE_SCATTER_BUCKETS_KERNEL(SUFFIX, ProbT) \+ [[max_total_threads_per_threadgroup(256)]] \kernel void moe_scatter_buckets_##SUFFIX( \device const int* topk_ids [[buffer(0)]], \device const ProbT* topk_probs [[buffer(1)]], \@@ -235,6 +237,7 @@ DEFINE_MOE_SCATTER_BUCKETS_KERNEL(bf16, bfloat)// Map variants#define DEFINE_MOE_SCATTER_BUCKETS_MAP_KERNEL(SUFFIX, ProbT) \+ [[max_total_threads_per_threadgroup(256)]] \kernel void moe_scatter_buckets_map_##SUFFIX( \device const int* topk_ids [[buffer(0)]], \device const ProbT* topk_probs [[buffer(1)]], \diff --git a/crates/uzu/src/backends/metal/kernel/moe/tiles.metal b/crates/uzu/src/backends/metal/kernel/moe/tiles.metalindex ccda7f661..fccd11b52 100644--- a/crates/uzu/src/backends/metal/kernel/moe/tiles.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/tiles.metal@@ -5,6 +5,7 @@ using namespace metal;#define BM 16// Compute per-expert tile counts: tiles_e = ceil((seg_len)/BM)+[[max_total_threads_per_threadgroup(256)]]kernel void moe_tile_counts(device const uint* offsets [[buffer(0)]], // [E+1]device uint* tile_counts [[buffer(1)]], // [E]@@ -22,6 +23,7 @@ kernel void moe_tile_counts(// Single-threadgroup exclusive scan over tile_counts -> tile_row_offsets, also// writes total_tiles+[[max_total_threads_per_threadgroup(256)]]kernel void moe_tile_scan(device const uint* tile_counts [[buffer(0)]], // [E]device uint* tile_row_offsets [[buffer(1)]], // [E+1]@@ -75,6 +77,7 @@ kernel void moe_tile_scan(// Build flattened tile_map of length total_tiles; each entry is 3 uints:// [3*i+0]=expert_idx, [3*i+1]=seg_start, [3*i+2]=tile_m0+[[max_total_threads_per_threadgroup(256)]]kernel void moe_build_tile_map(device const uint* offsets [[buffer(0)]], // [E+1]device const uint* tile_row_offsets [[buffer(1)]], // [E+1]@@ -98,6 +101,7 @@ kernel void moe_build_tile_map(// Write MTLDispatchThreadgroupsIndirectArguments {x, y, z} where:// x = num_tiles_n (computed on CPU and passed in), y = total_tiles, z = 1+[[max_total_threads_per_threadgroup(1)]]kernel void moe_write_dispatch_args(device const uint* total_tiles_buf[[buffer(0)]], // [>=1], total_tiles_buf[0] = total_rowsdiff --git a/crates/uzu/src/backends/metal/kernel/moe/topk_select.metal b/crates/uzu/src/backends/metal/kernel/moe/topk_select.metaldeleted file mode 100644index 68cf8b73c..000000000--- a/crates/uzu/src/backends/metal/kernel/moe/topk_select.metal+++ /dev/null@@ -1,170 +0,0 @@-#include <metal_stdlib>-#include "../definitions.metal"-using namespace metal;--constant uint THREADS_PER_TG = 128;-constant uint MAX_EXPERTS = 512;-constant uint MAX_TOPK = 128;-constant float NEG_INF = -INFINITY;--template <typename LogitT, typename ProbT>-inline void moe_topk_select_impl(- device const LogitT* logits,- device int* topk_ids,- device ProbT* topk_probs,- uint T,- uint E,- uint K,- uint renorm,- ushort lid,- uint token,- threadgroup float* tg_logits,- threadgroup uint* tg_indices,- threadgroup float* reduce_tmp,- threadgroup uint* reduce_tmp_u,- threadgroup float* selected_vals,- threadgroup uint* selected_ids-) {- if (token >= T || K == 0) {- return;- }-- const uint effective_k = min(K, MAX_TOPK);-- for (uint i = lid; i < MAX_EXPERTS; i += THREADS_PER_TG) {- if (i < E) {- tg_logits[i] = float(logits[token * E + i]);- } else {- tg_logits[i] = NEG_INF;- }- tg_indices[i] = i;- }-- threadgroup_barrier(mem_flags::mem_threadgroup);-- const uint selections = min(effective_k, E);- for (uint sel = 0; sel < selections; ++sel) {- float local_best_val = NEG_INF;- uint local_best_idx = 0xFFFFFFFFu;-- for (uint i = lid; i < E; i += THREADS_PER_TG) {- float candidate = tg_logits[i];- if (candidate > local_best_val ||- (candidate == local_best_val && i < local_best_idx)) {- local_best_val = candidate;- local_best_idx = i;- }- }-- float max_val = threadgroup_cooperative_reduce_max<THREADS_PER_TG>(- local_best_val,- reduce_tmp,- lid- );-- uint candidate_id =- (local_best_val == max_val) ? local_best_idx : 0xFFFFFFFFu;- uint best_idx = threadgroup_cooperative_reduce_min<THREADS_PER_TG>(- candidate_id,- reduce_tmp_u,- lid- );-- if (lid == 0) {- selected_vals[sel] = max_val;- selected_ids[sel] = best_idx;- tg_logits[best_idx] = NEG_INF;- }- threadgroup_barrier(mem_flags::mem_threadgroup);- }-- threadgroup_barrier(mem_flags::mem_threadgroup);-- if (lid == 0) {- device int* out_ids = topk_ids + token * K;- device ProbT* out_probs = topk_probs + token * K;-- const uint written = selections;- for (uint i = 0; i < written; ++i) {- out_ids[i] = int(selected_ids[i]);- }- for (uint i = written; i < K; ++i) {- out_ids[i] = 0;- }-- if (renorm != 0 && written > 0) {- float max_logit = -INFINITY;- for (uint i = 0; i < written; ++i) {- max_logit = fmax(max_logit, selected_vals[i]);- }-- float exp_values[MAX_TOPK];- float sum_exp = 0.0f;- for (uint i = 0; i < written; ++i) {- float e_val = exp(selected_vals[i] - max_logit);- exp_values[i] = e_val;- sum_exp += e_val;- }-- float default_prob = 1.0f / float(written);- float inv_sum = (sum_exp > 0.0f) ? (1.0f / sum_exp) : default_prob;-- for (uint i = 0; i < written; ++i) {- float prob =- (sum_exp > 0.0f) ? (exp_values[i] * inv_sum) : default_prob;- out_probs[i] = ProbT(prob);- }- for (uint i = written; i < K; ++i) {- out_probs[i] = ProbT(0.0f);- }- } else {- for (uint i = 0; i < written; ++i) {- out_probs[i] = ProbT(selected_vals[i]);- }- for (uint i = written; i < K; ++i) {- out_probs[i] = ProbT(0.0f);- }- }- }-}--#define DEFINE_MOE_TOPK_KERNEL(LOGIT_T, PROB_T, SUFFIX) \- kernel void moe_topk_select_##SUFFIX( \- device const LOGIT_T* logits [[buffer(0)]], \- device int* topk_ids [[buffer(1)]], \- device PROB_T* topk_probs [[buffer(2)]], \- constant uint& T [[buffer(3)]], \- constant uint& E [[buffer(4)]], \- constant uint& K [[buffer(5)]], \- constant uint& renorm [[buffer(6)]], \- ushort lid [[thread_index_in_threadgroup]], \- uint3 tgpig [[threadgroup_position_in_grid]] \- ) { \- threadgroup float tg_logits[MAX_EXPERTS]; \- threadgroup uint tg_indices[MAX_EXPERTS]; \- threadgroup float reduce_tmp[THREADS_PER_TG]; \- threadgroup uint reduce_tmp_u[THREADS_PER_TG]; \- threadgroup float selected_vals[MAX_TOPK]; \- threadgroup uint selected_ids[MAX_TOPK]; \- moe_topk_select_impl<LOGIT_T, PROB_T>( \- logits, \- topk_ids, \- topk_probs, \- T, \- E, \- K, \- renorm, \- lid, \- tgpig.x, \- tg_logits, \- tg_indices, \- reduce_tmp, \- reduce_tmp_u, \- selected_vals, \- selected_ids \- ); \- }--DEFINE_MOE_TOPK_KERNEL(half, half, f16)-DEFINE_MOE_TOPK_KERNEL(float, half, f32)-DEFINE_MOE_TOPK_KERNEL(bfloat, bfloat, bf16)diff --git a/crates/uzu/src/backends/metal/kernel/pooling/pooling.metal b/crates/uzu/src/backends/metal/kernel/pooling/pooling.metalindex e2c0eb7f2..2ec92926c 100644--- a/crates/uzu/src/backends/metal/kernel/pooling/pooling.metal+++ b/crates/uzu/src/backends/metal/kernel/pooling/pooling.metal@@ -5,7 +5,8 @@ using namespace metal;// CLS pooling: Extract first token [batch, seq_len, hidden_dim] → [batch,// hidden_dim]template <typename T>-kernel void pool_cls(+[[kernel, max_total_threads_per_threadgroup(256)]]+void pool_cls(const device T* input [[buffer(0)]],device T* output [[buffer(1)]],constant int& seq_len [[buffer(2)]],@@ -25,7 +26,8 @@ kernel void pool_cls(// Mean pooling: Average across sequence [batch, seq_len, hidden_dim] → [batch,// hidden_dim]template <typename T>-kernel void pool_mean(+[[kernel, max_total_threads_per_threadgroup(256)]]+void pool_mean(const device T* input [[buffer(0)]],device T* output [[buffer(1)]],constant int& seq_len [[buffer(2)]],diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metalindex 89772b581..ce9591df8 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal@@ -1725,7 +1725,7 @@ template <const int BM = 32,const int BK = 32,const int BN = 32>-[[kernel]] void qmm(+[[kernel, max_total_threads_per_threadgroup(128)]] void qmm(const device uint32_t* w [[buffer(0)]],const device T* scales [[buffer(1)]],const device uint8_t* zero_points@@ -1777,7 +1777,7 @@ template <const int BM = 32,const int BK = 32,const int BN = 32>-[[kernel]] void qmm_transposed(+[[kernel, max_total_threads_per_threadgroup(128)]] void qmm_transposed(const device uint32_t* w [[buffer(0)]],const device T* scales [[buffer(1)]],const device uint8_t* zero_points@@ -1821,7 +1821,7 @@ template <}template <typename T, int group_size, int bits>-[[kernel]] void qmv(+[[kernel, max_total_threads_per_threadgroup(64)]] void qmv(const device uint32_t* w [[buffer(0)]],const device T* scales [[buffer(1)]],const device uint8_t* zero_points@@ -1852,7 +1852,7 @@ template <typename T, int group_size, int bits>}template <typename T, int group_size, int bits>-[[kernel]] void qvm(+[[kernel, max_total_threads_per_threadgroup(64)]] void qvm(const device uint32_t* w [[buffer(0)]],const device T* scales [[buffer(1)]],const device uint8_t* zero_points@@ -1883,7 +1883,7 @@ template <typename T, int group_size, int bits>}template <typename T, int group_size, int bits>-[[kernel]] void qmv_fast(+[[kernel, max_total_threads_per_threadgroup(64)]] void qmv_fast(const device uint32_t* w [[buffer(0)]],const device T* scales [[buffer(1)]],const device uint8_t* zero_points@@ -4435,7 +4435,7 @@ void qmv_mlp_fused_impl(}template <typename T, int group_size, int bits>-[[kernel]] void qmv_mlp_fused(+[[kernel, max_total_threads_per_threadgroup(64)]] void qmv_mlp_fused(const device uint32_t* w [[buffer(0)]],const device T* scales [[buffer(1)]],const device uint8_t* zero_points@@ -4796,7 +4796,7 @@ void qmm_mlp_fused_impl(}template <typename T, int group_size, int bits>-[[kernel]] void qmm_mlp_fused(+[[kernel, max_total_threads_per_threadgroup(128)]] void qmm_mlp_fused(const device uint32_t* w [[buffer(0)]],const device T* scales [[buffer(1)]],const device uint8_t* zero_pointsdiff --git a/crates/uzu/src/backends/metal/kernel/rope/rope.metal b/crates/uzu/src/backends/metal/kernel/rope/rope.metalindex 1f61c9b92..20f7f2244 100644--- a/crates/uzu/src/backends/metal/kernel/rope/rope.metal+++ b/crates/uzu/src/backends/metal/kernel/rope/rope.metal@@ -131,7 +131,7 @@ void applyRope(maxSequenceLength, \position)-generateKernels(applyRope)+generateKernels(32, applyRope)#undef outerArguments#undef innerArgumentsdiff --git a/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal b/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metalindex bdb7e0edb..12819218f 100644--- a/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal+++ b/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal@@ -4,7 +4,8 @@using namespace metal;template <typename T>-kernel void short_conv_pack_kernel(+[[kernel, max_total_threads_per_threadgroup(32)]]+void short_conv_pack_kernel(device const T* state_in [[buffer(0)]],device const T* in_proj [[buffer(1)]],device T* padded [[buffer(2)]],@@ -45,7 +46,8 @@ kernel void short_conv_pack_kernel(}template <typename T>-kernel void short_conv_prefill_kernel(+[[kernel, max_total_threads_per_threadgroup(32)]]+void short_conv_prefill_kernel(device const T* padded [[buffer(0)]],device const T* in_proj [[buffer(1)]],device const T* w [[buffer(2)]],@@ -111,7 +113,8 @@ kernel void short_conv_prefill_kernel(}template <typename T>-kernel void short_conv_decode_kernel(+[[kernel, max_total_threads_per_threadgroup(32)]]+void short_conv_decode_kernel(device const T* in_proj [[buffer(0)]],device const T* w [[buffer(1)]],device const T* b [[buffer(2)]],diff --git a/crates/uzu/src/backends/metal/kernel/sigmoid/sigmoid.metal b/crates/uzu/src/backends/metal/kernel/sigmoid/sigmoid.metalindex 8391a045e..a76de23c0 100644--- a/crates/uzu/src/backends/metal/kernel/sigmoid/sigmoid.metal+++ b/crates/uzu/src/backends/metal/kernel/sigmoid/sigmoid.metal@@ -4,7 +4,8 @@ using namespace metal;// Apply sigmoid element-wise to logitstemplate <typename T>-kernel void apply_sigmoid(+[[kernel, max_total_threads_per_threadgroup(256)]]+void apply_sigmoid(const device T* input [[buffer(0)]],device T* output [[buffer(1)]],constant int& total_elements [[buffer(2)]],diff --git a/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.metal b/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.metalindex 369dc79ea..f0d8c0646 100644--- a/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.metal+++ b/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.metal@@ -40,7 +40,8 @@ inline T apply_activation_fn(T x, int activation_type) {}template <typename T>-kernel void conv1d_pack_prefix_kernel(+[[kernel, max_total_threads_per_threadgroup(32)]]+void conv1d_pack_prefix_kernel(device const T* state_in [[buffer(0)]],device const T* x [[buffer(1)]],device T* padded [[buffer(2)]],@@ -70,7 +71,8 @@ kernel void conv1d_pack_prefix_kernel(}template <typename T>-kernel void conv1d_scan_kernel(+[[kernel, max_total_threads_per_threadgroup(32)]]+void conv1d_scan_kernel(device const T* padded [[buffer(0)]], // (prefix+suffix, channels)device const T* w [[buffer(1)]], // (channels, kernel)device const T* b [[buffer(2)]], // optional (channels)@@ -142,7 +144,8 @@ kernel void conv1d_scan_kernel(}template <typename T>-kernel void conv1d_decode_kernel(+[[kernel, max_total_threads_per_threadgroup(32)]]+void conv1d_decode_kernel(device const T* x [[buffer(0)]],device const T* w [[buffer(1)]],device const T* b [[buffer(2)]],diff --git a/crates/uzu/src/backends/metal/kernel/ssm/split_inproj.metal b/crates/uzu/src/backends/metal/kernel/ssm/split_inproj.metalindex ae0066ead..87562ff04 100644--- a/crates/uzu/src/backends/metal/kernel/ssm/split_inproj.metal+++ b/crates/uzu/src/backends/metal/kernel/ssm/split_inproj.metal@@ -4,7 +4,8 @@using namespace metal;template <typename T>-kernel void ssm_split_inproj_kernel(+[[kernel, max_total_threads_per_threadgroup(256)]]+void ssm_split_inproj_kernel(device const T* input [[buffer(0)]],device T* conv_out [[buffer(1)]],device T* z_out [[buffer(2)]],diff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal b/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metalindex 72c0bebbe..a56e0e974 100644--- a/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal+++ b/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal@@ -26,7 +26,8 @@ inline T softplus(T x) {}template <typename T>-kernel void ssd_prefill_kernel_64(+[[kernel, max_total_threads_per_threadgroup(32)]]+void ssd_prefill_kernel_64(device const T* x [[buffer(0)]], // (suffix, h, dh)device const T* dt_raw [[buffer(1)]], // (suffix, h) - raw dt valuesdevice const T* B [[buffer(2)]], // (suffix, g, n)@@ -164,7 +165,8 @@ instantiate_ssd_prefill_kernel_64(half, half);#undef instantiate_ssd_prefill_kernel_64template <typename T>-kernel void ssd_prefill_kernel(+[[kernel, max_total_threads_per_threadgroup(32)]]+void ssd_prefill_kernel(device const T* x [[buffer(0)]], // (suffix, h, dh)device const T* dt_raw [[buffer(1)]], // (suffix, h) - raw dt valuesdevice const T* B [[buffer(2)]], // (suffix, g, n)@@ -303,7 +305,8 @@ instantiate_ssd_prefill_kernel(half, half);#undef instantiate_ssd_prefill_kerneltemplate <typename T>-kernel void ssd_prefill_kernel_sequential(+[[kernel, max_total_threads_per_threadgroup(1024)]]+void ssd_prefill_kernel_sequential(device const T* x [[buffer(0)]], // (suffix, h, dh)device const T* dt_raw [[buffer(1)]], // (suffix, h) - raw dt valuesdevice const T* B [[buffer(2)]], // (suffix, g, n)diff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal b/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metalindex 085fc63eb..10ce2519b 100644--- a/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal+++ b/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal@@ -23,7 +23,8 @@ inline T softplus(T x) {}template <typename T>-kernel void ssd_update_kernel(+[[kernel, max_total_threads_per_threadgroup(1024)]]+void ssd_update_kernel(// Inputdevice const T* x [[buffer(0)]], // (b, h, dh)device const T* dt_raw [[buffer(1)]], // (b, h)diff --git a/crates/uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal b/crates/uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metalindex ca865bfb4..b1b791801 100644--- a/crates/uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal+++ b/crates/uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal@@ -26,7 +26,7 @@ void tensorAddBias(#define innerArguments (input, bias, output, numCols, length, position)-generateKernels(tensorAddBias)+generateKernels(32, tensorAddBias)#undef outerArguments#undef innerArgumentsdiff --git a/crates/uzu/src/backends/metal/kernel/tensor_add_swap/tensor_add_swap.metal b/crates/uzu/src/backends/metal/kernel/tensor_add_swap/tensor_add_swap.metalindex 2b7d32ab5..cfc551b1f 100644--- a/crates/uzu/src/backends/metal/kernel/tensor_add_swap/tensor_add_swap.metal+++ b/crates/uzu/src/backends/metal/kernel/tensor_add_swap/tensor_add_swap.metal@@ -26,7 +26,7 @@ void tensorAddSwap(#define innerArguments (skipBuffer, mainBuffer, length, position)-generateKernels(tensorAddSwap)+generateKernels(32, tensorAddSwap)#undef outerArguments#undef innerArguments\ No newline at end of filediff --git a/crates/uzu/src/backends/metal/kernel/tensor_copy/tensor_copy.metal b/crates/uzu/src/backends/metal/kernel/tensor_copy/tensor_copy.metalindex 6ba08dfd8..a6e11cf6e 100644--- a/crates/uzu/src/backends/metal/kernel/tensor_copy/tensor_copy.metal+++ b/crates/uzu/src/backends/metal/kernel/tensor_copy/tensor_copy.metal@@ -21,7 +21,7 @@ void tensorCopy(#define innerArguments (sourceBuffer, destinationBuffer, length, position)-generateKernels(tensorCopy)+generateKernels(32, tensorCopy)#undef outerArguments#undef innerArguments\ No newline at end of filediff --git a/crates/uzu/src/backends/metal/kernel/token_copy/token_copy.metal b/crates/uzu/src/backends/metal/kernel/token_copy/token_copy.metalindex cba725880..9b2a1a8e8 100644--- a/crates/uzu/src/backends/metal/kernel/token_copy/token_copy.metal+++ b/crates/uzu/src/backends/metal/kernel/token_copy/token_copy.metal@@ -3,6 +3,7 @@ using namespace metal;/// Copies a single sampled token (u32) to the token_ids buffer (u64)./// Used in async pipeline to pass sampled token to next forward pass.+[[max_total_threads_per_threadgroup(1)]]kernel void copy_sampled_token(device const uint32_t* src [[buffer(0)]],device uint64_t* dst [[buffer(1)]],@@ -15,6 +16,7 @@ kernel void copy_sampled_token(/// Copies a single sampled token (u32) to an indexed results buffer (u32)./// Used in async pipeline to store result for callback access.+[[max_total_threads_per_threadgroup(1)]]kernel void copy_token_to_results(device const uint32_t* src [[buffer(0)]],device uint32_t* dst [[buffer(1)]],