A high-performance inference engine for AI models
internal metal kernels (#256)
uuuvn260351dbca4b
diff --git a/crates/uzu/build/metal/ast.rs b/crates/uzu/build/metal/ast.rsindex 2eafc5bdb..5d7e49614 100644--- a/crates/uzu/build/metal/ast.rs+++ b/crates/uzu/build/metal/ast.rs@@ -323,6 +323,7 @@ impl MetalTemplateParameter {#[derive(Debug, Serialize, Deserialize)]pub struct MetalKernelInfo {+ pub public: bool,pub name: Box<str>,pub arguments: Box<[MetalArgument]>,pub variants: Option<Box<[MetalTemplateParameter]>>,@@ -357,10 +358,14 @@ impl MetalKernelInfo {self.arguments.iter().any(|a| matches!(a.argument_type(), Ok(MetalArgumentType::Simd)))}- pub fn to_kernel(&self) -> Kernel {+ pub fn to_kernel(&self) -> Option<Kernel> {+ if !self.public {+ return None;+ }+let mut indirect_flag = false;- Kernel {+ Some(Kernel {name: self.name.clone(),parameters: self.variants@@ -403,7 +408,7 @@ impl MetalKernelInfo {_ => None,}).collect(),- }+ })}}@@ -487,6 +492,8 @@ impl MetalKernelInfo {return Ok(None);}+ let public = annotations.iter().any(|(k, _)| k.as_ref() == "dsl.public");+let variants: Box<[_]> = annotations.iter().filter(|(k, _)| k.as_ref() == "dsl.variants")@@ -542,6 +549,7 @@ impl MetalKernelInfo {.collect::<anyhow::Result<Box<[_]>>>()?;Ok(Some(MetalKernelInfo {+ public,name,arguments,variants,diff --git a/crates/uzu/build/metal/bindgen.rs b/crates/uzu/build/metal/bindgen.rsindex 309c6bb7a..b5be8941d 100644--- a/crates/uzu/build/metal/bindgen.rs+++ b/crates/uzu/build/metal/bindgen.rs@@ -18,7 +18,7 @@ use crate::{pub fn bindgen(kernel: &MetalKernelInfo,specialize_indices: &SpecializeBaseIndices,-) -> anyhow::Result<(TokenStream, TokenStream)> {+) -> anyhow::Result<(TokenStream, Option<TokenStream>)> {let kernel_name = kernel.name.as_ref();let trait_name = format_ident!("{kernel_name}Kernel");let struct_name = format_ident!("{kernel_name}MetalKernel");@@ -305,14 +305,24 @@ pub fn bindgen(quote! {}};+ let (maybe_trait_impl, maybe_associate_backend, associated_type) = if kernel.public {+ (+ quote! {crate::backends::common::kernel::#trait_name for},+ quote! { type Backend = crate::backends::metal::Metal; },+ Some(quote! { type #trait_name = #struct_name; }),+ )+ } else {+ (quote! {}, quote! {}, None)+ };+let kernel = quote! {pub struct #struct_name {pipeline: Retained<ProtocolObject<dyn MTLComputePipelineState>>,#(#conditional_buffer_fields,)*}- impl crate::backends::common::kernel::#trait_name for #struct_name {- type Backend = crate::backends::metal::Metal;+ impl #maybe_trait_impl #struct_name {+ #maybe_associate_backendfn new(context: &MetalContext #(, #variants_extra_arguments)* #(, #specialize_args)*) -> Result<Self, MetalError> {let entry_name = #entry_name;@@ -331,7 +341,5 @@ pub fn bindgen(}};- let associated_type = quote! { type #trait_name = #struct_name; };-Ok((kernel, associated_type))}diff --git a/crates/uzu/build/metal/compiler.rs b/crates/uzu/build/metal/compiler.rsindex ef53773b3..41671d934 100644--- a/crates/uzu/build/metal/compiler.rs+++ b/crates/uzu/build/metal/compiler.rs@@ -45,7 +45,7 @@ impl ObjectInfo {.map(|s| s.to_string().into_boxed_str()).collect();- let kernels = self.kernels.iter().map(|ki| ki.to_kernel()).collect();+ let kernels = self.kernels.iter().filter_map(|ki| ki.to_kernel()).collect();(src_rel_path, kernels)}@@ -280,7 +280,9 @@ impl MetalCompiler {super::bindgen::bindgen(k, specialize_indices).with_context(|| format!("cannot generate bindings for {}", k.name))})- .collect::<anyhow::Result<(Vec<TokenStream>, Vec<TokenStream>)>>()?;+ .collect::<anyhow::Result<(Vec<TokenStream>, Vec<Option<TokenStream>>)>>()?;++ let associated_types = associated_types.into_iter().flatten().collect::<Vec<TokenStream>>();let tokens = quote! {use metal::{MTLBuffer, MTLComputeCommandEncoder, MTLComputePipelineState, MTLFunctionConstantValues, MTLSize};diff --git a/crates/uzu/src/backends/metal/kernel/activation/activation.metal b/crates/uzu/src/backends/metal/kernel/activation/activation.metalindex b4f6299b9..074a77e44 100644--- a/crates/uzu/src/backends/metal/kernel/activation/activation.metal+++ b/crates/uzu/src/backends/metal/kernel/activation/activation.metal@@ -5,7 +5,7 @@template <typename T>VARIANTS(T, half, float, bfloat)-KERNEL(Activation) (+PUBLIC KERNEL(Activation) (const device T* input OPTIONAL(!in_place),device T* output,const constant uint& n,diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metalindex 22853c4b5..cba3a797b 100644--- a/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal@@ -39,7 +39,7 @@ template <typename T, uint BK, uint BD>VARIANTS(T, float, half, bfloat)VARIANTS(BK, 16, 32)VARIANTS(BD, 64, 128, 256)-KERNEL(AttentionGemm)(+PUBLIC KERNEL(AttentionGemm)(const device T* q,const device T* k,const device T* v,diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metalindex b599a39a2..7b693dd5f 100644--- a/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal@@ -8,7 +8,7 @@template <typename T, uint HEAD_DIM>VARIANTS(T, float, half, bfloat)VARIANTS(HEAD_DIM, 64, 128, 256)-KERNEL(AttentionSinglePass)(+PUBLIC KERNEL(AttentionSinglePass)(const device T* queries,const device T* keys,const device T* values,diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metalindex cb7ab22ce..9db9a7e86 100644--- a/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal@@ -10,7 +10,7 @@template <typename T, uint HEAD_DIM>VARIANTS(T, float, half, bfloat)VARIANTS(HEAD_DIM, 64, 128, 256)-KERNEL(AttentionTwoPass1)(+PUBLIC KERNEL(AttentionTwoPass1)(const device T* queries,const device T* keys,const device T* values,@@ -192,7 +192,7 @@ KERNEL(AttentionTwoPass1)(template <typename T, uint HEAD_DIM>VARIANTS(T, float, half, bfloat)VARIANTS(HEAD_DIM, 64, 128, 256)-KERNEL(AttentionTwoPass2)(+PUBLIC KERNEL(AttentionTwoPass2)(const device float* partials,const device float* sums,const device float* maxs,diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_update_kv_cache.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_update_kv_cache.metalindex c8918aa12..e28f779b4 100644--- a/crates/uzu/src/backends/metal/kernel/attention/attention_update_kv_cache.metal+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_update_kv_cache.metal@@ -3,7 +3,7 @@template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(AttentionUpdateKVCache)(+PUBLIC KERNEL(AttentionUpdateKVCache)(const device T* rotated_keys OPTIONAL(!keys_in_place),const device T* qkv,device T* key_cache,diff --git a/crates/uzu/src/backends/metal/kernel/audio/audio.metal b/crates/uzu/src/backends/metal/kernel/audio/audio.metalindex 2fb00029a..aff2451b3 100644--- a/crates/uzu/src/backends/metal/kernel/audio/audio.metal+++ b/crates/uzu/src/backends/metal/kernel/audio/audio.metal@@ -483,7 +483,7 @@ void fsq_encode(template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(AudioFsqDecode)(+PUBLIC KERNEL(AudioFsqDecode)(device const int* tokens,device T* out,device const int* lengths,@@ -510,7 +510,7 @@ KERNEL(AudioFsqDecode)(template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(AudioLeakyRelu)(+PUBLIC KERNEL(AudioLeakyRelu)(device const T* input,device T* output,const constant int& n,@@ -522,7 +522,7 @@ KERNEL(AudioLeakyRelu)(template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(AudioTanh)(+PUBLIC KERNEL(AudioTanh)(device const T* input,device T* output,const constant int& n,@@ -533,7 +533,7 @@ KERNEL(AudioTanh)(template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(AudioAdd)(+PUBLIC KERNEL(AudioAdd)(device const T* a,device const T* b,device T* out,@@ -545,7 +545,7 @@ KERNEL(AudioAdd)(template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(AudioScale)(+PUBLIC KERNEL(AudioScale)(device const T* input,device T* output,const constant int& n,@@ -557,7 +557,7 @@ KERNEL(AudioScale)(template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(AudioCausalConv1d)(+PUBLIC KERNEL(AudioCausalConv1d)(device const T* input,device const T* weight,device const T* bias,@@ -590,7 +590,7 @@ KERNEL(AudioCausalConv1d)(template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(AudioCausalConvTranspose1d)(+PUBLIC KERNEL(AudioCausalConvTranspose1d)(device const T* input,device const T* weight,device const T* bias,@@ -625,7 +625,7 @@ KERNEL(AudioCausalConvTranspose1d)(template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(AudioHalfSnake)(+PUBLIC KERNEL(AudioHalfSnake)(device const T* input,device const T* alpha,device T* output,@@ -654,7 +654,7 @@ KERNEL(AudioHalfSnake)(template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(AudioClamp)(+PUBLIC KERNEL(AudioClamp)(device const T* input,device T* output,const constant int& n,@@ -667,7 +667,7 @@ KERNEL(AudioClamp)(template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(AudioConv1d)(+PUBLIC KERNEL(AudioConv1d)(device const T* input,device const T* weight,device const T* bias,@@ -708,7 +708,7 @@ KERNEL(AudioConv1d)(template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(AudioFsqEncode)(+PUBLIC KERNEL(AudioFsqEncode)(device const T* input,device int* tokens,device const int* lengths,diff --git a/crates/uzu/src/backends/metal/kernel/definitions.metal b/crates/uzu/src/backends/metal/kernel/definitions.metalindex 5f48dde72..d6f5a7c8a 100644--- a/crates/uzu/src/backends/metal/kernel/definitions.metal+++ b/crates/uzu/src/backends/metal/kernel/definitions.metal@@ -486,6 +486,7 @@ static T threadgroup_cooperative_reduce_min(#define VARIANTS(TYPENAME, ...) \DSL_META("dsl.variants", #TYPENAME, #__VA_ARGS__)+#define PUBLIC DSL_META("dsl.public")#define KERNEL(NAME) DSL_META("dsl.kernel") void NAME#define SPECIALIZE DSL_META("dsl.specialize")diff --git a/crates/uzu/src/backends/metal/kernel/embedding/full_precision_embedding.metal b/crates/uzu/src/backends/metal/kernel/embedding/full_precision_embedding.metalindex 56672f4e0..acc6e397e 100644--- a/crates/uzu/src/backends/metal/kernel/embedding/full_precision_embedding.metal+++ b/crates/uzu/src/backends/metal/kernel/embedding/full_precision_embedding.metal@@ -5,7 +5,7 @@template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(FullPrecisionEmbeddingLookup) (+PUBLIC KERNEL(FullPrecisionEmbeddingLookup) (const device uint64_t* token_ids, // [batch_size]const device T* weights, // [vocab_size, model_dim]device T* output, // [batch_size, model_dim]diff --git a/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal b/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metalindex 74cfc6113..4ee6a69a7 100644--- a/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal+++ b/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal@@ -9,7 +9,7 @@ enum QuantizationMode : uint {template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(QuantizedEmbeddingLookup) (+PUBLIC KERNEL(QuantizedEmbeddingLookup) (const device uint64_t* token_ids, // [batch_size]const device uint8_t* weights, // [vocab_size, model_dim/packing_divisor] packedconst device T* scales, // [vocab_size, num_groups]diff --git a/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal b/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metalindex 80fe21813..c4c379179 100644--- a/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal+++ b/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal@@ -12,7 +12,7 @@ void swap(device T* buffer, const uint sourceIdx, const uint destIdx) {template <typename T>VARIANTS(T, float, bfloat, half)-KERNEL(KVCacheUpdate) (+PUBLIC KERNEL(KVCacheUpdate) (device T* in_place_keys,device T* in_place_values,const constant uzu::kv_cache_update::Swap* swaps,diff --git a/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal b/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metalindex 74ad16bda..2c5ead11e 100644--- a/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal+++ b/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal@@ -118,7 +118,7 @@ VARIANTS(IN, float, half, bfloat)VARIANTS(SC, float, half, bfloat)VARIANTS(OUT, float, half, bfloat)VARIANTS(ACC, float)-KERNEL(LayerNorm) (+PUBLIC KERNEL(LayerNorm) (const device IN* input OPTIONAL(!in_place),const device SC* scales,device OUT* output,diff --git a/crates/uzu/src/backends/metal/kernel/mask_update/mask_update.metal b/crates/uzu/src/backends/metal/kernel/mask_update/mask_update.metalindex 6fe56237e..b9f61e282 100644--- a/crates/uzu/src/backends/metal/kernel/mask_update/mask_update.metal+++ b/crates/uzu/src/backends/metal/kernel/mask_update/mask_update.metal@@ -3,7 +3,7 @@template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(MaskUpdate) (+PUBLIC KERNEL(MaskUpdate) (device T* mask,constant int& unmask_col,constant int& mask_coldiff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.metalindex 8e606f9de..bbbe12039 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.metal+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.metal@@ -187,7 +187,7 @@ METAL_FUNC void gemm_impl(template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(MatmulGemm)(+PUBLIC KERNEL(MatmulGemm)(const device T* a,const device T* b,device T* d,diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/shaders/gemv.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemv/shaders/gemv.metalindex fef07c3a7..93894971d 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv/shaders/gemv.metal+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv/shaders/gemv.metal@@ -12,7 +12,7 @@ using namespace metal;template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(MatmulGemv)(+PUBLIC KERNEL(MatmulGemv)(const device T* matrix,const device T* input_vector,const device T* output_source OPTIONAL(apply_output_scale_and_accumulate),diff --git a/crates/uzu/src/backends/metal/kernel/matmul/split_k/shaders/steel_gemm_splitk.metal b/crates/uzu/src/backends/metal/kernel/matmul/split_k/shaders/steel_gemm_splitk.metalindex 806683221..bbea73ad3 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/split_k/shaders/steel_gemm_splitk.metal+++ b/crates/uzu/src/backends/metal/kernel/matmul/split_k/shaders/steel_gemm_splitk.metal@@ -261,7 +261,7 @@ METAL_FUNC void gemm_splitk_accum_impl(// DSL kernel entry points///////////////////////////////////////////////////////////////////////////////-KERNEL(MatmulSplitKPartialBfloat16)(+PUBLIC KERNEL(MatmulSplitKPartialBfloat16)(const device bfloat16_t* a,const device bfloat16_t* b,device float* c,@@ -300,7 +300,7 @@ KERNEL(MatmulSplitKPartialBfloat16)();}-KERNEL(MatmulSplitKAccumBfloat16)(+PUBLIC KERNEL(MatmulSplitKAccumBfloat16)(const device float* c_split,device bfloat16_t* d,const constant int& k_partitions,diff --git a/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal b/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metalindex 8a622144b..0e2215a54 100644--- a/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal+++ b/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal@@ -4,7 +4,7 @@template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(MlpGateActMul) (+PUBLIC KERNEL(MlpGateActMul) (const device T* fused_up,device T* hidden,const constant int& h,diff --git a/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal b/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metalindex 3d44d6b05..d86888590 100644--- a/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal@@ -8,7 +8,7 @@// Single-kernel fused: count all experts + scan to offsets// This kernel is launched with SINGLE threadgroup-KERNEL(MoeCountsOffsetsFused)(+PUBLIC KERNEL(MoeCountsOffsetsFused)(device const int* topk_ids,device uint* offsets, // output: exclusive scan [E+1]device uint* sum_k_out, // output: total count [1]diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_single.metalindex bab241e36..2607983e5 100644--- a/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal@@ -11,7 +11,7 @@// ============================================================================template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(MoeExpertsDecodeSinglePassA)(+PUBLIC KERNEL(MoeExpertsDecodeSinglePassA)(device const T* x,device const int* topk_ids,device const T* w13_all,@@ -122,7 +122,7 @@ KERNEL(MoeExpertsDecodeSinglePassA)(// ============================================================================template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(MoeExpertsDecodeSinglePassB)(+PUBLIC KERNEL(MoeExpertsDecodeSinglePassB)(device const float* hidden, // [K, d_ff]device const int* topk_ids, // [K]device const T* topk_probs, // [K]diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metalindex 9ac862664..c7400665f 100644--- a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal@@ -10,7 +10,7 @@ using namespace metal;// Grid: (h_blocks, rows, 1) where h_blocks = ceil(d_ff/4)template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(MoeExpertsDecodePassA)(+PUBLIC KERNEL(MoeExpertsDecodePassA)(device const T* x_perm, // [total_rows, d_model]device const uint* expert_offsets, // [E + 1]device const T* w13_all, // [E, 2*d_ff, d_model]@@ -131,7 +131,7 @@ KERNEL(MoeExpertsDecodePassA)(template <typename T, typename AccumT>VARIANTS(T, float, half, bfloat)VARIANTS(AccumT, float)-KERNEL(MoeExpertsDecodeDownFused2D)(+PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(device const float* hidden, // [total_rows, d_ff] - f32 from Pass Adevice const uint* row_expert_map, // [total_rows] - direct row->expert lookupdevice const T* w2_all, // [E, d_model, d_ff] - layoutdiff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metalindex 6b5d54b9b..92f9ae0c8 100644--- a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal@@ -34,7 +34,7 @@ constant uint PASSA_TG_PAD = 4; // Padding to avoid bank conflictstemplate <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(MoeExpertsPrefillPassA)(+PUBLIC KERNEL(MoeExpertsPrefillPassA)(device const T* x_perm,device const uint* expert_offsets,device const T* w13_all,@@ -390,7 +390,7 @@ constant uint PASSB_TG_PAD = 4; // Padding to avoid bank conflictstemplate <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(MoeExpertsPrefillPassB)(+PUBLIC KERNEL(MoeExpertsPrefillPassB)(device const float* hidden,device const uint* expert_offsets,device const T* w2_all,diff --git a/crates/uzu/src/backends/metal/kernel/moe/finalize.metal b/crates/uzu/src/backends/metal/kernel/moe/finalize.metalindex f26f8117d..435eb592a 100644--- a/crates/uzu/src/backends/metal/kernel/moe/finalize.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/finalize.metal@@ -6,7 +6,7 @@template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(MoeFinalize)(+PUBLIC KERNEL(MoeFinalize)(device const int* tok2row, // [T*K], -1 if droppeddevice const T* probs, // [T*K]device const T* y_partial, // [sum_k, d_model]diff --git a/crates/uzu/src/backends/metal/kernel/moe/gather.metal b/crates/uzu/src/backends/metal/kernel/moe/gather.metalindex 1ef96ec33..74f724df0 100644--- a/crates/uzu/src/backends/metal/kernel/moe/gather.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/gather.metal@@ -9,7 +9,7 @@template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(MoeGatherXPerm2D)(+PUBLIC KERNEL(MoeGatherXPerm2D)(device const T* x,device const int* bucketed_ids,device T* x_perm,@@ -70,7 +70,7 @@ KERNEL(MoeGatherXPerm2D)(template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(MoeGatherXPerm1D)(+PUBLIC KERNEL(MoeGatherXPerm1D)(device const T* x,device const int* bucketed_ids,device T* x_perm,diff --git a/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal b/crates/uzu/src/backends/metal/kernel/moe/router_topk.metalindex a4fcca27e..3c0a29155 100644--- a/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal@@ -10,7 +10,7 @@ constant float NEG_INF = -INFINITY;template <typename ScalarT>VARIANTS(ScalarT, half, bfloat, float)-KERNEL(MoeRouterTopK)(+PUBLIC KERNEL(MoeRouterTopK)(const device ScalarT* input,const device ScalarT* weight,const device ScalarT* bias,diff --git a/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal b/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metalindex ea6d92a79..b32d13f4f 100644--- a/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal@@ -144,7 +144,7 @@ inline void moe_scatter_buckets_impl(// 4A: Compute per-block base offsets from partials// partials layout: index = (block_id * num_tiles + tile_id) * TILE_E + te-KERNEL(MoeBlockBasesFromPartials)(+PUBLIC KERNEL(MoeBlockBasesFromPartials)(device const uint* partials,device uint* block_bases,device uint* block_alloc,@@ -186,7 +186,7 @@ KERNEL(MoeBlockBasesFromPartials)(template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(MoeScatterBuckets)(+PUBLIC KERNEL(MoeScatterBuckets)(device const int* topk_ids,device const T* topk_probs,device const uint* offsets,@@ -234,7 +234,7 @@ KERNEL(MoeScatterBuckets)(// Map variantstemplate <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(MoeScatterBucketsMap)(+PUBLIC KERNEL(MoeScatterBucketsMap)(device const int* topk_ids,device const T* topk_probs,device const uint* offsets,diff --git a/crates/uzu/src/backends/metal/kernel/moe/tiles_map.metal b/crates/uzu/src/backends/metal/kernel/moe/tiles_map.metalindex 5bc8cf5a3..05a4cd5a4 100644--- a/crates/uzu/src/backends/metal/kernel/moe/tiles_map.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/tiles_map.metal@@ -5,7 +5,7 @@#define BM 16// Compute per-expert tile counts: tiles_e = ceil((seg_len)/BM)-KERNEL(MoeTileCounts)(+PUBLIC KERNEL(MoeTileCounts)(device const uint* offsets, // [e+1]device uint* tile_counts, // [e]constant uint& e,@@ -20,7 +20,7 @@ KERNEL(MoeTileCounts)(// Single-threadgroup exclusive scan over tile_counts -> tile_row_offsets, also// writes total_tiles-KERNEL(MoeTileScan)(+PUBLIC KERNEL(MoeTileScan)(device const uint* tile_counts, // [e]device uint* tile_row_offsets, // [e+1]device uint* total_tiles_buf, // [>=2]@@ -73,7 +73,7 @@ KERNEL(MoeTileScan)(// Build flattened tile_map of length total_tiles; each entry is 3 uints:// [3*i+0]=expert_idx, [3*i+1]=seg_start, [3*i+2]=tile_m0-KERNEL(MoeBuildTileMap)(+PUBLIC KERNEL(MoeBuildTileMap)(device const uint* offsets, // [e+1]device const uint* tile_row_offsets, // [e+1]device const uint* tile_counts, // [e]@@ -94,7 +94,7 @@ KERNEL(MoeBuildTileMap)(// Write MTLDispatchThreadgroupsIndirectArguments {x, y, z} where:// x = num_tiles_n (computed on CPU and passed in), y = total_tiles, z = 1-KERNEL(MoeWriteDispatchArgs)(+PUBLIC KERNEL(MoeWriteDispatchArgs)(device const uint*total_tiles_buf, // [>=1], total_tiles_buf[0] = total_rowsdevice uint* dispatch_args, // [3] u32diff --git a/crates/uzu/src/backends/metal/kernel/moe/tiles_pass_a.metal b/crates/uzu/src/backends/metal/kernel/moe/tiles_pass_a.metalindex 28b130777..f0de182c1 100644--- a/crates/uzu/src/backends/metal/kernel/moe/tiles_pass_a.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/tiles_pass_a.metal@@ -4,7 +4,7 @@// === Helper kernels for indirect dispatch of Pass A ===// Count tiles per expert: tiles = (num_rows > 0) ? num_rows * h_blocks : 0-KERNEL(MoePassATileCounts)(+PUBLIC KERNEL(MoePassATileCounts)(device const uint* expert_offsets, // [e+1]device uint* tile_counts, // [e]constant uint& e,@@ -18,7 +18,7 @@ KERNEL(MoePassATileCounts)(}// Exclusive scan of tile_counts to get tile_offsets and total_tiles-KERNEL(MoePassATileScan)(+PUBLIC KERNEL(MoePassATileScan)(device const uint* tile_counts, // [e]device uint* tile_offsets, // [e+1]device uint* total_tiles, // [1]@@ -66,7 +66,7 @@ KERNEL(MoePassATileScan)(}// Build row→expert map: one thread per routed row-KERNEL(MoePassABuildRowMap)(+PUBLIC KERNEL(MoePassABuildRowMap)(device const uint* expert_offsets, // [e+1]device uint* row_expert_map, // [total_rows]constant uint& total_rows,@@ -90,7 +90,7 @@ KERNEL(MoePassABuildRowMap)(}// Build tile map entries from row→expert map-KERNEL(MoePassABuildTileMap)(+PUBLIC KERNEL(MoePassABuildTileMap)(device const uint* expert_offsets, // [E+1]device const uint* tile_offsets, // [E+1]device const uint* row_expert_map, // [total_rows]@@ -117,7 +117,7 @@ KERNEL(MoePassABuildTileMap)(}// Write dispatch args for indirect dispatch (reusable from tiled version)-KERNEL(MoePassAWriteDispatchArgs)(+PUBLIC KERNEL(MoePassAWriteDispatchArgs)(device const uint* total_tiles, // [1]device uint*dispatch_args, // [3] - MTLDispatchThreadgroupsIndirectArgumentsdiff --git a/crates/uzu/src/backends/metal/kernel/pooling/pooling.metal b/crates/uzu/src/backends/metal/kernel/pooling/pooling.metalindex 134c7915a..185ae7d16 100644--- a/crates/uzu/src/backends/metal/kernel/pooling/pooling.metal+++ b/crates/uzu/src/backends/metal/kernel/pooling/pooling.metal@@ -5,7 +5,7 @@// hidden_dim]template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(PoolingCls) (+PUBLIC KERNEL(PoolingCls) (const device T* input,device T* output,constant uint& seq_len,@@ -22,7 +22,7 @@ KERNEL(PoolingCls) (// hidden_dim]template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(PoolingMean) (+PUBLIC KERNEL(PoolingMean) (const device T* input,device T* output,constant uint& seq_len,diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm.metalindex b005e80a5..f020cc70b 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm.metal@@ -6,7 +6,7 @@ template <typename T, int GROUP_SIZE, int BITS>VARIANTS(T, float, half, bfloat)VARIANTS(GROUP_SIZE, 32, 64, 128)VARIANTS(BITS, 4, 8)-KERNEL(QuantizedMatmulQmm)(+PUBLIC KERNEL(QuantizedMatmulQmm)(const device uint32_t* w,const device T* scales,const device uint8_t* zero_points OPTIONAL(use_zero_points),diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metalindex f2b600578..7f26b135c 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal@@ -6,7 +6,7 @@ template <typename T, int GROUP_SIZE, int BITS>VARIANTS(T, float, half, bfloat)VARIANTS(GROUP_SIZE, 32, 64, 128)VARIANTS(BITS, 4, 8)-KERNEL(QuantizedMatmulQmmTransposed)(+PUBLIC KERNEL(QuantizedMatmulQmmTransposed)(const device uint32_t* w,const device T* scales,const device uint8_t* zero_points OPTIONAL(use_zero_points),diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metalindex f1508fa29..012359d6b 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metal@@ -6,7 +6,7 @@ template <typename T, int GROUP_SIZE, int BITS>VARIANTS(T, bfloat)VARIANTS(GROUP_SIZE, 64, 128)VARIANTS(BITS, 4, 8)-KERNEL(QuantizedMatmulQmmTransposed64x64)(+PUBLIC KERNEL(QuantizedMatmulQmmTransposed64x64)(const device uint32_t* w,const device T* scales,const device uint8_t* zero_points OPTIONAL(use_zero_points),diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metalindex 1729a2d54..ea76f8f7c 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal@@ -6,7 +6,7 @@ template <typename T, int GROUP_SIZE, int BITS>VARIANTS(T, float, half, bfloat)VARIANTS(GROUP_SIZE, 32, 64, 128)VARIANTS(BITS, 4, 8)-KERNEL(QuantizedMatmulQmv)(+PUBLIC KERNEL(QuantizedMatmulQmv)(const device uint32_t* w,const device T* scales,const device uint8_t* zero_points OPTIONAL(use_zero_points),diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metalindex fd34f0b56..756cb624a 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal@@ -6,7 +6,7 @@ template <typename T, int GROUP_SIZE, int BITS>VARIANTS(T, float, half, bfloat)VARIANTS(GROUP_SIZE, 32, 64, 128)VARIANTS(BITS, 4, 8)-KERNEL(QuantizedMatmulQmvFast)(+PUBLIC KERNEL(QuantizedMatmulQmvFast)(const device uint32_t* w,const device T* scales,const device uint8_t* zero_points OPTIONAL(use_zero_points),diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qvm.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qvm.metalindex 85d5fa688..e12ac5063 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qvm.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qvm.metal@@ -6,7 +6,7 @@ template <typename T, int GROUP_SIZE, int BITS>VARIANTS(T, float, half, bfloat)VARIANTS(GROUP_SIZE, 32, 64, 128)VARIANTS(BITS, 4, 8)-KERNEL(QuantizedMatmulQvm)(+PUBLIC KERNEL(QuantizedMatmulQvm)(const device uint32_t* w,const device T* scales,const device uint8_t* zero_points OPTIONAL(use_zero_points),diff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/qk_norm.metal b/crates/uzu/src/backends/metal/kernel/rms_norm/qk_norm.metalindex b8b36e953..10cb61900 100644--- a/crates/uzu/src/backends/metal/kernel/rms_norm/qk_norm.metal+++ b/crates/uzu/src/backends/metal/kernel/rms_norm/qk_norm.metal@@ -16,7 +16,7 @@ VARIANTS(InputT, float, half, bfloat)VARIANTS(ScaleT, float, half, bfloat)VARIANTS(OutputT, float, half, bfloat)VARIANTS(AccumT, float, half)-KERNEL(QKNorm)(+PUBLIC KERNEL(QKNorm)(const device InputT* qkv_input OPTIONAL(!in_place),const device ScaleT* scales,device OutputT* qkv_output,diff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metalindex 5d774af77..f846ad0d2 100644--- a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal+++ b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal@@ -12,7 +12,7 @@ VARIANTS(InputT, float, half, bfloat)VARIANTS(ScaleT, float, half, bfloat)VARIANTS(OutputT, float, half, bfloat)VARIANTS(AccumT, float, half)-KERNEL(RMSNorm)(+PUBLIC KERNEL(RMSNorm)(const device InputT* input OPTIONAL(!in_place),const device ScaleT* scales,device OutputT* output,diff --git a/crates/uzu/src/backends/metal/kernel/rope/rope.metal b/crates/uzu/src/backends/metal/kernel/rope/rope.metalindex 3eab900a6..bb4cc6217 100644--- a/crates/uzu/src/backends/metal/kernel/rope/rope.metal+++ b/crates/uzu/src/backends/metal/kernel/rope/rope.metal@@ -20,7 +20,7 @@ inline T applyRopeTransform(template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(Rope)(+PUBLIC KERNEL(Rope)(device const T* qkv, // [suffix_len, (num_heads + 2*num_groups) * head_dim]device const T* cosines, // [max_seq_len, head_dim]device const T* sines, // [max_seq_len, head_dim]diff --git a/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal b/crates/uzu/src/backends/metal/kernel/sampling/argmax.metalindex a72dd6f4d..89d317024 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal+++ b/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal@@ -154,7 +154,7 @@ static ArgmaxPair threadgroup_raking_argmax(template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(ArgmaxSingle)(+PUBLIC KERNEL(ArgmaxSingle)(const device T* logits_data,device uint* final_tokens,constant uint& batch_size,@@ -196,7 +196,7 @@ KERNEL(ArgmaxSingle)(template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(ArgmaxMain)(+PUBLIC KERNEL(ArgmaxMain)(const device T* logits_data,device ArgmaxPair* partial_results,constant uint& batch_size,@@ -240,7 +240,7 @@ KERNEL(ArgmaxMain)(}}-KERNEL(ArgmaxFinal)(+PUBLIC KERNEL(ArgmaxFinal)(const device ArgmaxPair* partial_results,device uint* final_tokens,constant uint& batch_size,diff --git a/crates/uzu/src/backends/metal/kernel/sampling/bitmask.metal b/crates/uzu/src/backends/metal/kernel/sampling/bitmask.metalindex 61ea51c5f..010fa690b 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/bitmask.metal+++ b/crates/uzu/src/backends/metal/kernel/sampling/bitmask.metal@@ -7,7 +7,7 @@template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(Bitmask) (+PUBLIC KERNEL(Bitmask) (device const T* logits OPTIONAL(!in_place),device const uint32_t* bitmask,device T* processed_logits,diff --git a/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metal b/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metalindex 02aa010ac..349e49cbe 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metal+++ b/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metal@@ -8,7 +8,7 @@template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(Gumbel) (+PUBLIC KERNEL(Gumbel) (device const T* logits OPTIONAL(!in_place),device const uint64_t* batch_seeds,device T* processed_logits,diff --git a/crates/uzu/src/backends/metal/kernel/sampling/min_p.metal b/crates/uzu/src/backends/metal/kernel/sampling/min_p.metalindex b00e7ac51..327195b5a 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/min_p.metal+++ b/crates/uzu/src/backends/metal/kernel/sampling/min_p.metal@@ -6,7 +6,7 @@template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(MinP) (+PUBLIC KERNEL(MinP) (device const T* logits OPTIONAL(!in_place),device T* processed_logits,threadgroup float shared_reduce_buffer[BLOCK_SIZE_IN_SIMDS],diff --git a/crates/uzu/src/backends/metal/kernel/sampling/temperature.metal b/crates/uzu/src/backends/metal/kernel/sampling/temperature.metalindex 2855d93ba..cba9ff056 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/temperature.metal+++ b/crates/uzu/src/backends/metal/kernel/sampling/temperature.metal@@ -6,7 +6,7 @@template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(Temperature) (+PUBLIC KERNEL(Temperature) (device const T* logits OPTIONAL(!in_place),device T* processed_logits,constant uint& batch_size,diff --git a/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal b/crates/uzu/src/backends/metal/kernel/sampling/top_k.metalindex 505074c37..5113ee31c 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal+++ b/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal@@ -7,7 +7,7 @@template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(TopK)(+PUBLIC KERNEL(TopK)(device const T* logits OPTIONAL(!in_place),device T* processed_logits,threadgroup float shared_reduce_buffer[BLOCK_SIZE_IN_SIMDS],diff --git a/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal b/crates/uzu/src/backends/metal/kernel/sampling/top_p.metalindex 3e68e12dd..f94ddb033 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal+++ b/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal@@ -8,7 +8,7 @@template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(TopP) (+PUBLIC KERNEL(TopP) (device const T* logits OPTIONAL(!in_place),device T* processed_logits,threadgroup float shared_reduce_buffer[BLOCK_SIZE_IN_SIMDS],diff --git a/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal b/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metalindex f26c71edb..d4088a527 100644--- a/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal+++ b/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal@@ -3,7 +3,7 @@template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(ShortConvPack)(+PUBLIC KERNEL(ShortConvPack)(device const T* state_in,device const T* in_proj,device T* padded,@@ -34,7 +34,7 @@ KERNEL(ShortConvPack)(template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(ShortConvPrefill)(+PUBLIC KERNEL(ShortConvPrefill)(device const T* padded,device const T* in_proj,device const T* w,@@ -95,7 +95,7 @@ KERNEL(ShortConvPrefill)(template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(ShortConvDecode)(+PUBLIC KERNEL(ShortConvDecode)(device const T* in_proj,device const T* w,device const T* b OPTIONAL(has_bias),@@ -154,7 +154,7 @@ KERNEL(ShortConvDecode)(template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(ShortConvTrie)(+PUBLIC KERNEL(ShortConvTrie)(device const T* in_proj,device const T* w,device const T* b OPTIONAL(has_bias),diff --git a/crates/uzu/src/backends/metal/kernel/sigmoid/sigmoid.metal b/crates/uzu/src/backends/metal/kernel/sigmoid/sigmoid.metalindex 890c41e8d..69d5c400a 100644--- a/crates/uzu/src/backends/metal/kernel/sigmoid/sigmoid.metal+++ b/crates/uzu/src/backends/metal/kernel/sigmoid/sigmoid.metal@@ -3,7 +3,7 @@template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(Sigmoid) (+PUBLIC KERNEL(Sigmoid) (const device T* input,device T* output,const constant uint& total_elements,diff --git a/crates/uzu/src/backends/metal/kernel/ssm/conv1d.metal b/crates/uzu/src/backends/metal/kernel/ssm/conv1d.metalindex 2987bc1c9..1ff690c80 100644--- a/crates/uzu/src/backends/metal/kernel/ssm/conv1d.metal+++ b/crates/uzu/src/backends/metal/kernel/ssm/conv1d.metal@@ -4,7 +4,7 @@template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(Conv1dPack)(+PUBLIC KERNEL(Conv1dPack)(device const T* state_in,device const T* x,device T* padded,@@ -28,7 +28,7 @@ KERNEL(Conv1dPack)(template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(Conv1dDecode)(+PUBLIC KERNEL(Conv1dDecode)(device const T* x,device const T* w,device const T* b OPTIONAL(has_bias),@@ -105,7 +105,7 @@ KERNEL(Conv1dDecode)(template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(Conv1dScan)(+PUBLIC KERNEL(Conv1dScan)(device const T* padded, // (prefix+suffix, channels)device const T* w, // (channels, kernel)device const T* b OPTIONAL(has_bias), // optional (channels)diff --git a/crates/uzu/src/backends/metal/kernel/ssm/split_inproj.metal b/crates/uzu/src/backends/metal/kernel/ssm/split_inproj.metalindex a4949bb9e..b6c6709ad 100644--- a/crates/uzu/src/backends/metal/kernel/ssm/split_inproj.metal+++ b/crates/uzu/src/backends/metal/kernel/ssm/split_inproj.metal@@ -3,7 +3,7 @@template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(SplitInProj)(+PUBLIC KERNEL(SplitInProj)(device const T* input,device T* conv_out,device T* z_out,diff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal b/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metalindex 904574cb5..db7910523 100644--- a/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal+++ b/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal@@ -9,7 +9,7 @@ constant ushort SSM_PREFILL_MAX_STATE = 256;template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(SSDPrefill64)(+PUBLIC KERNEL(SSDPrefill64)(device const T* x, // (suffix, h, dh)device const T* dt_raw, // (suffix, h) - raw dt valuesdevice const T* b, // (suffix, g, n)@@ -107,7 +107,7 @@ KERNEL(SSDPrefill64)(template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(SSDPrefill)(+PUBLIC KERNEL(SSDPrefill)(device const T* x, // (suffix, h, dh)device const T* dt_raw, // (suffix, h) - raw dt valuesdevice const T* b, // (suffix, g, n)@@ -207,7 +207,7 @@ KERNEL(SSDPrefill)(template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(SSDPrefillSequential)(+PUBLIC KERNEL(SSDPrefillSequential)(device const T* x, // (suffix, h, dh)device const T* dt_raw, // (suffix, h) - raw dt valuesdevice const T* b, // (suffix, g, n)diff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal b/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metalindex 03e629219..33857f7fb 100644--- a/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal+++ b/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal@@ -6,7 +6,7 @@ using namespace metal;template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(SSDUpdate)(+PUBLIC KERNEL(SSDUpdate)(// Inputdevice const T* x, // (b, h, dh)device const T* dt_raw, // (b, h)diff --git a/crates/uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal b/crates/uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metalindex 4ef13b3eb..48e6dcb6f 100644--- a/crates/uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal+++ b/crates/uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal@@ -3,7 +3,7 @@template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(TensorAddBias)(+PUBLIC KERNEL(TensorAddBias)(const device T* input OPTIONAL(!in_place),const device T* bias,device T* output,diff --git a/crates/uzu/src/backends/metal/kernel/tensor_add_swap/tensor_add_swap.metal b/crates/uzu/src/backends/metal/kernel/tensor_add_swap/tensor_add_swap.metalindex deb5f2de9..3524e2e77 100644--- a/crates/uzu/src/backends/metal/kernel/tensor_add_swap/tensor_add_swap.metal+++ b/crates/uzu/src/backends/metal/kernel/tensor_add_swap/tensor_add_swap.metal@@ -3,7 +3,7 @@template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(TensorAddSwap)(+PUBLIC KERNEL(TensorAddSwap)(device T* skip_buffer,device T* main_buffer,constant uint& length,diff --git a/crates/uzu/src/backends/metal/kernel/tensor_copy/tensor_copy.metal b/crates/uzu/src/backends/metal/kernel/tensor_copy/tensor_copy.metalindex 1bc45bf28..d619ed30b 100644--- a/crates/uzu/src/backends/metal/kernel/tensor_copy/tensor_copy.metal+++ b/crates/uzu/src/backends/metal/kernel/tensor_copy/tensor_copy.metal@@ -3,7 +3,7 @@template <typename T>VARIANTS(T, float, half, bfloat)-KERNEL(TensorCopy)(+PUBLIC KERNEL(TensorCopy)(const device T* src_buffer,device T* dst_buffer,constant uint& length,diff --git a/crates/uzu/src/backends/metal/kernel/token_copy/token_copy.metal b/crates/uzu/src/backends/metal/kernel/token_copy/token_copy.metalindex ea916f2e9..f4ebbf08f 100644--- a/crates/uzu/src/backends/metal/kernel/token_copy/token_copy.metal+++ b/crates/uzu/src/backends/metal/kernel/token_copy/token_copy.metal@@ -3,12 +3,18 @@/// Copies a single sampled token (u32) to the token_ids buffer (u64)./// Used in async pipeline to pass sampled token to next forward pass.-KERNEL(TokenCopySampled)(device const uint32_t* src, device uint64_t* dst) {+PUBLIC KERNEL(TokenCopySampled)(+ device const uint32_t* src,+ device uint64_t* dst+) {dst[0] = static_cast<uint64_t>(src[0]);}/// Copies a single sampled token (u32) to an indexed results buffer (u32)./// Used in async pipeline to store result for callback access.-KERNEL(TokenCopyToResults)(device const uint32_t* src, device uint32_t* dst) {+PUBLIC KERNEL(TokenCopyToResults)(+ device const uint32_t* src,+ device uint32_t* dst+) {dst[0] = src[0];}