Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

internal metal kernels (#256)

uuuvn260351dbca4b
diff --git a/crates/uzu/build/metal/ast.rs b/crates/uzu/build/metal/ast.rs
index 2eafc5bdb..5d7e49614 100644
--- a/crates/uzu/build/metal/ast.rs
+++ b/crates/uzu/build/metal/ast.rs
@@ -323,6 +323,7 @@ impl MetalTemplateParameter {
#[derive(Debug, Serialize, Deserialize)]
pub struct MetalKernelInfo {
+ pub public: bool,
pub name: Box<str>,
pub arguments: Box<[MetalArgument]>,
pub variants: Option<Box<[MetalTemplateParameter]>>,
@@ -357,10 +358,14 @@ impl MetalKernelInfo {
self.arguments.iter().any(|a| matches!(a.argument_type(), Ok(MetalArgumentType::Simd)))
}
- pub fn to_kernel(&self) -> Kernel {
+ pub fn to_kernel(&self) -> Option<Kernel> {
+ if !self.public {
+ return None;
+ }
+
let mut indirect_flag = false;
- Kernel {
+ Some(Kernel {
name: self.name.clone(),
parameters: self
.variants
@@ -403,7 +408,7 @@ impl MetalKernelInfo {
_ => None,
})
.collect(),
- }
+ })
}
}
@@ -487,6 +492,8 @@ impl MetalKernelInfo {
return Ok(None);
}
+ let public = annotations.iter().any(|(k, _)| k.as_ref() == "dsl.public");
+
let variants: Box<[_]> = annotations
.iter()
.filter(|(k, _)| k.as_ref() == "dsl.variants")
@@ -542,6 +549,7 @@ impl MetalKernelInfo {
.collect::<anyhow::Result<Box<[_]>>>()?;
Ok(Some(MetalKernelInfo {
+ public,
name,
arguments,
variants,
diff --git a/crates/uzu/build/metal/bindgen.rs b/crates/uzu/build/metal/bindgen.rs
index 309c6bb7a..b5be8941d 100644
--- a/crates/uzu/build/metal/bindgen.rs
+++ b/crates/uzu/build/metal/bindgen.rs
@@ -18,7 +18,7 @@ use crate::{
pub fn bindgen(
kernel: &MetalKernelInfo,
specialize_indices: &SpecializeBaseIndices,
-) -> anyhow::Result<(TokenStream, TokenStream)> {
+) -> anyhow::Result<(TokenStream, Option<TokenStream>)> {
let kernel_name = kernel.name.as_ref();
let trait_name = format_ident!("{kernel_name}Kernel");
let struct_name = format_ident!("{kernel_name}MetalKernel");
@@ -305,14 +305,24 @@ pub fn bindgen(
quote! {}
};
+ let (maybe_trait_impl, maybe_associate_backend, associated_type) = if kernel.public {
+ (
+ quote! {crate::backends::common::kernel::#trait_name for},
+ quote! { type Backend = crate::backends::metal::Metal; },
+ Some(quote! { type #trait_name = #struct_name; }),
+ )
+ } else {
+ (quote! {}, quote! {}, None)
+ };
+
let kernel = quote! {
pub struct #struct_name {
pipeline: Retained<ProtocolObject<dyn MTLComputePipelineState>>,
#(#conditional_buffer_fields,)*
}
- impl crate::backends::common::kernel::#trait_name for #struct_name {
- type Backend = crate::backends::metal::Metal;
+ impl #maybe_trait_impl #struct_name {
+ #maybe_associate_backend
fn new(context: &MetalContext #(, #variants_extra_arguments)* #(, #specialize_args)*) -> Result<Self, MetalError> {
let entry_name = #entry_name;
@@ -331,7 +341,5 @@ pub fn bindgen(
}
};
- let associated_type = quote! { type #trait_name = #struct_name; };
-
Ok((kernel, associated_type))
}
diff --git a/crates/uzu/build/metal/compiler.rs b/crates/uzu/build/metal/compiler.rs
index ef53773b3..41671d934 100644
--- a/crates/uzu/build/metal/compiler.rs
+++ b/crates/uzu/build/metal/compiler.rs
@@ -45,7 +45,7 @@ impl ObjectInfo {
.map(|s| s.to_string().into_boxed_str())
.collect();
- let kernels = self.kernels.iter().map(|ki| ki.to_kernel()).collect();
+ let kernels = self.kernels.iter().filter_map(|ki| ki.to_kernel()).collect();
(src_rel_path, kernels)
}
@@ -280,7 +280,9 @@ impl MetalCompiler {
super::bindgen::bindgen(k, specialize_indices)
.with_context(|| format!("cannot generate bindings for {}", k.name))
})
- .collect::<anyhow::Result<(Vec<TokenStream>, Vec<TokenStream>)>>()?;
+ .collect::<anyhow::Result<(Vec<TokenStream>, Vec<Option<TokenStream>>)>>()?;
+
+ let associated_types = associated_types.into_iter().flatten().collect::<Vec<TokenStream>>();
let tokens = quote! {
use metal::{MTLBuffer, MTLComputeCommandEncoder, MTLComputePipelineState, MTLFunctionConstantValues, MTLSize};
diff --git a/crates/uzu/src/backends/metal/kernel/activation/activation.metal b/crates/uzu/src/backends/metal/kernel/activation/activation.metal
index b4f6299b9..074a77e44 100644
--- a/crates/uzu/src/backends/metal/kernel/activation/activation.metal
+++ b/crates/uzu/src/backends/metal/kernel/activation/activation.metal
@@ -5,7 +5,7 @@
template <typename T>
VARIANTS(T, half, float, bfloat)
-KERNEL(Activation) (
+PUBLIC KERNEL(Activation) (
const device T* input OPTIONAL(!in_place),
device T* output,
const constant uint& n,
diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal
index 22853c4b5..cba3a797b 100644
--- a/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal
+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal
@@ -39,7 +39,7 @@ template <typename T, uint BK, uint BD>
VARIANTS(T, float, half, bfloat)
VARIANTS(BK, 16, 32)
VARIANTS(BD, 64, 128, 256)
-KERNEL(AttentionGemm)(
+PUBLIC KERNEL(AttentionGemm)(
const device T* q,
const device T* k,
const device T* v,
diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal
index b599a39a2..7b693dd5f 100644
--- a/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal
+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal
@@ -8,7 +8,7 @@
template <typename T, uint HEAD_DIM>
VARIANTS(T, float, half, bfloat)
VARIANTS(HEAD_DIM, 64, 128, 256)
-KERNEL(AttentionSinglePass)(
+PUBLIC KERNEL(AttentionSinglePass)(
const device T* queries,
const device T* keys,
const device T* values,
diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal
index cb7ab22ce..9db9a7e86 100644
--- a/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal
+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal
@@ -10,7 +10,7 @@
template <typename T, uint HEAD_DIM>
VARIANTS(T, float, half, bfloat)
VARIANTS(HEAD_DIM, 64, 128, 256)
-KERNEL(AttentionTwoPass1)(
+PUBLIC KERNEL(AttentionTwoPass1)(
const device T* queries,
const device T* keys,
const device T* values,
@@ -192,7 +192,7 @@ KERNEL(AttentionTwoPass1)(
template <typename T, uint HEAD_DIM>
VARIANTS(T, float, half, bfloat)
VARIANTS(HEAD_DIM, 64, 128, 256)
-KERNEL(AttentionTwoPass2)(
+PUBLIC KERNEL(AttentionTwoPass2)(
const device float* partials,
const device float* sums,
const device float* maxs,
diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_update_kv_cache.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_update_kv_cache.metal
index c8918aa12..e28f779b4 100644
--- a/crates/uzu/src/backends/metal/kernel/attention/attention_update_kv_cache.metal
+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_update_kv_cache.metal
@@ -3,7 +3,7 @@
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(AttentionUpdateKVCache)(
+PUBLIC KERNEL(AttentionUpdateKVCache)(
const device T* rotated_keys OPTIONAL(!keys_in_place),
const device T* qkv,
device T* key_cache,
diff --git a/crates/uzu/src/backends/metal/kernel/audio/audio.metal b/crates/uzu/src/backends/metal/kernel/audio/audio.metal
index 2fb00029a..aff2451b3 100644
--- a/crates/uzu/src/backends/metal/kernel/audio/audio.metal
+++ b/crates/uzu/src/backends/metal/kernel/audio/audio.metal
@@ -483,7 +483,7 @@ void fsq_encode(
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(AudioFsqDecode)(
+PUBLIC KERNEL(AudioFsqDecode)(
device const int* tokens,
device T* out,
device const int* lengths,
@@ -510,7 +510,7 @@ KERNEL(AudioFsqDecode)(
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(AudioLeakyRelu)(
+PUBLIC KERNEL(AudioLeakyRelu)(
device const T* input,
device T* output,
const constant int& n,
@@ -522,7 +522,7 @@ KERNEL(AudioLeakyRelu)(
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(AudioTanh)(
+PUBLIC KERNEL(AudioTanh)(
device const T* input,
device T* output,
const constant int& n,
@@ -533,7 +533,7 @@ KERNEL(AudioTanh)(
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(AudioAdd)(
+PUBLIC KERNEL(AudioAdd)(
device const T* a,
device const T* b,
device T* out,
@@ -545,7 +545,7 @@ KERNEL(AudioAdd)(
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(AudioScale)(
+PUBLIC KERNEL(AudioScale)(
device const T* input,
device T* output,
const constant int& n,
@@ -557,7 +557,7 @@ KERNEL(AudioScale)(
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(AudioCausalConv1d)(
+PUBLIC KERNEL(AudioCausalConv1d)(
device const T* input,
device const T* weight,
device const T* bias,
@@ -590,7 +590,7 @@ KERNEL(AudioCausalConv1d)(
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(AudioCausalConvTranspose1d)(
+PUBLIC KERNEL(AudioCausalConvTranspose1d)(
device const T* input,
device const T* weight,
device const T* bias,
@@ -625,7 +625,7 @@ KERNEL(AudioCausalConvTranspose1d)(
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(AudioHalfSnake)(
+PUBLIC KERNEL(AudioHalfSnake)(
device const T* input,
device const T* alpha,
device T* output,
@@ -654,7 +654,7 @@ KERNEL(AudioHalfSnake)(
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(AudioClamp)(
+PUBLIC KERNEL(AudioClamp)(
device const T* input,
device T* output,
const constant int& n,
@@ -667,7 +667,7 @@ KERNEL(AudioClamp)(
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(AudioConv1d)(
+PUBLIC KERNEL(AudioConv1d)(
device const T* input,
device const T* weight,
device const T* bias,
@@ -708,7 +708,7 @@ KERNEL(AudioConv1d)(
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(AudioFsqEncode)(
+PUBLIC KERNEL(AudioFsqEncode)(
device const T* input,
device int* tokens,
device const int* lengths,
diff --git a/crates/uzu/src/backends/metal/kernel/definitions.metal b/crates/uzu/src/backends/metal/kernel/definitions.metal
index 5f48dde72..d6f5a7c8a 100644
--- a/crates/uzu/src/backends/metal/kernel/definitions.metal
+++ b/crates/uzu/src/backends/metal/kernel/definitions.metal
@@ -486,6 +486,7 @@ static T threadgroup_cooperative_reduce_min(
#define VARIANTS(TYPENAME, ...) \
DSL_META("dsl.variants", #TYPENAME, #__VA_ARGS__)
+#define PUBLIC DSL_META("dsl.public")
#define KERNEL(NAME) DSL_META("dsl.kernel") void NAME
#define SPECIALIZE DSL_META("dsl.specialize")
diff --git a/crates/uzu/src/backends/metal/kernel/embedding/full_precision_embedding.metal b/crates/uzu/src/backends/metal/kernel/embedding/full_precision_embedding.metal
index 56672f4e0..acc6e397e 100644
--- a/crates/uzu/src/backends/metal/kernel/embedding/full_precision_embedding.metal
+++ b/crates/uzu/src/backends/metal/kernel/embedding/full_precision_embedding.metal
@@ -5,7 +5,7 @@
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(FullPrecisionEmbeddingLookup) (
+PUBLIC KERNEL(FullPrecisionEmbeddingLookup) (
const device uint64_t* token_ids, // [batch_size]
const device T* weights, // [vocab_size, model_dim]
device T* output, // [batch_size, model_dim]
diff --git a/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal b/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal
index 74cfc6113..4ee6a69a7 100644
--- a/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal
+++ b/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal
@@ -9,7 +9,7 @@ enum QuantizationMode : uint {
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(QuantizedEmbeddingLookup) (
+PUBLIC KERNEL(QuantizedEmbeddingLookup) (
const device uint64_t* token_ids, // [batch_size]
const device uint8_t* weights, // [vocab_size, model_dim/packing_divisor] packed
const device T* scales, // [vocab_size, num_groups]
diff --git a/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal b/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal
index 80fe21813..c4c379179 100644
--- a/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal
+++ b/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal
@@ -12,7 +12,7 @@ void swap(device T* buffer, const uint sourceIdx, const uint destIdx) {
template <typename T>
VARIANTS(T, float, bfloat, half)
-KERNEL(KVCacheUpdate) (
+PUBLIC KERNEL(KVCacheUpdate) (
device T* in_place_keys,
device T* in_place_values,
const constant uzu::kv_cache_update::Swap* swaps,
diff --git a/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal b/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal
index 74ad16bda..2c5ead11e 100644
--- a/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal
+++ b/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal
@@ -118,7 +118,7 @@ VARIANTS(IN, float, half, bfloat)
VARIANTS(SC, float, half, bfloat)
VARIANTS(OUT, float, half, bfloat)
VARIANTS(ACC, float)
-KERNEL(LayerNorm) (
+PUBLIC KERNEL(LayerNorm) (
const device IN* input OPTIONAL(!in_place),
const device SC* scales,
device OUT* output,
diff --git a/crates/uzu/src/backends/metal/kernel/mask_update/mask_update.metal b/crates/uzu/src/backends/metal/kernel/mask_update/mask_update.metal
index 6fe56237e..b9f61e282 100644
--- a/crates/uzu/src/backends/metal/kernel/mask_update/mask_update.metal
+++ b/crates/uzu/src/backends/metal/kernel/mask_update/mask_update.metal
@@ -3,7 +3,7 @@
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(MaskUpdate) (
+PUBLIC KERNEL(MaskUpdate) (
device T* mask,
constant int& unmask_col,
constant int& mask_col
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.metal
index 8e606f9de..bbbe12039 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.metal
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.metal
@@ -187,7 +187,7 @@ METAL_FUNC void gemm_impl(
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(MatmulGemm)(
+PUBLIC KERNEL(MatmulGemm)(
const device T* a,
const device T* b,
device T* d,
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/shaders/gemv.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemv/shaders/gemv.metal
index fef07c3a7..93894971d 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv/shaders/gemv.metal
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv/shaders/gemv.metal
@@ -12,7 +12,7 @@ using namespace metal;
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(MatmulGemv)(
+PUBLIC KERNEL(MatmulGemv)(
const device T* matrix,
const device T* input_vector,
const device T* output_source OPTIONAL(apply_output_scale_and_accumulate),
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/split_k/shaders/steel_gemm_splitk.metal b/crates/uzu/src/backends/metal/kernel/matmul/split_k/shaders/steel_gemm_splitk.metal
index 806683221..bbea73ad3 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/split_k/shaders/steel_gemm_splitk.metal
+++ b/crates/uzu/src/backends/metal/kernel/matmul/split_k/shaders/steel_gemm_splitk.metal
@@ -261,7 +261,7 @@ METAL_FUNC void gemm_splitk_accum_impl(
// DSL kernel entry points
///////////////////////////////////////////////////////////////////////////////
-KERNEL(MatmulSplitKPartialBfloat16)(
+PUBLIC KERNEL(MatmulSplitKPartialBfloat16)(
const device bfloat16_t* a,
const device bfloat16_t* b,
device float* c,
@@ -300,7 +300,7 @@ KERNEL(MatmulSplitKPartialBfloat16)(
);
}
-KERNEL(MatmulSplitKAccumBfloat16)(
+PUBLIC KERNEL(MatmulSplitKAccumBfloat16)(
const device float* c_split,
device bfloat16_t* d,
const constant int& k_partitions,
diff --git a/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal b/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal
index 8a622144b..0e2215a54 100644
--- a/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal
+++ b/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal
@@ -4,7 +4,7 @@
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(MlpGateActMul) (
+PUBLIC KERNEL(MlpGateActMul) (
const device T* fused_up,
device T* hidden,
const constant int& h,
diff --git a/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal b/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal
index 3d44d6b05..d86888590 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal
@@ -8,7 +8,7 @@
// Single-kernel fused: count all experts + scan to offsets
// This kernel is launched with SINGLE threadgroup
-KERNEL(MoeCountsOffsetsFused)(
+PUBLIC KERNEL(MoeCountsOffsetsFused)(
device const int* topk_ids,
device uint* offsets, // output: exclusive scan [E+1]
device uint* sum_k_out, // output: total count [1]
diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal
index bab241e36..2607983e5 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal
@@ -11,7 +11,7 @@
// ============================================================================
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(MoeExpertsDecodeSinglePassA)(
+PUBLIC KERNEL(MoeExpertsDecodeSinglePassA)(
device const T* x,
device const int* topk_ids,
device const T* w13_all,
@@ -122,7 +122,7 @@ KERNEL(MoeExpertsDecodeSinglePassA)(
// ============================================================================
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(MoeExpertsDecodeSinglePassB)(
+PUBLIC KERNEL(MoeExpertsDecodeSinglePassB)(
device const float* hidden, // [K, d_ff]
device const int* topk_ids, // [K]
device const T* topk_probs, // [K]
diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal
index 9ac862664..c7400665f 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal
@@ -10,7 +10,7 @@ using namespace metal;
// Grid: (h_blocks, rows, 1) where h_blocks = ceil(d_ff/4)
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(MoeExpertsDecodePassA)(
+PUBLIC KERNEL(MoeExpertsDecodePassA)(
device const T* x_perm, // [total_rows, d_model]
device const uint* expert_offsets, // [E + 1]
device const T* w13_all, // [E, 2*d_ff, d_model]
@@ -131,7 +131,7 @@ KERNEL(MoeExpertsDecodePassA)(
template <typename T, typename AccumT>
VARIANTS(T, float, half, bfloat)
VARIANTS(AccumT, float)
-KERNEL(MoeExpertsDecodeDownFused2D)(
+PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(
device const float* hidden, // [total_rows, d_ff] - f32 from Pass A
device const uint* row_expert_map, // [total_rows] - direct row->expert lookup
device const T* w2_all, // [E, d_model, d_ff] - layout
diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal
index 6b5d54b9b..92f9ae0c8 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal
@@ -34,7 +34,7 @@ constant uint PASSA_TG_PAD = 4; // Padding to avoid bank conflicts
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(MoeExpertsPrefillPassA)(
+PUBLIC KERNEL(MoeExpertsPrefillPassA)(
device const T* x_perm,
device const uint* expert_offsets,
device const T* w13_all,
@@ -390,7 +390,7 @@ constant uint PASSB_TG_PAD = 4; // Padding to avoid bank conflicts
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(MoeExpertsPrefillPassB)(
+PUBLIC KERNEL(MoeExpertsPrefillPassB)(
device const float* hidden,
device const uint* expert_offsets,
device const T* w2_all,
diff --git a/crates/uzu/src/backends/metal/kernel/moe/finalize.metal b/crates/uzu/src/backends/metal/kernel/moe/finalize.metal
index f26f8117d..435eb592a 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/finalize.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/finalize.metal
@@ -6,7 +6,7 @@
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(MoeFinalize)(
+PUBLIC KERNEL(MoeFinalize)(
device const int* tok2row, // [T*K], -1 if dropped
device const T* probs, // [T*K]
device const T* y_partial, // [sum_k, d_model]
diff --git a/crates/uzu/src/backends/metal/kernel/moe/gather.metal b/crates/uzu/src/backends/metal/kernel/moe/gather.metal
index 1ef96ec33..74f724df0 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/gather.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/gather.metal
@@ -9,7 +9,7 @@
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(MoeGatherXPerm2D)(
+PUBLIC KERNEL(MoeGatherXPerm2D)(
device const T* x,
device const int* bucketed_ids,
device T* x_perm,
@@ -70,7 +70,7 @@ KERNEL(MoeGatherXPerm2D)(
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(MoeGatherXPerm1D)(
+PUBLIC KERNEL(MoeGatherXPerm1D)(
device const T* x,
device const int* bucketed_ids,
device T* x_perm,
diff --git a/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal b/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal
index a4fcca27e..3c0a29155 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal
@@ -10,7 +10,7 @@ constant float NEG_INF = -INFINITY;
template <typename ScalarT>
VARIANTS(ScalarT, half, bfloat, float)
-KERNEL(MoeRouterTopK)(
+PUBLIC KERNEL(MoeRouterTopK)(
const device ScalarT* input,
const device ScalarT* weight,
const device ScalarT* bias,
diff --git a/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal b/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal
index ea6d92a79..b32d13f4f 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal
@@ -144,7 +144,7 @@ inline void moe_scatter_buckets_impl(
// 4A: Compute per-block base offsets from partials
// partials layout: index = (block_id * num_tiles + tile_id) * TILE_E + te
-KERNEL(MoeBlockBasesFromPartials)(
+PUBLIC KERNEL(MoeBlockBasesFromPartials)(
device const uint* partials,
device uint* block_bases,
device uint* block_alloc,
@@ -186,7 +186,7 @@ KERNEL(MoeBlockBasesFromPartials)(
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(MoeScatterBuckets)(
+PUBLIC KERNEL(MoeScatterBuckets)(
device const int* topk_ids,
device const T* topk_probs,
device const uint* offsets,
@@ -234,7 +234,7 @@ KERNEL(MoeScatterBuckets)(
// Map variants
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(MoeScatterBucketsMap)(
+PUBLIC KERNEL(MoeScatterBucketsMap)(
device const int* topk_ids,
device const T* topk_probs,
device const uint* offsets,
diff --git a/crates/uzu/src/backends/metal/kernel/moe/tiles_map.metal b/crates/uzu/src/backends/metal/kernel/moe/tiles_map.metal
index 5bc8cf5a3..05a4cd5a4 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/tiles_map.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/tiles_map.metal
@@ -5,7 +5,7 @@
#define BM 16
// Compute per-expert tile counts: tiles_e = ceil((seg_len)/BM)
-KERNEL(MoeTileCounts)(
+PUBLIC KERNEL(MoeTileCounts)(
device const uint* offsets, // [e+1]
device uint* tile_counts, // [e]
constant uint& e,
@@ -20,7 +20,7 @@ KERNEL(MoeTileCounts)(
// Single-threadgroup exclusive scan over tile_counts -> tile_row_offsets, also
// writes total_tiles
-KERNEL(MoeTileScan)(
+PUBLIC KERNEL(MoeTileScan)(
device const uint* tile_counts, // [e]
device uint* tile_row_offsets, // [e+1]
device uint* total_tiles_buf, // [>=2]
@@ -73,7 +73,7 @@ KERNEL(MoeTileScan)(
// Build flattened tile_map of length total_tiles; each entry is 3 uints:
// [3*i+0]=expert_idx, [3*i+1]=seg_start, [3*i+2]=tile_m0
-KERNEL(MoeBuildTileMap)(
+PUBLIC KERNEL(MoeBuildTileMap)(
device const uint* offsets, // [e+1]
device const uint* tile_row_offsets, // [e+1]
device const uint* tile_counts, // [e]
@@ -94,7 +94,7 @@ KERNEL(MoeBuildTileMap)(
// Write MTLDispatchThreadgroupsIndirectArguments {x, y, z} where:
// x = num_tiles_n (computed on CPU and passed in), y = total_tiles, z = 1
-KERNEL(MoeWriteDispatchArgs)(
+PUBLIC KERNEL(MoeWriteDispatchArgs)(
device const uint*
total_tiles_buf, // [>=1], total_tiles_buf[0] = total_rows
device uint* dispatch_args, // [3] u32
diff --git a/crates/uzu/src/backends/metal/kernel/moe/tiles_pass_a.metal b/crates/uzu/src/backends/metal/kernel/moe/tiles_pass_a.metal
index 28b130777..f0de182c1 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/tiles_pass_a.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/tiles_pass_a.metal
@@ -4,7 +4,7 @@
// === Helper kernels for indirect dispatch of Pass A ===
// Count tiles per expert: tiles = (num_rows > 0) ? num_rows * h_blocks : 0
-KERNEL(MoePassATileCounts)(
+PUBLIC KERNEL(MoePassATileCounts)(
device const uint* expert_offsets, // [e+1]
device uint* tile_counts, // [e]
constant uint& e,
@@ -18,7 +18,7 @@ KERNEL(MoePassATileCounts)(
}
// Exclusive scan of tile_counts to get tile_offsets and total_tiles
-KERNEL(MoePassATileScan)(
+PUBLIC KERNEL(MoePassATileScan)(
device const uint* tile_counts, // [e]
device uint* tile_offsets, // [e+1]
device uint* total_tiles, // [1]
@@ -66,7 +66,7 @@ KERNEL(MoePassATileScan)(
}
// Build row→expert map: one thread per routed row
-KERNEL(MoePassABuildRowMap)(
+PUBLIC KERNEL(MoePassABuildRowMap)(
device const uint* expert_offsets, // [e+1]
device uint* row_expert_map, // [total_rows]
constant uint& total_rows,
@@ -90,7 +90,7 @@ KERNEL(MoePassABuildRowMap)(
}
// Build tile map entries from row→expert map
-KERNEL(MoePassABuildTileMap)(
+PUBLIC KERNEL(MoePassABuildTileMap)(
device const uint* expert_offsets, // [E+1]
device const uint* tile_offsets, // [E+1]
device const uint* row_expert_map, // [total_rows]
@@ -117,7 +117,7 @@ KERNEL(MoePassABuildTileMap)(
}
// Write dispatch args for indirect dispatch (reusable from tiled version)
-KERNEL(MoePassAWriteDispatchArgs)(
+PUBLIC KERNEL(MoePassAWriteDispatchArgs)(
device const uint* total_tiles, // [1]
device uint*
dispatch_args, // [3] - MTLDispatchThreadgroupsIndirectArguments
diff --git a/crates/uzu/src/backends/metal/kernel/pooling/pooling.metal b/crates/uzu/src/backends/metal/kernel/pooling/pooling.metal
index 134c7915a..185ae7d16 100644
--- a/crates/uzu/src/backends/metal/kernel/pooling/pooling.metal
+++ b/crates/uzu/src/backends/metal/kernel/pooling/pooling.metal
@@ -5,7 +5,7 @@
// hidden_dim]
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(PoolingCls) (
+PUBLIC KERNEL(PoolingCls) (
const device T* input,
device T* output,
constant uint& seq_len,
@@ -22,7 +22,7 @@ KERNEL(PoolingCls) (
// hidden_dim]
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(PoolingMean) (
+PUBLIC KERNEL(PoolingMean) (
const device T* input,
device T* output,
constant uint& seq_len,
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm.metal
index b005e80a5..f020cc70b 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm.metal
@@ -6,7 +6,7 @@ template <typename T, int GROUP_SIZE, int BITS>
VARIANTS(T, float, half, bfloat)
VARIANTS(GROUP_SIZE, 32, 64, 128)
VARIANTS(BITS, 4, 8)
-KERNEL(QuantizedMatmulQmm)(
+PUBLIC KERNEL(QuantizedMatmulQmm)(
const device uint32_t* w,
const device T* scales,
const device uint8_t* zero_points OPTIONAL(use_zero_points),
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal
index f2b600578..7f26b135c 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal
@@ -6,7 +6,7 @@ template <typename T, int GROUP_SIZE, int BITS>
VARIANTS(T, float, half, bfloat)
VARIANTS(GROUP_SIZE, 32, 64, 128)
VARIANTS(BITS, 4, 8)
-KERNEL(QuantizedMatmulQmmTransposed)(
+PUBLIC KERNEL(QuantizedMatmulQmmTransposed)(
const device uint32_t* w,
const device T* scales,
const device uint8_t* zero_points OPTIONAL(use_zero_points),
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metal
index f1508fa29..012359d6b 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metal
@@ -6,7 +6,7 @@ template <typename T, int GROUP_SIZE, int BITS>
VARIANTS(T, bfloat)
VARIANTS(GROUP_SIZE, 64, 128)
VARIANTS(BITS, 4, 8)
-KERNEL(QuantizedMatmulQmmTransposed64x64)(
+PUBLIC KERNEL(QuantizedMatmulQmmTransposed64x64)(
const device uint32_t* w,
const device T* scales,
const device uint8_t* zero_points OPTIONAL(use_zero_points),
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal
index 1729a2d54..ea76f8f7c 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal
@@ -6,7 +6,7 @@ template <typename T, int GROUP_SIZE, int BITS>
VARIANTS(T, float, half, bfloat)
VARIANTS(GROUP_SIZE, 32, 64, 128)
VARIANTS(BITS, 4, 8)
-KERNEL(QuantizedMatmulQmv)(
+PUBLIC KERNEL(QuantizedMatmulQmv)(
const device uint32_t* w,
const device T* scales,
const device uint8_t* zero_points OPTIONAL(use_zero_points),
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal
index fd34f0b56..756cb624a 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal
@@ -6,7 +6,7 @@ template <typename T, int GROUP_SIZE, int BITS>
VARIANTS(T, float, half, bfloat)
VARIANTS(GROUP_SIZE, 32, 64, 128)
VARIANTS(BITS, 4, 8)
-KERNEL(QuantizedMatmulQmvFast)(
+PUBLIC KERNEL(QuantizedMatmulQmvFast)(
const device uint32_t* w,
const device T* scales,
const device uint8_t* zero_points OPTIONAL(use_zero_points),
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qvm.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qvm.metal
index 85d5fa688..e12ac5063 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qvm.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qvm.metal
@@ -6,7 +6,7 @@ template <typename T, int GROUP_SIZE, int BITS>
VARIANTS(T, float, half, bfloat)
VARIANTS(GROUP_SIZE, 32, 64, 128)
VARIANTS(BITS, 4, 8)
-KERNEL(QuantizedMatmulQvm)(
+PUBLIC KERNEL(QuantizedMatmulQvm)(
const device uint32_t* w,
const device T* scales,
const device uint8_t* zero_points OPTIONAL(use_zero_points),
diff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/qk_norm.metal b/crates/uzu/src/backends/metal/kernel/rms_norm/qk_norm.metal
index b8b36e953..10cb61900 100644
--- a/crates/uzu/src/backends/metal/kernel/rms_norm/qk_norm.metal
+++ b/crates/uzu/src/backends/metal/kernel/rms_norm/qk_norm.metal
@@ -16,7 +16,7 @@ VARIANTS(InputT, float, half, bfloat)
VARIANTS(ScaleT, float, half, bfloat)
VARIANTS(OutputT, float, half, bfloat)
VARIANTS(AccumT, float, half)
-KERNEL(QKNorm)(
+PUBLIC KERNEL(QKNorm)(
const device InputT* qkv_input OPTIONAL(!in_place),
const device ScaleT* scales,
device OutputT* qkv_output,
diff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
index 5d774af77..f846ad0d2 100644
--- a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
+++ b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
@@ -12,7 +12,7 @@ VARIANTS(InputT, float, half, bfloat)
VARIANTS(ScaleT, float, half, bfloat)
VARIANTS(OutputT, float, half, bfloat)
VARIANTS(AccumT, float, half)
-KERNEL(RMSNorm)(
+PUBLIC KERNEL(RMSNorm)(
const device InputT* input OPTIONAL(!in_place),
const device ScaleT* scales,
device OutputT* output,
diff --git a/crates/uzu/src/backends/metal/kernel/rope/rope.metal b/crates/uzu/src/backends/metal/kernel/rope/rope.metal
index 3eab900a6..bb4cc6217 100644
--- a/crates/uzu/src/backends/metal/kernel/rope/rope.metal
+++ b/crates/uzu/src/backends/metal/kernel/rope/rope.metal
@@ -20,7 +20,7 @@ inline T applyRopeTransform(
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(Rope)(
+PUBLIC KERNEL(Rope)(
device const T* qkv, // [suffix_len, (num_heads + 2*num_groups) * head_dim]
device const T* cosines, // [max_seq_len, head_dim]
device const T* sines, // [max_seq_len, head_dim]
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal b/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal
index a72dd6f4d..89d317024 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal
@@ -154,7 +154,7 @@ static ArgmaxPair threadgroup_raking_argmax(
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(ArgmaxSingle)(
+PUBLIC KERNEL(ArgmaxSingle)(
const device T* logits_data,
device uint* final_tokens,
constant uint& batch_size,
@@ -196,7 +196,7 @@ KERNEL(ArgmaxSingle)(
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(ArgmaxMain)(
+PUBLIC KERNEL(ArgmaxMain)(
const device T* logits_data,
device ArgmaxPair* partial_results,
constant uint& batch_size,
@@ -240,7 +240,7 @@ KERNEL(ArgmaxMain)(
}
}
-KERNEL(ArgmaxFinal)(
+PUBLIC KERNEL(ArgmaxFinal)(
const device ArgmaxPair* partial_results,
device uint* final_tokens,
constant uint& batch_size,
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/bitmask.metal b/crates/uzu/src/backends/metal/kernel/sampling/bitmask.metal
index 61ea51c5f..010fa690b 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/bitmask.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/bitmask.metal
@@ -7,7 +7,7 @@
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(Bitmask) (
+PUBLIC KERNEL(Bitmask) (
device const T* logits OPTIONAL(!in_place),
device const uint32_t* bitmask,
device T* processed_logits,
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metal b/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metal
index 02aa010ac..349e49cbe 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metal
@@ -8,7 +8,7 @@
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(Gumbel) (
+PUBLIC KERNEL(Gumbel) (
device const T* logits OPTIONAL(!in_place),
device const uint64_t* batch_seeds,
device T* processed_logits,
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/min_p.metal b/crates/uzu/src/backends/metal/kernel/sampling/min_p.metal
index b00e7ac51..327195b5a 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/min_p.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/min_p.metal
@@ -6,7 +6,7 @@
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(MinP) (
+PUBLIC KERNEL(MinP) (
device const T* logits OPTIONAL(!in_place),
device T* processed_logits,
threadgroup float shared_reduce_buffer[BLOCK_SIZE_IN_SIMDS],
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/temperature.metal b/crates/uzu/src/backends/metal/kernel/sampling/temperature.metal
index 2855d93ba..cba9ff056 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/temperature.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/temperature.metal
@@ -6,7 +6,7 @@
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(Temperature) (
+PUBLIC KERNEL(Temperature) (
device const T* logits OPTIONAL(!in_place),
device T* processed_logits,
constant uint& batch_size,
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal b/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal
index 505074c37..5113ee31c 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal
@@ -7,7 +7,7 @@
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(TopK)(
+PUBLIC KERNEL(TopK)(
device const T* logits OPTIONAL(!in_place),
device T* processed_logits,
threadgroup float shared_reduce_buffer[BLOCK_SIZE_IN_SIMDS],
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal b/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal
index 3e68e12dd..f94ddb033 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal
@@ -8,7 +8,7 @@
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(TopP) (
+PUBLIC KERNEL(TopP) (
device const T* logits OPTIONAL(!in_place),
device T* processed_logits,
threadgroup float shared_reduce_buffer[BLOCK_SIZE_IN_SIMDS],
diff --git a/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal b/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal
index f26c71edb..d4088a527 100644
--- a/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal
+++ b/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal
@@ -3,7 +3,7 @@
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(ShortConvPack)(
+PUBLIC KERNEL(ShortConvPack)(
device const T* state_in,
device const T* in_proj,
device T* padded,
@@ -34,7 +34,7 @@ KERNEL(ShortConvPack)(
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(ShortConvPrefill)(
+PUBLIC KERNEL(ShortConvPrefill)(
device const T* padded,
device const T* in_proj,
device const T* w,
@@ -95,7 +95,7 @@ KERNEL(ShortConvPrefill)(
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(ShortConvDecode)(
+PUBLIC KERNEL(ShortConvDecode)(
device const T* in_proj,
device const T* w,
device const T* b OPTIONAL(has_bias),
@@ -154,7 +154,7 @@ KERNEL(ShortConvDecode)(
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(ShortConvTrie)(
+PUBLIC KERNEL(ShortConvTrie)(
device const T* in_proj,
device const T* w,
device const T* b OPTIONAL(has_bias),
diff --git a/crates/uzu/src/backends/metal/kernel/sigmoid/sigmoid.metal b/crates/uzu/src/backends/metal/kernel/sigmoid/sigmoid.metal
index 890c41e8d..69d5c400a 100644
--- a/crates/uzu/src/backends/metal/kernel/sigmoid/sigmoid.metal
+++ b/crates/uzu/src/backends/metal/kernel/sigmoid/sigmoid.metal
@@ -3,7 +3,7 @@
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(Sigmoid) (
+PUBLIC KERNEL(Sigmoid) (
const device T* input,
device T* output,
const constant uint& total_elements,
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/conv1d.metal b/crates/uzu/src/backends/metal/kernel/ssm/conv1d.metal
index 2987bc1c9..1ff690c80 100644
--- a/crates/uzu/src/backends/metal/kernel/ssm/conv1d.metal
+++ b/crates/uzu/src/backends/metal/kernel/ssm/conv1d.metal
@@ -4,7 +4,7 @@
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(Conv1dPack)(
+PUBLIC KERNEL(Conv1dPack)(
device const T* state_in,
device const T* x,
device T* padded,
@@ -28,7 +28,7 @@ KERNEL(Conv1dPack)(
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(Conv1dDecode)(
+PUBLIC KERNEL(Conv1dDecode)(
device const T* x,
device const T* w,
device const T* b OPTIONAL(has_bias),
@@ -105,7 +105,7 @@ KERNEL(Conv1dDecode)(
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(Conv1dScan)(
+PUBLIC KERNEL(Conv1dScan)(
device const T* padded, // (prefix+suffix, channels)
device const T* w, // (channels, kernel)
device const T* b OPTIONAL(has_bias), // optional (channels)
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/split_inproj.metal b/crates/uzu/src/backends/metal/kernel/ssm/split_inproj.metal
index a4949bb9e..b6c6709ad 100644
--- a/crates/uzu/src/backends/metal/kernel/ssm/split_inproj.metal
+++ b/crates/uzu/src/backends/metal/kernel/ssm/split_inproj.metal
@@ -3,7 +3,7 @@
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(SplitInProj)(
+PUBLIC KERNEL(SplitInProj)(
device const T* input,
device T* conv_out,
device T* z_out,
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal b/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal
index 904574cb5..db7910523 100644
--- a/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal
+++ b/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal
@@ -9,7 +9,7 @@ constant ushort SSM_PREFILL_MAX_STATE = 256;
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(SSDPrefill64)(
+PUBLIC KERNEL(SSDPrefill64)(
device const T* x, // (suffix, h, dh)
device const T* dt_raw, // (suffix, h) - raw dt values
device const T* b, // (suffix, g, n)
@@ -107,7 +107,7 @@ KERNEL(SSDPrefill64)(
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(SSDPrefill)(
+PUBLIC KERNEL(SSDPrefill)(
device const T* x, // (suffix, h, dh)
device const T* dt_raw, // (suffix, h) - raw dt values
device const T* b, // (suffix, g, n)
@@ -207,7 +207,7 @@ KERNEL(SSDPrefill)(
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(SSDPrefillSequential)(
+PUBLIC KERNEL(SSDPrefillSequential)(
device const T* x, // (suffix, h, dh)
device const T* dt_raw, // (suffix, h) - raw dt values
device const T* b, // (suffix, g, n)
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal b/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal
index 03e629219..33857f7fb 100644
--- a/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal
+++ b/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal
@@ -6,7 +6,7 @@ using namespace metal;
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(SSDUpdate)(
+PUBLIC KERNEL(SSDUpdate)(
// Input
device const T* x, // (b, h, dh)
device const T* dt_raw, // (b, h)
diff --git a/crates/uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal b/crates/uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal
index 4ef13b3eb..48e6dcb6f 100644
--- a/crates/uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal
+++ b/crates/uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal
@@ -3,7 +3,7 @@
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(TensorAddBias)(
+PUBLIC KERNEL(TensorAddBias)(
const device T* input OPTIONAL(!in_place),
const device T* bias,
device T* output,
diff --git a/crates/uzu/src/backends/metal/kernel/tensor_add_swap/tensor_add_swap.metal b/crates/uzu/src/backends/metal/kernel/tensor_add_swap/tensor_add_swap.metal
index deb5f2de9..3524e2e77 100644
--- a/crates/uzu/src/backends/metal/kernel/tensor_add_swap/tensor_add_swap.metal
+++ b/crates/uzu/src/backends/metal/kernel/tensor_add_swap/tensor_add_swap.metal
@@ -3,7 +3,7 @@
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(TensorAddSwap)(
+PUBLIC KERNEL(TensorAddSwap)(
device T* skip_buffer,
device T* main_buffer,
constant uint& length,
diff --git a/crates/uzu/src/backends/metal/kernel/tensor_copy/tensor_copy.metal b/crates/uzu/src/backends/metal/kernel/tensor_copy/tensor_copy.metal
index 1bc45bf28..d619ed30b 100644
--- a/crates/uzu/src/backends/metal/kernel/tensor_copy/tensor_copy.metal
+++ b/crates/uzu/src/backends/metal/kernel/tensor_copy/tensor_copy.metal
@@ -3,7 +3,7 @@
template <typename T>
VARIANTS(T, float, half, bfloat)
-KERNEL(TensorCopy)(
+PUBLIC KERNEL(TensorCopy)(
const device T* src_buffer,
device T* dst_buffer,
constant uint& length,
diff --git a/crates/uzu/src/backends/metal/kernel/token_copy/token_copy.metal b/crates/uzu/src/backends/metal/kernel/token_copy/token_copy.metal
index ea916f2e9..f4ebbf08f 100644
--- a/crates/uzu/src/backends/metal/kernel/token_copy/token_copy.metal
+++ b/crates/uzu/src/backends/metal/kernel/token_copy/token_copy.metal
@@ -3,12 +3,18 @@
/// Copies a single sampled token (u32) to the token_ids buffer (u64).
/// Used in async pipeline to pass sampled token to next forward pass.
-KERNEL(TokenCopySampled)(device const uint32_t* src, device uint64_t* dst) {
+PUBLIC KERNEL(TokenCopySampled)(
+ device const uint32_t* src,
+ device uint64_t* dst
+) {
dst[0] = static_cast<uint64_t>(src[0]);
}
/// Copies a single sampled token (u32) to an indexed results buffer (u32).
/// Used in async pipeline to store result for callback access.
-KERNEL(TokenCopyToResults)(device const uint32_t* src, device uint32_t* dst) {
+PUBLIC KERNEL(TokenCopyToResults)(
+ device const uint32_t* src,
+ device uint32_t* dst
+) {
dst[0] = src[0];
}