A high-performance inference engine for AI models
Metal common cleanup (#310)
uuuvn08cbb23b9f75
diff --git a/crates/uzu/build/metal/ast.rs b/crates/uzu/build/metal/ast.rsindex c8e789d6a..d91893a63 100644--- a/crates/uzu/build/metal/ast.rs+++ b/crates/uzu/build/metal/ast.rs@@ -42,8 +42,6 @@ pub enum MetalAstKind {StringLiteral {value: Box<str>,},- MetalKernelAttr,- MetalMaxTotalThreadsPerThreadGroupAttr,Other,}@@ -570,63 +568,3 @@ impl MetalKernelInfo {}))}}--pub fn validate_raw_kernel(node: &MetalAstNode) -> anyhow::Result<()> {- let (node, is_template) = if matches!(node.kind, MetalAstKind::FunctionTemplateDecl) {- let inner_fn = node.inner.iter().find(|c| {- matches!(- c.kind,- MetalAstKind::FunctionDecl {- name: _- }- )- });- match inner_fn {- Some(n) => (n, true),- None => return Ok(()),- }- } else if matches!(node.kind, MetalAstKind::FunctionDecl { .. }) {- (node, false)- } else {- return Ok(());- };-- let MetalAstKind::FunctionDecl {- name,- } = &node.kind- else {- return Ok(());- };-- let mut has_metal_kernel_attr = false;- let mut has_max_threads_attr = false;- let mut has_dsl_kernel = false;-- for child in &node.inner {- match &child.kind {- MetalAstKind::MetalKernelAttr => has_metal_kernel_attr = true,- MetalAstKind::MetalMaxTotalThreadsPerThreadGroupAttr => has_max_threads_attr = true,- MetalAstKind::AnnotateAttr => {- if let Ok(annotation) = annotation_from_ast_node(child.clone()) {- if annotation.first().map(|s| s.as_ref()) == Some("dsl.kernel") {- has_dsl_kernel = true;- }- }- },- _ => (),- }- }-- if has_metal_kernel_attr && !has_dsl_kernel && !has_max_threads_attr {- let kind = if is_template {- "template"- } else {- "function"- };- bail!(- "kernel {kind} '{name}' is missing `[[max_total_threads_per_threadgroup(N)]]` attribute! it **MUST** be present and correct!"- );- }-- Ok(())-}diff --git a/crates/uzu/build/metal/toolchain.rs b/crates/uzu/build/metal/toolchain.rsindex e0a792d92..b5262ab8a 100644--- a/crates/uzu/build/metal/toolchain.rs+++ b/crates/uzu/build/metal/toolchain.rs@@ -10,7 +10,7 @@ use anyhow::{Context, bail};use tempfile::NamedTempFile;use tokio::{io::AsyncWriteExt, process::Command};-use super::ast::{MetalAstKind, MetalAstNode, MetalKernelInfo, validate_raw_kernel};+use super::ast::{MetalAstKind, MetalAstNode, MetalKernelInfo};#[derive(Debug)]pub enum MetalSdk {@@ -192,10 +192,6 @@ impl MetalToolchain {let source_contents = fs::read_to_string(path).context("cannot read source file")?;- for node in &ast_root.inner {- validate_raw_kernel(node).context("validation of legacy (non-dsl) kernel failed")?;- }-let kernel_infos = ast_root.inner.into_iter()diff --git a/crates/uzu/build/metal/wrapper.rs b/crates/uzu/build/metal/wrapper.rsindex 32e781924..d31d3a580 100644--- a/crates/uzu/build/metal/wrapper.rs+++ b/crates/uzu/build/metal/wrapper.rs@@ -230,7 +230,7 @@ fn kernel_wrappers(MetalArgumentType::Threads(_) => {format!("__dsl_thread_idx.{}", thread_axis_letters.next().unwrap())},- MetalArgumentType::ThreadContext => "ThreadContext { .simdgroup_index = __dsl_simd_lane_idx, .threadgroup_index = __dsl_simd_group_idx, .simdgroup_size = __dsl_simd_group_size, .threadgroup_size = __dsl_simd_group_size * __dsl_simd_group_per_threadgroup, .simdgroups_per_threadgroup = __dsl_simd_group_per_threadgroup }".into(),+ MetalArgumentType::ThreadContext => "ThreadContext { .simdgroup_index = __dsl_simd_lane_idx, .threadgroup_index = __dsl_simd_group_idx, .simdgroup_size = __dsl_simd_group_size, .simdgroups_per_threadgroup = __dsl_simd_group_per_threadgroup }".into(),}).collect::<Vec<_>>().join(", ")diff --git a/crates/uzu/src/backends/cpu/kernel/mod.rs b/crates/uzu/src/backends/cpu/kernel/mod.rsindex fd116b887..91bd75de1 100644--- a/crates/uzu/src/backends/cpu/kernel/mod.rs+++ b/crates/uzu/src/backends/cpu/kernel/mod.rs@@ -16,7 +16,6 @@ mod rms_norm;mod rope;mod sampling;mod short_conv;-mod sigmoid;mod ssm;mod tensor_add_bias;mod tensor_add_scale;diff --git a/crates/uzu/src/backends/cpu/kernel/sigmoid/mod.rs b/crates/uzu/src/backends/cpu/kernel/sigmoid/mod.rsdeleted file mode 100644index 33caeb2aa..000000000--- a/crates/uzu/src/backends/cpu/kernel/sigmoid/mod.rs+++ /dev/null@@ -1 +0,0 @@-pub mod sigmoid;diff --git a/crates/uzu/src/backends/cpu/kernel/sigmoid/sigmoid.rs b/crates/uzu/src/backends/cpu/kernel/sigmoid/sigmoid.rsdeleted file mode 100644index ba1ad5b59..000000000--- a/crates/uzu/src/backends/cpu/kernel/sigmoid/sigmoid.rs+++ /dev/null@@ -1,21 +0,0 @@-use dsl::kernel;-use half::{bf16, f16};-use num_traits::Float;--use crate::ArrayElement;--#[kernel(Sigmoid)]-#[variants(T, f32, f16, bf16)]-pub fn sigmoid<T: ArrayElement + Float>(- input: *const T,- output: *mut T,- total_elements: u32,-) {- for i in 0..total_elements as usize {- unsafe {- let input_value = (*input.add(i)).to_f32().unwrap();- let value = 1.0 / (1.0 + (-input_value).exp());- *output.add(i) = T::from(value).unwrap();- }- }-}diff --git a/crates/uzu/src/backends/metal/kernel/activation/activations.h b/crates/uzu/src/backends/metal/kernel/activation/activations.hindex 979a5c255..b09b0b45d 100644--- a/crates/uzu/src/backends/metal/kernel/activation/activations.h+++ b/crates/uzu/src/backends/metal/kernel/activation/activations.h@@ -1,5 +1,4 @@-#ifndef __UZU_ACTIVATIONS_H__-#define __UZU_ACTIVATIONS_H__+#pragma once#include <metal_stdlib>#include "activation_type.h"@@ -54,5 +53,3 @@ inline T activate(T x, ActivationType type) {return x;}}--#endif // __UZU_ACTIVATIONS_H__\ No newline at end of filediff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metalindex acf5ff8d6..ee1f0a074 100644--- a/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal@@ -434,7 +434,9 @@ PUBLIC KERNEL(AttentionGemm)(for (short id = 0; id < HEAD_DIM_GRID_COLS; id++) {METAL_PRAGMA_UNROLLfor (short ik = 0; ik < KEY_GRID_COLS; ik++) {- IF_CONSTEXPR(BD == 128) { simdgroup_barrier(mem_flags::mem_none); }+ if constexpr (BD == 128) {+ simdgroup_barrier(mem_flags::mem_none);+ }const short kk = ik * SIMDGROUP_BLOCK_SIZE;const short dd = id * SIMDGROUP_BLOCK_SIZE;@@ -444,7 +446,9 @@ PUBLIC KERNEL(AttentionGemm)([value_shared_offset + kk * value_leading_dimension + dd]);- IF_CONSTEXPR(BD == 128) { simdgroup_barrier(mem_flags::mem_none); }+ if constexpr (BD == 128) {+ simdgroup_barrier(mem_flags::mem_none);+ }SimdgroupMultiplyAccumulateType::multiply_accumulate(output_fragment.multiply_accumulate_at(0, id),diff --git a/crates/uzu/src/backends/metal/kernel/attention/mask.h b/crates/uzu/src/backends/metal/kernel/attention/mask.hindex 83b696dc1..e553eabc0 100644--- a/crates/uzu/src/backends/metal/kernel/attention/mask.h+++ b/crates/uzu/src/backends/metal/kernel/attention/mask.h@@ -6,7 +6,7 @@using namespace uzu::ring;using namespace uzu::trie;-bool should_use_key(+static inline bool should_use_key(const constant RingParams& ring_params,const device TrieNode* trie,const constant uint& sliding_window_size,@@ -19,4 +19,60 @@ bool should_use_key(const bool is_causal,const bool is_trie,const bool is_sliding_window-);+) {+ bool use_key = true;++ uint key_position;++ if (i >= prefix_length) {+ // We're in the suffix+ uint key_position_in_suffix = i - prefix_length;++ if (is_trie) {+ // Trie, get position from the flat trie buffer+ TrieNode key_trie = trie[key_position_in_suffix];++ key_position = suffix_position + key_trie.height;++ if (is_causal) {+ use_key &=+ q_seq_idx >= key_trie.trie_start && q_seq_idx <= key_trie.trie_end;+ }+ } else {+ // Non-trie, position equals index in the suffix+ key_position = suffix_position + key_position_in_suffix;++ if (is_causal) {+ use_key &= key_position_in_suffix <= q_seq_idx;+ }+ }++ } else {+ // We're in the prefix+ if (is_kv_cache_ring) {+ // Ring, calculate position in the ring+ key_position =+ (prefix_length + i - ring_params.ring_offset) % prefix_length;+ // Ring also skips entries that aren't filled yet+ use_key &= key_position < ring_params.ring_length;+ } else {+ // Non-ring, position equals to the index in the kvcache buffer+ key_position = i;+ }+ }++ if (is_sliding_window) {+ if (is_causal) {+ use_key &= key_position <= query_position &&+ (query_position - key_position) < sliding_window_size;+ } else {+ if (key_position <= query_position) {+ use_key &= (query_position - key_position) <= (sliding_window_size / 2);+ } else {+ use_key &= (key_position - query_position) <= (sliding_window_size / 2);+ }+ }+ }++ return use_key;+}diff --git a/crates/uzu/src/backends/metal/kernel/attention/mask.metal b/crates/uzu/src/backends/metal/kernel/attention/mask.metaldeleted file mode 100644index b3260231c..000000000--- a/crates/uzu/src/backends/metal/kernel/attention/mask.metal+++ /dev/null@@ -1,76 +0,0 @@-#include "../generated/ring.h"-#include "../generated/trie.h"--using namespace uzu::ring;-using namespace uzu::trie;--bool should_use_key(- const constant RingParams& ring_params,- const device TrieNode* trie,- const constant uint& sliding_window_size,- const uint q_seq_idx,- const uint prefix_length,- const uint suffix_position,- const uint query_position,- const uint i,- const bool is_kv_cache_ring,- const bool is_causal,- const bool is_trie,- const bool is_sliding_window-) {- bool use_key = true;-- uint key_position;-- if (i >= prefix_length) {- // We're in the suffix- uint key_position_in_suffix = i - prefix_length;-- if (is_trie) {- // Trie, get position from the flat trie buffer- TrieNode key_trie = trie[key_position_in_suffix];-- key_position = suffix_position + key_trie.height;-- if (is_causal) {- use_key &=- q_seq_idx >= key_trie.trie_start && q_seq_idx <= key_trie.trie_end;- }- } else {- // Non-trie, position equals index in the suffix- key_position = suffix_position + key_position_in_suffix;-- if (is_causal) {- use_key &= key_position_in_suffix <= q_seq_idx;- }- }-- } else {- // We're in the prefix- if (is_kv_cache_ring) {- // Ring, calculate position in the ring- key_position =- (prefix_length + i - ring_params.ring_offset) % prefix_length;- // Ring also skips entries that aren't filled yet- use_key &= key_position < ring_params.ring_length;- } else {- // Non-ring, position equals to the index in the kvcache buffer- key_position = i;- }- }-- if (is_sliding_window) {- if (is_causal) {- use_key &= key_position <= query_position &&- (query_position - key_position) < sliding_window_size;- } else {- if (key_position <= query_position) {- use_key &= (query_position - key_position) <= (sliding_window_size / 2);- } else {- use_key &= (key_position - query_position) <= (sliding_window_size / 2);- }- }- }-- return use_key;-}diff --git a/crates/uzu/src/backends/metal/kernel/audio/norm_ncs.metal b/crates/uzu/src/backends/metal/kernel/audio/norm_ncs.metalindex 46d59eb11..241597b59 100644--- a/crates/uzu/src/backends/metal/kernel/audio/norm_ncs.metal+++ b/crates/uzu/src/backends/metal/kernel/audio/norm_ncs.metal@@ -48,13 +48,9 @@ void norm_ncs(}}- const float sum =- threadgroup_cooperative_reduce_sum<AUDIO_NORM_NCS_BLOCK_SIZE>(- partial_sum,- shared_mean,- (ushort)lid,- thread_context- );+ const float sum = threadgroup_cooperative_reduce<+ SimdReduceSum<float>,+ AUDIO_NORM_NCS_BLOCK_SIZE>(partial_sum, shared_mean, thread_context);const float mean = (subtract_mean != 0) ? (sum / (float)channels) : 0.0f;float partial_variance = 0.0f;@@ -65,13 +61,13 @@ void norm_ncs(partial_variance += centered * centered;}- const float variance_sum =- threadgroup_cooperative_reduce_sum<AUDIO_NORM_NCS_BLOCK_SIZE>(- partial_variance,- shared_variance,- (ushort)lid,- thread_context- );+ const float variance_sum = threadgroup_cooperative_reduce<+ SimdReduceSum<float>,+ AUDIO_NORM_NCS_BLOCK_SIZE>(+ partial_variance,+ shared_variance,+ thread_context+ );const float inv_std = rsqrt(variance_sum / (float)channels + epsilon);for (uint c = lid; c < (uint)channels; c += AUDIO_NORM_NCS_BLOCK_SIZE) {diff --git a/crates/uzu/src/backends/metal/kernel/common/bf16.h b/crates/uzu/src/backends/metal/kernel/common/bf16.hdeleted file mode 100644index 1fafab2b1..000000000--- a/crates/uzu/src/backends/metal/kernel/common/bf16.h+++ /dev/null@@ -1,14 +0,0 @@-#pragma once--#include <metal_stdlib>--using namespace metal;--typedef bfloat bfloat16_t;-inline uint16_t bfloat16_to_uint16(const bfloat16_t x) {- return as_type<uint16_t>(x);-}--inline bfloat16_t uint16_to_bfloat16(const uint16_t x) {- return as_type<bfloat16_t>(x);-}diff --git a/crates/uzu/src/backends/metal/kernel/common/bf16_math.h b/crates/uzu/src/backends/metal/kernel/common/bf16_math.hdeleted file mode 100644index 639e04eed..000000000--- a/crates/uzu/src/backends/metal/kernel/common/bf16_math.h+++ /dev/null@@ -1,436 +0,0 @@-#pragma once--///////////////////////////////////////////////////////////////////////////////-// Metal math for bfloat16-///////////////////////////////////////////////////////////////////////////////--/*--Following the Metal Shading Language Specification (Metal 3.1)--"bfloat is an extended itypeing point type that only allows implicit conversion- to a type of greater itypeing point rank. While bfloat can be implicitly- converted to itype, it cannot be implicitly converted to half, and neither- itype nor half can be implicitly converted to bfloat."--Further, as far as I can tell, the stdlib math/simd functions are not defined-for bfloat and calling with an argument of type bfloat will result in that-argument getting implicitly converted to itype which then returns an output-that is (likely) a itype which cannot be implicitly converted into a bfloat--This leads to situations where-bfloat a = 5.0bf;-bfloat b = metal::abs(a); // this will throw an error since abs return itype-bfloat c = static_cast<bfloat>(metal::abs(a)); // this is fine--For the moment, I will be adding overloaded instantiations of the math-functions to accordingly automatically handle the casting--*/--#define instantiate_metal_math_funcs(itype, otype, ctype, mfast) \- \- METAL_FUNC otype abs(itype x) { \- return static_cast<otype>(__metal_fabs(static_cast<ctype>(x), mfast)); \- } \- METAL_FUNC otype acos(itype x) { \- return static_cast<otype>(__metal_acos(static_cast<ctype>(x), mfast)); \- } \- METAL_FUNC otype acosh(itype x) { \- return static_cast<otype>(__metal_acosh(static_cast<ctype>(x), mfast)); \- } \- METAL_FUNC otype asin(itype x) { \- return static_cast<otype>(__metal_asin(static_cast<ctype>(x), mfast)); \- } \- METAL_FUNC otype asinh(itype x) { \- return static_cast<otype>(__metal_asinh(static_cast<ctype>(x), mfast)); \- } \- METAL_FUNC otype atan(itype y_over_x) { \- return static_cast<otype>( \- __metal_atan(static_cast<ctype>(y_over_x), mfast) \- ); \- } \- METAL_FUNC otype atan2(itype y, itype x) { \- return static_cast<otype>( \- __metal_atan2(static_cast<ctype>(y), static_cast<ctype>(x), mfast) \- ); \- } \- METAL_FUNC otype atanh(itype x) { \- return static_cast<otype>(__metal_atanh(static_cast<ctype>(x), mfast)); \- } \- METAL_FUNC otype ceil(itype x) { \- return static_cast<otype>(__metal_ceil(static_cast<ctype>(x), mfast)); \- } \- METAL_FUNC otype cos(itype x) { \- return static_cast<otype>(__metal_cos(static_cast<ctype>(x), mfast)); \- } \- METAL_FUNC otype cosh(itype x) { \- return static_cast<otype>(__metal_cosh(static_cast<ctype>(x), mfast)); \- } \- METAL_FUNC otype cospi(itype x) { \- return static_cast<otype>(__metal_cospi(static_cast<ctype>(x), mfast)); \- } \- METAL_FUNC otype divide(itype x, itype y) { \- return static_cast<otype>( \- __metal_divide(static_cast<ctype>(x), static_cast<ctype>(y), mfast) \- ); \- } \- METAL_FUNC otype exp(itype x) { \- return static_cast<otype>(__metal_exp(static_cast<ctype>(x), mfast)); \- } \- METAL_FUNC otype exp10(itype x) { \- return static_cast<otype>(__metal_exp10(static_cast<ctype>(x), mfast)); \- } \- METAL_FUNC otype exp2(itype x) { \- return static_cast<otype>(__metal_exp2(static_cast<ctype>(x), mfast)); \- } \- METAL_FUNC otype fabs(itype x) { \- return static_cast<otype>(__metal_fabs(static_cast<ctype>(x), mfast)); \- } \- METAL_FUNC otype fdim(itype x, itype y) { \- ctype t = static_cast<ctype>(x - y); \- return static_cast<otype>(select(t, ctype(0), t < ctype(0) || x == y)); \- } \- METAL_FUNC otype floor(itype x) { \- return static_cast<otype>(__metal_floor(static_cast<ctype>(x), mfast)); \- } \- METAL_FUNC otype fma(itype x, itype y, itype z) { \- return static_cast<otype>(__metal_fma( \- static_cast<ctype>(x), \- static_cast<ctype>(y), \- static_cast<ctype>(z) \- )); \- } \- METAL_FUNC otype fmax(itype x, itype y) { \- return static_cast<otype>( \- __metal_fmax(static_cast<ctype>(x), static_cast<ctype>(y), mfast) \- ); \- } \- METAL_FUNC otype fmax3(itype x, itype y, itype z) { \- return static_cast<otype>(__metal_fmax3( \- static_cast<ctype>(x), \- static_cast<ctype>(y), \- static_cast<ctype>(z), \- mfast \- )); \- } \- METAL_FUNC otype fmedian3(itype x, itype y, itype z) { \- return static_cast<otype>(__metal_fmedian3( \- static_cast<ctype>(x), \- static_cast<ctype>(y), \- static_cast<ctype>(z), \- mfast \- )); \- } \- METAL_FUNC otype fmin(itype x, itype y) { \- return static_cast<otype>( \- __metal_fmin(static_cast<ctype>(x), static_cast<ctype>(y), mfast) \- ); \- } \- METAL_FUNC otype fmin3(itype x, itype y, itype z) { \- return static_cast<otype>(__metal_fmin3( \- static_cast<ctype>(x), \- static_cast<ctype>(y), \- static_cast<ctype>(z), \- mfast \- )); \- } \- METAL_FUNC otype fmod(itype x, itype y) { \- return static_cast<otype>( \- __metal_fmod(static_cast<ctype>(x), static_cast<ctype>(y), mfast) \- ); \- } \- METAL_FUNC otype fract(itype x) { \- return static_cast<otype>(__metal_fract(static_cast<ctype>(x), mfast)); \- } \- METAL_FUNC otype frexp(itype x, thread int& exp) { \- return static_cast<otype>(__metal_frexp(static_cast<ctype>(x), &exp)); \- } \- METAL_FUNC otype ldexp(itype x, int k) { \- return static_cast<otype>(__metal_ldexp(static_cast<ctype>(x), k, mfast)); \- } \- METAL_FUNC otype log(itype x) { \- return static_cast<otype>(__metal_log(static_cast<ctype>(x), mfast)); \- } \- METAL_FUNC otype log10(itype x) { \- return static_cast<otype>(__metal_log10(static_cast<ctype>(x), mfast)); \- } \- METAL_FUNC otype log2(itype x) { \- return static_cast<otype>(__metal_log2(static_cast<ctype>(x), mfast)); \- } \- METAL_FUNC otype max(itype x, itype y) { \- return static_cast<otype>( \- __metal_fmax(static_cast<ctype>(x), static_cast<ctype>(y), mfast) \- ); \- } \- METAL_FUNC otype max3(itype x, itype y, itype z) { \- return static_cast<otype>(__metal_fmax3( \- static_cast<ctype>(x), \- static_cast<ctype>(y), \- static_cast<ctype>(z), \- mfast \- )); \- } \- METAL_FUNC otype median3(itype x, itype y, itype z) { \- return static_cast<otype>(__metal_fmedian3( \- static_cast<ctype>(x), \- static_cast<ctype>(y), \- static_cast<ctype>(z), \- mfast \- )); \- } \- METAL_FUNC otype min(itype x, itype y) { \- return static_cast<otype>( \- __metal_fmin(static_cast<ctype>(x), static_cast<ctype>(y), mfast) \- ); \- } \- METAL_FUNC otype min3(itype x, itype y, itype z) { \- return static_cast<otype>(__metal_fmin3( \- static_cast<ctype>(x), \- static_cast<ctype>(y), \- static_cast<ctype>(z), \- mfast \- )); \- } \- METAL_FUNC otype nextafter(itype x, itype y) { \- return static_cast<otype>( \- __metal_nextafter(static_cast<ctype>(x), static_cast<ctype>(y)) \- ); \- } \- METAL_FUNC otype pow(itype x, itype y) { \- return static_cast<otype>( \- __metal_pow(static_cast<ctype>(x), static_cast<ctype>(y), mfast) \- ); \- } \- METAL_FUNC otype powr(itype x, itype y) { \- return static_cast<otype>( \- __metal_powr(static_cast<ctype>(x), static_cast<ctype>(y), mfast) \- ); \- } \- METAL_FUNC otype rint(itype x) { \- return static_cast<otype>(__metal_rint(static_cast<ctype>(x), mfast)); \- } \- METAL_FUNC otype round(itype x) { \- return static_cast<otype>(__metal_round(static_cast<ctype>(x), mfast)); \- } \- METAL_FUNC otype rsqrt(itype x) { \- return static_cast<otype>(__metal_rsqrt(static_cast<ctype>(x), mfast)); \- } \- METAL_FUNC otype sin(itype x) { \- return static_cast<otype>(__metal_sin(static_cast<ctype>(x), mfast)); \- } \- METAL_FUNC otype sinh(itype x) { \- return static_cast<otype>(__metal_sinh(static_cast<ctype>(x), mfast)); \- } \- METAL_FUNC otype sinpi(itype x) { \- return static_cast<otype>(__metal_sinpi(static_cast<ctype>(x), mfast)); \- } \- METAL_FUNC otype sqrt(itype x) { \- return static_cast<otype>(__metal_sqrt(static_cast<ctype>(x), mfast)); \- } \- METAL_FUNC otype tan(itype x) { \- return static_cast<otype>(__metal_tan(static_cast<ctype>(x), mfast)); \- } \- METAL_FUNC otype tanh(itype x) { \- return static_cast<otype>(__metal_tanh(static_cast<ctype>(x), mfast)); \- } \- METAL_FUNC otype tanpi(itype x) { \- return static_cast<otype>(__metal_tanpi(static_cast<ctype>(x), mfast)); \- } \- METAL_FUNC otype trunc(itype x) { \- return static_cast<otype>(__metal_trunc(static_cast<ctype>(x), mfast)); \- }--namespace metal {--instantiate_metal_math_funcs(- bfloat16_t,- bfloat16_t,- float,- __METAL_MAYBE_FAST_MATH__-);--namespace fast {--instantiate_metal_math_funcs(- bfloat16_t,- bfloat16_t,- float,- __METAL_FAST_MATH__-);--} // namespace fast--namespace precise {--instantiate_metal_math_funcs(- bfloat16_t,- bfloat16_t,- float,- __METAL_PRECISE_MATH__-);--} // namespace precise--} // namespace metal--///////////////////////////////////////////////////////////////////////////////-// Metal simd for bfloat16-///////////////////////////////////////////////////////////////////////////////--#define instantiate_metal_simd_comm_funcs( \- itype, \- otype, \- ctype, \- itype_to_ctype, \- ctype_to_otype \-) \- \- METAL_FUNC otype simd_broadcast(itype data, ushort broadcast_lane_id) { \- return ctype_to_otype( \- __metal_simd_broadcast(itype_to_ctype(data), broadcast_lane_id) \- ); \- } \- \- METAL_FUNC otype simd_shuffle(itype data, ushort simd_lane_id) { \- return ctype_to_otype( \- __metal_simd_shuffle(itype_to_ctype(data), simd_lane_id) \- ); \- } \- \- METAL_FUNC otype simd_shuffle_and_fill_down( \- itype data, \- itype filling_data, \- ushort delta, \- ushort modulo \- ) { \- return ctype_to_otype(__metal_simd_shuffle_and_fill_down( \- itype_to_ctype(data), \- itype_to_ctype(filling_data), \- delta, \- modulo \- )); \- } \- \- METAL_FUNC otype \- simd_shuffle_and_fill_down(itype data, itype filling_data, ushort delta) { \- return ctype_to_otype(__metal_simd_shuffle_and_fill_down( \- itype_to_ctype(data), \- itype_to_ctype(filling_data), \- delta, \- __metal_get_simdgroup_size(ushort()) \- )); \- } \- \- METAL_FUNC otype simd_shuffle_and_fill_up( \- itype data, \- itype filling_data, \- ushort delta, \- ushort modulo \- ) { \- return ctype_to_otype(__metal_simd_shuffle_and_fill_up( \- itype_to_ctype(data), \- itype_to_ctype(filling_data), \- delta, \- modulo \- )); \- } \- \- METAL_FUNC otype \- simd_shuffle_and_fill_up(itype data, itype filling_data, ushort delta) { \- return ctype_to_otype(__metal_simd_shuffle_and_fill_up( \- itype_to_ctype(data), \- itype_to_ctype(filling_data), \- delta, \- __metal_get_simdgroup_size(ushort()) \- )); \- } \- \- METAL_FUNC otype simd_shuffle_down(itype data, ushort delta) { \- return ctype_to_otype( \- __metal_simd_shuffle_down(itype_to_ctype(data), delta) \- ); \- } \- \- METAL_FUNC otype simd_shuffle_rotate_down(itype data, ushort delta) { \- return ctype_to_otype( \- __metal_simd_shuffle_rotate_down(itype_to_ctype(data), delta) \- ); \- } \- \- METAL_FUNC otype simd_shuffle_rotate_up(itype data, ushort delta) { \- return ctype_to_otype( \- __metal_simd_shuffle_rotate_up(itype_to_ctype(data), delta) \- ); \- } \- \- METAL_FUNC otype simd_shuffle_up(itype data, ushort delta) { \- return ctype_to_otype( \- __metal_simd_shuffle_up(itype_to_ctype(data), delta) \- ); \- } \- \- METAL_FUNC otype simd_shuffle_xor(itype data, ushort mask) { \- return ctype_to_otype( \- __metal_simd_shuffle_xor(itype_to_ctype(data), mask) \- ); \- }--#define instantiate_metal_simd_reduction_funcs(itype, otype, ctype) \- \- METAL_FUNC otype simd_max(itype data) { \- return static_cast<otype>(__metal_simd_max(static_cast<ctype>(data))); \- } \- \- METAL_FUNC otype simd_min(itype data) { \- return static_cast<otype>(__metal_simd_min(static_cast<ctype>(data))); \- } \- \- METAL_FUNC otype simd_prefix_exclusive_product(itype data) { \- return static_cast<otype>( \- __metal_simd_prefix_exclusive_product(static_cast<ctype>(data)) \- ); \- } \- \- METAL_FUNC otype simd_prefix_exclusive_sum(itype data) { \- return static_cast<otype>( \- __metal_simd_prefix_exclusive_sum(static_cast<ctype>(data)) \- ); \- } \- \- METAL_FUNC otype simd_prefix_inclusive_product(itype data) { \- return static_cast<otype>( \- __metal_simd_prefix_inclusive_product(static_cast<ctype>(data)) \- ); \- } \- \- METAL_FUNC otype simd_prefix_inclusive_sum(itype data) { \- return static_cast<otype>( \- __metal_simd_prefix_inclusive_sum(static_cast<ctype>(data)) \- ); \- } \- \- METAL_FUNC otype simd_product(itype data) { \- return static_cast<otype>(__metal_simd_product(static_cast<ctype>(data))); \- } \- \- METAL_FUNC otype simd_sum(itype data) { \- return static_cast<otype>(__metal_simd_sum(static_cast<ctype>(data))); \- } \- \- METAL_FUNC otype simd_xor(itype data) { \- return static_cast<otype>(__metal_simd_xor(static_cast<ctype>(data))); \- }--namespace metal {--instantiate_metal_simd_comm_funcs(- bfloat16_t,- bfloat16_t,- uint16_t,- bfloat16_to_uint16,- uint16_to_bfloat16-);-instantiate_metal_simd_reduction_funcs(bfloat16_t, bfloat16_t, float);--} // namespace metaldiff --git a/crates/uzu/src/backends/metal/kernel/common/complex.h b/crates/uzu/src/backends/metal/kernel/common/complex.hdeleted file mode 100644index 84bbbeb11..000000000--- a/crates/uzu/src/backends/metal/kernel/common/complex.h+++ /dev/null@@ -1,170 +0,0 @@-#pragma once--#include <metal_stdlib>--using namespace metal;--struct complex64_t;--template <typename T>-static constexpr constant bool can_convert_to_complex64 =- !is_same_v<T, complex64_t> && is_convertible_v<T, float>;--template <typename T>-static constexpr constant bool can_convert_from_complex64 =- !is_same_v<T, complex64_t> &&- (is_convertible_v<float, T> || is_convertible_v<bfloat16_t, T>);--struct complex64_t {- float real;- float imag;-- // Constructors- constexpr complex64_t(float real, float imag) : real(real), imag(imag) {};- constexpr complex64_t() : real(0), imag(0) {};- constexpr complex64_t() threadgroup : real(0), imag(0) {};-- // Conversions to complex64_t- template <- typename T,- typename = typename enable_if<can_convert_to_complex64<T>>::type>- constexpr complex64_t(T x) thread : real(x), imag(0) {}-- template <- typename T,- typename = typename enable_if<can_convert_to_complex64<T>>::type>- constexpr complex64_t(T x) threadgroup : real(x), imag(0) {}-- template <- typename T,- typename = typename enable_if<can_convert_to_complex64<T>>::type>- constexpr complex64_t(T x) device : real(x), imag(0) {}-- template <- typename T,- typename = typename enable_if<can_convert_to_complex64<T>>::type>- constexpr complex64_t(T x) constant : real(x), imag(0) {}-- // Conversions from complex64_t- template <- typename T,- typename = typename enable_if<can_convert_from_complex64<T>>::type>- constexpr operator T() const thread {- return static_cast<T>(real);- }-- template <- typename T,- typename = typename enable_if<can_convert_from_complex64<T>>::type>- constexpr operator T() const threadgroup {- return static_cast<T>(real);- }-- template <- typename T,- typename = typename enable_if<can_convert_from_complex64<T>>::type>- constexpr operator T() const device {- return static_cast<T>(real);- }-- template <- typename T,- typename = typename enable_if<can_convert_from_complex64<T>>::type>- constexpr operator T() const constant {- return static_cast<T>(real);- }-};--constexpr complex64_t operator-(complex64_t x) { return {-x.real, -x.imag}; }--constexpr bool operator>=(complex64_t a, complex64_t b) {- return (a.real > b.real) || (a.real == b.real && a.imag >= b.imag);-}--constexpr bool operator>(complex64_t a, complex64_t b) {- return (a.real > b.real) || (a.real == b.real && a.imag > b.imag);-}--constexpr bool operator<=(complex64_t a, complex64_t b) {- return operator>=(b, a);-}--constexpr bool operator<(complex64_t a, complex64_t b) {- return operator>(b, a);-}--constexpr bool operator==(complex64_t a, complex64_t b) {- return a.real == b.real && a.imag == b.imag;-}--constexpr complex64_t operator+(complex64_t a, complex64_t b) {- return {a.real + b.real, a.imag + b.imag};-}--constexpr thread complex64_t& operator+=(thread complex64_t& a, complex64_t b) {- a.real += b.real;- a.imag += b.imag;- return a;-}--constexpr threadgroup complex64_t& operator+=(- threadgroup complex64_t& a,- complex64_t b-) {- a.real += b.real;- a.imag += b.imag;- return a;-}--constexpr device complex64_t& operator+=(device complex64_t& a, complex64_t b) {- a.real += b.real;- a.imag += b.imag;- return a;-}--constexpr complex64_t operator+(float a, complex64_t b) {- return {a + b.real, b.imag};-}-constexpr complex64_t operator+(complex64_t a, float b) {- return {a.real + b, a.imag};-}--constexpr complex64_t operator-(complex64_t a, complex64_t b) {- return {a.real - b.real, a.imag - b.imag};-}-constexpr complex64_t operator-(float a, complex64_t b) {- return {a - b.real, -b.imag};-}-constexpr complex64_t operator-(complex64_t a, float b) {- return {a.real - b, a.imag};-}--constexpr complex64_t operator*(complex64_t a, complex64_t b) {- return {a.real * b.real - a.imag * b.imag, a.real * b.imag + a.imag * b.real};-}--constexpr complex64_t operator/(complex64_t a, complex64_t b) {- auto denom = b.real * b.real + b.imag * b.imag;- auto x = a.real * b.real + a.imag * b.imag;- auto y = a.imag * b.real - a.real * b.imag;- return {x / denom, y / denom};-}--constexpr complex64_t operator/(float a, complex64_t b) {- auto denom = b.real * b.real + b.imag * b.imag;- auto x = a * b.real;- auto y = -a * b.imag;- return {x / denom, y / denom};-}--constexpr complex64_t operator%(complex64_t a, complex64_t b) {- auto real = a.real - (b.real * static_cast<int64_t>(a.real / b.real));- auto imag = a.imag - (b.imag * static_cast<int64_t>(a.imag / b.imag));- if (real != 0 && (real < 0 != b.real < 0)) {- real += b.real;- }- if (imag != 0 && (imag < 0 != b.imag < 0)) {- imag += b.imag;- }- return {real, imag};-}diff --git a/crates/uzu/src/backends/metal/kernel/common/defines.h b/crates/uzu/src/backends/metal/kernel/common/defines.hindex 97e2fcfd3..602984747 100644--- a/crates/uzu/src/backends/metal/kernel/common/defines.h+++ b/crates/uzu/src/backends/metal/kernel/common/defines.h@@ -3,16 +3,5 @@#define METAL_CONST static constant constexpr#define METAL_PRAGMA_UNROLL _Pragma("clang loop unroll(full)")#define METAL_SIMD_SIZE 32-METAL_FUNC int pow2(int n) { return 1 << n; }--METAL_CONST int MAX_REDUCE_SPECIALIZED_DIMS = 4;-METAL_CONST int REDUCE_N_READS = 4;-METAL_CONST int REDUCE_N_WRITES = 4;-METAL_CONST int SOFTMAX_N_READS = 4;-METAL_CONST int RMS_N_READS = 4;-METAL_CONST int RMS_LOOPED_LIMIT = 4096;-#define instantiate_kernel(name, func, ...) \- template [[host_name( \- name \- )]] [[kernel]] decltype(func<__VA_ARGS__>) func<__VA_ARGS__>;+METAL_FUNC int pow2(int n) { return 1 << n; }diff --git a/crates/uzu/src/backends/metal/kernel/common/dsl.h b/crates/uzu/src/backends/metal/kernel/common/dsl.hindex 3bda99637..11922d052 100644--- a/crates/uzu/src/backends/metal/kernel/common/dsl.h+++ b/crates/uzu/src/backends/metal/kernel/common/dsl.h@@ -1,13 +1,8 @@#pragma once#include <metal_stdlib>-using namespace metal;-#if defined(__cpp_if_constexpr)-#define IF_CONSTEXPR(cond) if constexpr (cond)-#else-#define IF_CONSTEXPR(cond) if (cond)-#endif+using namespace metal;#ifdef DSL_ANALYZE#define DSL_META(...) [[clang::annotate("", __VA_ARGS__)]]@@ -30,38 +25,3 @@ using namespace metal;#define AXIS(TDS, TPG) DSL_META("dsl.axis", DSL_XSTR(TDS), DSL_XSTR(TPG))#define GROUPS(EXPR) DSL_META("dsl.groups", DSL_XSTR(EXPR))#define THREADS(EXPR) DSL_META("dsl.threads", DSL_XSTR(EXPR))--#define generateKernel( \- max_threads, \- functionName, \- scalarType, \- outerArgs, \- innerArgs \-) \- [[max_total_threads_per_threadgroup(max_threads)]] \- kernel void functionName##_##scalarType outerArgs { \- functionName innerArgs; \- }--#define generateKernels(max_threads, functionName) \- generateKernel( \- max_threads, \- functionName, \- float, \- outerArguments(float), \- innerArguments \- ); \- generateKernel( \- max_threads, \- functionName, \- bfloat, \- outerArguments(bfloat), \- innerArguments \- ); \- generateKernel( \- max_threads, \- functionName, \- half, \- outerArguments(half), \- innerArguments \- );diff --git a/crates/uzu/src/backends/metal/kernel/common/logging.h b/crates/uzu/src/backends/metal/kernel/common/logging.hdeleted file mode 100644index 4d06cd85a..000000000--- a/crates/uzu/src/backends/metal/kernel/common/logging.h+++ /dev/null@@ -1,20 +0,0 @@-#pragma once--#if defined(__METAL_VERSION__) && (__METAL_VERSION__ >= 320)-#include <metal_logging>--using os_log = metal::os_log;--#else--struct os_log {- constexpr os_log(constant char*, constant char*) constant {}-- template <typename... Args>- void log_debug(constant char*, Args...) const {}-- template <typename... Args>- void log_debug(constant char*, Args...) const constant {}-};--#endif\ No newline at end of filediff --git a/crates/uzu/src/backends/metal/kernel/common/mlp_epilogue.h b/crates/uzu/src/backends/metal/kernel/common/mlp_epilogue.hdeleted file mode 100644index 936a01fe8..000000000--- a/crates/uzu/src/backends/metal/kernel/common/mlp_epilogue.h+++ /dev/null@@ -1,76 +0,0 @@-#pragma once--#include <metal_stdlib>-using namespace metal;--// MLP Fusion Function Constants-// These control whether the kernel computes paired up/gate projections-// and applies activation fusion (SwiGLU/GeGLU style).-//-// Index allocation:-// 50 - MLP_FUSED (bool): enable paired computation + fused activation-// 51 - MLP_HIDDEN_DIM (uint): offset to gate weights (= hidden_dim)-// 52 - MLP_ACTIVATION (uint): 0=SiLU, 1=GELU--constant bool MLP_FUSED [[function_constant(50)]];-constant uint MLP_HIDDEN_DIM [[function_constant(51)]];-constant uint MLP_ACTIVATION [[function_constant(52)]];--// Activation type enum for readability-enum MlpActivationType : uint {- MLP_ACT_SILU = 0,- MLP_ACT_GELU = 1,-};--// SiLU activation: x * sigmoid(x)-template <typename T>-inline T mlp_silu(T x) {- float xf = float(x);- float result = xf / (1.0f + exp(-xf));- return T(result);-}--// GELU approximation (tanh-based)-template <typename T>-inline T mlp_gelu(T x) {- constexpr float k0 = 0.044715f;- constexpr float k1 = 0.7978845608f; // sqrt(2/pi)- float xf = float(x);- float t = k1 * (xf + k0 * xf * xf * xf);- float result = 0.5f * xf * (1.0f + metal::precise::tanh(t));- return T(result);-}--// Apply activation based on MLP_ACTIVATION constant-template <typename T>-inline T mlp_activate(T x) {- if (MLP_ACTIVATION == MLP_ACT_SILU) {- return mlp_silu(x);- }- return mlp_gelu(x);-}--// Fused MLP epilogue: up * activation(gate)-// Used in SwiGLU (SiLU) and GeGLU (GELU) style MLPs-template <typename T>-inline T mlp_fused_epilogue(T up, T gate) {- T activated_gate = mlp_activate(gate);- return T(float(up) * float(activated_gate));-}--// Fused MLP epilogue with float accumulators-inline float mlp_fused_epilogue_f32(float up, float gate) {- float activated_gate;- if (MLP_ACTIVATION == MLP_ACT_SILU) {- activated_gate = up / (1.0f + exp(-gate));- // Note: SwiGLU is gate_act * up, not up_act * gate- // Correcting: activated_gate = gate / (1.0f + exp(-gate))- activated_gate = gate / (1.0f + exp(-gate));- } else {- constexpr float k0 = 0.044715f;- constexpr float k1 = 0.7978845608f;- float t = k1 * (gate + k0 * gate * gate * gate);- activated_gate = 0.5f * gate * (1.0f + metal::precise::tanh(t));- }- return up * activated_gate;-}diff --git a/crates/uzu/src/backends/metal/kernel/common/tensor_view.h b/crates/uzu/src/backends/metal/kernel/common/tensor_view.hindex ef5e977fd..96030c488 100644--- a/crates/uzu/src/backends/metal/kernel/common/tensor_view.h+++ b/crates/uzu/src/backends/metal/kernel/common/tensor_view.h@@ -67,54 +67,3 @@ struct TensorView3D {device T& operator()(uint i, uint j, uint k) const { return at(i, j, k); }};--template <typename T>-struct TensorView4D {- device T* data;- uint dim0, dim1, dim2, dim3;- uint stride0, stride1, stride2, stride3;-- TensorView4D(device T* ptr) : data(ptr) {}-- TensorView4D(device T* ptr, int d0, int d1, int d2, int d3)- : data(ptr), dim0(d0), dim1(d1), dim2(d2), dim3(d3) {- stride0 = d1 * d2 * d3;- stride1 = d2 * d3;- stride2 = d3;- stride3 = 1;- }-- TensorView4D(- device T* ptr,- int d0,- int d1,- int d2,- int d3,- int s0,- int s1,- int s2,- int s3- )- : data(ptr), dim0(d0), dim1(d1), dim2(d2), dim3(d3), stride0(s0),- stride1(s1), stride2(s2), stride3(s3) {}-- thread TensorView4D& shaped(int d0, int d1, int d2, int d3) {- dim0 = d0;- dim1 = d1;- dim2 = d2;- dim3 = d3;- stride0 = d1 * d2 * d3;- stride1 = d2 * d3;- stride2 = d3;- stride3 = 1;- return *this;- }-- device T& at(uint i, uint j, uint k, uint l) const {- return data[i * stride0 + j * stride1 + k * stride2 + l * stride3];- }-- device T& operator()(uint i, uint j, uint k, uint l) const {- return at(i, j, k, l);- }-};diff --git a/crates/uzu/src/backends/metal/kernel/common/thread_context.h b/crates/uzu/src/backends/metal/kernel/common/thread_context.hindex 9a3f827a5..f0fa45ae7 100644--- a/crates/uzu/src/backends/metal/kernel/common/thread_context.h+++ b/crates/uzu/src/backends/metal/kernel/common/thread_context.h@@ -7,6 +7,5 @@ struct ThreadContext {uint simdgroup_index;uint threadgroup_index;uint simdgroup_size;- uint threadgroup_size;uint simdgroups_per_threadgroup;};diff --git a/crates/uzu/src/backends/metal/kernel/common/thread_functions.h b/crates/uzu/src/backends/metal/kernel/common/thread_functions.hdeleted file mode 100644index 5185a9831..000000000--- a/crates/uzu/src/backends/metal/kernel/common/thread_functions.h+++ /dev/null@@ -1,54 +0,0 @@-#pragma once--#include <metal_stdlib>-using namespace metal;--template <ushort LENGTH, typename T>-static inline T thread_prefix_inclusive_sum(thread T (&values)[LENGTH]) {- for (ushort i = 1; i < LENGTH; i++) {- values[i] += values[i - 1];- }- return values[LENGTH - 1];-}--template <ushort LENGTH, typename T>-static inline T thread_prefix_inclusive_sum(threadgroup T* values) {- for (ushort i = 1; i < LENGTH; i++) {- values[i] += values[i - 1];- }- return values[LENGTH - 1];-}--template <ushort LENGTH, typename T>-static inline T thread_prefix_exclusive_sum(thread T (&values)[LENGTH]) {- T inclusive_prefix = thread_prefix_inclusive_sum<LENGTH>(values);- for (ushort i = LENGTH - 1; i > 0; i--) {- values[i] = values[i - 1];- }- values[0] = 0;- return inclusive_prefix;-}--template <ushort LENGTH, typename T>-static inline T thread_prefix_exclusive_sum(threadgroup T* values) {- T inclusive_prefix = thread_prefix_inclusive_sum<LENGTH>(values);- for (ushort i = LENGTH - 1; i > 0; i--) {- values[i] = values[i - 1];- }- values[0] = 0;- return inclusive_prefix;-}--template <ushort LENGTH, typename T>-static inline void thread_uniform_add(thread T (&values)[LENGTH], T uni) {- for (ushort i = 0; i < LENGTH; i++) {- values[i] += uni;- }-}--template <ushort LENGTH, typename T>-static inline void thread_uniform_add(threadgroup T* values, T uni) {- for (ushort i = 0; i < LENGTH; i++) {- values[i] += uni;- }-}diff --git a/crates/uzu/src/backends/metal/kernel/common/threadgroup_reduce.h b/crates/uzu/src/backends/metal/kernel/common/threadgroup_reduce.hindex 4475cc46d..34a0bf229 100644--- a/crates/uzu/src/backends/metal/kernel/common/threadgroup_reduce.h+++ b/crates/uzu/src/backends/metal/kernel/common/threadgroup_reduce.h@@ -1,230 +1,70 @@#pragma once-#include "thread_context.h"#include <metal_stdlib>-using namespace metal;--template <ushort BLOCK_SIZE, typename T>-static T threadgroup_raking_prefix_exclusive_sum(- T value,- threadgroup T* shared,- const ushort lid-) {- shared[lid] = value;- threadgroup_barrier(mem_flags::mem_threadgroup);-- if (lid < 32) {- const short values_per_thread = BLOCK_SIZE / 32;- const short first_index = lid * values_per_thread;- for (short i = first_index + 1; i < first_index + values_per_thread; i++) {- shared[i] += shared[i - 1];- }- T partial_sum = shared[first_index + values_per_thread - 1];- for (short i = first_index + values_per_thread - 1; i > first_index; i--) {- shared[i] = shared[i - 1];- }- shared[first_index] = 0;-- T prefix = simd_prefix_exclusive_sum(partial_sum);-- for (short i = first_index; i < first_index + values_per_thread; i++) {- shared[i] += prefix;- }- }- threadgroup_barrier(mem_flags::mem_threadgroup);-- const T result = shared[lid];- threadgroup_barrier(mem_flags::mem_threadgroup);- return result;-}--template <ushort BLOCK_SIZE, typename T>-static T threadgroup_raking_reduce_sum(- T value,- threadgroup T* shared,- const ushort lid-) {- shared[lid] = value;- threadgroup_barrier(mem_flags::mem_threadgroup);-- if (lid < 32) {- const short values_per_thread = BLOCK_SIZE / 32;- const short first_index = lid * values_per_thread;-- T thread_sum = shared[first_index];- for (short i = first_index + 1; i < first_index + values_per_thread; i++) {- thread_sum += shared[i];- }-- T total_sum = simd_sum(thread_sum);-- if (lid == 0) {- shared[0] = total_sum;- }- }-- threadgroup_barrier(mem_flags::mem_threadgroup);-- const T result = shared[0];- threadgroup_barrier(mem_flags::mem_threadgroup);- return result;-}--template <ushort BLOCK_SIZE, typename T>-static T threadgroup_raking_reduce_max(- T value,- threadgroup T* shared,- const ushort lid-) {- shared[lid] = value;- threadgroup_barrier(mem_flags::mem_threadgroup);-- if (lid < 32) {- const short values_per_thread = BLOCK_SIZE / 32;- const short first_index = lid * values_per_thread;-- T thread_max = shared[first_index];- for (short i = first_index + 1; i < first_index + values_per_thread; i++) {- thread_max = max(thread_max, shared[i]);- }-- T total_max = simd_max(thread_max);-- if (lid == 0) {- shared[0] = total_max;- }- }-- threadgroup_barrier(mem_flags::mem_threadgroup);-- const T result = shared[0];- threadgroup_barrier(mem_flags::mem_threadgroup);- return result;-}--template <ushort BLOCK_SIZE, typename T>-static T threadgroup_raking_reduce_min(- T value,- threadgroup T* shared,- const ushort lid-) {- shared[lid] = value;- threadgroup_barrier(mem_flags::mem_threadgroup);-- if (lid < 32) {- const short values_per_thread = BLOCK_SIZE / 32;- const short first_index = lid * values_per_thread;-- T thread_min = shared[first_index];- for (short i = first_index + 1; i < first_index + values_per_thread; i++) {- thread_min = min(thread_min, shared[i]);- }-- T total_min = simd_min(thread_min);-- if (lid == 0) {- shared[0] = total_min;- }- }- threadgroup_barrier(mem_flags::mem_threadgroup);-- const T result = shared[0];- threadgroup_barrier(mem_flags::mem_threadgroup);- return result;-}--template <ushort BLOCK_SIZE, typename T>-static T threadgroup_cooperative_reduce_sum(- T value,- threadgroup T* shared,- const ushort lid,- const thread ThreadContext& thread_context-) {- T local_sum = simd_sum(value);-- if (thread_context.simdgroup_index == 0) {- shared[thread_context.threadgroup_index] = local_sum;- }-- threadgroup_barrier(mem_flags::mem_threadgroup);-- T total_sum = T(0);- const ushort num_simd_groups =- (BLOCK_SIZE + thread_context.simdgroup_size - 1) /- thread_context.simdgroup_size;- if (lid < num_simd_groups) {- total_sum = shared[lid];- }- total_sum = simd_sum(total_sum);-- if (lid == 0) {- shared[0] = total_sum;- }- threadgroup_barrier(mem_flags::mem_threadgroup);+#include "thread_context.h"- const T result = shared[0];- threadgroup_barrier(mem_flags::mem_threadgroup);- return result;-}+using namespace metal;-template <ushort BLOCK_SIZE, typename T>-static T threadgroup_cooperative_reduce_max(- T value,- threadgroup T* shared,- const ushort lid,+template <typename T>+struct SimdReduceSum {+ using value_type = T;+ static constant constexpr T identity = static_cast<T>(0);+ static T simd_reduce(T x) { return simd_sum(x); }+};++template <typename T>+struct SimdReduceMax {+ using value_type = T;+ static constant constexpr T identity = numeric_limits<T>::has_infinity+ ? -numeric_limits<T>::infinity()+ : numeric_limits<T>::lowest();+ static T simd_reduce(T x) { return simd_max(x); }+};++template <typename T>+struct SimdReduceMin {+ using value_type = T;+ static constant constexpr T identity = numeric_limits<T>::has_infinity+ ? numeric_limits<T>::infinity()+ : numeric_limits<T>::max();+ static T simd_reduce(T x) { return simd_min(x); }+};++template <typename Op, ushort BLOCK_SIZE>+static typename Op::value_type threadgroup_cooperative_reduce(+ typename Op::value_type value,+ threadgroup typename Op::value_type* shared,const thread ThreadContext& thread_context) {- T local_max = simd_max(value);-+ // Phase 1: reduce within each simdgroup+ typename Op::value_type local = Op::simd_reduce(value);if (thread_context.simdgroup_index == 0) {- shared[thread_context.threadgroup_index] = local_max;- }-- threadgroup_barrier(mem_flags::mem_threadgroup);-- T total_max = lid < (BLOCK_SIZE + thread_context.simdgroup_size - 1) /- thread_context.simdgroup_size- ? shared[lid]- : T(-INFINITY);- total_max = simd_max(total_max);-- if (lid == 0) {- shared[0] = total_max;+ shared[thread_context.threadgroup_index] = local;}- threadgroup_barrier(mem_flags::mem_threadgroup);- const T result = shared[0];threadgroup_barrier(mem_flags::mem_threadgroup);- return result;-}-template <ushort BLOCK_SIZE, typename T>-static T threadgroup_cooperative_reduce_min(- T value,- threadgroup T* shared,- const ushort lid,- const thread ThreadContext& thread_context-) {- T local_min = simd_min(value);+ // Phase 2: first simdgroup reduces across simdgropus+ if (thread_context.threadgroup_index == 0) {+ typename Op::value_type total =+ thread_context.simdgroup_index <+ (BLOCK_SIZE + thread_context.simdgroup_size - 1) /+ thread_context.simdgroup_size+ ? shared[thread_context.simdgroup_index]+ : Op::identity;+ total = Op::simd_reduce(total);- if (thread_context.simdgroup_index == 0) {- shared[thread_context.threadgroup_index] = local_min;+ if (thread_context.simdgroup_index == 0) {+ shared[0] = total;+ }}threadgroup_barrier(mem_flags::mem_threadgroup);- T total_min = lid < (BLOCK_SIZE + thread_context.simdgroup_size - 1) /- thread_context.simdgroup_size- ? shared[lid]- : T(INFINITY);- total_min = simd_min(total_min);+ // Phase 3: broadcast+ const typename Op::value_type result = shared[0];- if (lid == 0) {- shared[0] = total_min;- }threadgroup_barrier(mem_flags::mem_threadgroup);- const T result = shared[0];- threadgroup_barrier(mem_flags::mem_threadgroup);return result;}diff --git a/crates/uzu/src/backends/metal/kernel/common/utils.h b/crates/uzu/src/backends/metal/kernel/common/utils.hdeleted file mode 100644index 71a185205..000000000--- a/crates/uzu/src/backends/metal/kernel/common/utils.h+++ /dev/null@@ -1,469 +0,0 @@-#pragma once--#include <metal_math>--#include "bf16.h"-#include "bf16_math.h"-#include "complex.h"-#include "defines.h"-#include "logging.h"--typedef half float16_t;--// Work per thread values for different types.-template <typename U>-struct WorkPerThread {- static_assert(sizeof(U) <= 8, "Type too large");- static constexpr int constant n = 8 / sizeof(U);-};--///////////////////////////////////////////////////////////////////////////////-// Type limits utils-///////////////////////////////////////////////////////////////////////////////--template <typename U>-struct Limits {- static const constant U max = metal::numeric_limits<U>::max();- static const constant U min = metal::numeric_limits<U>::min();- static const constant U finite_max = metal::numeric_limits<U>::max();- static const constant U finite_min = metal::numeric_limits<U>::min();-};--#define instantiate_default_limit(type) \- template <> \- struct Limits<type> { \- static constexpr constant type max = metal::numeric_limits<type>::max(); \- static constexpr constant type min = metal::numeric_limits<type>::min(); \- static constexpr constant type finite_max = \- metal::numeric_limits<type>::max(); \- static constexpr constant type finite_min = \- metal::numeric_limits<type>::min(); \- };--instantiate_default_limit(uint8_t);-instantiate_default_limit(uint16_t);-instantiate_default_limit(uint32_t);-instantiate_default_limit(uint64_t);-instantiate_default_limit(int8_t);-instantiate_default_limit(int16_t);-instantiate_default_limit(int32_t);-instantiate_default_limit(int64_t);--#define instantiate_float_limit(type) \- template <> \- struct Limits<type> { \- static constexpr constant type max = \- metal::numeric_limits<type>::infinity(); \- static constexpr constant type min = \- -metal::numeric_limits<type>::infinity(); \- static constexpr constant type finite_max = \- metal::numeric_limits<type>::max(); \- static constexpr constant type finite_min = \- -metal::numeric_limits<type>::max(); \- };--instantiate_float_limit(half);-instantiate_float_limit(float);-instantiate_float_limit(bfloat16_t);--template <>-struct Limits<bool> {- static constexpr constant bool max = true;- static constexpr constant bool min = false;-};--template <>-struct Limits<complex64_t> {- static constexpr constant complex64_t max = complex64_t(- metal::numeric_limits<float>::infinity(),- metal::numeric_limits<float>::infinity()- );- static constexpr constant complex64_t min = complex64_t(- -metal::numeric_limits<float>::infinity(),- -metal::numeric_limits<float>::infinity()- );-};--///////////////////////////////////////////////////////////////////////////////-// Indexing utils-///////////////////////////////////////////////////////////////////////////////--///////////////////////////////////////////////////////////////////////////////-// Single Array with generic dims--template <typename IdxT = int64_t>-METAL_FUNC IdxT elem_to_loc(- IdxT elem,- constant const int* shape,- constant const int64_t* strides,- int ndim-) {- IdxT loc = 0;- for (int i = ndim - 1; i >= 0 && elem > 0; --i) {- loc += (elem % shape[i]) * IdxT(strides[i]);- elem /= shape[i];- }- return loc;-}--// Non templated version to handle arbitrary dims-template <typename IdxT = int64_t>-METAL_FUNC IdxT elem_to_loc(- uint3 elem,- constant const int* shape,- constant const int64_t* strides,- int ndim-) {- IdxT loc =- elem.x * IdxT(strides[ndim - 1]) + elem.y * IdxT(strides[ndim - 2]);- for (int d = ndim - 3; d >= 0; --d) {- loc += (elem.z % shape[d]) * IdxT(strides[d]);- elem.z /= shape[d];- }- return loc;-}--///////////////////////////////////////////////////////////////////////////////-// Single Array with fixed N dims--template <typename IdxT = int64_t>-METAL_FUNC IdxT elem_to_loc_1(uint elem, constant const int64_t& stride) {- return elem * IdxT(stride);-}--template <typename IdxT = int64_t>-METAL_FUNC IdxT elem_to_loc_2(uint2 elem, constant const int64_t strides[2]) {- return elem.x * IdxT(strides[1]) + elem.y * IdxT(strides[0]);-}--template <typename IdxT = int64_t>-METAL_FUNC IdxT elem_to_loc_3(uint3 elem, constant const int64_t strides[3]) {- return elem.x * IdxT(strides[2]) + elem.y * IdxT(strides[1]) +- elem.z * IdxT(strides[0]);-}--///////////////////////////////////////////////////////////////////////////////-// Multiple Arrays with generic dims--template <typename IdxT = int64_t>-METAL_FUNC vec<IdxT, 2> elem_to_loc_2_nd(- uint3 elem,- constant const int* shape,- constant const int64_t* a_strides,- constant const int64_t* b_strides,- int ndim-) {- vec<IdxT, 2> loc = {- IdxT(- elem.x * IdxT(a_strides[ndim - 1]) +- IdxT(elem.y) * IdxT(a_strides[ndim - 2])- ),- IdxT(- elem.x * IdxT(b_strides[ndim - 1]) +- elem.y * IdxT(b_strides[ndim - 2])- )- };- for (int d = ndim - 3; d >= 0; --d) {- uint l = elem.z % shape[d];- loc.x += l * IdxT(a_strides[d]);- loc.y += l * IdxT(b_strides[d]);- elem.z /= shape[d];- }- return loc;-}--template <typename IdxT = int64_t>-METAL_FUNC vec<IdxT, 3> elem_to_loc_3_nd(- uint3 elem,- constant const int* shape,- constant const int64_t* a_strides,- constant const int64_t* b_strides,- constant const int64_t* c_strides,- int ndim-) {- vec<IdxT, 3> loc = {- IdxT(elem.x * IdxT(a_strides[ndim - 1])) +- IdxT(elem.y * IdxT(a_strides[ndim - 2])),- IdxT(elem.x * IdxT(b_strides[ndim - 1])) +- IdxT(elem.y * IdxT(b_strides[ndim - 2])),- IdxT(elem.x * IdxT(c_strides[ndim - 1])) +- IdxT(elem.y * IdxT(c_strides[ndim - 2]))- };- for (int d = ndim - 3; d >= 0; --d) {- uint l = elem.z % shape[d];- loc.x += l * IdxT(a_strides[d]);- loc.y += l * IdxT(b_strides[d]);- loc.z += l * IdxT(c_strides[d]);- elem.z /= shape[d];- }- return loc;-}--///////////////////////////////////////////////////////////////////////////////-// Elem to loc in a loop utils-///////////////////////////////////////////////////////////////////////////////--template <int DIM, typename OffsetT = size_t, bool General = true>-struct LoopedElemToLoc {- int dim;- LoopedElemToLoc<DIM - 1, OffsetT, General> inner_looper;- OffsetT offset{0};- int index{0};-- LoopedElemToLoc(int dim) : dim(dim), inner_looper(dim - 1) {}-- void next(const constant int* shape, const constant int64_t* strides) {- if (dim == 0) {- return;- }- index++;- offset += OffsetT(strides[dim - 1]);- if (index >= shape[dim - 1]) {- index = 0;- inner_looper.next(shape, strides);- offset = inner_looper.offset;- }- }-- void next(int n, const constant int* shape, const constant int64_t* strides) {- if (dim == 0) {- return;- }- index += n;- offset += n * OffsetT(strides[dim - 1]);-- if (index >= shape[dim - 1]) {- int extra = index - shape[dim - 1];- if (extra >= shape[dim - 1]) {- inner_looper.next(1 + extra / shape[dim - 1], shape, strides);- extra = extra % shape[dim - 1];- } else {- inner_looper.next(shape, strides);- }- index = 0;- offset = inner_looper.offset;- if (extra > 0) {- next(extra, shape, strides);- }- }- }-- OffsetT location() { return offset; }-};--template <typename OffsetT>-struct LoopedElemToLoc<1, OffsetT, true> {- int dim;- OffsetT offset{0};- uint index{0};-- LoopedElemToLoc(int dim) : dim(dim) {}-- void next(const constant int* shape, const constant int64_t* strides) {- index++;- if (dim > 1) {- offset = elem_to_loc<OffsetT>(index, shape, strides, dim);- } else {- offset += OffsetT(strides[0]);- }- }-- void next(int n, const constant int* shape, const constant int64_t* strides) {- index += n;- if (dim > 1) {- offset = elem_to_loc<OffsetT>(index, shape, strides, dim);- } else {- offset = index * OffsetT(strides[0]);- }- }-- OffsetT location() { return offset; }-};--template <typename OffsetT>-struct LoopedElemToLoc<1, OffsetT, false> {- OffsetT offset{0};-- LoopedElemToLoc(int) {}-- void next(const constant int*, const constant int64_t* strides) {- offset += OffsetT(strides[0]);- }-- void next(int n, const constant int*, const constant int64_t* strides) {- offset += n * OffsetT(strides[0]);- }-- OffsetT location() { return offset; }-};--///////////////////////////////////////////////////////////////////////////////-// Calculation utils-///////////////////////////////////////////////////////////////////////////////--/** Compute ceil((float)N/(float)M) */-template <typename T, typename U>-inline T ceildiv(T N, U M) {- return (N + M - 1) / M;-}--// https://docs.oracle.com/cd/E19957-01/806-3568/ncg_goldberg.html#1202-inline float log1p(float x) {- float xp1 = 1.0f + x;- if (xp1 == Limits<float>::max) {- return Limits<float>::max;- }- if (xp1 == 1.0f) {- return x;- }-- return x * (metal::log(xp1) / (xp1 - 1.0f));-}--inline bfloat16_t log1p(bfloat16_t x) {- float xp1 = 1.0f + static_cast<float>(x);- if (xp1 == Limits<float>::max) {- return Limits<bfloat16_t>::max;- }- if (xp1 == 1.0f) {- return x;- }-- return bfloat16_t(x * (metal::log(xp1) / (xp1 - 1.0f)));-}--inline complex64_t log1p(complex64_t in) {- float x = in.real;- float y = in.imag;- float zabs = metal::precise::sqrt(x * x + y * y);- float theta = metal::atan2(y, x + 1);- if (zabs < 0.5f) {- float r = x * (2 + x) + y * y;- if (r == 0) { // handle underflow- return {x, theta};- }- return {0.5f * log1p(r), theta};- } else {- auto z0 = metal::sqrt((x + 1) * (x + 1) + y * y);- return {metal::log(z0), theta};- }-}--///////////////////////////////////////////////////////////////////////////////-// SIMD shuffle ops-///////////////////////////////////////////////////////////////////////////////--inline uint64_t simd_shuffle_down(uint64_t data, uint16_t delta) {- return as_type<uint64_t>(- metal::simd_shuffle_down(as_type<uint2>(data), delta)- );-}--inline int64_t simd_shuffle_down(int64_t data, uint16_t delta) {- return as_type<int64_t>(- metal::simd_shuffle_down(as_type<uint2>(data), delta)- );-}--inline bool simd_shuffle_down(bool data, uint16_t delta) {- return simd_shuffle_down(static_cast<uint32_t>(data), delta);-}--inline complex64_t simd_shuffle_down(complex64_t data, uint16_t delta) {- return complex64_t(- simd_shuffle_down(data.real, delta),- simd_shuffle_down(data.imag, delta)- );-}--inline uint64_t simd_shuffle_up(uint64_t data, uint16_t delta) {- return as_type<uint64_t>(metal::simd_shuffle_up(as_type<uint2>(data), delta));-}--inline int64_t simd_shuffle_up(int64_t data, uint16_t delta) {- return as_type<int64_t>(metal::simd_shuffle_up(as_type<uint2>(data), delta));-}--inline bool simd_shuffle_up(bool data, uint16_t delta) {- return simd_shuffle_up(static_cast<uint32_t>(data), delta);-}--inline complex64_t simd_shuffle_up(complex64_t data, uint16_t delta) {- return complex64_t(- simd_shuffle_up(data.real, delta),- simd_shuffle_up(data.imag, delta)- );-}--inline uint64_t simd_shuffle_and_fill_up(- uint64_t data,- uint64_t filling,- uint16_t delta-) {- return as_type<uint64_t>(metal::simd_shuffle_and_fill_up(- as_type<uint2>(data),- as_type<uint2>(filling),- delta- ));-}--inline int64_t simd_shuffle_and_fill_up(- int64_t data,- int64_t filling,- uint16_t delta-) {- return as_type<int64_t>(metal::simd_shuffle_and_fill_up(- as_type<uint2>(data),- as_type<uint2>(filling),- delta- ));-}--inline bool simd_shuffle_and_fill_up(bool data, bool filling, uint16_t delta) {- return simd_shuffle_and_fill_up(- static_cast<uint32_t>(data),- static_cast<uint32_t>(filling),- delta- );-}--inline complex64_t simd_shuffle_and_fill_up(- complex64_t data,- complex64_t filling,- uint16_t delta-) {- return complex64_t(- simd_shuffle_and_fill_up(data.real, filling.real, delta),- simd_shuffle_and_fill_up(data.imag, filling.imag, delta)- );-}--inline uint64_t simd_shuffle(uint64_t data, uint16_t lane) {- return as_type<uint64_t>(metal::simd_shuffle(as_type<uint2>(data), lane));-}--inline int64_t simd_shuffle(int64_t data, uint16_t lane) {- return as_type<int64_t>(metal::simd_shuffle(as_type<uint2>(data), lane));-}--inline bool simd_shuffle(bool data, uint16_t lane) {- return simd_shuffle(static_cast<uint32_t>(data), lane);-}--inline complex64_t simd_shuffle(complex64_t data, uint16_t lane) {- return complex64_t(- simd_shuffle(data.real, lane),- simd_shuffle(data.imag, lane)- );-}--// std::conditional is not included with Metal-template <bool condition, typename T, typename U>-struct ConditionalType {- using type = U;-};--template <typename T, typename U>-struct ConditionalType<true, T, U> {- using type = T;-};diff --git a/crates/uzu/src/backends/metal/kernel/delta_net/norm_gate.metal b/crates/uzu/src/backends/metal/kernel/delta_net/norm_gate.metalindex ba45c4018..2a7020b36 100644--- a/crates/uzu/src/backends/metal/kernel/delta_net/norm_gate.metal+++ b/crates/uzu/src/backends/metal/kernel/delta_net/norm_gate.metal@@ -42,12 +42,12 @@ PUBLIC KERNEL(DeltaNetNormGate)(// RMSNorm: cooperative sum of squaresfloat o_sq = active ? o_i * o_i : 0.0f;- float o_sumsq = threadgroup_cooperative_reduce_sum<HEAD_V_DIM>(- o_sq,- shared_scratch,- tid,- thread_context- );+ float o_sumsq =+ threadgroup_cooperative_reduce<SimdReduceSum<float>, HEAD_V_DIM>(+ o_sq,+ shared_scratch,+ thread_context+ );float inv_rms = rsqrt(o_sumsq / float(head_v_dim) + norm_epsilon);// Apply norm + SiLU gate (in-place)diff --git a/crates/uzu/src/backends/metal/kernel/delta_net/update.metal b/crates/uzu/src/backends/metal/kernel/delta_net/update.metalindex 7fdd882a4..a12e283a4 100644--- a/crates/uzu/src/backends/metal/kernel/delta_net/update.metal+++ b/crates/uzu/src/backends/metal/kernel/delta_net/update.metal@@ -62,21 +62,21 @@ PUBLIC KERNEL(DeltaNetUpdate)(threadgroup_barrier(mem_flags::mem_threadgroup);float q_partial = (tid < HEAD_K_DIM) ? shared_q[tid] * shared_q[tid] : 0.0f;- float q_norm_sq = threadgroup_cooperative_reduce_sum<UPDATE_THREADS>(- q_partial,- shared_scratch,- tid,- thread_context- );+ float q_norm_sq =+ threadgroup_cooperative_reduce<SimdReduceSum<float>, UPDATE_THREADS>(+ q_partial,+ shared_scratch,+ thread_context+ );float q_inv_norm = rsqrt(q_norm_sq + 1e-6f);float k_partial = (tid < HEAD_K_DIM) ? shared_k[tid] * shared_k[tid] : 0.0f;- float k_norm_sq = threadgroup_cooperative_reduce_sum<UPDATE_THREADS>(- k_partial,- shared_scratch,- tid,- thread_context- );+ float k_norm_sq =+ threadgroup_cooperative_reduce<SimdReduceSum<float>, UPDATE_THREADS>(+ k_partial,+ shared_scratch,+ thread_context+ );float k_inv_norm = rsqrt(k_norm_sq + 1e-6f);float q_scale = rsqrt(float(HEAD_K_DIM));@@ -87,12 +87,12 @@ PUBLIC KERNEL(DeltaNetUpdate)(threadgroup_barrier(mem_flags::mem_threadgroup);float kq_p = (tid < HEAD_K_DIM) ? shared_k[tid] * shared_q[tid] : 0.0f;- float kq_dot = threadgroup_cooperative_reduce_sum<UPDATE_THREADS>(- kq_p,- shared_scratch,- tid,- thread_context- );+ float kq_dot =+ threadgroup_cooperative_reduce<SimdReduceSum<float>, UPDATE_THREADS>(+ kq_p,+ shared_scratch,+ thread_context+ );float beta_raw = float(in_proj[conv_dim + value_dim + hv_idx]);float beta = 1.0f / (1.0f + fast::exp(-beta_raw));@@ -139,12 +139,12 @@ PUBLIC KERNEL(DeltaNetUpdate)(// RMSNorm + SiLU gate — only ki_part==0 contributesfloat o_sq = (active && ki_part == 0) ? o_i * o_i : 0.0f;- float o_sumsq = threadgroup_cooperative_reduce_sum<UPDATE_THREADS>(- o_sq,- shared_scratch,- tid,- thread_context- );+ float o_sumsq =+ threadgroup_cooperative_reduce<SimdReduceSum<float>, UPDATE_THREADS>(+ o_sq,+ shared_scratch,+ thread_context+ );float inv_rms = rsqrt(o_sumsq / float(head_v_dim) + norm_epsilon);if (active && ki_part == 0) {diff --git a/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal b/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metalindex 08758f523..92c946f22 100644--- a/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal+++ b/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal@@ -36,12 +36,12 @@ void layer_norm_core(}// Reduce to get total mean- AccumT total_sum = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(- partial_sum,- shared_mean,- thread_in_row,- thread_context- );+ AccumT total_sum =+ threadgroup_cooperative_reduce<SimdReduceSum<AccumT>, BLOCK_SIZE>(+ partial_sum,+ shared_mean,+ thread_context+ );AccumT mean = total_sum / static_cast<AccumT>(element_count);// Compute variance: sum of squared deviations@@ -58,12 +58,12 @@ void layer_norm_core(}// Reduce to get total variance- AccumT total_var = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(- partial_var_sum,- shared_variance,- thread_in_row,- thread_context- );+ AccumT total_var =+ threadgroup_cooperative_reduce<SimdReduceSum<AccumT>, BLOCK_SIZE>(+ partial_var_sum,+ shared_variance,+ thread_context+ );AccumT variance = total_var / static_cast<AccumT>(element_count);AccumT inv_std = rsqrt(variance + static_cast<AccumT>(epsilon));diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/defines.h b/crates/uzu/src/backends/metal/kernel/matmul/common/defines.hindex c441e1f58..1b56d03b3 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/common/defines.h+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/defines.h@@ -33,11 +33,6 @@ struct PointerElementTypeImpl<threadgroup T*> {template <typename T>using PointerElementType = typename PointerElementTypeImpl<T>::type;-template <typename T>-struct AccumulatorHelper {- typedef float AccumulatorType;-};-template <typename OutT, typename InT>struct TransformNone {TransformNone(float = 1.0f, float = 0.0f) {}@@ -46,16 +41,6 @@ struct TransformNone {METAL_FUNC OutT apply(InT x, OutT) const { return static_cast<OutT>(x); }};-template <typename OutT, typename InT>-struct TransformAdd {- TransformAdd(float = 1.0f, float = 1.0f) {}-- static METAL_FUNC OutT apply(InT x) { return static_cast<OutT>(x); }- METAL_FUNC OutT apply(InT x, OutT c) const {- return static_cast<OutT>(x) + c;- }-};-template <typename OutT, typename InT>struct TransformScaleAccumulate {const float alpha;diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemm.metalindex 9f6d548fb..f790479f0 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm.metal+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm.metal@@ -1,4 +1,3 @@-#include "../common/utils.h"#include "../common/dsl.h"#include "../common/thread_context.h"diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemv.metalindex 3187bee36..b08eb3e73 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv.metal+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv.metal@@ -1,4 +1,6 @@-#include "../common/utils.h"+#include <metal_stdlib>++#include "../common/defines.h"#include "../common/dsl.h"#include "../common/thread_context.h"diff --git a/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal b/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metalindex 571d408ce..8b10d91f3 100644--- a/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal@@ -5,6 +5,40 @@#include "../common/thread_context.h"#include "../common/threadgroup_reduce.h"+template <ushort BLOCK_SIZE, typename T>+static T threadgroup_raking_prefix_exclusive_sum(+ T value,+ threadgroup T* shared,+ const ushort lid+) {+ shared[lid] = value;+ threadgroup_barrier(mem_flags::mem_threadgroup);++ if (lid < 32) {+ const short values_per_thread = BLOCK_SIZE / 32;+ const short first_index = lid * values_per_thread;+ for (short i = first_index + 1; i < first_index + values_per_thread; i++) {+ shared[i] += shared[i - 1];+ }+ T partial_sum = shared[first_index + values_per_thread - 1];+ for (short i = first_index + values_per_thread - 1; i > first_index; i--) {+ shared[i] = shared[i - 1];+ }+ shared[first_index] = 0;++ T prefix = simd_prefix_exclusive_sum(partial_sum);++ for (short i = first_index; i < first_index + values_per_thread; i++) {+ shared[i] += prefix;+ }+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);++ const T result = shared[lid];+ threadgroup_barrier(mem_flags::mem_threadgroup);+ return result;+}+#define BLOCK_SIZE 128#define TILE_E 512@@ -100,12 +134,12 @@ PUBLIC KERNEL(MoeCountsOffsetsFused)(offsets[base + lid] = prefix_global;}- uint block_sum = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(- v,- reduce_shared,- (ushort)lid,- thread_context- );+ uint block_sum =+ threadgroup_cooperative_reduce<SimdReduceSum<uint>, BLOCK_SIZE>(+ v,+ reduce_shared,+ thread_context+ );if (lid == 0) {carry += block_sum;}diff --git a/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal b/crates/uzu/src/backends/metal/kernel/moe/router_topk.metalindex 181d9afa0..cbb1a161e 100644--- a/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal@@ -96,20 +96,20 @@ PUBLIC KERNEL(MoeRouterTopK)(}}- float max_val = threadgroup_cooperative_reduce_max<THREADS_PER_TG>(- local_best,- reduce_tmp,- lid,- thread_context- );+ float max_val =+ threadgroup_cooperative_reduce<SimdReduceMax<float>, THREADS_PER_TG>(+ local_best,+ reduce_tmp,+ thread_context+ );uint candidate_id = (local_best == max_val) ? local_idx : 0xFFFFFFFFu;- uint best_idx = threadgroup_cooperative_reduce_min<THREADS_PER_TG>(- candidate_id,- reduce_tmp_u,- lid,- thread_context- );+ uint best_idx =+ threadgroup_cooperative_reduce<SimdReduceMin<uint>, THREADS_PER_TG>(+ candidate_id,+ reduce_tmp_u,+ thread_context+ );if (lid == 0) {shared_best_idx[0] = best_idx;diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm.metalindex f16fba9bb..bcdc3b970 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm.metal@@ -1,6 +1,6 @@#include <metal_stdlib>#include "../common/dsl.h"-#include "quant_matmul.metal"+#include "quant_matmul.h"template <typename T, int GROUP_SIZE, int BITS>VARIANTS(T, float, half, bfloat)diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metalindex c324e936f..84b0932b9 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal@@ -1,6 +1,6 @@#include <metal_stdlib>#include "../common/dsl.h"-#include "quant_matmul.metal"+#include "quant_matmul.h"template <typename T, int GROUP_SIZE, int BITS>VARIANTS(T, float, half, bfloat)diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metalindex 61d6fc85f..b4af07b58 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metal@@ -1,6 +1,6 @@#include <metal_stdlib>#include "../common/dsl.h"-#include "quant_matmul.metal"+#include "quant_matmul.h"template <typename T, int GROUP_SIZE, int BITS>VARIANTS(T, bfloat)diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_wide.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_wide.metalindex 6118a70fe..a066b3022 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_wide.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_wide.metal@@ -1,6 +1,6 @@#include <metal_stdlib>#include "../common/dsl.h"-#include "quant_matmul.metal"+#include "quant_matmul.h"// Wide QMM tile: BM=64, BK=32, BN=64.// Processes 4x more output elements per threadgroup than the 32x32 variant.diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metalindex 6bcfc03b9..527711d3a 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal@@ -1,6 +1,6 @@#include <metal_stdlib>#include "../common/dsl.h"-#include "quant_matmul.metal"+#include "quant_matmul.h"template <typename T, int GROUP_SIZE, int BITS>VARIANTS(T, float, half, bfloat)diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metalindex c8d0a8b35..db8eda84f 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal@@ -1,6 +1,6 @@#include <metal_stdlib>#include "../common/dsl.h"-#include "quant_matmul.metal"+#include "quant_matmul.h"template <typename T, int GROUP_SIZE, int BITS>VARIANTS(T, float, half, bfloat)diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.hsimilarity index 100%rename from crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metalrename to crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.hdiff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qvm.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qvm.metalindex 3c389f961..27356c4a1 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qvm.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qvm.metal@@ -1,6 +1,6 @@#include <metal_stdlib>#include "../common/dsl.h"-#include "quant_matmul.metal"+#include "quant_matmul.h"template <typename T, int GROUP_SIZE, int BITS>VARIANTS(T, float, half, bfloat)diff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metalindex d08730f49..79e1e57b9 100644--- a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal+++ b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal@@ -56,12 +56,12 @@ PUBLIC KERNEL(RMSNorm)(}// Compute total sum across threadgroup- AccumT total_sum = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(- partial_sum,- shared_sum,- thread_in_row,- thread_context- );+ AccumT total_sum =+ threadgroup_cooperative_reduce<SimdReduceSum<AccumT>, BLOCK_SIZE>(+ partial_sum,+ shared_sum,+ thread_context+ );// Compute RMS norm factorAccumT mean_square =diff --git a/crates/uzu/src/backends/metal/kernel/rng.metal b/crates/uzu/src/backends/metal/kernel/rng.hsimilarity index 97%rename from crates/uzu/src/backends/metal/kernel/rng.metalrename to crates/uzu/src/backends/metal/kernel/rng.hindex 97979277f..8715ebc7d 100644--- a/crates/uzu/src/backends/metal/kernel/rng.metal+++ b/crates/uzu/src/backends/metal/kernel/rng.h@@ -1,8 +1,6 @@-#ifndef rng_metal-#define rng_metal+#pragma once#include <metal_stdlib>-using namespace metal;/* ───────────── Philox4x32-10 RNG ───────────── */@@ -126,5 +124,3 @@ inline uint philox_next(thread PhiloxState* state) {inline float uniform_float(thread PhiloxState* state) {return float(philox_next(state)) * (1.0f / 4294967296.0f); /* (0,1) */}--#endif /* rng_metal */diff --git a/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal b/crates/uzu/src/backends/metal/kernel/sampling/argmax.metalindex 497ffa642..852a43700 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal+++ b/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal@@ -2,6 +2,7 @@#include <metal_atomic>#include "../common/dsl.h"#include "../common/thread_context.h"+#include "../common/threadgroup_reduce.h"#include "argmax.h"@@ -20,59 +21,20 @@ ArgmaxPair argmax_combine(ArgmaxPair a, ArgmaxPair b) {return argmax_is_better(a, b) ? a : b;}-//------------------------------------------------------------------------------------------------//-// Cooperative threadgroup argmax reduction (2-level hierarchical)-template <ushort BLOCK_SIZE_PARAM>-static ArgmaxPair threadgroup_cooperative_argmax(- ArgmaxPair value,- threadgroup ArgmaxPair* shared,- const ushort lid,- const thread ThreadContext& thread_context-) {- // Reduce within simdgroup using manual shuffle operations- ArgmaxPair local_result = value;- for (ushort offset = 16; offset > 0; offset /= 2) {- ArgmaxPair other = {- simd_shuffle_down(local_result.value, offset),- simd_shuffle_down(local_result.index, offset)- };- local_result = argmax_combine(local_result, other);- }-- // First thread in each simdgroup writes to shared memory- if (thread_context.simdgroup_index == 0) {- shared[thread_context.threadgroup_index] = local_result;- }-- // Synchronize across the threadgroup- threadgroup_barrier(mem_flags::mem_threadgroup);-- // Reduce across simdgroups- const ushort num_simd_groups = (BLOCK_SIZE_PARAM + 31) / 32;- ArgmaxPair total_result = ARGMAX_INIT;- if (lid < num_simd_groups) {- total_result = shared[lid];- }-- // Final simdgroup reduction- for (ushort offset = 16; offset > 0; offset /= 2) {- ArgmaxPair other = {- simd_shuffle_down(total_result.value, offset),- simd_shuffle_down(total_result.index, offset)- };- total_result = argmax_combine(total_result, other);- }-- // Broadcast the result to all threads- if (lid == 0) {- shared[0] = total_result;+struct SimdReduceArgmax {+ using value_type = ArgmaxPair;+ static constant constexpr ArgmaxPair identity = ARGMAX_INIT;+ static ArgmaxPair simd_reduce(ArgmaxPair x) {+ for (ushort offset = 16; offset > 0; offset /= 2) {+ ArgmaxPair other = {+ simd_shuffle_down(x.value, offset),+ simd_shuffle_down(x.index, offset)+ };+ x = argmax_combine(x, other);+ }+ return x;}- threadgroup_barrier(mem_flags::mem_threadgroup);-- const ArgmaxPair result = shared[0];- threadgroup_barrier(mem_flags::mem_threadgroup);- return result;-}+};template <ushort GRAIN_SIZE_PARAM>static inline ArgmaxPair thread_argmax_reduce(@@ -226,12 +188,12 @@ PUBLIC KERNEL(ArgmaxMain)();ArgmaxPair thread_result = thread_argmax_reduce<GRAIN_SIZE>(values);- ArgmaxPair group_result = threadgroup_cooperative_argmax<BLOCK_SIZE>(- thread_result,- shared,- local_id,- thread_context- );+ ArgmaxPair group_result =+ threadgroup_cooperative_reduce<SimdReduceArgmax, BLOCK_SIZE>(+ thread_result,+ shared,+ thread_context+ );if (local_id == 0) {uint vocab_groups_per_batch =@@ -267,12 +229,12 @@ PUBLIC KERNEL(ArgmaxFinal)(argmax_combine(thread_result, partial_results[partial_offset + i]);}- ArgmaxPair result = threadgroup_cooperative_argmax<BLOCK_SIZE>(- thread_result,- shared,- local_id,- thread_context- );+ ArgmaxPair result =+ threadgroup_cooperative_reduce<SimdReduceArgmax, BLOCK_SIZE>(+ thread_result,+ shared,+ thread_context+ );if (local_id == 0) {final_tokens[batch_idx] = result.index;diff --git a/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metal b/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metalindex 6fb5ce481..cecb7587f 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metal+++ b/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metal@@ -1,6 +1,6 @@#include <metal_stdlib>#include "../common/dsl.h"-#include "../rng.metal"+#include "../rng.h"#define BLOCK_SIZE 1024#define GRAIN_SIZE 64diff --git a/crates/uzu/src/backends/metal/kernel/sampling/min_p.metal b/crates/uzu/src/backends/metal/kernel/sampling/min_p.metalindex 9b3962188..36b988cec 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/min_p.metal+++ b/crates/uzu/src/backends/metal/kernel/sampling/min_p.metal@@ -33,12 +33,12 @@ PUBLIC KERNEL(MinP) (float logit_value = float(logits[batch_start + i]);local_max = fmax(local_max, logit_value);}- float max_logit = threadgroup_cooperative_reduce_max<BLOCK_SIZE>(- local_max,- shared_reduce_buffer,- thread_idx,- thread_context- );+ float max_logit =+ threadgroup_cooperative_reduce<SimdReduceMax<float>, BLOCK_SIZE>(+ local_max,+ shared_reduce_buffer,+ thread_context+ );// Then the threshold is just max_logit + log(min_p), mask everything strictly// below itdiff --git a/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal b/crates/uzu/src/backends/metal/kernel/sampling/top_k.metalindex c4c08b604..7a27876aa 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal+++ b/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal@@ -40,18 +40,18 @@ PUBLIC KERNEL(TopK)(logit_value > -INFINITY);}- float max_logit = threadgroup_cooperative_reduce_max<BLOCK_SIZE>(- local_max,- shared_reduce_buffer,- thread_idx,- thread_context- );- float min_logit = threadgroup_cooperative_reduce_min<BLOCK_SIZE>(- local_min,- shared_reduce_buffer,- thread_idx,- thread_context- );+ float max_logit =+ threadgroup_cooperative_reduce<SimdReduceMax<float>, BLOCK_SIZE>(+ local_max,+ shared_reduce_buffer,+ thread_context+ );+ float min_logit =+ threadgroup_cooperative_reduce<SimdReduceMin<float>, BLOCK_SIZE>(+ local_min,+ shared_reduce_buffer,+ thread_context+ );// Do the binary search on the thresholdfloat low = min_logit;float high = max_logit;@@ -65,12 +65,12 @@ PUBLIC KERNEL(TopK)(float logit_value = float(logits[batch_start + i]);local_num_above_threshold += select(0, 1, logit_value >= threshold);}- uint num_above_threshold = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(- local_num_above_threshold,- (threadgroup uint*)shared_reduce_buffer,- thread_idx,- thread_context- );+ uint num_above_threshold =+ threadgroup_cooperative_reduce<SimdReduceSum<uint>, BLOCK_SIZE>(+ local_num_above_threshold,+ (threadgroup uint*)shared_reduce_buffer,+ thread_context+ );// Update binary searchif (num_above_threshold == top_k) {diff --git a/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal b/crates/uzu/src/backends/metal/kernel/sampling/top_p.metalindex 1ab64c2b7..a4b9ea0d8 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal+++ b/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal@@ -41,18 +41,18 @@ PUBLIC KERNEL(TopP) (logit_value > -INFINITY);}- float max_logit = threadgroup_cooperative_reduce_max<BLOCK_SIZE>(- local_max,- shared_reduce_buffer,- thread_idx,- thread_context- );- float min_logit = threadgroup_cooperative_reduce_min<BLOCK_SIZE>(- local_min,- shared_reduce_buffer,- thread_idx,- thread_context- );+ float max_logit =+ threadgroup_cooperative_reduce<SimdReduceMax<float>, BLOCK_SIZE>(+ local_max,+ shared_reduce_buffer,+ thread_context+ );+ float min_logit =+ threadgroup_cooperative_reduce<SimdReduceMin<float>, BLOCK_SIZE>(+ local_min,+ shared_reduce_buffer,+ thread_context+ );// Find denominator for softmaxfloat local_sum = 0.0f;@@ -65,12 +65,12 @@ PUBLIC KERNEL(TopP) (logit_value > -INFINITY);}- float total_sum = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(- local_sum,- shared_reduce_buffer,- thread_idx,- thread_context- );+ float total_sum =+ threadgroup_cooperative_reduce<SimdReduceSum<float>, BLOCK_SIZE>(+ local_sum,+ shared_reduce_buffer,+ thread_context+ );// Do the binary search on the thresholdfloat target_mass = top_p * total_sum;@@ -94,18 +94,18 @@ PUBLIC KERNEL(TopP) (select(INFINITY, logit_mass, logit_value >= threshold));}- float sum_above_threshold = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(- local_sum_above_threshold,- shared_reduce_buffer,- thread_idx,- thread_context- );- float min_above_threshold = threadgroup_cooperative_reduce_min<BLOCK_SIZE>(- local_min_above_threshold,- shared_reduce_buffer,- thread_idx,- thread_context- );+ float sum_above_threshold =+ threadgroup_cooperative_reduce<SimdReduceSum<float>, BLOCK_SIZE>(+ local_sum_above_threshold,+ shared_reduce_buffer,+ thread_context+ );+ float min_above_threshold =+ threadgroup_cooperative_reduce<SimdReduceMin<float>, BLOCK_SIZE>(+ local_min_above_threshold,+ shared_reduce_buffer,+ thread_context+ );// Early exitif (sum_above_threshold >= target_mass &&diff --git a/crates/uzu/src/backends/metal/kernel/sigmoid/sigmoid.metal b/crates/uzu/src/backends/metal/kernel/sigmoid/sigmoid.metaldeleted file mode 100644index e4adb869f..000000000--- a/crates/uzu/src/backends/metal/kernel/sigmoid/sigmoid.metal+++ /dev/null@@ -1,14 +0,0 @@-#include <metal_stdlib>-#include "../common/dsl.h"--template <typename T>-VARIANTS(T, float, half, bfloat)-PUBLIC KERNEL(Sigmoid) (- const device T* input,- device T* output,- const constant uint& total_elements,- uint tid AXIS(total_elements, 256)-) {- float x = float(input[tid]);- output[tid] = T(1.0f / (1.0f + exp(-x)));-}\ No newline at end of filediff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssm_common.h b/crates/uzu/src/backends/metal/kernel/ssm/ssm_common.hindex 41845ba0d..1884fc210 100644--- a/crates/uzu/src/backends/metal/kernel/ssm/ssm_common.h+++ b/crates/uzu/src/backends/metal/kernel/ssm/ssm_common.h@@ -1,3 +1,5 @@+#pragma once+#include <metal_stdlib>template <typename T>@@ -7,4 +9,4 @@ inline T softplus(T x) {return x;}return static_cast<T>(log(1.0f + fast::exp(xf)));-}\ No newline at end of file+}diff --git a/crates/uzu/tests/unit/kernel/mod.rs b/crates/uzu/tests/unit/kernel/mod.rsindex 92faaafff..7e47c316a 100644--- a/crates/uzu/tests/unit/kernel/mod.rs+++ b/crates/uzu/tests/unit/kernel/mod.rs@@ -18,7 +18,6 @@ mod rms_norm_test;mod rope_test;mod sampling;mod short_conv;-mod sigmoid_test;mod tensor_add_bias_test;mod tensor_add_scale_test;mod tensor_add_swap_test;diff --git a/crates/uzu/tests/unit/kernel/sigmoid_test.rs b/crates/uzu/tests/unit/kernel/sigmoid_test.rsdeleted file mode 100644index ca4f467a9..000000000--- a/crates/uzu/tests/unit/kernel/sigmoid_test.rs+++ /dev/null@@ -1,176 +0,0 @@-use std::{- fmt::{Debug, Display},- ops::{Deref, DerefMut},-};--use half::{bf16, f16};-use num_traits::Float;-use uzu::{- ArrayContextExt, ArrayElement, DataType,- backends::{- common::{Backend, Context, Encoder, Kernels, kernel::SigmoidKernel},- cpu::Cpu,- },-};--use crate::common::assert::assert_eq_float;--struct Input<T: ArrayElement + Float> {- data: Box<[T]>,-}--fn get_input<T: ArrayElement + Float>(n: usize) -> Vec<T> {- (0..n).map(|i| T::from((i as f32 * 0.3) - 5.0).unwrap()).collect()-}--fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {- let context = B::Context::new().expect("Failed to create Context");-- let kernel = <<B as Backend>::Kernels as Kernels>::SigmoidKernel::new(&context, T::data_type())- .expect("Failed to create SigmoidKernel");-- let n = input.data.len();- let input_array = context.create_array_from(&[n], &input.data, "");- let output_array = context.create_array_uninitialized(&[n], T::data_type(), "");-- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");- kernel.encode(- input_array.buffer().borrow().deref(),- output_array.buffer().borrow_mut().deref_mut(),- n as u32,- &mut encoder,- );- encoder.end_encoding().submit().wait_until_completed().unwrap();-- output_array.as_slice().to_vec()-}--fn get_test_data_basic<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {- let data = get_input::<T>(128);- let input = Input {- data: data.into_boxed_slice(),- };- let expected = get_output::<T, Cpu>(&input);- (input, expected)-}--fn get_test_data_large<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {- let data = get_input::<T>(4096);- let input = Input {- data: data.into_boxed_slice(),- };- let expected = get_output::<T, Cpu>(&input);- (input, expected)-}--fn get_test_data_edge<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {- // Values at sigmoid boundaries: very negative, zero, very positive- let values: Vec<T> = [-100.0f32, -10.0, -1.0, 0.0, 1.0, 10.0, 100.0].iter().map(|&v| T::from(v).unwrap()).collect();- let input = Input {- data: values.into_boxed_slice(),- };- let expected = get_output::<T, Cpu>(&input);- (input, expected)-}--fn test_basic<T: ArrayElement + Float + Debug + Display>() {- let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {- 0.02f32- } else {- 1e-5- };- let (input, expected) = get_test_data_basic::<T>();- for_each_non_cpu_backend!(|B| {- let output = get_output::<T, B>(&input);- assert_eq_float::<T>(- &expected,- &output,- eps,- &format!("Sigmoid basic test failed for backend {}", std::any::type_name::<B>()),- );- });-}--fn test_large<T: ArrayElement + Float + Debug + Display>() {- let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {- 0.02f32- } else {- 1e-5- };- let (input, expected) = get_test_data_large::<T>();- for_each_non_cpu_backend!(|B| {- let output = get_output::<T, B>(&input);- assert_eq_float::<T>(- &expected,- &output,- eps,- &format!("Sigmoid large test failed for backend {}", std::any::type_name::<B>()),- );- });-}--fn test_edge<T: ArrayElement + Float + Debug + Display>() {- let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {- 0.02f32- } else {- 1e-5- };- let (input, expected) = get_test_data_edge::<T>();- for_each_non_cpu_backend!(|B| {- let output = get_output::<T, B>(&input);- assert_eq_float::<T>(- &expected,- &output,- eps,- &format!("Sigmoid edge test failed for backend {}", std::any::type_name::<B>()),- );- });-}--// basic tests-#[test]-fn test_basic_f32() {- test_basic::<f32>();-}--#[test]-fn test_basic_f16() {- test_basic::<f16>();-}--#[test]-fn test_basic_bf16() {- test_basic::<bf16>();-}--// large tests-#[test]-fn test_large_f32() {- test_large::<f32>();-}--#[test]-fn test_large_f16() {- test_large::<f16>();-}--#[test]-fn test_large_bf16() {- test_large::<bf16>();-}--// edge tests-#[test]-fn test_edge_f32() {- test_edge::<f32>();-}--#[test]-fn test_edge_f16() {- test_edge::<f16>();-}--#[test]-fn test_edge_bf16() {- test_edge::<bf16>();-}