Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Metal common cleanup (#310)

uuuvn08cbb23b9f75
diff --git a/crates/uzu/build/metal/ast.rs b/crates/uzu/build/metal/ast.rs
index c8e789d6a..d91893a63 100644
--- a/crates/uzu/build/metal/ast.rs
+++ b/crates/uzu/build/metal/ast.rs
@@ -42,8 +42,6 @@ pub enum MetalAstKind {
StringLiteral {
value: Box<str>,
},
- MetalKernelAttr,
- MetalMaxTotalThreadsPerThreadGroupAttr,
Other,
}
@@ -570,63 +568,3 @@ impl MetalKernelInfo {
}))
}
}
-
-pub fn validate_raw_kernel(node: &MetalAstNode) -> anyhow::Result<()> {
- let (node, is_template) = if matches!(node.kind, MetalAstKind::FunctionTemplateDecl) {
- let inner_fn = node.inner.iter().find(|c| {
- matches!(
- c.kind,
- MetalAstKind::FunctionDecl {
- name: _
- }
- )
- });
- match inner_fn {
- Some(n) => (n, true),
- None => return Ok(()),
- }
- } else if matches!(node.kind, MetalAstKind::FunctionDecl { .. }) {
- (node, false)
- } else {
- return Ok(());
- };
-
- let MetalAstKind::FunctionDecl {
- name,
- } = &node.kind
- else {
- return Ok(());
- };
-
- let mut has_metal_kernel_attr = false;
- let mut has_max_threads_attr = false;
- let mut has_dsl_kernel = false;
-
- for child in &node.inner {
- match &child.kind {
- MetalAstKind::MetalKernelAttr => has_metal_kernel_attr = true,
- MetalAstKind::MetalMaxTotalThreadsPerThreadGroupAttr => has_max_threads_attr = true,
- MetalAstKind::AnnotateAttr => {
- if let Ok(annotation) = annotation_from_ast_node(child.clone()) {
- if annotation.first().map(|s| s.as_ref()) == Some("dsl.kernel") {
- has_dsl_kernel = true;
- }
- }
- },
- _ => (),
- }
- }
-
- if has_metal_kernel_attr && !has_dsl_kernel && !has_max_threads_attr {
- let kind = if is_template {
- "template"
- } else {
- "function"
- };
- bail!(
- "kernel {kind} '{name}' is missing `[[max_total_threads_per_threadgroup(N)]]` attribute! it **MUST** be present and correct!"
- );
- }
-
- Ok(())
-}
diff --git a/crates/uzu/build/metal/toolchain.rs b/crates/uzu/build/metal/toolchain.rs
index e0a792d92..b5262ab8a 100644
--- a/crates/uzu/build/metal/toolchain.rs
+++ b/crates/uzu/build/metal/toolchain.rs
@@ -10,7 +10,7 @@ use anyhow::{Context, bail};
use tempfile::NamedTempFile;
use tokio::{io::AsyncWriteExt, process::Command};
-use super::ast::{MetalAstKind, MetalAstNode, MetalKernelInfo, validate_raw_kernel};
+use super::ast::{MetalAstKind, MetalAstNode, MetalKernelInfo};
#[derive(Debug)]
pub enum MetalSdk {
@@ -192,10 +192,6 @@ impl MetalToolchain {
let source_contents = fs::read_to_string(path).context("cannot read source file")?;
- for node in &ast_root.inner {
- validate_raw_kernel(node).context("validation of legacy (non-dsl) kernel failed")?;
- }
-
let kernel_infos = ast_root
.inner
.into_iter()
diff --git a/crates/uzu/build/metal/wrapper.rs b/crates/uzu/build/metal/wrapper.rs
index 32e781924..d31d3a580 100644
--- a/crates/uzu/build/metal/wrapper.rs
+++ b/crates/uzu/build/metal/wrapper.rs
@@ -230,7 +230,7 @@ fn kernel_wrappers(
MetalArgumentType::Threads(_) => {
format!("__dsl_thread_idx.{}", thread_axis_letters.next().unwrap())
},
- MetalArgumentType::ThreadContext => "ThreadContext { .simdgroup_index = __dsl_simd_lane_idx, .threadgroup_index = __dsl_simd_group_idx, .simdgroup_size = __dsl_simd_group_size, .threadgroup_size = __dsl_simd_group_size * __dsl_simd_group_per_threadgroup, .simdgroups_per_threadgroup = __dsl_simd_group_per_threadgroup }".into(),
+ MetalArgumentType::ThreadContext => "ThreadContext { .simdgroup_index = __dsl_simd_lane_idx, .threadgroup_index = __dsl_simd_group_idx, .simdgroup_size = __dsl_simd_group_size, .simdgroups_per_threadgroup = __dsl_simd_group_per_threadgroup }".into(),
})
.collect::<Vec<_>>()
.join(", ")
diff --git a/crates/uzu/src/backends/cpu/kernel/mod.rs b/crates/uzu/src/backends/cpu/kernel/mod.rs
index fd116b887..91bd75de1 100644
--- a/crates/uzu/src/backends/cpu/kernel/mod.rs
+++ b/crates/uzu/src/backends/cpu/kernel/mod.rs
@@ -16,7 +16,6 @@ mod rms_norm;
mod rope;
mod sampling;
mod short_conv;
-mod sigmoid;
mod ssm;
mod tensor_add_bias;
mod tensor_add_scale;
diff --git a/crates/uzu/src/backends/cpu/kernel/sigmoid/mod.rs b/crates/uzu/src/backends/cpu/kernel/sigmoid/mod.rs
deleted file mode 100644
index 33caeb2aa..000000000
--- a/crates/uzu/src/backends/cpu/kernel/sigmoid/mod.rs
+++ /dev/null
@@ -1 +0,0 @@
-pub mod sigmoid;
diff --git a/crates/uzu/src/backends/cpu/kernel/sigmoid/sigmoid.rs b/crates/uzu/src/backends/cpu/kernel/sigmoid/sigmoid.rs
deleted file mode 100644
index ba1ad5b59..000000000
--- a/crates/uzu/src/backends/cpu/kernel/sigmoid/sigmoid.rs
+++ /dev/null
@@ -1,21 +0,0 @@
-use dsl::kernel;
-use half::{bf16, f16};
-use num_traits::Float;
-
-use crate::ArrayElement;
-
-#[kernel(Sigmoid)]
-#[variants(T, f32, f16, bf16)]
-pub fn sigmoid<T: ArrayElement + Float>(
- input: *const T,
- output: *mut T,
- total_elements: u32,
-) {
- for i in 0..total_elements as usize {
- unsafe {
- let input_value = (*input.add(i)).to_f32().unwrap();
- let value = 1.0 / (1.0 + (-input_value).exp());
- *output.add(i) = T::from(value).unwrap();
- }
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/activation/activations.h b/crates/uzu/src/backends/metal/kernel/activation/activations.h
index 979a5c255..b09b0b45d 100644
--- a/crates/uzu/src/backends/metal/kernel/activation/activations.h
+++ b/crates/uzu/src/backends/metal/kernel/activation/activations.h
@@ -1,5 +1,4 @@
-#ifndef __UZU_ACTIVATIONS_H__
-#define __UZU_ACTIVATIONS_H__
+#pragma once
#include <metal_stdlib>
#include "activation_type.h"
@@ -54,5 +53,3 @@ inline T activate(T x, ActivationType type) {
return x;
}
}
-
-#endif // __UZU_ACTIVATIONS_H__
\ No newline at end of file
diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal
index acf5ff8d6..ee1f0a074 100644
--- a/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal
+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal
@@ -434,7 +434,9 @@ PUBLIC KERNEL(AttentionGemm)(
for (short id = 0; id < HEAD_DIM_GRID_COLS; id++) {
METAL_PRAGMA_UNROLL
for (short ik = 0; ik < KEY_GRID_COLS; ik++) {
- IF_CONSTEXPR(BD == 128) { simdgroup_barrier(mem_flags::mem_none); }
+ if constexpr (BD == 128) {
+ simdgroup_barrier(mem_flags::mem_none);
+ }
const short kk = ik * SIMDGROUP_BLOCK_SIZE;
const short dd = id * SIMDGROUP_BLOCK_SIZE;
@@ -444,7 +446,9 @@ PUBLIC KERNEL(AttentionGemm)(
[value_shared_offset + kk * value_leading_dimension + dd]
);
- IF_CONSTEXPR(BD == 128) { simdgroup_barrier(mem_flags::mem_none); }
+ if constexpr (BD == 128) {
+ simdgroup_barrier(mem_flags::mem_none);
+ }
SimdgroupMultiplyAccumulateType::multiply_accumulate(
output_fragment.multiply_accumulate_at(0, id),
diff --git a/crates/uzu/src/backends/metal/kernel/attention/mask.h b/crates/uzu/src/backends/metal/kernel/attention/mask.h
index 83b696dc1..e553eabc0 100644
--- a/crates/uzu/src/backends/metal/kernel/attention/mask.h
+++ b/crates/uzu/src/backends/metal/kernel/attention/mask.h
@@ -6,7 +6,7 @@
using namespace uzu::ring;
using namespace uzu::trie;
-bool should_use_key(
+static inline bool should_use_key(
const constant RingParams& ring_params,
const device TrieNode* trie,
const constant uint& sliding_window_size,
@@ -19,4 +19,60 @@ bool should_use_key(
const bool is_causal,
const bool is_trie,
const bool is_sliding_window
-);
+) {
+ bool use_key = true;
+
+ uint key_position;
+
+ if (i >= prefix_length) {
+ // We're in the suffix
+ uint key_position_in_suffix = i - prefix_length;
+
+ if (is_trie) {
+ // Trie, get position from the flat trie buffer
+ TrieNode key_trie = trie[key_position_in_suffix];
+
+ key_position = suffix_position + key_trie.height;
+
+ if (is_causal) {
+ use_key &=
+ q_seq_idx >= key_trie.trie_start && q_seq_idx <= key_trie.trie_end;
+ }
+ } else {
+ // Non-trie, position equals index in the suffix
+ key_position = suffix_position + key_position_in_suffix;
+
+ if (is_causal) {
+ use_key &= key_position_in_suffix <= q_seq_idx;
+ }
+ }
+
+ } else {
+ // We're in the prefix
+ if (is_kv_cache_ring) {
+ // Ring, calculate position in the ring
+ key_position =
+ (prefix_length + i - ring_params.ring_offset) % prefix_length;
+ // Ring also skips entries that aren't filled yet
+ use_key &= key_position < ring_params.ring_length;
+ } else {
+ // Non-ring, position equals to the index in the kvcache buffer
+ key_position = i;
+ }
+ }
+
+ if (is_sliding_window) {
+ if (is_causal) {
+ use_key &= key_position <= query_position &&
+ (query_position - key_position) < sliding_window_size;
+ } else {
+ if (key_position <= query_position) {
+ use_key &= (query_position - key_position) <= (sliding_window_size / 2);
+ } else {
+ use_key &= (key_position - query_position) <= (sliding_window_size / 2);
+ }
+ }
+ }
+
+ return use_key;
+}
diff --git a/crates/uzu/src/backends/metal/kernel/attention/mask.metal b/crates/uzu/src/backends/metal/kernel/attention/mask.metal
deleted file mode 100644
index b3260231c..000000000
--- a/crates/uzu/src/backends/metal/kernel/attention/mask.metal
+++ /dev/null
@@ -1,76 +0,0 @@
-#include "../generated/ring.h"
-#include "../generated/trie.h"
-
-using namespace uzu::ring;
-using namespace uzu::trie;
-
-bool should_use_key(
- const constant RingParams& ring_params,
- const device TrieNode* trie,
- const constant uint& sliding_window_size,
- const uint q_seq_idx,
- const uint prefix_length,
- const uint suffix_position,
- const uint query_position,
- const uint i,
- const bool is_kv_cache_ring,
- const bool is_causal,
- const bool is_trie,
- const bool is_sliding_window
-) {
- bool use_key = true;
-
- uint key_position;
-
- if (i >= prefix_length) {
- // We're in the suffix
- uint key_position_in_suffix = i - prefix_length;
-
- if (is_trie) {
- // Trie, get position from the flat trie buffer
- TrieNode key_trie = trie[key_position_in_suffix];
-
- key_position = suffix_position + key_trie.height;
-
- if (is_causal) {
- use_key &=
- q_seq_idx >= key_trie.trie_start && q_seq_idx <= key_trie.trie_end;
- }
- } else {
- // Non-trie, position equals index in the suffix
- key_position = suffix_position + key_position_in_suffix;
-
- if (is_causal) {
- use_key &= key_position_in_suffix <= q_seq_idx;
- }
- }
-
- } else {
- // We're in the prefix
- if (is_kv_cache_ring) {
- // Ring, calculate position in the ring
- key_position =
- (prefix_length + i - ring_params.ring_offset) % prefix_length;
- // Ring also skips entries that aren't filled yet
- use_key &= key_position < ring_params.ring_length;
- } else {
- // Non-ring, position equals to the index in the kvcache buffer
- key_position = i;
- }
- }
-
- if (is_sliding_window) {
- if (is_causal) {
- use_key &= key_position <= query_position &&
- (query_position - key_position) < sliding_window_size;
- } else {
- if (key_position <= query_position) {
- use_key &= (query_position - key_position) <= (sliding_window_size / 2);
- } else {
- use_key &= (key_position - query_position) <= (sliding_window_size / 2);
- }
- }
- }
-
- return use_key;
-}
diff --git a/crates/uzu/src/backends/metal/kernel/audio/norm_ncs.metal b/crates/uzu/src/backends/metal/kernel/audio/norm_ncs.metal
index 46d59eb11..241597b59 100644
--- a/crates/uzu/src/backends/metal/kernel/audio/norm_ncs.metal
+++ b/crates/uzu/src/backends/metal/kernel/audio/norm_ncs.metal
@@ -48,13 +48,9 @@ void norm_ncs(
}
}
- const float sum =
- threadgroup_cooperative_reduce_sum<AUDIO_NORM_NCS_BLOCK_SIZE>(
- partial_sum,
- shared_mean,
- (ushort)lid,
- thread_context
- );
+ const float sum = threadgroup_cooperative_reduce<
+ SimdReduceSum<float>,
+ AUDIO_NORM_NCS_BLOCK_SIZE>(partial_sum, shared_mean, thread_context);
const float mean = (subtract_mean != 0) ? (sum / (float)channels) : 0.0f;
float partial_variance = 0.0f;
@@ -65,13 +61,13 @@ void norm_ncs(
partial_variance += centered * centered;
}
- const float variance_sum =
- threadgroup_cooperative_reduce_sum<AUDIO_NORM_NCS_BLOCK_SIZE>(
- partial_variance,
- shared_variance,
- (ushort)lid,
- thread_context
- );
+ const float variance_sum = threadgroup_cooperative_reduce<
+ SimdReduceSum<float>,
+ AUDIO_NORM_NCS_BLOCK_SIZE>(
+ partial_variance,
+ shared_variance,
+ thread_context
+ );
const float inv_std = rsqrt(variance_sum / (float)channels + epsilon);
for (uint c = lid; c < (uint)channels; c += AUDIO_NORM_NCS_BLOCK_SIZE) {
diff --git a/crates/uzu/src/backends/metal/kernel/common/bf16.h b/crates/uzu/src/backends/metal/kernel/common/bf16.h
deleted file mode 100644
index 1fafab2b1..000000000
--- a/crates/uzu/src/backends/metal/kernel/common/bf16.h
+++ /dev/null
@@ -1,14 +0,0 @@
-#pragma once
-
-#include <metal_stdlib>
-
-using namespace metal;
-
-typedef bfloat bfloat16_t;
-inline uint16_t bfloat16_to_uint16(const bfloat16_t x) {
- return as_type<uint16_t>(x);
-}
-
-inline bfloat16_t uint16_to_bfloat16(const uint16_t x) {
- return as_type<bfloat16_t>(x);
-}
diff --git a/crates/uzu/src/backends/metal/kernel/common/bf16_math.h b/crates/uzu/src/backends/metal/kernel/common/bf16_math.h
deleted file mode 100644
index 639e04eed..000000000
--- a/crates/uzu/src/backends/metal/kernel/common/bf16_math.h
+++ /dev/null
@@ -1,436 +0,0 @@
-#pragma once
-
-///////////////////////////////////////////////////////////////////////////////
-// Metal math for bfloat16
-///////////////////////////////////////////////////////////////////////////////
-
-/*
-
-Following the Metal Shading Language Specification (Metal 3.1)
-
-"bfloat is an extended itypeing point type that only allows implicit conversion
- to a type of greater itypeing point rank. While bfloat can be implicitly
- converted to itype, it cannot be implicitly converted to half, and neither
- itype nor half can be implicitly converted to bfloat."
-
-Further, as far as I can tell, the stdlib math/simd functions are not defined
-for bfloat and calling with an argument of type bfloat will result in that
-argument getting implicitly converted to itype which then returns an output
-that is (likely) a itype which cannot be implicitly converted into a bfloat
-
-This leads to situations where
-bfloat a = 5.0bf;
-bfloat b = metal::abs(a); // this will throw an error since abs return itype
-bfloat c = static_cast<bfloat>(metal::abs(a)); // this is fine
-
-For the moment, I will be adding overloaded instantiations of the math
-functions to accordingly automatically handle the casting
-
-*/
-
-#define instantiate_metal_math_funcs(itype, otype, ctype, mfast) \
- \
- METAL_FUNC otype abs(itype x) { \
- return static_cast<otype>(__metal_fabs(static_cast<ctype>(x), mfast)); \
- } \
- METAL_FUNC otype acos(itype x) { \
- return static_cast<otype>(__metal_acos(static_cast<ctype>(x), mfast)); \
- } \
- METAL_FUNC otype acosh(itype x) { \
- return static_cast<otype>(__metal_acosh(static_cast<ctype>(x), mfast)); \
- } \
- METAL_FUNC otype asin(itype x) { \
- return static_cast<otype>(__metal_asin(static_cast<ctype>(x), mfast)); \
- } \
- METAL_FUNC otype asinh(itype x) { \
- return static_cast<otype>(__metal_asinh(static_cast<ctype>(x), mfast)); \
- } \
- METAL_FUNC otype atan(itype y_over_x) { \
- return static_cast<otype>( \
- __metal_atan(static_cast<ctype>(y_over_x), mfast) \
- ); \
- } \
- METAL_FUNC otype atan2(itype y, itype x) { \
- return static_cast<otype>( \
- __metal_atan2(static_cast<ctype>(y), static_cast<ctype>(x), mfast) \
- ); \
- } \
- METAL_FUNC otype atanh(itype x) { \
- return static_cast<otype>(__metal_atanh(static_cast<ctype>(x), mfast)); \
- } \
- METAL_FUNC otype ceil(itype x) { \
- return static_cast<otype>(__metal_ceil(static_cast<ctype>(x), mfast)); \
- } \
- METAL_FUNC otype cos(itype x) { \
- return static_cast<otype>(__metal_cos(static_cast<ctype>(x), mfast)); \
- } \
- METAL_FUNC otype cosh(itype x) { \
- return static_cast<otype>(__metal_cosh(static_cast<ctype>(x), mfast)); \
- } \
- METAL_FUNC otype cospi(itype x) { \
- return static_cast<otype>(__metal_cospi(static_cast<ctype>(x), mfast)); \
- } \
- METAL_FUNC otype divide(itype x, itype y) { \
- return static_cast<otype>( \
- __metal_divide(static_cast<ctype>(x), static_cast<ctype>(y), mfast) \
- ); \
- } \
- METAL_FUNC otype exp(itype x) { \
- return static_cast<otype>(__metal_exp(static_cast<ctype>(x), mfast)); \
- } \
- METAL_FUNC otype exp10(itype x) { \
- return static_cast<otype>(__metal_exp10(static_cast<ctype>(x), mfast)); \
- } \
- METAL_FUNC otype exp2(itype x) { \
- return static_cast<otype>(__metal_exp2(static_cast<ctype>(x), mfast)); \
- } \
- METAL_FUNC otype fabs(itype x) { \
- return static_cast<otype>(__metal_fabs(static_cast<ctype>(x), mfast)); \
- } \
- METAL_FUNC otype fdim(itype x, itype y) { \
- ctype t = static_cast<ctype>(x - y); \
- return static_cast<otype>(select(t, ctype(0), t < ctype(0) || x == y)); \
- } \
- METAL_FUNC otype floor(itype x) { \
- return static_cast<otype>(__metal_floor(static_cast<ctype>(x), mfast)); \
- } \
- METAL_FUNC otype fma(itype x, itype y, itype z) { \
- return static_cast<otype>(__metal_fma( \
- static_cast<ctype>(x), \
- static_cast<ctype>(y), \
- static_cast<ctype>(z) \
- )); \
- } \
- METAL_FUNC otype fmax(itype x, itype y) { \
- return static_cast<otype>( \
- __metal_fmax(static_cast<ctype>(x), static_cast<ctype>(y), mfast) \
- ); \
- } \
- METAL_FUNC otype fmax3(itype x, itype y, itype z) { \
- return static_cast<otype>(__metal_fmax3( \
- static_cast<ctype>(x), \
- static_cast<ctype>(y), \
- static_cast<ctype>(z), \
- mfast \
- )); \
- } \
- METAL_FUNC otype fmedian3(itype x, itype y, itype z) { \
- return static_cast<otype>(__metal_fmedian3( \
- static_cast<ctype>(x), \
- static_cast<ctype>(y), \
- static_cast<ctype>(z), \
- mfast \
- )); \
- } \
- METAL_FUNC otype fmin(itype x, itype y) { \
- return static_cast<otype>( \
- __metal_fmin(static_cast<ctype>(x), static_cast<ctype>(y), mfast) \
- ); \
- } \
- METAL_FUNC otype fmin3(itype x, itype y, itype z) { \
- return static_cast<otype>(__metal_fmin3( \
- static_cast<ctype>(x), \
- static_cast<ctype>(y), \
- static_cast<ctype>(z), \
- mfast \
- )); \
- } \
- METAL_FUNC otype fmod(itype x, itype y) { \
- return static_cast<otype>( \
- __metal_fmod(static_cast<ctype>(x), static_cast<ctype>(y), mfast) \
- ); \
- } \
- METAL_FUNC otype fract(itype x) { \
- return static_cast<otype>(__metal_fract(static_cast<ctype>(x), mfast)); \
- } \
- METAL_FUNC otype frexp(itype x, thread int& exp) { \
- return static_cast<otype>(__metal_frexp(static_cast<ctype>(x), &exp)); \
- } \
- METAL_FUNC otype ldexp(itype x, int k) { \
- return static_cast<otype>(__metal_ldexp(static_cast<ctype>(x), k, mfast)); \
- } \
- METAL_FUNC otype log(itype x) { \
- return static_cast<otype>(__metal_log(static_cast<ctype>(x), mfast)); \
- } \
- METAL_FUNC otype log10(itype x) { \
- return static_cast<otype>(__metal_log10(static_cast<ctype>(x), mfast)); \
- } \
- METAL_FUNC otype log2(itype x) { \
- return static_cast<otype>(__metal_log2(static_cast<ctype>(x), mfast)); \
- } \
- METAL_FUNC otype max(itype x, itype y) { \
- return static_cast<otype>( \
- __metal_fmax(static_cast<ctype>(x), static_cast<ctype>(y), mfast) \
- ); \
- } \
- METAL_FUNC otype max3(itype x, itype y, itype z) { \
- return static_cast<otype>(__metal_fmax3( \
- static_cast<ctype>(x), \
- static_cast<ctype>(y), \
- static_cast<ctype>(z), \
- mfast \
- )); \
- } \
- METAL_FUNC otype median3(itype x, itype y, itype z) { \
- return static_cast<otype>(__metal_fmedian3( \
- static_cast<ctype>(x), \
- static_cast<ctype>(y), \
- static_cast<ctype>(z), \
- mfast \
- )); \
- } \
- METAL_FUNC otype min(itype x, itype y) { \
- return static_cast<otype>( \
- __metal_fmin(static_cast<ctype>(x), static_cast<ctype>(y), mfast) \
- ); \
- } \
- METAL_FUNC otype min3(itype x, itype y, itype z) { \
- return static_cast<otype>(__metal_fmin3( \
- static_cast<ctype>(x), \
- static_cast<ctype>(y), \
- static_cast<ctype>(z), \
- mfast \
- )); \
- } \
- METAL_FUNC otype nextafter(itype x, itype y) { \
- return static_cast<otype>( \
- __metal_nextafter(static_cast<ctype>(x), static_cast<ctype>(y)) \
- ); \
- } \
- METAL_FUNC otype pow(itype x, itype y) { \
- return static_cast<otype>( \
- __metal_pow(static_cast<ctype>(x), static_cast<ctype>(y), mfast) \
- ); \
- } \
- METAL_FUNC otype powr(itype x, itype y) { \
- return static_cast<otype>( \
- __metal_powr(static_cast<ctype>(x), static_cast<ctype>(y), mfast) \
- ); \
- } \
- METAL_FUNC otype rint(itype x) { \
- return static_cast<otype>(__metal_rint(static_cast<ctype>(x), mfast)); \
- } \
- METAL_FUNC otype round(itype x) { \
- return static_cast<otype>(__metal_round(static_cast<ctype>(x), mfast)); \
- } \
- METAL_FUNC otype rsqrt(itype x) { \
- return static_cast<otype>(__metal_rsqrt(static_cast<ctype>(x), mfast)); \
- } \
- METAL_FUNC otype sin(itype x) { \
- return static_cast<otype>(__metal_sin(static_cast<ctype>(x), mfast)); \
- } \
- METAL_FUNC otype sinh(itype x) { \
- return static_cast<otype>(__metal_sinh(static_cast<ctype>(x), mfast)); \
- } \
- METAL_FUNC otype sinpi(itype x) { \
- return static_cast<otype>(__metal_sinpi(static_cast<ctype>(x), mfast)); \
- } \
- METAL_FUNC otype sqrt(itype x) { \
- return static_cast<otype>(__metal_sqrt(static_cast<ctype>(x), mfast)); \
- } \
- METAL_FUNC otype tan(itype x) { \
- return static_cast<otype>(__metal_tan(static_cast<ctype>(x), mfast)); \
- } \
- METAL_FUNC otype tanh(itype x) { \
- return static_cast<otype>(__metal_tanh(static_cast<ctype>(x), mfast)); \
- } \
- METAL_FUNC otype tanpi(itype x) { \
- return static_cast<otype>(__metal_tanpi(static_cast<ctype>(x), mfast)); \
- } \
- METAL_FUNC otype trunc(itype x) { \
- return static_cast<otype>(__metal_trunc(static_cast<ctype>(x), mfast)); \
- }
-
-namespace metal {
-
-instantiate_metal_math_funcs(
- bfloat16_t,
- bfloat16_t,
- float,
- __METAL_MAYBE_FAST_MATH__
-);
-
-namespace fast {
-
-instantiate_metal_math_funcs(
- bfloat16_t,
- bfloat16_t,
- float,
- __METAL_FAST_MATH__
-);
-
-} // namespace fast
-
-namespace precise {
-
-instantiate_metal_math_funcs(
- bfloat16_t,
- bfloat16_t,
- float,
- __METAL_PRECISE_MATH__
-);
-
-} // namespace precise
-
-} // namespace metal
-
-///////////////////////////////////////////////////////////////////////////////
-// Metal simd for bfloat16
-///////////////////////////////////////////////////////////////////////////////
-
-#define instantiate_metal_simd_comm_funcs( \
- itype, \
- otype, \
- ctype, \
- itype_to_ctype, \
- ctype_to_otype \
-) \
- \
- METAL_FUNC otype simd_broadcast(itype data, ushort broadcast_lane_id) { \
- return ctype_to_otype( \
- __metal_simd_broadcast(itype_to_ctype(data), broadcast_lane_id) \
- ); \
- } \
- \
- METAL_FUNC otype simd_shuffle(itype data, ushort simd_lane_id) { \
- return ctype_to_otype( \
- __metal_simd_shuffle(itype_to_ctype(data), simd_lane_id) \
- ); \
- } \
- \
- METAL_FUNC otype simd_shuffle_and_fill_down( \
- itype data, \
- itype filling_data, \
- ushort delta, \
- ushort modulo \
- ) { \
- return ctype_to_otype(__metal_simd_shuffle_and_fill_down( \
- itype_to_ctype(data), \
- itype_to_ctype(filling_data), \
- delta, \
- modulo \
- )); \
- } \
- \
- METAL_FUNC otype \
- simd_shuffle_and_fill_down(itype data, itype filling_data, ushort delta) { \
- return ctype_to_otype(__metal_simd_shuffle_and_fill_down( \
- itype_to_ctype(data), \
- itype_to_ctype(filling_data), \
- delta, \
- __metal_get_simdgroup_size(ushort()) \
- )); \
- } \
- \
- METAL_FUNC otype simd_shuffle_and_fill_up( \
- itype data, \
- itype filling_data, \
- ushort delta, \
- ushort modulo \
- ) { \
- return ctype_to_otype(__metal_simd_shuffle_and_fill_up( \
- itype_to_ctype(data), \
- itype_to_ctype(filling_data), \
- delta, \
- modulo \
- )); \
- } \
- \
- METAL_FUNC otype \
- simd_shuffle_and_fill_up(itype data, itype filling_data, ushort delta) { \
- return ctype_to_otype(__metal_simd_shuffle_and_fill_up( \
- itype_to_ctype(data), \
- itype_to_ctype(filling_data), \
- delta, \
- __metal_get_simdgroup_size(ushort()) \
- )); \
- } \
- \
- METAL_FUNC otype simd_shuffle_down(itype data, ushort delta) { \
- return ctype_to_otype( \
- __metal_simd_shuffle_down(itype_to_ctype(data), delta) \
- ); \
- } \
- \
- METAL_FUNC otype simd_shuffle_rotate_down(itype data, ushort delta) { \
- return ctype_to_otype( \
- __metal_simd_shuffle_rotate_down(itype_to_ctype(data), delta) \
- ); \
- } \
- \
- METAL_FUNC otype simd_shuffle_rotate_up(itype data, ushort delta) { \
- return ctype_to_otype( \
- __metal_simd_shuffle_rotate_up(itype_to_ctype(data), delta) \
- ); \
- } \
- \
- METAL_FUNC otype simd_shuffle_up(itype data, ushort delta) { \
- return ctype_to_otype( \
- __metal_simd_shuffle_up(itype_to_ctype(data), delta) \
- ); \
- } \
- \
- METAL_FUNC otype simd_shuffle_xor(itype data, ushort mask) { \
- return ctype_to_otype( \
- __metal_simd_shuffle_xor(itype_to_ctype(data), mask) \
- ); \
- }
-
-#define instantiate_metal_simd_reduction_funcs(itype, otype, ctype) \
- \
- METAL_FUNC otype simd_max(itype data) { \
- return static_cast<otype>(__metal_simd_max(static_cast<ctype>(data))); \
- } \
- \
- METAL_FUNC otype simd_min(itype data) { \
- return static_cast<otype>(__metal_simd_min(static_cast<ctype>(data))); \
- } \
- \
- METAL_FUNC otype simd_prefix_exclusive_product(itype data) { \
- return static_cast<otype>( \
- __metal_simd_prefix_exclusive_product(static_cast<ctype>(data)) \
- ); \
- } \
- \
- METAL_FUNC otype simd_prefix_exclusive_sum(itype data) { \
- return static_cast<otype>( \
- __metal_simd_prefix_exclusive_sum(static_cast<ctype>(data)) \
- ); \
- } \
- \
- METAL_FUNC otype simd_prefix_inclusive_product(itype data) { \
- return static_cast<otype>( \
- __metal_simd_prefix_inclusive_product(static_cast<ctype>(data)) \
- ); \
- } \
- \
- METAL_FUNC otype simd_prefix_inclusive_sum(itype data) { \
- return static_cast<otype>( \
- __metal_simd_prefix_inclusive_sum(static_cast<ctype>(data)) \
- ); \
- } \
- \
- METAL_FUNC otype simd_product(itype data) { \
- return static_cast<otype>(__metal_simd_product(static_cast<ctype>(data))); \
- } \
- \
- METAL_FUNC otype simd_sum(itype data) { \
- return static_cast<otype>(__metal_simd_sum(static_cast<ctype>(data))); \
- } \
- \
- METAL_FUNC otype simd_xor(itype data) { \
- return static_cast<otype>(__metal_simd_xor(static_cast<ctype>(data))); \
- }
-
-namespace metal {
-
-instantiate_metal_simd_comm_funcs(
- bfloat16_t,
- bfloat16_t,
- uint16_t,
- bfloat16_to_uint16,
- uint16_to_bfloat16
-);
-instantiate_metal_simd_reduction_funcs(bfloat16_t, bfloat16_t, float);
-
-} // namespace metal
diff --git a/crates/uzu/src/backends/metal/kernel/common/complex.h b/crates/uzu/src/backends/metal/kernel/common/complex.h
deleted file mode 100644
index 84bbbeb11..000000000
--- a/crates/uzu/src/backends/metal/kernel/common/complex.h
+++ /dev/null
@@ -1,170 +0,0 @@
-#pragma once
-
-#include <metal_stdlib>
-
-using namespace metal;
-
-struct complex64_t;
-
-template <typename T>
-static constexpr constant bool can_convert_to_complex64 =
- !is_same_v<T, complex64_t> && is_convertible_v<T, float>;
-
-template <typename T>
-static constexpr constant bool can_convert_from_complex64 =
- !is_same_v<T, complex64_t> &&
- (is_convertible_v<float, T> || is_convertible_v<bfloat16_t, T>);
-
-struct complex64_t {
- float real;
- float imag;
-
- // Constructors
- constexpr complex64_t(float real, float imag) : real(real), imag(imag) {};
- constexpr complex64_t() : real(0), imag(0) {};
- constexpr complex64_t() threadgroup : real(0), imag(0) {};
-
- // Conversions to complex64_t
- template <
- typename T,
- typename = typename enable_if<can_convert_to_complex64<T>>::type>
- constexpr complex64_t(T x) thread : real(x), imag(0) {}
-
- template <
- typename T,
- typename = typename enable_if<can_convert_to_complex64<T>>::type>
- constexpr complex64_t(T x) threadgroup : real(x), imag(0) {}
-
- template <
- typename T,
- typename = typename enable_if<can_convert_to_complex64<T>>::type>
- constexpr complex64_t(T x) device : real(x), imag(0) {}
-
- template <
- typename T,
- typename = typename enable_if<can_convert_to_complex64<T>>::type>
- constexpr complex64_t(T x) constant : real(x), imag(0) {}
-
- // Conversions from complex64_t
- template <
- typename T,
- typename = typename enable_if<can_convert_from_complex64<T>>::type>
- constexpr operator T() const thread {
- return static_cast<T>(real);
- }
-
- template <
- typename T,
- typename = typename enable_if<can_convert_from_complex64<T>>::type>
- constexpr operator T() const threadgroup {
- return static_cast<T>(real);
- }
-
- template <
- typename T,
- typename = typename enable_if<can_convert_from_complex64<T>>::type>
- constexpr operator T() const device {
- return static_cast<T>(real);
- }
-
- template <
- typename T,
- typename = typename enable_if<can_convert_from_complex64<T>>::type>
- constexpr operator T() const constant {
- return static_cast<T>(real);
- }
-};
-
-constexpr complex64_t operator-(complex64_t x) { return {-x.real, -x.imag}; }
-
-constexpr bool operator>=(complex64_t a, complex64_t b) {
- return (a.real > b.real) || (a.real == b.real && a.imag >= b.imag);
-}
-
-constexpr bool operator>(complex64_t a, complex64_t b) {
- return (a.real > b.real) || (a.real == b.real && a.imag > b.imag);
-}
-
-constexpr bool operator<=(complex64_t a, complex64_t b) {
- return operator>=(b, a);
-}
-
-constexpr bool operator<(complex64_t a, complex64_t b) {
- return operator>(b, a);
-}
-
-constexpr bool operator==(complex64_t a, complex64_t b) {
- return a.real == b.real && a.imag == b.imag;
-}
-
-constexpr complex64_t operator+(complex64_t a, complex64_t b) {
- return {a.real + b.real, a.imag + b.imag};
-}
-
-constexpr thread complex64_t& operator+=(thread complex64_t& a, complex64_t b) {
- a.real += b.real;
- a.imag += b.imag;
- return a;
-}
-
-constexpr threadgroup complex64_t& operator+=(
- threadgroup complex64_t& a,
- complex64_t b
-) {
- a.real += b.real;
- a.imag += b.imag;
- return a;
-}
-
-constexpr device complex64_t& operator+=(device complex64_t& a, complex64_t b) {
- a.real += b.real;
- a.imag += b.imag;
- return a;
-}
-
-constexpr complex64_t operator+(float a, complex64_t b) {
- return {a + b.real, b.imag};
-}
-constexpr complex64_t operator+(complex64_t a, float b) {
- return {a.real + b, a.imag};
-}
-
-constexpr complex64_t operator-(complex64_t a, complex64_t b) {
- return {a.real - b.real, a.imag - b.imag};
-}
-constexpr complex64_t operator-(float a, complex64_t b) {
- return {a - b.real, -b.imag};
-}
-constexpr complex64_t operator-(complex64_t a, float b) {
- return {a.real - b, a.imag};
-}
-
-constexpr complex64_t operator*(complex64_t a, complex64_t b) {
- return {a.real * b.real - a.imag * b.imag, a.real * b.imag + a.imag * b.real};
-}
-
-constexpr complex64_t operator/(complex64_t a, complex64_t b) {
- auto denom = b.real * b.real + b.imag * b.imag;
- auto x = a.real * b.real + a.imag * b.imag;
- auto y = a.imag * b.real - a.real * b.imag;
- return {x / denom, y / denom};
-}
-
-constexpr complex64_t operator/(float a, complex64_t b) {
- auto denom = b.real * b.real + b.imag * b.imag;
- auto x = a * b.real;
- auto y = -a * b.imag;
- return {x / denom, y / denom};
-}
-
-constexpr complex64_t operator%(complex64_t a, complex64_t b) {
- auto real = a.real - (b.real * static_cast<int64_t>(a.real / b.real));
- auto imag = a.imag - (b.imag * static_cast<int64_t>(a.imag / b.imag));
- if (real != 0 && (real < 0 != b.real < 0)) {
- real += b.real;
- }
- if (imag != 0 && (imag < 0 != b.imag < 0)) {
- imag += b.imag;
- }
- return {real, imag};
-}
diff --git a/crates/uzu/src/backends/metal/kernel/common/defines.h b/crates/uzu/src/backends/metal/kernel/common/defines.h
index 97e2fcfd3..602984747 100644
--- a/crates/uzu/src/backends/metal/kernel/common/defines.h
+++ b/crates/uzu/src/backends/metal/kernel/common/defines.h
@@ -3,16 +3,5 @@
#define METAL_CONST static constant constexpr
#define METAL_PRAGMA_UNROLL _Pragma("clang loop unroll(full)")
#define METAL_SIMD_SIZE 32
-METAL_FUNC int pow2(int n) { return 1 << n; }
-
-METAL_CONST int MAX_REDUCE_SPECIALIZED_DIMS = 4;
-METAL_CONST int REDUCE_N_READS = 4;
-METAL_CONST int REDUCE_N_WRITES = 4;
-METAL_CONST int SOFTMAX_N_READS = 4;
-METAL_CONST int RMS_N_READS = 4;
-METAL_CONST int RMS_LOOPED_LIMIT = 4096;
-#define instantiate_kernel(name, func, ...) \
- template [[host_name( \
- name \
- )]] [[kernel]] decltype(func<__VA_ARGS__>) func<__VA_ARGS__>;
+METAL_FUNC int pow2(int n) { return 1 << n; }
diff --git a/crates/uzu/src/backends/metal/kernel/common/dsl.h b/crates/uzu/src/backends/metal/kernel/common/dsl.h
index 3bda99637..11922d052 100644
--- a/crates/uzu/src/backends/metal/kernel/common/dsl.h
+++ b/crates/uzu/src/backends/metal/kernel/common/dsl.h
@@ -1,13 +1,8 @@
#pragma once
#include <metal_stdlib>
-using namespace metal;
-#if defined(__cpp_if_constexpr)
-#define IF_CONSTEXPR(cond) if constexpr (cond)
-#else
-#define IF_CONSTEXPR(cond) if (cond)
-#endif
+using namespace metal;
#ifdef DSL_ANALYZE
#define DSL_META(...) [[clang::annotate("", __VA_ARGS__)]]
@@ -30,38 +25,3 @@ using namespace metal;
#define AXIS(TDS, TPG) DSL_META("dsl.axis", DSL_XSTR(TDS), DSL_XSTR(TPG))
#define GROUPS(EXPR) DSL_META("dsl.groups", DSL_XSTR(EXPR))
#define THREADS(EXPR) DSL_META("dsl.threads", DSL_XSTR(EXPR))
-
-#define generateKernel( \
- max_threads, \
- functionName, \
- scalarType, \
- outerArgs, \
- innerArgs \
-) \
- [[max_total_threads_per_threadgroup(max_threads)]] \
- kernel void functionName##_##scalarType outerArgs { \
- functionName innerArgs; \
- }
-
-#define generateKernels(max_threads, functionName) \
- generateKernel( \
- max_threads, \
- functionName, \
- float, \
- outerArguments(float), \
- innerArguments \
- ); \
- generateKernel( \
- max_threads, \
- functionName, \
- bfloat, \
- outerArguments(bfloat), \
- innerArguments \
- ); \
- generateKernel( \
- max_threads, \
- functionName, \
- half, \
- outerArguments(half), \
- innerArguments \
- );
diff --git a/crates/uzu/src/backends/metal/kernel/common/logging.h b/crates/uzu/src/backends/metal/kernel/common/logging.h
deleted file mode 100644
index 4d06cd85a..000000000
--- a/crates/uzu/src/backends/metal/kernel/common/logging.h
+++ /dev/null
@@ -1,20 +0,0 @@
-#pragma once
-
-#if defined(__METAL_VERSION__) && (__METAL_VERSION__ >= 320)
-#include <metal_logging>
-
-using os_log = metal::os_log;
-
-#else
-
-struct os_log {
- constexpr os_log(constant char*, constant char*) constant {}
-
- template <typename... Args>
- void log_debug(constant char*, Args...) const {}
-
- template <typename... Args>
- void log_debug(constant char*, Args...) const constant {}
-};
-
-#endif
\ No newline at end of file
diff --git a/crates/uzu/src/backends/metal/kernel/common/mlp_epilogue.h b/crates/uzu/src/backends/metal/kernel/common/mlp_epilogue.h
deleted file mode 100644
index 936a01fe8..000000000
--- a/crates/uzu/src/backends/metal/kernel/common/mlp_epilogue.h
+++ /dev/null
@@ -1,76 +0,0 @@
-#pragma once
-
-#include <metal_stdlib>
-using namespace metal;
-
-// MLP Fusion Function Constants
-// These control whether the kernel computes paired up/gate projections
-// and applies activation fusion (SwiGLU/GeGLU style).
-//
-// Index allocation:
-// 50 - MLP_FUSED (bool): enable paired computation + fused activation
-// 51 - MLP_HIDDEN_DIM (uint): offset to gate weights (= hidden_dim)
-// 52 - MLP_ACTIVATION (uint): 0=SiLU, 1=GELU
-
-constant bool MLP_FUSED [[function_constant(50)]];
-constant uint MLP_HIDDEN_DIM [[function_constant(51)]];
-constant uint MLP_ACTIVATION [[function_constant(52)]];
-
-// Activation type enum for readability
-enum MlpActivationType : uint {
- MLP_ACT_SILU = 0,
- MLP_ACT_GELU = 1,
-};
-
-// SiLU activation: x * sigmoid(x)
-template <typename T>
-inline T mlp_silu(T x) {
- float xf = float(x);
- float result = xf / (1.0f + exp(-xf));
- return T(result);
-}
-
-// GELU approximation (tanh-based)
-template <typename T>
-inline T mlp_gelu(T x) {
- constexpr float k0 = 0.044715f;
- constexpr float k1 = 0.7978845608f; // sqrt(2/pi)
- float xf = float(x);
- float t = k1 * (xf + k0 * xf * xf * xf);
- float result = 0.5f * xf * (1.0f + metal::precise::tanh(t));
- return T(result);
-}
-
-// Apply activation based on MLP_ACTIVATION constant
-template <typename T>
-inline T mlp_activate(T x) {
- if (MLP_ACTIVATION == MLP_ACT_SILU) {
- return mlp_silu(x);
- }
- return mlp_gelu(x);
-}
-
-// Fused MLP epilogue: up * activation(gate)
-// Used in SwiGLU (SiLU) and GeGLU (GELU) style MLPs
-template <typename T>
-inline T mlp_fused_epilogue(T up, T gate) {
- T activated_gate = mlp_activate(gate);
- return T(float(up) * float(activated_gate));
-}
-
-// Fused MLP epilogue with float accumulators
-inline float mlp_fused_epilogue_f32(float up, float gate) {
- float activated_gate;
- if (MLP_ACTIVATION == MLP_ACT_SILU) {
- activated_gate = up / (1.0f + exp(-gate));
- // Note: SwiGLU is gate_act * up, not up_act * gate
- // Correcting: activated_gate = gate / (1.0f + exp(-gate))
- activated_gate = gate / (1.0f + exp(-gate));
- } else {
- constexpr float k0 = 0.044715f;
- constexpr float k1 = 0.7978845608f;
- float t = k1 * (gate + k0 * gate * gate * gate);
- activated_gate = 0.5f * gate * (1.0f + metal::precise::tanh(t));
- }
- return up * activated_gate;
-}
diff --git a/crates/uzu/src/backends/metal/kernel/common/tensor_view.h b/crates/uzu/src/backends/metal/kernel/common/tensor_view.h
index ef5e977fd..96030c488 100644
--- a/crates/uzu/src/backends/metal/kernel/common/tensor_view.h
+++ b/crates/uzu/src/backends/metal/kernel/common/tensor_view.h
@@ -67,54 +67,3 @@ struct TensorView3D {
device T& operator()(uint i, uint j, uint k) const { return at(i, j, k); }
};
-
-template <typename T>
-struct TensorView4D {
- device T* data;
- uint dim0, dim1, dim2, dim3;
- uint stride0, stride1, stride2, stride3;
-
- TensorView4D(device T* ptr) : data(ptr) {}
-
- TensorView4D(device T* ptr, int d0, int d1, int d2, int d3)
- : data(ptr), dim0(d0), dim1(d1), dim2(d2), dim3(d3) {
- stride0 = d1 * d2 * d3;
- stride1 = d2 * d3;
- stride2 = d3;
- stride3 = 1;
- }
-
- TensorView4D(
- device T* ptr,
- int d0,
- int d1,
- int d2,
- int d3,
- int s0,
- int s1,
- int s2,
- int s3
- )
- : data(ptr), dim0(d0), dim1(d1), dim2(d2), dim3(d3), stride0(s0),
- stride1(s1), stride2(s2), stride3(s3) {}
-
- thread TensorView4D& shaped(int d0, int d1, int d2, int d3) {
- dim0 = d0;
- dim1 = d1;
- dim2 = d2;
- dim3 = d3;
- stride0 = d1 * d2 * d3;
- stride1 = d2 * d3;
- stride2 = d3;
- stride3 = 1;
- return *this;
- }
-
- device T& at(uint i, uint j, uint k, uint l) const {
- return data[i * stride0 + j * stride1 + k * stride2 + l * stride3];
- }
-
- device T& operator()(uint i, uint j, uint k, uint l) const {
- return at(i, j, k, l);
- }
-};
diff --git a/crates/uzu/src/backends/metal/kernel/common/thread_context.h b/crates/uzu/src/backends/metal/kernel/common/thread_context.h
index 9a3f827a5..f0fa45ae7 100644
--- a/crates/uzu/src/backends/metal/kernel/common/thread_context.h
+++ b/crates/uzu/src/backends/metal/kernel/common/thread_context.h
@@ -7,6 +7,5 @@ struct ThreadContext {
uint simdgroup_index;
uint threadgroup_index;
uint simdgroup_size;
- uint threadgroup_size;
uint simdgroups_per_threadgroup;
};
diff --git a/crates/uzu/src/backends/metal/kernel/common/thread_functions.h b/crates/uzu/src/backends/metal/kernel/common/thread_functions.h
deleted file mode 100644
index 5185a9831..000000000
--- a/crates/uzu/src/backends/metal/kernel/common/thread_functions.h
+++ /dev/null
@@ -1,54 +0,0 @@
-#pragma once
-
-#include <metal_stdlib>
-using namespace metal;
-
-template <ushort LENGTH, typename T>
-static inline T thread_prefix_inclusive_sum(thread T (&values)[LENGTH]) {
- for (ushort i = 1; i < LENGTH; i++) {
- values[i] += values[i - 1];
- }
- return values[LENGTH - 1];
-}
-
-template <ushort LENGTH, typename T>
-static inline T thread_prefix_inclusive_sum(threadgroup T* values) {
- for (ushort i = 1; i < LENGTH; i++) {
- values[i] += values[i - 1];
- }
- return values[LENGTH - 1];
-}
-
-template <ushort LENGTH, typename T>
-static inline T thread_prefix_exclusive_sum(thread T (&values)[LENGTH]) {
- T inclusive_prefix = thread_prefix_inclusive_sum<LENGTH>(values);
- for (ushort i = LENGTH - 1; i > 0; i--) {
- values[i] = values[i - 1];
- }
- values[0] = 0;
- return inclusive_prefix;
-}
-
-template <ushort LENGTH, typename T>
-static inline T thread_prefix_exclusive_sum(threadgroup T* values) {
- T inclusive_prefix = thread_prefix_inclusive_sum<LENGTH>(values);
- for (ushort i = LENGTH - 1; i > 0; i--) {
- values[i] = values[i - 1];
- }
- values[0] = 0;
- return inclusive_prefix;
-}
-
-template <ushort LENGTH, typename T>
-static inline void thread_uniform_add(thread T (&values)[LENGTH], T uni) {
- for (ushort i = 0; i < LENGTH; i++) {
- values[i] += uni;
- }
-}
-
-template <ushort LENGTH, typename T>
-static inline void thread_uniform_add(threadgroup T* values, T uni) {
- for (ushort i = 0; i < LENGTH; i++) {
- values[i] += uni;
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/common/threadgroup_reduce.h b/crates/uzu/src/backends/metal/kernel/common/threadgroup_reduce.h
index 4475cc46d..34a0bf229 100644
--- a/crates/uzu/src/backends/metal/kernel/common/threadgroup_reduce.h
+++ b/crates/uzu/src/backends/metal/kernel/common/threadgroup_reduce.h
@@ -1,230 +1,70 @@
#pragma once
-#include "thread_context.h"
#include <metal_stdlib>
-using namespace metal;
-
-template <ushort BLOCK_SIZE, typename T>
-static T threadgroup_raking_prefix_exclusive_sum(
- T value,
- threadgroup T* shared,
- const ushort lid
-) {
- shared[lid] = value;
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- if (lid < 32) {
- const short values_per_thread = BLOCK_SIZE / 32;
- const short first_index = lid * values_per_thread;
- for (short i = first_index + 1; i < first_index + values_per_thread; i++) {
- shared[i] += shared[i - 1];
- }
- T partial_sum = shared[first_index + values_per_thread - 1];
- for (short i = first_index + values_per_thread - 1; i > first_index; i--) {
- shared[i] = shared[i - 1];
- }
- shared[first_index] = 0;
-
- T prefix = simd_prefix_exclusive_sum(partial_sum);
-
- for (short i = first_index; i < first_index + values_per_thread; i++) {
- shared[i] += prefix;
- }
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- const T result = shared[lid];
- threadgroup_barrier(mem_flags::mem_threadgroup);
- return result;
-}
-
-template <ushort BLOCK_SIZE, typename T>
-static T threadgroup_raking_reduce_sum(
- T value,
- threadgroup T* shared,
- const ushort lid
-) {
- shared[lid] = value;
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- if (lid < 32) {
- const short values_per_thread = BLOCK_SIZE / 32;
- const short first_index = lid * values_per_thread;
-
- T thread_sum = shared[first_index];
- for (short i = first_index + 1; i < first_index + values_per_thread; i++) {
- thread_sum += shared[i];
- }
-
- T total_sum = simd_sum(thread_sum);
-
- if (lid == 0) {
- shared[0] = total_sum;
- }
- }
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- const T result = shared[0];
- threadgroup_barrier(mem_flags::mem_threadgroup);
- return result;
-}
-
-template <ushort BLOCK_SIZE, typename T>
-static T threadgroup_raking_reduce_max(
- T value,
- threadgroup T* shared,
- const ushort lid
-) {
- shared[lid] = value;
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- if (lid < 32) {
- const short values_per_thread = BLOCK_SIZE / 32;
- const short first_index = lid * values_per_thread;
-
- T thread_max = shared[first_index];
- for (short i = first_index + 1; i < first_index + values_per_thread; i++) {
- thread_max = max(thread_max, shared[i]);
- }
-
- T total_max = simd_max(thread_max);
-
- if (lid == 0) {
- shared[0] = total_max;
- }
- }
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- const T result = shared[0];
- threadgroup_barrier(mem_flags::mem_threadgroup);
- return result;
-}
-
-template <ushort BLOCK_SIZE, typename T>
-static T threadgroup_raking_reduce_min(
- T value,
- threadgroup T* shared,
- const ushort lid
-) {
- shared[lid] = value;
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- if (lid < 32) {
- const short values_per_thread = BLOCK_SIZE / 32;
- const short first_index = lid * values_per_thread;
-
- T thread_min = shared[first_index];
- for (short i = first_index + 1; i < first_index + values_per_thread; i++) {
- thread_min = min(thread_min, shared[i]);
- }
-
- T total_min = simd_min(thread_min);
-
- if (lid == 0) {
- shared[0] = total_min;
- }
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- const T result = shared[0];
- threadgroup_barrier(mem_flags::mem_threadgroup);
- return result;
-}
-
-template <ushort BLOCK_SIZE, typename T>
-static T threadgroup_cooperative_reduce_sum(
- T value,
- threadgroup T* shared,
- const ushort lid,
- const thread ThreadContext& thread_context
-) {
- T local_sum = simd_sum(value);
-
- if (thread_context.simdgroup_index == 0) {
- shared[thread_context.threadgroup_index] = local_sum;
- }
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- T total_sum = T(0);
- const ushort num_simd_groups =
- (BLOCK_SIZE + thread_context.simdgroup_size - 1) /
- thread_context.simdgroup_size;
- if (lid < num_simd_groups) {
- total_sum = shared[lid];
- }
- total_sum = simd_sum(total_sum);
-
- if (lid == 0) {
- shared[0] = total_sum;
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
+#include "thread_context.h"
- const T result = shared[0];
- threadgroup_barrier(mem_flags::mem_threadgroup);
- return result;
-}
+using namespace metal;
-template <ushort BLOCK_SIZE, typename T>
-static T threadgroup_cooperative_reduce_max(
- T value,
- threadgroup T* shared,
- const ushort lid,
+template <typename T>
+struct SimdReduceSum {
+ using value_type = T;
+ static constant constexpr T identity = static_cast<T>(0);
+ static T simd_reduce(T x) { return simd_sum(x); }
+};
+
+template <typename T>
+struct SimdReduceMax {
+ using value_type = T;
+ static constant constexpr T identity = numeric_limits<T>::has_infinity
+ ? -numeric_limits<T>::infinity()
+ : numeric_limits<T>::lowest();
+ static T simd_reduce(T x) { return simd_max(x); }
+};
+
+template <typename T>
+struct SimdReduceMin {
+ using value_type = T;
+ static constant constexpr T identity = numeric_limits<T>::has_infinity
+ ? numeric_limits<T>::infinity()
+ : numeric_limits<T>::max();
+ static T simd_reduce(T x) { return simd_min(x); }
+};
+
+template <typename Op, ushort BLOCK_SIZE>
+static typename Op::value_type threadgroup_cooperative_reduce(
+ typename Op::value_type value,
+ threadgroup typename Op::value_type* shared,
const thread ThreadContext& thread_context
) {
- T local_max = simd_max(value);
-
+ // Phase 1: reduce within each simdgroup
+ typename Op::value_type local = Op::simd_reduce(value);
if (thread_context.simdgroup_index == 0) {
- shared[thread_context.threadgroup_index] = local_max;
- }
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- T total_max = lid < (BLOCK_SIZE + thread_context.simdgroup_size - 1) /
- thread_context.simdgroup_size
- ? shared[lid]
- : T(-INFINITY);
- total_max = simd_max(total_max);
-
- if (lid == 0) {
- shared[0] = total_max;
+ shared[thread_context.threadgroup_index] = local;
}
- threadgroup_barrier(mem_flags::mem_threadgroup);
- const T result = shared[0];
threadgroup_barrier(mem_flags::mem_threadgroup);
- return result;
-}
-template <ushort BLOCK_SIZE, typename T>
-static T threadgroup_cooperative_reduce_min(
- T value,
- threadgroup T* shared,
- const ushort lid,
- const thread ThreadContext& thread_context
-) {
- T local_min = simd_min(value);
+ // Phase 2: first simdgroup reduces across simdgropus
+ if (thread_context.threadgroup_index == 0) {
+ typename Op::value_type total =
+ thread_context.simdgroup_index <
+ (BLOCK_SIZE + thread_context.simdgroup_size - 1) /
+ thread_context.simdgroup_size
+ ? shared[thread_context.simdgroup_index]
+ : Op::identity;
+ total = Op::simd_reduce(total);
- if (thread_context.simdgroup_index == 0) {
- shared[thread_context.threadgroup_index] = local_min;
+ if (thread_context.simdgroup_index == 0) {
+ shared[0] = total;
+ }
}
threadgroup_barrier(mem_flags::mem_threadgroup);
- T total_min = lid < (BLOCK_SIZE + thread_context.simdgroup_size - 1) /
- thread_context.simdgroup_size
- ? shared[lid]
- : T(INFINITY);
- total_min = simd_min(total_min);
+ // Phase 3: broadcast
+ const typename Op::value_type result = shared[0];
- if (lid == 0) {
- shared[0] = total_min;
- }
threadgroup_barrier(mem_flags::mem_threadgroup);
- const T result = shared[0];
- threadgroup_barrier(mem_flags::mem_threadgroup);
return result;
}
diff --git a/crates/uzu/src/backends/metal/kernel/common/utils.h b/crates/uzu/src/backends/metal/kernel/common/utils.h
deleted file mode 100644
index 71a185205..000000000
--- a/crates/uzu/src/backends/metal/kernel/common/utils.h
+++ /dev/null
@@ -1,469 +0,0 @@
-#pragma once
-
-#include <metal_math>
-
-#include "bf16.h"
-#include "bf16_math.h"
-#include "complex.h"
-#include "defines.h"
-#include "logging.h"
-
-typedef half float16_t;
-
-// Work per thread values for different types.
-template <typename U>
-struct WorkPerThread {
- static_assert(sizeof(U) <= 8, "Type too large");
- static constexpr int constant n = 8 / sizeof(U);
-};
-
-///////////////////////////////////////////////////////////////////////////////
-// Type limits utils
-///////////////////////////////////////////////////////////////////////////////
-
-template <typename U>
-struct Limits {
- static const constant U max = metal::numeric_limits<U>::max();
- static const constant U min = metal::numeric_limits<U>::min();
- static const constant U finite_max = metal::numeric_limits<U>::max();
- static const constant U finite_min = metal::numeric_limits<U>::min();
-};
-
-#define instantiate_default_limit(type) \
- template <> \
- struct Limits<type> { \
- static constexpr constant type max = metal::numeric_limits<type>::max(); \
- static constexpr constant type min = metal::numeric_limits<type>::min(); \
- static constexpr constant type finite_max = \
- metal::numeric_limits<type>::max(); \
- static constexpr constant type finite_min = \
- metal::numeric_limits<type>::min(); \
- };
-
-instantiate_default_limit(uint8_t);
-instantiate_default_limit(uint16_t);
-instantiate_default_limit(uint32_t);
-instantiate_default_limit(uint64_t);
-instantiate_default_limit(int8_t);
-instantiate_default_limit(int16_t);
-instantiate_default_limit(int32_t);
-instantiate_default_limit(int64_t);
-
-#define instantiate_float_limit(type) \
- template <> \
- struct Limits<type> { \
- static constexpr constant type max = \
- metal::numeric_limits<type>::infinity(); \
- static constexpr constant type min = \
- -metal::numeric_limits<type>::infinity(); \
- static constexpr constant type finite_max = \
- metal::numeric_limits<type>::max(); \
- static constexpr constant type finite_min = \
- -metal::numeric_limits<type>::max(); \
- };
-
-instantiate_float_limit(half);
-instantiate_float_limit(float);
-instantiate_float_limit(bfloat16_t);
-
-template <>
-struct Limits<bool> {
- static constexpr constant bool max = true;
- static constexpr constant bool min = false;
-};
-
-template <>
-struct Limits<complex64_t> {
- static constexpr constant complex64_t max = complex64_t(
- metal::numeric_limits<float>::infinity(),
- metal::numeric_limits<float>::infinity()
- );
- static constexpr constant complex64_t min = complex64_t(
- -metal::numeric_limits<float>::infinity(),
- -metal::numeric_limits<float>::infinity()
- );
-};
-
-///////////////////////////////////////////////////////////////////////////////
-// Indexing utils
-///////////////////////////////////////////////////////////////////////////////
-
-///////////////////////////////////////////////////////////////////////////////
-// Single Array with generic dims
-
-template <typename IdxT = int64_t>
-METAL_FUNC IdxT elem_to_loc(
- IdxT elem,
- constant const int* shape,
- constant const int64_t* strides,
- int ndim
-) {
- IdxT loc = 0;
- for (int i = ndim - 1; i >= 0 && elem > 0; --i) {
- loc += (elem % shape[i]) * IdxT(strides[i]);
- elem /= shape[i];
- }
- return loc;
-}
-
-// Non templated version to handle arbitrary dims
-template <typename IdxT = int64_t>
-METAL_FUNC IdxT elem_to_loc(
- uint3 elem,
- constant const int* shape,
- constant const int64_t* strides,
- int ndim
-) {
- IdxT loc =
- elem.x * IdxT(strides[ndim - 1]) + elem.y * IdxT(strides[ndim - 2]);
- for (int d = ndim - 3; d >= 0; --d) {
- loc += (elem.z % shape[d]) * IdxT(strides[d]);
- elem.z /= shape[d];
- }
- return loc;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// Single Array with fixed N dims
-
-template <typename IdxT = int64_t>
-METAL_FUNC IdxT elem_to_loc_1(uint elem, constant const int64_t& stride) {
- return elem * IdxT(stride);
-}
-
-template <typename IdxT = int64_t>
-METAL_FUNC IdxT elem_to_loc_2(uint2 elem, constant const int64_t strides[2]) {
- return elem.x * IdxT(strides[1]) + elem.y * IdxT(strides[0]);
-}
-
-template <typename IdxT = int64_t>
-METAL_FUNC IdxT elem_to_loc_3(uint3 elem, constant const int64_t strides[3]) {
- return elem.x * IdxT(strides[2]) + elem.y * IdxT(strides[1]) +
- elem.z * IdxT(strides[0]);
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// Multiple Arrays with generic dims
-
-template <typename IdxT = int64_t>
-METAL_FUNC vec<IdxT, 2> elem_to_loc_2_nd(
- uint3 elem,
- constant const int* shape,
- constant const int64_t* a_strides,
- constant const int64_t* b_strides,
- int ndim
-) {
- vec<IdxT, 2> loc = {
- IdxT(
- elem.x * IdxT(a_strides[ndim - 1]) +
- IdxT(elem.y) * IdxT(a_strides[ndim - 2])
- ),
- IdxT(
- elem.x * IdxT(b_strides[ndim - 1]) +
- elem.y * IdxT(b_strides[ndim - 2])
- )
- };
- for (int d = ndim - 3; d >= 0; --d) {
- uint l = elem.z % shape[d];
- loc.x += l * IdxT(a_strides[d]);
- loc.y += l * IdxT(b_strides[d]);
- elem.z /= shape[d];
- }
- return loc;
-}
-
-template <typename IdxT = int64_t>
-METAL_FUNC vec<IdxT, 3> elem_to_loc_3_nd(
- uint3 elem,
- constant const int* shape,
- constant const int64_t* a_strides,
- constant const int64_t* b_strides,
- constant const int64_t* c_strides,
- int ndim
-) {
- vec<IdxT, 3> loc = {
- IdxT(elem.x * IdxT(a_strides[ndim - 1])) +
- IdxT(elem.y * IdxT(a_strides[ndim - 2])),
- IdxT(elem.x * IdxT(b_strides[ndim - 1])) +
- IdxT(elem.y * IdxT(b_strides[ndim - 2])),
- IdxT(elem.x * IdxT(c_strides[ndim - 1])) +
- IdxT(elem.y * IdxT(c_strides[ndim - 2]))
- };
- for (int d = ndim - 3; d >= 0; --d) {
- uint l = elem.z % shape[d];
- loc.x += l * IdxT(a_strides[d]);
- loc.y += l * IdxT(b_strides[d]);
- loc.z += l * IdxT(c_strides[d]);
- elem.z /= shape[d];
- }
- return loc;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// Elem to loc in a loop utils
-///////////////////////////////////////////////////////////////////////////////
-
-template <int DIM, typename OffsetT = size_t, bool General = true>
-struct LoopedElemToLoc {
- int dim;
- LoopedElemToLoc<DIM - 1, OffsetT, General> inner_looper;
- OffsetT offset{0};
- int index{0};
-
- LoopedElemToLoc(int dim) : dim(dim), inner_looper(dim - 1) {}
-
- void next(const constant int* shape, const constant int64_t* strides) {
- if (dim == 0) {
- return;
- }
- index++;
- offset += OffsetT(strides[dim - 1]);
- if (index >= shape[dim - 1]) {
- index = 0;
- inner_looper.next(shape, strides);
- offset = inner_looper.offset;
- }
- }
-
- void next(int n, const constant int* shape, const constant int64_t* strides) {
- if (dim == 0) {
- return;
- }
- index += n;
- offset += n * OffsetT(strides[dim - 1]);
-
- if (index >= shape[dim - 1]) {
- int extra = index - shape[dim - 1];
- if (extra >= shape[dim - 1]) {
- inner_looper.next(1 + extra / shape[dim - 1], shape, strides);
- extra = extra % shape[dim - 1];
- } else {
- inner_looper.next(shape, strides);
- }
- index = 0;
- offset = inner_looper.offset;
- if (extra > 0) {
- next(extra, shape, strides);
- }
- }
- }
-
- OffsetT location() { return offset; }
-};
-
-template <typename OffsetT>
-struct LoopedElemToLoc<1, OffsetT, true> {
- int dim;
- OffsetT offset{0};
- uint index{0};
-
- LoopedElemToLoc(int dim) : dim(dim) {}
-
- void next(const constant int* shape, const constant int64_t* strides) {
- index++;
- if (dim > 1) {
- offset = elem_to_loc<OffsetT>(index, shape, strides, dim);
- } else {
- offset += OffsetT(strides[0]);
- }
- }
-
- void next(int n, const constant int* shape, const constant int64_t* strides) {
- index += n;
- if (dim > 1) {
- offset = elem_to_loc<OffsetT>(index, shape, strides, dim);
- } else {
- offset = index * OffsetT(strides[0]);
- }
- }
-
- OffsetT location() { return offset; }
-};
-
-template <typename OffsetT>
-struct LoopedElemToLoc<1, OffsetT, false> {
- OffsetT offset{0};
-
- LoopedElemToLoc(int) {}
-
- void next(const constant int*, const constant int64_t* strides) {
- offset += OffsetT(strides[0]);
- }
-
- void next(int n, const constant int*, const constant int64_t* strides) {
- offset += n * OffsetT(strides[0]);
- }
-
- OffsetT location() { return offset; }
-};
-
-///////////////////////////////////////////////////////////////////////////////
-// Calculation utils
-///////////////////////////////////////////////////////////////////////////////
-
-/** Compute ceil((float)N/(float)M) */
-template <typename T, typename U>
-inline T ceildiv(T N, U M) {
- return (N + M - 1) / M;
-}
-
-// https://docs.oracle.com/cd/E19957-01/806-3568/ncg_goldberg.html#1202
-inline float log1p(float x) {
- float xp1 = 1.0f + x;
- if (xp1 == Limits<float>::max) {
- return Limits<float>::max;
- }
- if (xp1 == 1.0f) {
- return x;
- }
-
- return x * (metal::log(xp1) / (xp1 - 1.0f));
-}
-
-inline bfloat16_t log1p(bfloat16_t x) {
- float xp1 = 1.0f + static_cast<float>(x);
- if (xp1 == Limits<float>::max) {
- return Limits<bfloat16_t>::max;
- }
- if (xp1 == 1.0f) {
- return x;
- }
-
- return bfloat16_t(x * (metal::log(xp1) / (xp1 - 1.0f)));
-}
-
-inline complex64_t log1p(complex64_t in) {
- float x = in.real;
- float y = in.imag;
- float zabs = metal::precise::sqrt(x * x + y * y);
- float theta = metal::atan2(y, x + 1);
- if (zabs < 0.5f) {
- float r = x * (2 + x) + y * y;
- if (r == 0) { // handle underflow
- return {x, theta};
- }
- return {0.5f * log1p(r), theta};
- } else {
- auto z0 = metal::sqrt((x + 1) * (x + 1) + y * y);
- return {metal::log(z0), theta};
- }
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// SIMD shuffle ops
-///////////////////////////////////////////////////////////////////////////////
-
-inline uint64_t simd_shuffle_down(uint64_t data, uint16_t delta) {
- return as_type<uint64_t>(
- metal::simd_shuffle_down(as_type<uint2>(data), delta)
- );
-}
-
-inline int64_t simd_shuffle_down(int64_t data, uint16_t delta) {
- return as_type<int64_t>(
- metal::simd_shuffle_down(as_type<uint2>(data), delta)
- );
-}
-
-inline bool simd_shuffle_down(bool data, uint16_t delta) {
- return simd_shuffle_down(static_cast<uint32_t>(data), delta);
-}
-
-inline complex64_t simd_shuffle_down(complex64_t data, uint16_t delta) {
- return complex64_t(
- simd_shuffle_down(data.real, delta),
- simd_shuffle_down(data.imag, delta)
- );
-}
-
-inline uint64_t simd_shuffle_up(uint64_t data, uint16_t delta) {
- return as_type<uint64_t>(metal::simd_shuffle_up(as_type<uint2>(data), delta));
-}
-
-inline int64_t simd_shuffle_up(int64_t data, uint16_t delta) {
- return as_type<int64_t>(metal::simd_shuffle_up(as_type<uint2>(data), delta));
-}
-
-inline bool simd_shuffle_up(bool data, uint16_t delta) {
- return simd_shuffle_up(static_cast<uint32_t>(data), delta);
-}
-
-inline complex64_t simd_shuffle_up(complex64_t data, uint16_t delta) {
- return complex64_t(
- simd_shuffle_up(data.real, delta),
- simd_shuffle_up(data.imag, delta)
- );
-}
-
-inline uint64_t simd_shuffle_and_fill_up(
- uint64_t data,
- uint64_t filling,
- uint16_t delta
-) {
- return as_type<uint64_t>(metal::simd_shuffle_and_fill_up(
- as_type<uint2>(data),
- as_type<uint2>(filling),
- delta
- ));
-}
-
-inline int64_t simd_shuffle_and_fill_up(
- int64_t data,
- int64_t filling,
- uint16_t delta
-) {
- return as_type<int64_t>(metal::simd_shuffle_and_fill_up(
- as_type<uint2>(data),
- as_type<uint2>(filling),
- delta
- ));
-}
-
-inline bool simd_shuffle_and_fill_up(bool data, bool filling, uint16_t delta) {
- return simd_shuffle_and_fill_up(
- static_cast<uint32_t>(data),
- static_cast<uint32_t>(filling),
- delta
- );
-}
-
-inline complex64_t simd_shuffle_and_fill_up(
- complex64_t data,
- complex64_t filling,
- uint16_t delta
-) {
- return complex64_t(
- simd_shuffle_and_fill_up(data.real, filling.real, delta),
- simd_shuffle_and_fill_up(data.imag, filling.imag, delta)
- );
-}
-
-inline uint64_t simd_shuffle(uint64_t data, uint16_t lane) {
- return as_type<uint64_t>(metal::simd_shuffle(as_type<uint2>(data), lane));
-}
-
-inline int64_t simd_shuffle(int64_t data, uint16_t lane) {
- return as_type<int64_t>(metal::simd_shuffle(as_type<uint2>(data), lane));
-}
-
-inline bool simd_shuffle(bool data, uint16_t lane) {
- return simd_shuffle(static_cast<uint32_t>(data), lane);
-}
-
-inline complex64_t simd_shuffle(complex64_t data, uint16_t lane) {
- return complex64_t(
- simd_shuffle(data.real, lane),
- simd_shuffle(data.imag, lane)
- );
-}
-
-// std::conditional is not included with Metal
-template <bool condition, typename T, typename U>
-struct ConditionalType {
- using type = U;
-};
-
-template <typename T, typename U>
-struct ConditionalType<true, T, U> {
- using type = T;
-};
diff --git a/crates/uzu/src/backends/metal/kernel/delta_net/norm_gate.metal b/crates/uzu/src/backends/metal/kernel/delta_net/norm_gate.metal
index ba45c4018..2a7020b36 100644
--- a/crates/uzu/src/backends/metal/kernel/delta_net/norm_gate.metal
+++ b/crates/uzu/src/backends/metal/kernel/delta_net/norm_gate.metal
@@ -42,12 +42,12 @@ PUBLIC KERNEL(DeltaNetNormGate)(
// RMSNorm: cooperative sum of squares
float o_sq = active ? o_i * o_i : 0.0f;
- float o_sumsq = threadgroup_cooperative_reduce_sum<HEAD_V_DIM>(
- o_sq,
- shared_scratch,
- tid,
- thread_context
- );
+ float o_sumsq =
+ threadgroup_cooperative_reduce<SimdReduceSum<float>, HEAD_V_DIM>(
+ o_sq,
+ shared_scratch,
+ thread_context
+ );
float inv_rms = rsqrt(o_sumsq / float(head_v_dim) + norm_epsilon);
// Apply norm + SiLU gate (in-place)
diff --git a/crates/uzu/src/backends/metal/kernel/delta_net/update.metal b/crates/uzu/src/backends/metal/kernel/delta_net/update.metal
index 7fdd882a4..a12e283a4 100644
--- a/crates/uzu/src/backends/metal/kernel/delta_net/update.metal
+++ b/crates/uzu/src/backends/metal/kernel/delta_net/update.metal
@@ -62,21 +62,21 @@ PUBLIC KERNEL(DeltaNetUpdate)(
threadgroup_barrier(mem_flags::mem_threadgroup);
float q_partial = (tid < HEAD_K_DIM) ? shared_q[tid] * shared_q[tid] : 0.0f;
- float q_norm_sq = threadgroup_cooperative_reduce_sum<UPDATE_THREADS>(
- q_partial,
- shared_scratch,
- tid,
- thread_context
- );
+ float q_norm_sq =
+ threadgroup_cooperative_reduce<SimdReduceSum<float>, UPDATE_THREADS>(
+ q_partial,
+ shared_scratch,
+ thread_context
+ );
float q_inv_norm = rsqrt(q_norm_sq + 1e-6f);
float k_partial = (tid < HEAD_K_DIM) ? shared_k[tid] * shared_k[tid] : 0.0f;
- float k_norm_sq = threadgroup_cooperative_reduce_sum<UPDATE_THREADS>(
- k_partial,
- shared_scratch,
- tid,
- thread_context
- );
+ float k_norm_sq =
+ threadgroup_cooperative_reduce<SimdReduceSum<float>, UPDATE_THREADS>(
+ k_partial,
+ shared_scratch,
+ thread_context
+ );
float k_inv_norm = rsqrt(k_norm_sq + 1e-6f);
float q_scale = rsqrt(float(HEAD_K_DIM));
@@ -87,12 +87,12 @@ PUBLIC KERNEL(DeltaNetUpdate)(
threadgroup_barrier(mem_flags::mem_threadgroup);
float kq_p = (tid < HEAD_K_DIM) ? shared_k[tid] * shared_q[tid] : 0.0f;
- float kq_dot = threadgroup_cooperative_reduce_sum<UPDATE_THREADS>(
- kq_p,
- shared_scratch,
- tid,
- thread_context
- );
+ float kq_dot =
+ threadgroup_cooperative_reduce<SimdReduceSum<float>, UPDATE_THREADS>(
+ kq_p,
+ shared_scratch,
+ thread_context
+ );
float beta_raw = float(in_proj[conv_dim + value_dim + hv_idx]);
float beta = 1.0f / (1.0f + fast::exp(-beta_raw));
@@ -139,12 +139,12 @@ PUBLIC KERNEL(DeltaNetUpdate)(
// RMSNorm + SiLU gate — only ki_part==0 contributes
float o_sq = (active && ki_part == 0) ? o_i * o_i : 0.0f;
- float o_sumsq = threadgroup_cooperative_reduce_sum<UPDATE_THREADS>(
- o_sq,
- shared_scratch,
- tid,
- thread_context
- );
+ float o_sumsq =
+ threadgroup_cooperative_reduce<SimdReduceSum<float>, UPDATE_THREADS>(
+ o_sq,
+ shared_scratch,
+ thread_context
+ );
float inv_rms = rsqrt(o_sumsq / float(head_v_dim) + norm_epsilon);
if (active && ki_part == 0) {
diff --git a/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal b/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal
index 08758f523..92c946f22 100644
--- a/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal
+++ b/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal
@@ -36,12 +36,12 @@ void layer_norm_core(
}
// Reduce to get total mean
- AccumT total_sum = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(
- partial_sum,
- shared_mean,
- thread_in_row,
- thread_context
- );
+ AccumT total_sum =
+ threadgroup_cooperative_reduce<SimdReduceSum<AccumT>, BLOCK_SIZE>(
+ partial_sum,
+ shared_mean,
+ thread_context
+ );
AccumT mean = total_sum / static_cast<AccumT>(element_count);
// Compute variance: sum of squared deviations
@@ -58,12 +58,12 @@ void layer_norm_core(
}
// Reduce to get total variance
- AccumT total_var = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(
- partial_var_sum,
- shared_variance,
- thread_in_row,
- thread_context
- );
+ AccumT total_var =
+ threadgroup_cooperative_reduce<SimdReduceSum<AccumT>, BLOCK_SIZE>(
+ partial_var_sum,
+ shared_variance,
+ thread_context
+ );
AccumT variance = total_var / static_cast<AccumT>(element_count);
AccumT inv_std = rsqrt(variance + static_cast<AccumT>(epsilon));
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/defines.h b/crates/uzu/src/backends/metal/kernel/matmul/common/defines.h
index c441e1f58..1b56d03b3 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/common/defines.h
+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/defines.h
@@ -33,11 +33,6 @@ struct PointerElementTypeImpl<threadgroup T*> {
template <typename T>
using PointerElementType = typename PointerElementTypeImpl<T>::type;
-template <typename T>
-struct AccumulatorHelper {
- typedef float AccumulatorType;
-};
-
template <typename OutT, typename InT>
struct TransformNone {
TransformNone(float = 1.0f, float = 0.0f) {}
@@ -46,16 +41,6 @@ struct TransformNone {
METAL_FUNC OutT apply(InT x, OutT) const { return static_cast<OutT>(x); }
};
-template <typename OutT, typename InT>
-struct TransformAdd {
- TransformAdd(float = 1.0f, float = 1.0f) {}
-
- static METAL_FUNC OutT apply(InT x) { return static_cast<OutT>(x); }
- METAL_FUNC OutT apply(InT x, OutT c) const {
- return static_cast<OutT>(x) + c;
- }
-};
-
template <typename OutT, typename InT>
struct TransformScaleAccumulate {
const float alpha;
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemm.metal
index 9f6d548fb..f790479f0 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm.metal
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm.metal
@@ -1,4 +1,3 @@
-#include "../common/utils.h"
#include "../common/dsl.h"
#include "../common/thread_context.h"
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemv.metal
index 3187bee36..b08eb3e73 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv.metal
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv.metal
@@ -1,4 +1,6 @@
-#include "../common/utils.h"
+#include <metal_stdlib>
+
+#include "../common/defines.h"
#include "../common/dsl.h"
#include "../common/thread_context.h"
diff --git a/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal b/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal
index 571d408ce..8b10d91f3 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal
@@ -5,6 +5,40 @@
#include "../common/thread_context.h"
#include "../common/threadgroup_reduce.h"
+template <ushort BLOCK_SIZE, typename T>
+static T threadgroup_raking_prefix_exclusive_sum(
+ T value,
+ threadgroup T* shared,
+ const ushort lid
+) {
+ shared[lid] = value;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ if (lid < 32) {
+ const short values_per_thread = BLOCK_SIZE / 32;
+ const short first_index = lid * values_per_thread;
+ for (short i = first_index + 1; i < first_index + values_per_thread; i++) {
+ shared[i] += shared[i - 1];
+ }
+ T partial_sum = shared[first_index + values_per_thread - 1];
+ for (short i = first_index + values_per_thread - 1; i > first_index; i--) {
+ shared[i] = shared[i - 1];
+ }
+ shared[first_index] = 0;
+
+ T prefix = simd_prefix_exclusive_sum(partial_sum);
+
+ for (short i = first_index; i < first_index + values_per_thread; i++) {
+ shared[i] += prefix;
+ }
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ const T result = shared[lid];
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ return result;
+}
+
#define BLOCK_SIZE 128
#define TILE_E 512
@@ -100,12 +134,12 @@ PUBLIC KERNEL(MoeCountsOffsetsFused)(
offsets[base + lid] = prefix_global;
}
- uint block_sum = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(
- v,
- reduce_shared,
- (ushort)lid,
- thread_context
- );
+ uint block_sum =
+ threadgroup_cooperative_reduce<SimdReduceSum<uint>, BLOCK_SIZE>(
+ v,
+ reduce_shared,
+ thread_context
+ );
if (lid == 0) {
carry += block_sum;
}
diff --git a/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal b/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal
index 181d9afa0..cbb1a161e 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal
@@ -96,20 +96,20 @@ PUBLIC KERNEL(MoeRouterTopK)(
}
}
- float max_val = threadgroup_cooperative_reduce_max<THREADS_PER_TG>(
- local_best,
- reduce_tmp,
- lid,
- thread_context
- );
+ float max_val =
+ threadgroup_cooperative_reduce<SimdReduceMax<float>, THREADS_PER_TG>(
+ local_best,
+ reduce_tmp,
+ thread_context
+ );
uint candidate_id = (local_best == max_val) ? local_idx : 0xFFFFFFFFu;
- uint best_idx = threadgroup_cooperative_reduce_min<THREADS_PER_TG>(
- candidate_id,
- reduce_tmp_u,
- lid,
- thread_context
- );
+ uint best_idx =
+ threadgroup_cooperative_reduce<SimdReduceMin<uint>, THREADS_PER_TG>(
+ candidate_id,
+ reduce_tmp_u,
+ thread_context
+ );
if (lid == 0) {
shared_best_idx[0] = best_idx;
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm.metal
index f16fba9bb..bcdc3b970 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm.metal
@@ -1,6 +1,6 @@
#include <metal_stdlib>
#include "../common/dsl.h"
-#include "quant_matmul.metal"
+#include "quant_matmul.h"
template <typename T, int GROUP_SIZE, int BITS>
VARIANTS(T, float, half, bfloat)
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal
index c324e936f..84b0932b9 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal
@@ -1,6 +1,6 @@
#include <metal_stdlib>
#include "../common/dsl.h"
-#include "quant_matmul.metal"
+#include "quant_matmul.h"
template <typename T, int GROUP_SIZE, int BITS>
VARIANTS(T, float, half, bfloat)
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metal
index 61d6fc85f..b4af07b58 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metal
@@ -1,6 +1,6 @@
#include <metal_stdlib>
#include "../common/dsl.h"
-#include "quant_matmul.metal"
+#include "quant_matmul.h"
template <typename T, int GROUP_SIZE, int BITS>
VARIANTS(T, bfloat)
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_wide.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_wide.metal
index 6118a70fe..a066b3022 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_wide.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_wide.metal
@@ -1,6 +1,6 @@
#include <metal_stdlib>
#include "../common/dsl.h"
-#include "quant_matmul.metal"
+#include "quant_matmul.h"
// Wide QMM tile: BM=64, BK=32, BN=64.
// Processes 4x more output elements per threadgroup than the 32x32 variant.
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal
index 6bcfc03b9..527711d3a 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal
@@ -1,6 +1,6 @@
#include <metal_stdlib>
#include "../common/dsl.h"
-#include "quant_matmul.metal"
+#include "quant_matmul.h"
template <typename T, int GROUP_SIZE, int BITS>
VARIANTS(T, float, half, bfloat)
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal
index c8d0a8b35..db8eda84f 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal
@@ -1,6 +1,6 @@
#include <metal_stdlib>
#include "../common/dsl.h"
-#include "quant_matmul.metal"
+#include "quant_matmul.h"
template <typename T, int GROUP_SIZE, int BITS>
VARIANTS(T, float, half, bfloat)
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.h
similarity index 100%
rename from crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal
rename to crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.h
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qvm.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qvm.metal
index 3c389f961..27356c4a1 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qvm.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qvm.metal
@@ -1,6 +1,6 @@
#include <metal_stdlib>
#include "../common/dsl.h"
-#include "quant_matmul.metal"
+#include "quant_matmul.h"
template <typename T, int GROUP_SIZE, int BITS>
VARIANTS(T, float, half, bfloat)
diff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
index d08730f49..79e1e57b9 100644
--- a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
+++ b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
@@ -56,12 +56,12 @@ PUBLIC KERNEL(RMSNorm)(
}
// Compute total sum across threadgroup
- AccumT total_sum = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(
- partial_sum,
- shared_sum,
- thread_in_row,
- thread_context
- );
+ AccumT total_sum =
+ threadgroup_cooperative_reduce<SimdReduceSum<AccumT>, BLOCK_SIZE>(
+ partial_sum,
+ shared_sum,
+ thread_context
+ );
// Compute RMS norm factor
AccumT mean_square =
diff --git a/crates/uzu/src/backends/metal/kernel/rng.metal b/crates/uzu/src/backends/metal/kernel/rng.h
similarity index 97%
rename from crates/uzu/src/backends/metal/kernel/rng.metal
rename to crates/uzu/src/backends/metal/kernel/rng.h
index 97979277f..8715ebc7d 100644
--- a/crates/uzu/src/backends/metal/kernel/rng.metal
+++ b/crates/uzu/src/backends/metal/kernel/rng.h
@@ -1,8 +1,6 @@
-#ifndef rng_metal
-#define rng_metal
+#pragma once
#include <metal_stdlib>
-using namespace metal;
/* ───────────── Philox4x32-10 RNG ───────────── */
@@ -126,5 +124,3 @@ inline uint philox_next(thread PhiloxState* state) {
inline float uniform_float(thread PhiloxState* state) {
return float(philox_next(state)) * (1.0f / 4294967296.0f); /* (0,1) */
}
-
-#endif /* rng_metal */
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal b/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal
index 497ffa642..852a43700 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal
@@ -2,6 +2,7 @@
#include <metal_atomic>
#include "../common/dsl.h"
#include "../common/thread_context.h"
+#include "../common/threadgroup_reduce.h"
#include "argmax.h"
@@ -20,59 +21,20 @@ ArgmaxPair argmax_combine(ArgmaxPair a, ArgmaxPair b) {
return argmax_is_better(a, b) ? a : b;
}
-//------------------------------------------------------------------------------------------------//
-// Cooperative threadgroup argmax reduction (2-level hierarchical)
-template <ushort BLOCK_SIZE_PARAM>
-static ArgmaxPair threadgroup_cooperative_argmax(
- ArgmaxPair value,
- threadgroup ArgmaxPair* shared,
- const ushort lid,
- const thread ThreadContext& thread_context
-) {
- // Reduce within simdgroup using manual shuffle operations
- ArgmaxPair local_result = value;
- for (ushort offset = 16; offset > 0; offset /= 2) {
- ArgmaxPair other = {
- simd_shuffle_down(local_result.value, offset),
- simd_shuffle_down(local_result.index, offset)
- };
- local_result = argmax_combine(local_result, other);
- }
-
- // First thread in each simdgroup writes to shared memory
- if (thread_context.simdgroup_index == 0) {
- shared[thread_context.threadgroup_index] = local_result;
- }
-
- // Synchronize across the threadgroup
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // Reduce across simdgroups
- const ushort num_simd_groups = (BLOCK_SIZE_PARAM + 31) / 32;
- ArgmaxPair total_result = ARGMAX_INIT;
- if (lid < num_simd_groups) {
- total_result = shared[lid];
- }
-
- // Final simdgroup reduction
- for (ushort offset = 16; offset > 0; offset /= 2) {
- ArgmaxPair other = {
- simd_shuffle_down(total_result.value, offset),
- simd_shuffle_down(total_result.index, offset)
- };
- total_result = argmax_combine(total_result, other);
- }
-
- // Broadcast the result to all threads
- if (lid == 0) {
- shared[0] = total_result;
+struct SimdReduceArgmax {
+ using value_type = ArgmaxPair;
+ static constant constexpr ArgmaxPair identity = ARGMAX_INIT;
+ static ArgmaxPair simd_reduce(ArgmaxPair x) {
+ for (ushort offset = 16; offset > 0; offset /= 2) {
+ ArgmaxPair other = {
+ simd_shuffle_down(x.value, offset),
+ simd_shuffle_down(x.index, offset)
+ };
+ x = argmax_combine(x, other);
+ }
+ return x;
}
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- const ArgmaxPair result = shared[0];
- threadgroup_barrier(mem_flags::mem_threadgroup);
- return result;
-}
+};
template <ushort GRAIN_SIZE_PARAM>
static inline ArgmaxPair thread_argmax_reduce(
@@ -226,12 +188,12 @@ PUBLIC KERNEL(ArgmaxMain)(
);
ArgmaxPair thread_result = thread_argmax_reduce<GRAIN_SIZE>(values);
- ArgmaxPair group_result = threadgroup_cooperative_argmax<BLOCK_SIZE>(
- thread_result,
- shared,
- local_id,
- thread_context
- );
+ ArgmaxPair group_result =
+ threadgroup_cooperative_reduce<SimdReduceArgmax, BLOCK_SIZE>(
+ thread_result,
+ shared,
+ thread_context
+ );
if (local_id == 0) {
uint vocab_groups_per_batch =
@@ -267,12 +229,12 @@ PUBLIC KERNEL(ArgmaxFinal)(
argmax_combine(thread_result, partial_results[partial_offset + i]);
}
- ArgmaxPair result = threadgroup_cooperative_argmax<BLOCK_SIZE>(
- thread_result,
- shared,
- local_id,
- thread_context
- );
+ ArgmaxPair result =
+ threadgroup_cooperative_reduce<SimdReduceArgmax, BLOCK_SIZE>(
+ thread_result,
+ shared,
+ thread_context
+ );
if (local_id == 0) {
final_tokens[batch_idx] = result.index;
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metal b/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metal
index 6fb5ce481..cecb7587f 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metal
@@ -1,6 +1,6 @@
#include <metal_stdlib>
#include "../common/dsl.h"
-#include "../rng.metal"
+#include "../rng.h"
#define BLOCK_SIZE 1024
#define GRAIN_SIZE 64
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/min_p.metal b/crates/uzu/src/backends/metal/kernel/sampling/min_p.metal
index 9b3962188..36b988cec 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/min_p.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/min_p.metal
@@ -33,12 +33,12 @@ PUBLIC KERNEL(MinP) (
float logit_value = float(logits[batch_start + i]);
local_max = fmax(local_max, logit_value);
}
- float max_logit = threadgroup_cooperative_reduce_max<BLOCK_SIZE>(
- local_max,
- shared_reduce_buffer,
- thread_idx,
- thread_context
- );
+ float max_logit =
+ threadgroup_cooperative_reduce<SimdReduceMax<float>, BLOCK_SIZE>(
+ local_max,
+ shared_reduce_buffer,
+ thread_context
+ );
// Then the threshold is just max_logit + log(min_p), mask everything strictly
// below it
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal b/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal
index c4c08b604..7a27876aa 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal
@@ -40,18 +40,18 @@ PUBLIC KERNEL(TopK)(
logit_value > -INFINITY
);
}
- float max_logit = threadgroup_cooperative_reduce_max<BLOCK_SIZE>(
- local_max,
- shared_reduce_buffer,
- thread_idx,
- thread_context
- );
- float min_logit = threadgroup_cooperative_reduce_min<BLOCK_SIZE>(
- local_min,
- shared_reduce_buffer,
- thread_idx,
- thread_context
- );
+ float max_logit =
+ threadgroup_cooperative_reduce<SimdReduceMax<float>, BLOCK_SIZE>(
+ local_max,
+ shared_reduce_buffer,
+ thread_context
+ );
+ float min_logit =
+ threadgroup_cooperative_reduce<SimdReduceMin<float>, BLOCK_SIZE>(
+ local_min,
+ shared_reduce_buffer,
+ thread_context
+ );
// Do the binary search on the threshold
float low = min_logit;
float high = max_logit;
@@ -65,12 +65,12 @@ PUBLIC KERNEL(TopK)(
float logit_value = float(logits[batch_start + i]);
local_num_above_threshold += select(0, 1, logit_value >= threshold);
}
- uint num_above_threshold = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(
- local_num_above_threshold,
- (threadgroup uint*)shared_reduce_buffer,
- thread_idx,
- thread_context
- );
+ uint num_above_threshold =
+ threadgroup_cooperative_reduce<SimdReduceSum<uint>, BLOCK_SIZE>(
+ local_num_above_threshold,
+ (threadgroup uint*)shared_reduce_buffer,
+ thread_context
+ );
// Update binary search
if (num_above_threshold == top_k) {
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal b/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal
index 1ab64c2b7..a4b9ea0d8 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal
@@ -41,18 +41,18 @@ PUBLIC KERNEL(TopP) (
logit_value > -INFINITY
);
}
- float max_logit = threadgroup_cooperative_reduce_max<BLOCK_SIZE>(
- local_max,
- shared_reduce_buffer,
- thread_idx,
- thread_context
- );
- float min_logit = threadgroup_cooperative_reduce_min<BLOCK_SIZE>(
- local_min,
- shared_reduce_buffer,
- thread_idx,
- thread_context
- );
+ float max_logit =
+ threadgroup_cooperative_reduce<SimdReduceMax<float>, BLOCK_SIZE>(
+ local_max,
+ shared_reduce_buffer,
+ thread_context
+ );
+ float min_logit =
+ threadgroup_cooperative_reduce<SimdReduceMin<float>, BLOCK_SIZE>(
+ local_min,
+ shared_reduce_buffer,
+ thread_context
+ );
// Find denominator for softmax
float local_sum = 0.0f;
@@ -65,12 +65,12 @@ PUBLIC KERNEL(TopP) (
logit_value > -INFINITY
);
}
- float total_sum = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(
- local_sum,
- shared_reduce_buffer,
- thread_idx,
- thread_context
- );
+ float total_sum =
+ threadgroup_cooperative_reduce<SimdReduceSum<float>, BLOCK_SIZE>(
+ local_sum,
+ shared_reduce_buffer,
+ thread_context
+ );
// Do the binary search on the threshold
float target_mass = top_p * total_sum;
@@ -94,18 +94,18 @@ PUBLIC KERNEL(TopP) (
select(INFINITY, logit_mass, logit_value >= threshold)
);
}
- float sum_above_threshold = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(
- local_sum_above_threshold,
- shared_reduce_buffer,
- thread_idx,
- thread_context
- );
- float min_above_threshold = threadgroup_cooperative_reduce_min<BLOCK_SIZE>(
- local_min_above_threshold,
- shared_reduce_buffer,
- thread_idx,
- thread_context
- );
+ float sum_above_threshold =
+ threadgroup_cooperative_reduce<SimdReduceSum<float>, BLOCK_SIZE>(
+ local_sum_above_threshold,
+ shared_reduce_buffer,
+ thread_context
+ );
+ float min_above_threshold =
+ threadgroup_cooperative_reduce<SimdReduceMin<float>, BLOCK_SIZE>(
+ local_min_above_threshold,
+ shared_reduce_buffer,
+ thread_context
+ );
// Early exit
if (sum_above_threshold >= target_mass &&
diff --git a/crates/uzu/src/backends/metal/kernel/sigmoid/sigmoid.metal b/crates/uzu/src/backends/metal/kernel/sigmoid/sigmoid.metal
deleted file mode 100644
index e4adb869f..000000000
--- a/crates/uzu/src/backends/metal/kernel/sigmoid/sigmoid.metal
+++ /dev/null
@@ -1,14 +0,0 @@
-#include <metal_stdlib>
-#include "../common/dsl.h"
-
-template <typename T>
-VARIANTS(T, float, half, bfloat)
-PUBLIC KERNEL(Sigmoid) (
- const device T* input,
- device T* output,
- const constant uint& total_elements,
- uint tid AXIS(total_elements, 256)
-) {
- float x = float(input[tid]);
- output[tid] = T(1.0f / (1.0f + exp(-x)));
-}
\ No newline at end of file
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssm_common.h b/crates/uzu/src/backends/metal/kernel/ssm/ssm_common.h
index 41845ba0d..1884fc210 100644
--- a/crates/uzu/src/backends/metal/kernel/ssm/ssm_common.h
+++ b/crates/uzu/src/backends/metal/kernel/ssm/ssm_common.h
@@ -1,3 +1,5 @@
+#pragma once
+
#include <metal_stdlib>
template <typename T>
@@ -7,4 +9,4 @@ inline T softplus(T x) {
return x;
}
return static_cast<T>(log(1.0f + fast::exp(xf)));
-}
\ No newline at end of file
+}
diff --git a/crates/uzu/tests/unit/kernel/mod.rs b/crates/uzu/tests/unit/kernel/mod.rs
index 92faaafff..7e47c316a 100644
--- a/crates/uzu/tests/unit/kernel/mod.rs
+++ b/crates/uzu/tests/unit/kernel/mod.rs
@@ -18,7 +18,6 @@ mod rms_norm_test;
mod rope_test;
mod sampling;
mod short_conv;
-mod sigmoid_test;
mod tensor_add_bias_test;
mod tensor_add_scale_test;
mod tensor_add_swap_test;
diff --git a/crates/uzu/tests/unit/kernel/sigmoid_test.rs b/crates/uzu/tests/unit/kernel/sigmoid_test.rs
deleted file mode 100644
index ca4f467a9..000000000
--- a/crates/uzu/tests/unit/kernel/sigmoid_test.rs
+++ /dev/null
@@ -1,176 +0,0 @@
-use std::{
- fmt::{Debug, Display},
- ops::{Deref, DerefMut},
-};
-
-use half::{bf16, f16};
-use num_traits::Float;
-use uzu::{
- ArrayContextExt, ArrayElement, DataType,
- backends::{
- common::{Backend, Context, Encoder, Kernels, kernel::SigmoidKernel},
- cpu::Cpu,
- },
-};
-
-use crate::common::assert::assert_eq_float;
-
-struct Input<T: ArrayElement + Float> {
- data: Box<[T]>,
-}
-
-fn get_input<T: ArrayElement + Float>(n: usize) -> Vec<T> {
- (0..n).map(|i| T::from((i as f32 * 0.3) - 5.0).unwrap()).collect()
-}
-
-fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
- let context = B::Context::new().expect("Failed to create Context");
-
- let kernel = <<B as Backend>::Kernels as Kernels>::SigmoidKernel::new(&context, T::data_type())
- .expect("Failed to create SigmoidKernel");
-
- let n = input.data.len();
- let input_array = context.create_array_from(&[n], &input.data, "");
- let output_array = context.create_array_uninitialized(&[n], T::data_type(), "");
-
- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
- kernel.encode(
- input_array.buffer().borrow().deref(),
- output_array.buffer().borrow_mut().deref_mut(),
- n as u32,
- &mut encoder,
- );
- encoder.end_encoding().submit().wait_until_completed().unwrap();
-
- output_array.as_slice().to_vec()
-}
-
-fn get_test_data_basic<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
- let data = get_input::<T>(128);
- let input = Input {
- data: data.into_boxed_slice(),
- };
- let expected = get_output::<T, Cpu>(&input);
- (input, expected)
-}
-
-fn get_test_data_large<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
- let data = get_input::<T>(4096);
- let input = Input {
- data: data.into_boxed_slice(),
- };
- let expected = get_output::<T, Cpu>(&input);
- (input, expected)
-}
-
-fn get_test_data_edge<T: ArrayElement + Float>() -> (Input<T>, Vec<T>) {
- // Values at sigmoid boundaries: very negative, zero, very positive
- let values: Vec<T> = [-100.0f32, -10.0, -1.0, 0.0, 1.0, 10.0, 100.0].iter().map(|&v| T::from(v).unwrap()).collect();
- let input = Input {
- data: values.into_boxed_slice(),
- };
- let expected = get_output::<T, Cpu>(&input);
- (input, expected)
-}
-
-fn test_basic<T: ArrayElement + Float + Debug + Display>() {
- let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
- 0.02f32
- } else {
- 1e-5
- };
- let (input, expected) = get_test_data_basic::<T>();
- for_each_non_cpu_backend!(|B| {
- let output = get_output::<T, B>(&input);
- assert_eq_float::<T>(
- &expected,
- &output,
- eps,
- &format!("Sigmoid basic test failed for backend {}", std::any::type_name::<B>()),
- );
- });
-}
-
-fn test_large<T: ArrayElement + Float + Debug + Display>() {
- let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
- 0.02f32
- } else {
- 1e-5
- };
- let (input, expected) = get_test_data_large::<T>();
- for_each_non_cpu_backend!(|B| {
- let output = get_output::<T, B>(&input);
- assert_eq_float::<T>(
- &expected,
- &output,
- eps,
- &format!("Sigmoid large test failed for backend {}", std::any::type_name::<B>()),
- );
- });
-}
-
-fn test_edge<T: ArrayElement + Float + Debug + Display>() {
- let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
- 0.02f32
- } else {
- 1e-5
- };
- let (input, expected) = get_test_data_edge::<T>();
- for_each_non_cpu_backend!(|B| {
- let output = get_output::<T, B>(&input);
- assert_eq_float::<T>(
- &expected,
- &output,
- eps,
- &format!("Sigmoid edge test failed for backend {}", std::any::type_name::<B>()),
- );
- });
-}
-
-// basic tests
-#[test]
-fn test_basic_f32() {
- test_basic::<f32>();
-}
-
-#[test]
-fn test_basic_f16() {
- test_basic::<f16>();
-}
-
-#[test]
-fn test_basic_bf16() {
- test_basic::<bf16>();
-}
-
-// large tests
-#[test]
-fn test_large_f32() {
- test_large::<f32>();
-}
-
-#[test]
-fn test_large_f16() {
- test_large::<f16>();
-}
-
-#[test]
-fn test_large_bf16() {
- test_large::<bf16>();
-}
-
-// edge tests
-#[test]
-fn test_edge_f32() {
- test_edge::<f32>();
-}
-
-#[test]
-fn test_edge_f16() {
- test_edge::<f16>();
-}
-
-#[test]
-fn test_edge_bf16() {
- test_edge::<bf16>();
-}