Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

shaders refactoring

eugenec7b328fb4bb8
diff --git a/crates/uzu/build/metal/ast.rs b/crates/uzu/build/metal/ast.rs
index 5d7e49614..8bbc95f8b 100644
--- a/crates/uzu/build/metal/ast.rs
+++ b/crates/uzu/build/metal/ast.rs
@@ -119,7 +119,7 @@ pub enum MetalArgumentType {
Axis(Box<str>, Box<str>),
Groups(MetalGroupsType),
Threads(Box<str>),
- Simd,
+ ThreadContext,
}
#[derive(Debug, Serialize, Deserialize)]
@@ -251,8 +251,8 @@ impl MetalArgument {
},
_ => bail!("unknown annotation: {annotation_key}"),
}
- } else if self.c_type.as_ref() == "Simd" || self.c_type.as_ref() == "const Simd" {
- Ok(MetalArgumentType::Simd)
+ } else if self.c_type.as_ref() == "ThreadContext" || self.c_type.as_ref() == "const ThreadContext" {
+ Ok(MetalArgumentType::ThreadContext)
} else if self.c_type.contains("device") && self.c_type.contains('*') && !self.c_type.contains('&') {
Ok(MetalArgumentType::Buffer(if self.c_type.contains("const") {
MetalBufferAccess::Read
@@ -354,8 +354,8 @@ impl MetalKernelInfo {
self.arguments.iter().any(|a| matches!(a.argument_type(), Ok(MetalArgumentType::Threads(_))))
}
- pub fn has_simd(&self) -> bool {
- self.arguments.iter().any(|a| matches!(a.argument_type(), Ok(MetalArgumentType::Simd)))
+ pub fn has_thread_context(&self) -> bool {
+ self.arguments.iter().any(|a| matches!(a.argument_type(), Ok(MetalArgumentType::ThreadContext)))
}
pub fn to_kernel(&self) -> Option<Kernel> {
diff --git a/crates/uzu/build/metal/wrapper.rs b/crates/uzu/build/metal/wrapper.rs
index b02c373c4..6e63c71df 100644
--- a/crates/uzu/build/metal/wrapper.rs
+++ b/crates/uzu/build/metal/wrapper.rs
@@ -170,7 +170,7 @@ fn kernel_wrappers(
wrapper_arguments.push("uint3 __dsl_thread_idx [[thread_position_in_threadgroup]]".into());
}
- if kernel.has_simd() {
+ if kernel.has_thread_context() {
wrapper_arguments.push("uint __dsl_simd_lane_idx [[thread_index_in_simdgroup]]".into());
wrapper_arguments.push("uint __dsl_simd_group_idx [[simdgroup_index_in_threadgroup]]".into());
wrapper_arguments.push("uint __dsl_simd_group_size [[threads_per_simdgroup]]".into());
@@ -210,7 +210,7 @@ fn kernel_wrappers(
MetalArgumentType::Threads(_) => {
format!("__dsl_thread_idx.{}", thread_axis_letters.next().unwrap())
},
- MetalArgumentType::Simd => "Simd { .lane_idx = __dsl_simd_lane_idx, .group_idx = __dsl_simd_group_idx, .group_size = __dsl_simd_group_size, .groups_per_threadgroup = __dsl_simd_group_per_threadgroup }".into(),
+ MetalArgumentType::ThreadContext => "ThreadContext { .simdgroup_index = __dsl_simd_lane_idx, .threadgroup_index = __dsl_simd_group_idx, .simdgroup_size = __dsl_simd_group_size, .threadgroup_size = __dsl_simd_group_size * __dsl_simd_group_per_threadgroup, .simdgroups_per_threadgroup = __dsl_simd_group_per_threadgroup }".into(),
})
.collect::<Vec<_>>()
.join(", ")
diff --git a/crates/uzu/src/backends/common/gpu_types/matmul.rs b/crates/uzu/src/backends/common/gpu_types/matmul.rs
index d54a635a0..96713f917 100644
--- a/crates/uzu/src/backends/common/gpu_types/matmul.rs
+++ b/crates/uzu/src/backends/common/gpu_types/matmul.rs
@@ -1,22 +1,16 @@
-//! GEMM kernel parameter structs.
-//!
-//! Field names use uppercase for matrix dimensions (M, N, K) to match
-//! standard BLAS/GEMM convention and generated shader headers.
-
#![allow(non_snake_case)]
-/// Main GEMM parameters passed to backend kernels as constant buffer.
#[repr(C)]
#[derive(Debug, Default, Copy, Clone)]
-pub struct GEMMParams {
+pub struct GemmParams {
pub M: i32,
pub N: i32,
pub K: i32,
- pub lda: i32,
- pub ldb: i32,
- pub ldd: i32,
- pub tiles_n: i32,
- pub tiles_m: i32,
+ pub leading_dimension_a: i32,
+ pub leading_dimension_b: i32,
+ pub leading_dimension_d: i32,
+ pub threadgroups_per_column: i32,
+ pub threadgroups_per_row: i32,
pub swizzle_log: i32,
- pub gemm_k_iterations_aligned: i32,
+ pub aligned_inner_iterations: i32,
}
diff --git a/crates/uzu/src/backends/common/kernel/matmul/full_precision.rs b/crates/uzu/src/backends/common/kernel/matmul/full_precision.rs
index b22a6aaaf..5cf2ad669 100644
--- a/crates/uzu/src/backends/common/kernel/matmul/full_precision.rs
+++ b/crates/uzu/src/backends/common/kernel/matmul/full_precision.rs
@@ -62,9 +62,9 @@ impl<B: Backend<Kernels: MatmulKernels<FullPrecisionMatmulKernel = Self>>> FullP
batch: arguments.batch as i32,
input_dim: arguments.input_dim as i32,
output_dim: arguments.output_dim as i32,
- lda: arguments.input_dim as i32,
- ldb: arguments.input_dim as i32,
- ldd: arguments.output_dim as i32,
+ leading_dimension_a: arguments.input_dim as i32,
+ leading_dimension_b: arguments.input_dim as i32,
+ leading_dimension_d: arguments.output_dim as i32,
transpose_b: true,
};
diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemm/dispatch_descriptor.rs b/crates/uzu/src/backends/common/kernel/matmul/gemm/dispatch_descriptor.rs
index a4cbbad71..21ce06389 100644
--- a/crates/uzu/src/backends/common/kernel/matmul/gemm/dispatch_descriptor.rs
+++ b/crates/uzu/src/backends/common/kernel/matmul/gemm/dispatch_descriptor.rs
@@ -3,7 +3,7 @@ use crate::{
DataType,
backends::common::{
Backend,
- gpu_types::GEMMParams,
+ gpu_types::GemmParams,
kernel::matmul::{MatmulArguments, MatmulError},
},
};
@@ -11,7 +11,7 @@ use crate::{
#[derive(Debug, Clone)]
pub struct GemmDispatchDescriptor {
pub specialization: GemmSpecialization,
- pub params: GEMMParams,
+ pub params: GemmParams,
pub threadgroups: GridSize,
}
@@ -31,25 +31,25 @@ impl GemmDispatchDescriptor {
let n = arguments.output_dim;
let k = arguments.input_dim;
- let tiles_n = (n + config.block_cols - 1) / config.block_cols;
- let tiles_m = (m + config.block_rows - 1) / config.block_rows;
+ let threadgroups_per_row = (n + config.block_cols - 1) / config.block_cols;
+ let threadgroups_per_column = (m + config.block_rows - 1) / config.block_rows;
let swizzle_log = config.swizzle_log2;
- let tile_swizzle = 1 << swizzle_log;
- let tm_swizzled = (tiles_m + tile_swizzle - 1) / tile_swizzle;
- let tn_swizzled = tiles_n * tile_swizzle;
+ let swizzle_stride = 1_i32 << swizzle_log;
+ let tm_swizzled = (threadgroups_per_column + swizzle_stride - 1) / swizzle_stride;
+ let tn_swizzled = threadgroups_per_row * swizzle_stride;
- let params = GEMMParams {
+ let params = GemmParams {
M: m,
N: n,
K: k,
- lda: arguments.lda,
- ldb: arguments.ldb,
- ldd: arguments.ldd,
- tiles_n,
- tiles_m,
+ leading_dimension_a: arguments.leading_dimension_a,
+ leading_dimension_b: arguments.leading_dimension_b,
+ leading_dimension_d: arguments.leading_dimension_d,
+ threadgroups_per_row,
+ threadgroups_per_column,
swizzle_log,
- gemm_k_iterations_aligned: k / config.block_depth,
+ aligned_inner_iterations: k / config.block_depth,
};
let threadgroups = GridSize {
diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemm/kernel.rs b/crates/uzu/src/backends/common/kernel/matmul/gemm/kernel.rs
index 7e6d673ff..a1502a1bd 100644
--- a/crates/uzu/src/backends/common/kernel/matmul/gemm/kernel.rs
+++ b/crates/uzu/src/backends/common/kernel/matmul/gemm/kernel.rs
@@ -50,8 +50,8 @@ impl<B: Backend> GemmKernel<B> {
config.block_rows as u32,
config.block_cols as u32,
config.block_depth as u32,
- config.warps_per_row as u32,
- config.warps_per_col as u32,
+ config.simdgroups_per_row as u32,
+ config.simdgroups_per_column as u32,
config.align_m,
config.align_n,
config.align_k,
diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemm/specialization.rs b/crates/uzu/src/backends/common/kernel/matmul/gemm/specialization.rs
index da1cd1ed4..42e60d822 100644
--- a/crates/uzu/src/backends/common/kernel/matmul/gemm/specialization.rs
+++ b/crates/uzu/src/backends/common/kernel/matmul/gemm/specialization.rs
@@ -8,8 +8,8 @@ pub struct GemmSpecialization {
pub block_rows: i32,
pub block_cols: i32,
pub block_depth: i32,
- pub warps_per_row: u64,
- pub warps_per_col: u64,
+ pub simdgroups_per_row: u64,
+ pub simdgroups_per_column: u64,
pub swizzle_log2: i32,
pub align_m: bool,
pub align_n: bool,
@@ -24,8 +24,8 @@ impl GemmSpecialization {
block_rows: 64,
block_cols: 32,
block_depth: 32,
- warps_per_row: 2,
- warps_per_col: 2,
+ simdgroups_per_row: 2,
+ simdgroups_per_column: 2,
swizzle_log2: 0,
align_m: false,
align_n: true,
@@ -35,8 +35,8 @@ impl GemmSpecialization {
block_rows: 64,
block_cols: 32,
block_depth: 32,
- warps_per_row: 2,
- warps_per_col: 2,
+ simdgroups_per_row: 2,
+ simdgroups_per_column: 2,
swizzle_log2: 0,
align_m: true,
align_n: true,
@@ -46,8 +46,8 @@ impl GemmSpecialization {
block_rows: 64,
block_cols: 64,
block_depth: 16,
- warps_per_row: 2,
- warps_per_col: 2,
+ simdgroups_per_row: 2,
+ simdgroups_per_column: 2,
swizzle_log2: 0,
align_m: false,
align_n: true,
@@ -57,8 +57,8 @@ impl GemmSpecialization {
block_rows: 64,
block_cols: 64,
block_depth: 16,
- warps_per_row: 2,
- warps_per_col: 2,
+ simdgroups_per_row: 2,
+ simdgroups_per_column: 2,
swizzle_log2: 0,
align_m: true,
align_n: false,
@@ -68,8 +68,8 @@ impl GemmSpecialization {
block_rows: 64,
block_cols: 64,
block_depth: 16,
- warps_per_row: 2,
- warps_per_col: 2,
+ simdgroups_per_row: 2,
+ simdgroups_per_column: 2,
swizzle_log2: 0,
align_m: true,
align_n: true,
@@ -79,8 +79,8 @@ impl GemmSpecialization {
block_rows: 64,
block_cols: 64,
block_depth: 16,
- warps_per_row: 1,
- warps_per_col: 2,
+ simdgroups_per_row: 1,
+ simdgroups_per_column: 2,
swizzle_log2: 0,
align_m: true,
align_n: true,
@@ -92,8 +92,8 @@ impl GemmSpecialization {
block_rows: 64,
block_cols: 64,
block_depth: 16,
- warps_per_row: 2,
- warps_per_col: 2,
+ simdgroups_per_row: 2,
+ simdgroups_per_column: 2,
swizzle_log2: 0,
align_m: true,
align_n: true,
@@ -103,8 +103,8 @@ impl GemmSpecialization {
block_rows: 64,
block_cols: 64,
block_depth: 16,
- warps_per_row: 2,
- warps_per_col: 2,
+ simdgroups_per_row: 2,
+ simdgroups_per_column: 2,
swizzle_log2: 0,
align_m: false,
align_n: true,
@@ -116,8 +116,8 @@ impl GemmSpecialization {
block_rows: 32,
block_cols: 64,
block_depth: 16,
- warps_per_row: 2,
- warps_per_col: 2,
+ simdgroups_per_row: 2,
+ simdgroups_per_column: 2,
swizzle_log2: 0,
align_m: false,
align_n: true,
@@ -127,8 +127,8 @@ impl GemmSpecialization {
block_rows: 32,
block_cols: 64,
block_depth: 16,
- warps_per_row: 2,
- warps_per_col: 2,
+ simdgroups_per_row: 2,
+ simdgroups_per_column: 2,
swizzle_log2: 0,
align_m: true,
align_n: true,
@@ -148,7 +148,7 @@ impl GemmSpecialization {
let is_float32 = matches!(data_type, DataType::F32);
let prefer_half_or_tf32 = !is_float32 || B::Context::tf32_enabled();
- let (block_rows, block_cols, block_depth, warps_per_row, warps_per_col, swizzle_log2) =
+ let (block_rows, block_cols, block_depth, simdgroups_per_row, simdgroups_per_column, swizzle_log2) =
if context.is_high_performance() && overall_work_elements >= (1_i64 << 20) {
if prefer_half_or_tf32 {
if 2 * std::cmp::max(arguments.batch, arguments.output_dim) > arguments.input_dim {
@@ -183,8 +183,8 @@ impl GemmSpecialization {
block_rows,
block_cols,
block_depth,
- warps_per_row,
- warps_per_col,
+ simdgroups_per_row,
+ simdgroups_per_column,
swizzle_log2,
align_m: (m % block_rows) == 0,
align_n: (n % block_cols) == 0,
diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemv/dispatch_descriptor.rs b/crates/uzu/src/backends/common/kernel/matmul/gemv/dispatch_descriptor.rs
index 77e77df26..e45ac57bf 100644
--- a/crates/uzu/src/backends/common/kernel/matmul/gemv/dispatch_descriptor.rs
+++ b/crates/uzu/src/backends/common/kernel/matmul/gemv/dispatch_descriptor.rs
@@ -95,9 +95,9 @@ impl GemvDispatchDescriptor {
let input_dimension = arguments.input_dim;
let matrix_leading_dim = if matrix_is_rhs {
- arguments.ldb
+ arguments.leading_dimension_b
} else {
- arguments.lda
+ arguments.leading_dimension_a
};
let batch_rows = arguments.batch;
diff --git a/crates/uzu/src/backends/common/kernel/matmul/matmul_arguments.rs b/crates/uzu/src/backends/common/kernel/matmul/matmul_arguments.rs
index 4981baff7..a432f4e0b 100644
--- a/crates/uzu/src/backends/common/kernel/matmul/matmul_arguments.rs
+++ b/crates/uzu/src/backends/common/kernel/matmul/matmul_arguments.rs
@@ -14,8 +14,8 @@ pub struct MatmulArguments<'a, B: Backend> {
pub input_dim: i32,
/// N dimension - output_dim (cols of B, cols of D)
pub output_dim: i32,
- pub lda: i32,
- pub ldb: i32,
- pub ldd: i32,
+ pub leading_dimension_a: i32,
+ pub leading_dimension_b: i32,
+ pub leading_dimension_d: i32,
pub transpose_b: bool,
}
diff --git a/crates/uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs b/crates/uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs
index f9043cb45..302d77480 100644
--- a/crates/uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs
+++ b/crates/uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs
@@ -15,19 +15,19 @@ unsafe fn matmul_gemm_compute_row<T: ArrayElement + Float>(
row: usize,
n: usize,
k: usize,
- lda: usize,
- ldb: usize,
- ldd: usize,
+ leading_dimension_a: usize,
+ leading_dimension_b: usize,
+ leading_dimension_d: usize,
) {
unsafe {
for col in 0..n {
let mut acc: f32 = 0.0;
for i in 0..k {
- let a_val = *a.0.add(row * lda + i);
- let b_val = *b.0.add(col * ldb + i);
+ let a_val = *a.0.add(row * leading_dimension_a + i);
+ let b_val = *b.0.add(col * leading_dimension_b + i);
acc = acc + a_val.to_f32().unwrap() * b_val.to_f32().unwrap();
}
- *d.0.add(row * ldd + col) = T::from(acc).unwrap();
+ *d.0.add(row * leading_dimension_d + col) = T::from(acc).unwrap();
}
}
}
@@ -39,9 +39,9 @@ fn matmul_gemm_impl<T: ArrayElement + Float>(
m: usize,
n: usize,
k: usize,
- lda: usize,
- ldb: usize,
- ldd: usize,
+ leading_dimension_a: usize,
+ leading_dimension_b: usize,
+ leading_dimension_d: usize,
) {
let num_threads = std::thread::available_parallelism().map(|n| n.get()).unwrap_or(1);
let a = SendPtr(a);
@@ -65,7 +65,17 @@ fn matmul_gemm_impl<T: ArrayElement + Float>(
s.spawn(move || unsafe {
for row in start..end {
- matmul_gemm_compute_row(a, b, d, row, n, k, lda, ldb, ldd);
+ matmul_gemm_compute_row(
+ a,
+ b,
+ d,
+ row,
+ n,
+ k,
+ leading_dimension_a,
+ leading_dimension_b,
+ leading_dimension_d,
+ );
}
});
@@ -81,7 +91,7 @@ pub fn matmul_gemm<T: ArrayElement + Float>(
a: *const T,
b: *const T,
d: *mut T,
- params: &[crate::backends::common::gpu_types::matmul::GEMMParams],
+ params: &[crate::backends::common::gpu_types::matmul::GemmParams],
#[allow(unused)] group_count_x: u32,
#[allow(unused)] group_count_y: u32,
#[allow(unused)]
@@ -95,10 +105,10 @@ pub fn matmul_gemm<T: ArrayElement + Float>(
block_depth: u32,
#[allow(unused)]
#[specialize]
- warps_per_row: u32,
+ simdgroups_per_row: u32,
#[allow(unused)]
#[specialize]
- warps_per_col: u32,
+ simdgroups_per_column: u32,
#[allow(unused)]
#[specialize]
align_m: bool,
@@ -118,9 +128,9 @@ pub fn matmul_gemm<T: ArrayElement + Float>(
p.M as usize,
p.N as usize,
p.K as usize,
- p.lda as usize,
- p.ldb as usize,
- p.ldd as usize,
+ p.leading_dimension_a as usize,
+ p.leading_dimension_b as usize,
+ p.leading_dimension_d as usize,
);
}
}
diff --git a/crates/uzu/src/backends/metal/generated/matmul.h b/crates/uzu/src/backends/metal/generated/matmul.h
index 2f5fe3580..86d45c6ec 100644
--- a/crates/uzu/src/backends/metal/generated/matmul.h
+++ b/crates/uzu/src/backends/metal/generated/matmul.h
@@ -14,19 +14,18 @@ namespace matmul {
#include <stdint.h>
#endif
-/**Main GEMM parameters passed to backend kernels as constant buffer. */
typedef struct {
int32_t M;
int32_t N;
int32_t K;
- int32_t lda;
- int32_t ldb;
- int32_t ldd;
- int32_t tiles_n;
- int32_t tiles_m;
+ int32_t leading_dimension_a;
+ int32_t leading_dimension_b;
+ int32_t leading_dimension_d;
+ int32_t threadgroups_per_column;
+ int32_t threadgroups_per_row;
int32_t swizzle_log;
- int32_t gemm_k_iterations_aligned;
-} GEMMParams;
+ int32_t aligned_inner_iterations;
+} GemmParams;
#ifdef __METAL_VERSION__
} // namespace matmul
diff --git a/crates/uzu/src/backends/metal/kernel/activation/activation.metal b/crates/uzu/src/backends/metal/kernel/activation/activation.metal
index 074a77e44..a815a2722 100644
--- a/crates/uzu/src/backends/metal/kernel/activation/activation.metal
+++ b/crates/uzu/src/backends/metal/kernel/activation/activation.metal
@@ -1,5 +1,5 @@
#include <metal_stdlib>
-#include "../definitions.metal"
+#include "../common/dsl.h"
#include "activation.h"
diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal
index cba3a797b..fbe1a533b 100644
--- a/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal
+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal
@@ -1,7 +1,8 @@
#include <metal_stdlib>
-#include "../definitions.metal"
+#include "../common/dsl.h"
+#include "../common/thread_context.h"
#include "../matmul/common/loader.h"
-#include "../matmul/common/mma.h"
+#include "../matmul/common/simdgroup_fragment.h"
#include "attention.h"
using namespace metal;
@@ -17,8 +18,8 @@ struct TransformScale {
template <typename T>
METAL_FUNC T row_reduce_max(T v) {
- // BaseMMAFrag::get_coord mapping groups lanes for a row as:
- // {lane, lane^1, lane^8, (lane^1)^8}. Reduce in two steps.
+ // SimdgroupMultiplyAccumulate::get_lane_coordinates mapping groups lanes for
+ // a row as: {lane, lane^1, lane^8, (lane^1)^8}. Reduce in two steps.
v = metal::max(v, simd_shuffle_xor(v, 1));
v = metal::max(v, simd_shuffle_xor(v, 8));
return v;
@@ -31,9 +32,9 @@ METAL_FUNC T row_reduce_sum(T v) {
return v;
}
-#define BQ 32
-#define WM 4
-#define WN 1
+#define BLOCK_QUERY_ROWS 32
+#define SIMDGROUPS_PER_ROW 4
+#define SIMDGROUPS_PER_COLUMN 1
template <typename T, uint BK, uint BD>
VARIANTS(T, float, half, bfloat)
@@ -55,17 +56,17 @@ PUBLIC KERNEL(AttentionGemm)(
const bool do_causal SPECIALIZE,
const bool has_mask SPECIALIZE,
const bool has_sinks SPECIALIZE,
- threadgroup T q_smem[BQ * (BD + 16 / sizeof(T))],
+ threadgroup T q_smem[BLOCK_QUERY_ROWS * (BD + 16 / sizeof(T))],
threadgroup T kv_smem[BK * (BD + 16 / sizeof(T))],
- const Simd simd,
- const uint tgid_x GROUPS(suffix_length.div_ceil(BQ)),
+ const ThreadContext simd,
+ const uint tgid_x GROUPS(suffix_length.div_ceil(BLOCK_QUERY_ROWS)),
const uint tgid_y GROUPS(num_heads),
const uint tgid_z GROUPS(1),
const uint lid THREADS(128)
) {
// -------------------------------------------------------------------------
// Pointer setup (all strides are in elements)
- // tgid_x: query tile index (BQ rows)
+ // tgid_x: query tile index (BLOCK_QUERY_ROWS rows)
// tgid_y: query head index
// tgid_z: batch index (currently 1 in uzu, but kept for completeness)
const uint batch_idx = tgid_z;
@@ -73,7 +74,7 @@ PUBLIC KERNEL(AttentionGemm)(
const uint q_tile_idx = tgid_x;
q += batch_idx * params.q_strides[0] + head_idx * params.q_strides[1] +
- q_tile_idx * int64_t(BQ) * params.q_strides[2];
+ q_tile_idx * int64_t(BLOCK_QUERY_ROWS) * params.q_strides[2];
const int kv_head_idx = int(tgid_y) / params.gqa_factor;
k += batch_idx * params.k_strides[0] +
@@ -82,7 +83,7 @@ PUBLIC KERNEL(AttentionGemm)(
int64_t(kv_head_idx) * params.v_strides[1];
o += batch_idx * params.o_strides[0] + head_idx * params.o_strides[1] +
- q_tile_idx * int64_t(BQ) * params.o_strides[2];
+ q_tile_idx * int64_t(BLOCK_QUERY_ROWS) * params.o_strides[2];
if (has_mask) {
mask += batch_idx * mask_params.m_strides[0] +
@@ -91,115 +92,162 @@ PUBLIC KERNEL(AttentionGemm)(
// -------------------------------------------------------------------------
// Threadgroup memory
- constexpr short padQ = 16 / sizeof(T);
- constexpr short padK = 16 / sizeof(T);
- constexpr short padV = 16 / sizeof(T);
+ constexpr short query_padding = 16 / sizeof(T);
+ constexpr short key_padding = 16 / sizeof(T);
+ constexpr short value_padding = 16 / sizeof(T);
- constexpr short LDQ_tgp = BD + padQ;
- constexpr short LDK_tgp = BD + padK; // K stored as [BK, BD] row-major
- constexpr short LDV_tgp = BD + padV;
+ constexpr short query_leading_dimension = BD + query_padding;
+ constexpr short key_leading_dimension = BD + key_padding;
+ constexpr short value_leading_dimension = BD + value_padding;
- threadgroup T* Qs = q_smem;
- threadgroup T* Ks = kv_smem;
- threadgroup T* Vs = kv_smem;
+ threadgroup T* query_shared = q_smem;
+ threadgroup T* key_shared = kv_smem;
+ threadgroup T* value_shared = kv_smem;
//
// -------------------------------------------------------------------------
// Block loaders
- using QBlockLoader = BlockLoader<
+ using QueryLoader = ThreadgroupLoader<
T,
- /*BROWS=*/BQ,
- /*BCOLS=*/BD,
- /*dst_ld=*/LDQ_tgp,
- /*reduction_dim=*/1,
- /*tgp_size=*/WM * WN * 32>;
+ BLOCK_QUERY_ROWS,
+ BD,
+ query_leading_dimension,
+ 1,
+ SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * 32>;
- using KBlockLoader = BlockLoader<
+ using KeyLoader = ThreadgroupLoader<
T,
- /*BROWS=*/BK,
- /*BCOLS=*/BD,
- /*dst_ld=*/LDK_tgp,
- /*reduction_dim=*/0,
- /*tgp_size=*/WM * WN * 32>;
+ BK,
+ BD,
+ key_leading_dimension,
+ 0,
+ SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * 32>;
- using VBlockLoader = BlockLoader<
+ using ValueLoader = ThreadgroupLoader<
T,
- /*BROWS=*/BK,
- /*BCOLS=*/BD,
- /*dst_ld=*/LDV_tgp,
- /*reduction_dim=*/0,
- /*tgp_size=*/WM * WN * 32>;
-
- const int q_src_ld = int(params.q_strides[2]);
- const int k_src_ld = int(params.k_strides[2]);
- const int v_src_ld = int(params.v_strides[2]);
-
- thread QBlockLoader loader_q(q, q_src_ld, Qs, simd.group_idx, simd.lane_idx);
- thread KBlockLoader loader_k(k, k_src_ld, Ks, simd.group_idx, simd.lane_idx);
- thread VBlockLoader loader_v(v, v_src_ld, Vs, simd.group_idx, simd.lane_idx);
+ BK,
+ BD,
+ value_leading_dimension,
+ 0,
+ SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * 32>;
+
+ const int query_source_stride = int(params.q_strides[2]);
+ const int key_source_stride = int(params.k_strides[2]);
+ const int value_source_stride = int(params.v_strides[2]);
+
+ thread QueryLoader query_loader(
+ q,
+ query_source_stride,
+ query_shared,
+ simd.threadgroup_index,
+ simd.simdgroup_index
+ );
+ thread KeyLoader key_loader(
+ k,
+ key_source_stride,
+ key_shared,
+ simd.threadgroup_index,
+ simd.simdgroup_index
+ );
+ thread ValueLoader value_loader(
+ v,
+ value_source_stride,
+ value_shared,
+ simd.threadgroup_index,
+ simd.simdgroup_index
+ );
TransformScale<T> ts(static_cast<T>(params.scale * M_LOG2E_F));
// -------------------------------------------------------------------------
// MMA tiles
- constexpr short kFragSize = 8;
+ constexpr short SIMDGROUP_BLOCK_SIZE = 8;
using AccumType = float;
- using MMAFrag_acc_t = BaseMMAFrag<AccumType, kFragSize, kFragSize>;
+ using SimdgroupMultiplyAccumulateType = SimdgroupMultiplyAccumulate<
+ AccumType,
+ SIMDGROUP_BLOCK_SIZE,
+ SIMDGROUP_BLOCK_SIZE>;
- constexpr int kNWarps = WM * WN;
+ constexpr int SIMDGROUPS_PER_THREADGROUP =
+ SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN;
static_assert(
- BQ >= (kNWarps * kFragSize) && BQ % (kNWarps * kFragSize) == 0,
+ BLOCK_QUERY_ROWS >= (SIMDGROUPS_PER_THREADGROUP * SIMDGROUP_BLOCK_SIZE) &&
+ BLOCK_QUERY_ROWS %
+ (SIMDGROUPS_PER_THREADGROUP * SIMDGROUP_BLOCK_SIZE) ==
+ 0,
"Each simdgroup must host at least 1 simdgroup matrix along Q sequence."
);
- // Q seq frags per warp (we keep TQ == 1 for the 32-row block layout)
- constexpr int TQ = BQ / (kNWarps * kFragSize);
- // KV sequence frags
- constexpr int TK = BK / kFragSize;
- // HeadDim frags
- constexpr int TD = BD / kFragSize;
-
- static_assert(TQ == 1, "Expected TQ == 1");
-
- MMATile<AccumType, TQ, 1, MMAFrag_acc_t> Qtile;
- MMATile<AccumType, 1, TK, MMAFrag_acc_t> Ktile; // represents K^T slice
- MMATile<AccumType, TQ, TK, MMAFrag_acc_t> Stile;
- MMATile<AccumType, 1, 1, MMAFrag_acc_t> Vtile;
- MMATile<AccumType, TQ, TD, MMAFrag_acc_t> Otile;
-
- Otile.clear();
+ // Q sequence multiply-accumulate blocks per simdgroup (QUERY_GRID_ROWS == 1
+ // for the 32-row block layout)
+ constexpr int QUERY_GRID_ROWS =
+ BLOCK_QUERY_ROWS / (SIMDGROUPS_PER_THREADGROUP * SIMDGROUP_BLOCK_SIZE);
+ constexpr int KEY_GRID_COLS = BK / SIMDGROUP_BLOCK_SIZE;
+ constexpr int HEAD_DIM_GRID_COLS = BD / SIMDGROUP_BLOCK_SIZE;
+
+ static_assert(QUERY_GRID_ROWS == 1, "Expected QUERY_GRID_ROWS == 1");
+
+ SimdgroupFragment<
+ AccumType,
+ QUERY_GRID_ROWS,
+ 1,
+ SimdgroupMultiplyAccumulateType>
+ query_fragment;
+ SimdgroupFragment<
+ AccumType,
+ 1,
+ KEY_GRID_COLS,
+ SimdgroupMultiplyAccumulateType>
+ key_fragment;
+ SimdgroupFragment<
+ AccumType,
+ QUERY_GRID_ROWS,
+ KEY_GRID_COLS,
+ SimdgroupMultiplyAccumulateType>
+ score_fragment;
+ SimdgroupFragment<AccumType, 1, 1, SimdgroupMultiplyAccumulateType>
+ value_fragment;
+ SimdgroupFragment<
+ AccumType,
+ QUERY_GRID_ROWS,
+ HEAD_DIM_GRID_COLS,
+ SimdgroupMultiplyAccumulateType>
+ output_fragment;
+
+ output_fragment.clear();
// -------------------------------------------------------------------------
// Lane coordinates and pointer offsets
- const short2 simd_coord = MMAFrag_acc_t::get_coord(simd.lane_idx);
- const short sm = simd_coord.y;
- const short sn = simd_coord.x;
+ const short2 lane_coordinates =
+ SimdgroupMultiplyAccumulateType::get_lane_coordinates(
+ simd.simdgroup_index
+ );
+ const short lane_row = lane_coordinates.y;
+ const short lane_col = lane_coordinates.x;
- const short tm = kFragSize * TQ * short(simd.group_idx);
+ const short simdgroup_row_base =
+ SIMDGROUP_BLOCK_SIZE * QUERY_GRID_ROWS * short(simd.threadgroup_index);
- // Qs is row-major [BQ, BD]
- const short Qs_offset = (tm + sm) * LDQ_tgp + sn;
- constexpr short Qs_tile_stride = kFragSize;
+ const short query_shared_offset =
+ (simdgroup_row_base + lane_row) * query_leading_dimension + lane_col;
+ constexpr short query_tile_stride = SIMDGROUP_BLOCK_SIZE;
- // Ks is row-major [BK, BD] but we load K^T by swapping strides:
- // B_str_k = 1, B_str_n = LDK_tgp => offset = sn * LDK_tgp + sm
- const short Ks_offset = sn * LDK_tgp + sm;
- constexpr short Ks_tile_stride =
- kFragSize; // advance along head-dim (contiguous)
+ const short key_shared_offset = lane_col * key_leading_dimension + lane_row;
+ constexpr short key_tile_stride = SIMDGROUP_BLOCK_SIZE;
- // Vs is row-major [BK, BD]
- const short Vs_offset = sm * LDV_tgp + sn;
+ const short value_shared_offset =
+ lane_row * value_leading_dimension + lane_col;
// -------------------------------------------------------------------------
// Load Q block once (and apply scaling)
threadgroup_barrier(mem_flags::mem_threadgroup);
if (!align_q && int(tgid_x) == params.nq_aligned) {
- loader_q.load_safe(short2(BD, params.q_rem));
+ query_loader.load_safe(short2(BD, params.q_rem));
} else {
- loader_q.load_unsafe();
+ query_loader.load_unsafe();
}
- loader_q.apply_inplace_op(ts);
+ query_loader.apply_inplace_op(ts);
// -------------------------------------------------------------------------
// Streaming softmax state for this row (shared across lanes in a row)
@@ -215,51 +263,57 @@ PUBLIC KERNEL(AttentionGemm)(
// Determine K block loop limit (causal can early-stop)
int kb_lim = params.nk;
if (do_causal) {
- const int q_max = (int(tgid_x) + 1) * BQ + params.q_off;
+ const int q_max = (int(tgid_x) + 1) * BLOCK_QUERY_ROWS + params.q_off;
kb_lim = (q_max + BK - 1) / BK;
kb_lim = min(params.nk, kb_lim);
}
- const int q_rel = int(tgid_x) * BQ + int(tm) + int(sm); // [0, q_len)
- const int q_abs = q_rel + params.q_off; // [0, k_len)
+ const int q_rel = int(tgid_x) * BLOCK_QUERY_ROWS + int(simdgroup_row_base) +
+ int(lane_row); // [0, q_len)
+ const int q_abs = q_rel + params.q_off; // [0, k_len)
// Loop over KV blocks
for (int kb = 0; kb < kb_lim; kb++) {
// Load K block
threadgroup_barrier(mem_flags::mem_threadgroup);
if (!align_k && kb == params.nk_aligned) {
- loader_k.load_safe(short2(BD, params.k_rem));
+ key_loader.load_safe(short2(BD, params.k_rem));
} else {
- loader_k.load_unsafe();
+ key_loader.load_unsafe();
}
// Compute S = Q @ K^T for this block
- Stile.clear();
+ score_fragment.clear();
threadgroup_barrier(mem_flags::mem_threadgroup);
- UZU_PRAGMA_UNROLL
- for (short dd = 0; dd < TD; dd++) {
+ METAL_PRAGMA_UNROLL
+ for (short dd = 0; dd < HEAD_DIM_GRID_COLS; dd++) {
simdgroup_barrier(mem_flags::mem_none);
- Qtile.template load<T, 1, 1, LDQ_tgp, 1>(
- &Qs[Qs_offset + dd * Qs_tile_stride]
+ query_fragment.template load<T, 1, 1, query_leading_dimension, 1>(
+ &query_shared[query_shared_offset + dd * query_tile_stride]
);
- Ktile.template load<T, 1, 1, 1, LDK_tgp>(
- &Ks[Ks_offset + dd * Ks_tile_stride]
+ key_fragment.template load<T, 1, 1, 1, key_leading_dimension>(
+ &key_shared[key_shared_offset + dd * key_tile_stride]
);
simdgroup_barrier(mem_flags::mem_none);
- tile_matmad(Stile, Qtile, Ktile, Stile);
+ tile_multiply_accumulate(
+ score_fragment,
+ query_fragment,
+ key_fragment,
+ score_fragment
+ );
}
// Mask out tail keys for the last (unaligned) K block
if (!align_k && kb == params.nk_aligned) {
const int k_rem = params.k_rem;
- UZU_PRAGMA_UNROLL
- for (short j = 0; j < TK; j++) {
- thread auto& frag = Stile.frag_at(0, j);
- const int col0 = int(sn) + int(j) * kFragSize;
+ METAL_PRAGMA_UNROLL
+ for (short j = 0; j < KEY_GRID_COLS; j++) {
+ thread auto& frag = score_fragment.multiply_accumulate_at(0, j);
+ const int col0 = int(lane_col) + int(j) * SIMDGROUP_BLOCK_SIZE;
if (col0 >= k_rem) {
frag[0] = neg_inf;
}
@@ -271,13 +325,14 @@ PUBLIC KERNEL(AttentionGemm)(
// Causal mask (only needed for the last few blocks near the diagonal)
if (do_causal) {
- const int tail_blocks = (BQ + BK - 1) / BK + int(!align_k);
+ const int tail_blocks = (BLOCK_QUERY_ROWS + BK - 1) / BK + int(!align_k);
const int tail_start = kb_lim - tail_blocks;
if (kb >= tail_start) {
- UZU_PRAGMA_UNROLL
- for (short j = 0; j < TK; j++) {
- thread auto& frag = Stile.frag_at(0, j);
- const int col_base = kb * BK + int(sn) + int(j) * kFragSize;
+ METAL_PRAGMA_UNROLL
+ for (short j = 0; j < KEY_GRID_COLS; j++) {
+ thread auto& frag = score_fragment.multiply_accumulate_at(0, j);
+ const int col_base =
+ kb * BK + int(lane_col) + int(j) * SIMDGROUP_BLOCK_SIZE;
if (q_abs < col_base) {
frag[0] = neg_inf;
}
@@ -293,10 +348,11 @@ PUBLIC KERNEL(AttentionGemm)(
const int64_t row_stride = mask_params.m_strides[2];
const int64_t row_base = int64_t(q_rel) * row_stride;
- UZU_PRAGMA_UNROLL
- for (short j = 0; j < TK; j++) {
- thread auto& frag = Stile.frag_at(0, j);
- const int col_base = kb * BK + int(sn) + int(j) * kFragSize;
+ METAL_PRAGMA_UNROLL
+ for (short j = 0; j < KEY_GRID_COLS; j++) {
+ thread auto& frag = score_fragment.multiply_accumulate_at(0, j);
+ const int col_base =
+ kb * BK + int(lane_col) + int(j) * SIMDGROUP_BLOCK_SIZE;
const int k0 = col_base;
const int k1 = col_base + 1;
@@ -317,9 +373,9 @@ PUBLIC KERNEL(AttentionGemm)(
// Load V block (overwriting K in shared memory)
threadgroup_barrier(mem_flags::mem_threadgroup);
if (!align_k && kb == params.nk_aligned) {
- loader_v.load_safe(short2(BD, params.k_rem));
+ value_loader.load_safe(short2(BD, params.k_rem));
} else {
- loader_v.load_unsafe();
+ value_loader.load_unsafe();
}
// -----------------------------------------------------------------------
@@ -327,9 +383,9 @@ PUBLIC KERNEL(AttentionGemm)(
// Row max for this block
AccumType block_max_local = -INFINITY;
- UZU_PRAGMA_UNROLL
- for (short j = 0; j < TK; j++) {
- const thread auto& frag = Stile.frag_at(0, j);
+ METAL_PRAGMA_UNROLL
+ for (short j = 0; j < KEY_GRID_COLS; j++) {
+ const thread auto& frag = score_fragment.multiply_accumulate_at(0, j);
block_max_local = metal::max(block_max_local, frag[0]);
block_max_local = metal::max(block_max_local, frag[1]);
}
@@ -344,9 +400,9 @@ PUBLIC KERNEL(AttentionGemm)(
// exp2(S - new_max) and row sum
AccumType block_sum_local = AccumType(0);
- UZU_PRAGMA_UNROLL
- for (short j = 0; j < TK; j++) {
- thread auto& frag = Stile.frag_at(0, j);
+ METAL_PRAGMA_UNROLL
+ for (short j = 0; j < KEY_GRID_COLS; j++) {
+ thread auto& frag = score_fragment.multiply_accumulate_at(0, j);
frag[0] = fast::exp2(frag[0] - new_max);
frag[1] = fast::exp2(frag[1] - new_max);
block_sum_local += frag[0] + frag[1];
@@ -355,50 +411,51 @@ PUBLIC KERNEL(AttentionGemm)(
sum_score += block_sum;
// Rescale output accumulator
- UZU_PRAGMA_UNROLL
- for (short id = 0; id < TD; id++) {
- thread auto& frag = Otile.frag_at(0, id);
+ METAL_PRAGMA_UNROLL
+ for (short id = 0; id < HEAD_DIM_GRID_COLS; id++) {
+ thread auto& frag = output_fragment.multiply_accumulate_at(0, id);
frag[0] *= factor;
frag[1] *= factor;
}
- // Accumulate output: Otile += Stile * Vblock
+ // Accumulate output: output_fragment += score_fragment * Vblock
threadgroup_barrier(mem_flags::mem_threadgroup);
- UZU_PRAGMA_UNROLL
- for (short id = 0; id < TD; id++) {
- UZU_PRAGMA_UNROLL
- for (short ik = 0; ik < TK; ik++) {
+ METAL_PRAGMA_UNROLL
+ for (short id = 0; id < HEAD_DIM_GRID_COLS; id++) {
+ METAL_PRAGMA_UNROLL
+ for (short ik = 0; ik < KEY_GRID_COLS; ik++) {
IF_CONSTEXPR(BD == 128) { simdgroup_barrier(mem_flags::mem_none); }
- const short kk = ik * kFragSize;
- const short dd = id * kFragSize;
+ const short kk = ik * SIMDGROUP_BLOCK_SIZE;
+ const short dd = id * SIMDGROUP_BLOCK_SIZE;
- Vtile.template load<T, 1, 1, LDV_tgp, 1>(
- &Vs[Vs_offset + kk * LDV_tgp + dd]
+ value_fragment.template load<T, 1, 1, value_leading_dimension, 1>(
+ &value_shared
+ [value_shared_offset + kk * value_leading_dimension + dd]
);
IF_CONSTEXPR(BD == 128) { simdgroup_barrier(mem_flags::mem_none); }
- MMAFrag_acc_t::mma(
- Otile.frag_at(0, id),
- Stile.frag_at(0, ik),
- Vtile.frag_at(0, 0),
- Otile.frag_at(0, id)
+ SimdgroupMultiplyAccumulateType::multiply_accumulate(
+ output_fragment.multiply_accumulate_at(0, id),
+ score_fragment.multiply_accumulate_at(0, ik),
+ value_fragment.multiply_accumulate_at(0, 0),
+ output_fragment.multiply_accumulate_at(0, id)
);
}
}
// Prepare for next iteration
- loader_k.next();
- loader_v.next();
+ key_loader.next();
+ value_loader.next();
}
// -------------------------------------------------------------------------
// Normalize output by sum_score (avoid div-by-zero for masked-out rows)
const AccumType inv_sum = AccumType(1) / sum_score;
- UZU_PRAGMA_UNROLL
- for (short id = 0; id < TD; id++) {
- thread auto& frag = Otile.frag_at(0, id);
+ METAL_PRAGMA_UNROLL
+ for (short id = 0; id < HEAD_DIM_GRID_COLS; id++) {
+ thread auto& frag = output_fragment.multiply_accumulate_at(0, id);
frag[0] *= inv_sum;
frag[1] *= inv_sum;
}
@@ -406,21 +463,23 @@ PUBLIC KERNEL(AttentionGemm)(
threadgroup_barrier(mem_flags::mem_none);
// Store results (O is row-major with row-stride params.o_strides[2])
- o += int64_t(tm + sm) * params.o_strides[2] + int64_t(sn);
+ o += int64_t(simdgroup_row_base + lane_row) * params.o_strides[2] +
+ int64_t(lane_col);
if (!align_q && int(tgid_x) == params.nq_aligned) {
- const short2 dst_tile_dims = short2(BD - sn, params.q_rem - (tm + sm));
+ const short2 dst_tile_dims =
+ short2(BD - lane_col, params.q_rem - (simdgroup_row_base + lane_row));
if (dst_tile_dims.x <= 0 || dst_tile_dims.y <= 0) {
return;
}
- Otile.template store_safe<T, 1, 1>(
+ output_fragment.template store_safe<T, 1, 1>(
o,
int(params.o_strides[2]),
dst_tile_dims
);
} else {
- Otile.template store<T, 1, 1>(o, int(params.o_strides[2]));
+ output_fragment.template store<T, 1, 1>(o, int(params.o_strides[2]));
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal
index 7b693dd5f..1a09cb6be 100644
--- a/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal
+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal
@@ -1,6 +1,7 @@
#include <metal_stdlib>
#include <metal_simdgroup>
-#include "../definitions.metal"
+#include "../common/dsl.h"
+#include "../common/thread_context.h"
#define SEQUENCE_BLOCK_SIZE 32
#define HEAD_BLOCK_SIZE 32
@@ -34,7 +35,7 @@ PUBLIC KERNEL(AttentionSinglePass)(
const bool has_mask SPECIALIZE,
const bool has_sinks SPECIALIZE,
const bool do_causal SPECIALIZE,
- const Simd simd,
+ const ThreadContext simd,
const uint head_idx GROUPS(num_heads),
const uint q_seq_idx GROUPS(suffix_length),
const uint tid THREADS(1024)
@@ -59,18 +60,22 @@ PUBLIC KERNEL(AttentionSinglePass)(
const uint q_offset = query_transposed ? tpg.x * q_seq_idx + head_idx
: head_idx * tpg.y + q_seq_idx;
- queries += q_offset * HEAD_DIM + simd.lane_idx * qk_elements_per_thread;
- keys += kv_head_idx * k_head_stride + simd.group_idx * k_seq_stride +
- simd.lane_idx * qk_elements_per_thread;
- values += kv_head_idx * v_head_stride + simd.group_idx * v_seq_stride +
- simd.lane_idx * value_elements_per_thread;
+ queries +=
+ q_offset * HEAD_DIM + simd.simdgroup_index * qk_elements_per_thread;
+ keys += kv_head_idx * k_head_stride + simd.threadgroup_index * k_seq_stride +
+ simd.simdgroup_index * qk_elements_per_thread;
+ values += kv_head_idx * v_head_stride +
+ simd.threadgroup_index * v_seq_stride +
+ simd.simdgroup_index * value_elements_per_thread;
if (float_mask) {
- fmask += head_idx * mask_head_stride + simd.group_idx * mask_kv_seq_stride +
+ fmask += head_idx * mask_head_stride +
+ simd.threadgroup_index * mask_kv_seq_stride +
q_seq_idx * mask_q_seq_stride;
}
- out += o_offset * value_dim + simd.group_idx * value_elements_per_thread;
+ out +=
+ o_offset * value_dim + simd.threadgroup_index * value_elements_per_thread;
// Read the query and 0 the output accumulator
for (int i = 0; i < qk_elements_per_thread; i++) {
@@ -82,7 +87,7 @@ PUBLIC KERNEL(AttentionSinglePass)(
U max_score = -INFINITY;
U sum_exp_score = 0;
- if (has_sinks && simd.group_idx == 0) {
+ if (has_sinks && simd.threadgroup_index == 0) {
const int num_q_heads = static_cast<int>(tpg.x);
int q_head_idx = head_idx % num_q_heads;
max_score = static_cast<U>(sinks[q_head_idx]);
@@ -90,7 +95,8 @@ PUBLIC KERNEL(AttentionSinglePass)(
}
// For each key
- for (uint i = simd.group_idx; i < sequence_length; i += SEQUENCE_BLOCK_SIZE) {
+ for (uint i = simd.threadgroup_index; i < sequence_length;
+ i += SEQUENCE_BLOCK_SIZE) {
bool use_key = true;
if (do_causal) {
use_key = i <= (sequence_length - tpg.y + q_seq_idx);
@@ -135,31 +141,35 @@ PUBLIC KERNEL(AttentionSinglePass)(
}
// Each thread has a partial part of the output so we need to combine them.
- if (simd.lane_idx == 0) {
- shared_max_scores[simd.group_idx] = max_score;
- shared_sum_exp_scores[simd.group_idx] = sum_exp_score;
+ if (simd.simdgroup_index == 0) {
+ shared_max_scores[simd.threadgroup_index] = max_score;
+ shared_sum_exp_scores[simd.threadgroup_index] = sum_exp_score;
}
threadgroup_barrier(mem_flags::mem_threadgroup);
- max_score = shared_max_scores[simd.lane_idx];
+ max_score = shared_max_scores[simd.simdgroup_index];
U new_max = simd_max(max_score);
U factor = fast::exp(max_score - new_max);
- sum_exp_score = simd_sum(shared_sum_exp_scores[simd.lane_idx] * factor);
+ sum_exp_score =
+ simd_sum(shared_sum_exp_scores[simd.simdgroup_index] * factor);
// Now we need to aggregate all the outputs
for (int i = 0; i < value_elements_per_thread; i++) {
- shared_outputs[simd.lane_idx * HEAD_BLOCK_SIZE + simd.group_idx] = o[i];
+ shared_outputs
+ [simd.simdgroup_index * HEAD_BLOCK_SIZE + simd.threadgroup_index] =
+ o[i];
threadgroup_barrier(mem_flags::mem_threadgroup);
- o[i] =
- simd_sum(
- shared_outputs[simd.group_idx * HEAD_BLOCK_SIZE + simd.lane_idx] *
- factor
- ) /
- sum_exp_score;
+ o[i] = simd_sum(
+ shared_outputs
+ [simd.threadgroup_index * HEAD_BLOCK_SIZE +
+ simd.simdgroup_index] *
+ factor
+ ) /
+ sum_exp_score;
threadgroup_barrier(mem_flags::mem_threadgroup);
}
// And write the output
- if (simd.lane_idx == 0) {
+ if (simd.simdgroup_index == 0) {
for (int i = 0; i < value_elements_per_thread; i++) {
out[i] = static_cast<T>(o[i]);
}
diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal
index 9db9a7e86..a47849b78 100644
--- a/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal
+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal
@@ -1,6 +1,7 @@
#include <metal_stdlib>
#include <metal_simdgroup>
-#include "../definitions.metal"
+#include "../common/dsl.h"
+#include "../common/thread_context.h"
#define HEAD_BLOCK_SIZE 32
#define SEQUENCE_BLOCK_SIZE_1 8
@@ -38,7 +39,7 @@ PUBLIC KERNEL(AttentionTwoPass1)(
threadgroup float shared_max_scores[SEQUENCE_BLOCK_SIZE_1],
threadgroup float shared_sum_exp_scores[SEQUENCE_BLOCK_SIZE_1],
threadgroup float shared_outputs[SEQUENCE_BLOCK_SIZE_1 * HEAD_BLOCK_SIZE],
- const Simd simd,
+ const ThreadContext simd,
const uint head_idx GROUPS(num_heads),
const uint q_seq_idx GROUPS(suffix_length),
const uint block_idx GROUPS(TOTAL_BLOCKS_COUNT),
@@ -66,19 +67,21 @@ PUBLIC KERNEL(AttentionTwoPass1)(
: head_idx * tpg.y + q_seq_idx; // Consistent with single-pass
const uint kv_head_idx = head_idx / gqa_factor;
- queries += q_offset * HEAD_DIM + simd.lane_idx * qk_elements_per_thread;
+ queries +=
+ q_offset * HEAD_DIM + simd.simdgroup_index * qk_elements_per_thread;
keys += kv_head_idx * k_head_stride +
- (block_idx * SEQUENCE_BLOCK_SIZE_1 + simd.group_idx) * k_seq_stride +
- simd.lane_idx * qk_elements_per_thread;
- values +=
- kv_head_idx * v_head_stride +
- (block_idx * SEQUENCE_BLOCK_SIZE_1 + simd.group_idx) * v_seq_stride +
- simd.lane_idx * value_elements_per_thread;
+ (block_idx * SEQUENCE_BLOCK_SIZE_1 + simd.threadgroup_index) *
+ k_seq_stride +
+ simd.simdgroup_index * qk_elements_per_thread;
+ values += kv_head_idx * v_head_stride +
+ (block_idx * SEQUENCE_BLOCK_SIZE_1 + simd.threadgroup_index) *
+ v_seq_stride +
+ simd.simdgroup_index * value_elements_per_thread;
out += o_offset * TOTAL_BLOCKS_COUNT * value_dim + block_idx * value_dim +
- simd.lane_idx * value_elements_per_thread;
+ simd.simdgroup_index * value_elements_per_thread;
if (float_mask) {
fmask += head_idx * mask_head_stride +
- (block_idx * SEQUENCE_BLOCK_SIZE_1 + simd.group_idx) *
+ (block_idx * SEQUENCE_BLOCK_SIZE_1 + simd.threadgroup_index) *
mask_kv_seq_stride +
q_seq_idx * mask_q_seq_stride;
}
@@ -95,7 +98,7 @@ PUBLIC KERNEL(AttentionTwoPass1)(
U max_score = -1e9;
U sum_exp_score = 0;
- if (has_sinks && block_idx == 0 && simd.group_idx == 0) {
+ if (has_sinks && block_idx == 0 && simd.threadgroup_index == 0) {
const uint num_q_heads = tpg.x;
int q_head_idx = head_idx % num_q_heads;
max_score = static_cast<U>(sinks[q_head_idx]);
@@ -103,7 +106,7 @@ PUBLIC KERNEL(AttentionTwoPass1)(
}
// For each key
- for (uint i = block_idx * SEQUENCE_BLOCK_SIZE_1 + simd.group_idx;
+ for (uint i = block_idx * SEQUENCE_BLOCK_SIZE_1 + simd.threadgroup_index;
i < sequence_length;
i += TOTAL_BLOCKS_COUNT * SEQUENCE_BLOCK_SIZE_1) {
bool use_key = true;
@@ -150,38 +153,42 @@ PUBLIC KERNEL(AttentionTwoPass1)(
}
// Each thread has a partial part of the output so we need to combine them.
- if (simd.lane_idx == 0) {
- shared_max_scores[simd.group_idx] = max_score;
- shared_sum_exp_scores[simd.group_idx] = sum_exp_score;
+ if (simd.simdgroup_index == 0) {
+ shared_max_scores[simd.threadgroup_index] = max_score;
+ shared_sum_exp_scores[simd.threadgroup_index] = sum_exp_score;
}
threadgroup_barrier(mem_flags::mem_threadgroup);
- max_score = (simd.lane_idx < SEQUENCE_BLOCK_SIZE_1)
- ? shared_max_scores[simd.lane_idx]
+ max_score = (simd.simdgroup_index < SEQUENCE_BLOCK_SIZE_1)
+ ? shared_max_scores[simd.simdgroup_index]
: -1e9;
U new_max = simd_max(max_score);
U factor = fast::exp(max_score - new_max);
- sum_exp_score = (simd.lane_idx < SEQUENCE_BLOCK_SIZE_1)
- ? shared_sum_exp_scores[simd.lane_idx]
+ sum_exp_score = (simd.simdgroup_index < SEQUENCE_BLOCK_SIZE_1)
+ ? shared_sum_exp_scores[simd.simdgroup_index]
: 0;
sum_exp_score = simd_sum(sum_exp_score * factor);
// Write the sum and new max
- if (simd.group_idx == 0) {
+ if (simd.threadgroup_index == 0) {
sums[0] = sum_exp_score;
maxs[0] = new_max;
}
// Now we need to aggregate all the outputs
for (uint i = 0; i < value_elements_per_thread; i++) {
- shared_outputs[simd.lane_idx * SEQUENCE_BLOCK_SIZE_1 + simd.group_idx] =
- o[i] * fast::exp(shared_max_scores[simd.group_idx] - new_max);
+ shared_outputs
+ [simd.simdgroup_index * SEQUENCE_BLOCK_SIZE_1 +
+ simd.threadgroup_index] =
+ o[i] *
+ fast::exp(shared_max_scores[simd.threadgroup_index] - new_max);
threadgroup_barrier(mem_flags::mem_threadgroup);
// And write the output
- if (simd.group_idx == 0) {
- U output = shared_outputs[simd.lane_idx * SEQUENCE_BLOCK_SIZE_1];
+ if (simd.threadgroup_index == 0) {
+ U output = shared_outputs[simd.simdgroup_index * SEQUENCE_BLOCK_SIZE_1];
for (uint j = 1; j < SEQUENCE_BLOCK_SIZE_1; j++) {
- output += shared_outputs[simd.lane_idx * SEQUENCE_BLOCK_SIZE_1 + j];
+ output +=
+ shared_outputs[simd.simdgroup_index * SEQUENCE_BLOCK_SIZE_1 + j];
}
out[i] = static_cast<T>(output);
}
@@ -200,7 +207,7 @@ PUBLIC KERNEL(AttentionTwoPass2)(
const constant uint& num_heads,
const constant uint& suffix_length,
threadgroup float shared_outputs[SEQUENCE_BLOCK_SIZE_2 * HEAD_BLOCK_SIZE],
- const Simd simd,
+ const ThreadContext simd,
const uint head_idx GROUPS(num_heads),
const uint q_seq_idx GROUPS(suffix_length),
const uint tid THREADS(1024)
@@ -214,17 +221,19 @@ PUBLIC KERNEL(AttentionTwoPass2)(
const uint o_offset = q_seq_idx * tpg.x + head_idx; // Our custom layout
partials += o_offset * TOTAL_BLOCKS_COUNT * HEAD_DIM +
- simd.group_idx * HEAD_DIM + simd.lane_idx * elements_per_thread;
+ simd.threadgroup_index * HEAD_DIM +
+ simd.simdgroup_index * elements_per_thread;
sums += o_offset * TOTAL_BLOCKS_COUNT;
maxs += o_offset * TOTAL_BLOCKS_COUNT;
- out += o_offset * HEAD_DIM +
- simd.group_idx * elements_per_thread; // Our custom output layout
+ out +=
+ o_offset * HEAD_DIM +
+ simd.threadgroup_index * elements_per_thread; // Our custom output layout
// First everybody reads the max and sum_exp
- U max_score = maxs[simd.lane_idx];
+ U max_score = maxs[simd.simdgroup_index];
U new_max = simd_max(max_score);
U factor = fast::exp(max_score - new_max);
- U sum_exp_score = simd_sum(sums[simd.lane_idx] * factor);
+ U sum_exp_score = simd_sum(sums[simd.simdgroup_index] * factor);
// Now read the block into registers and then use shared memory to transpose
// it
@@ -232,19 +241,22 @@ PUBLIC KERNEL(AttentionTwoPass2)(
o[i] = partials[i];
}
for (uint i = 0; i < elements_per_thread; i++) {
- shared_outputs[simd.lane_idx * HEAD_BLOCK_SIZE + simd.group_idx] = o[i];
+ shared_outputs
+ [simd.simdgroup_index * HEAD_BLOCK_SIZE + simd.threadgroup_index] =
+ o[i];
threadgroup_barrier(mem_flags::mem_threadgroup);
- o[i] =
- simd_sum(
- shared_outputs[simd.group_idx * HEAD_BLOCK_SIZE + simd.lane_idx] *
- factor
- ) /
- sum_exp_score;
+ o[i] = simd_sum(
+ shared_outputs
+ [simd.threadgroup_index * HEAD_BLOCK_SIZE +
+ simd.simdgroup_index] *
+ factor
+ ) /
+ sum_exp_score;
threadgroup_barrier(mem_flags::mem_threadgroup);
}
// And write the output
- if (simd.lane_idx == 0) {
+ if (simd.simdgroup_index == 0) {
for (uint i = 0; i < elements_per_thread; i++) {
out[i] = static_cast<T>(o[i]);
}
diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_update_kv_cache.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_update_kv_cache.metal
index e28f779b4..f27f2c3af 100644
--- a/crates/uzu/src/backends/metal/kernel/attention/attention_update_kv_cache.metal
+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_update_kv_cache.metal
@@ -1,5 +1,5 @@
#include <metal_stdlib>
-#include "../definitions.metal"
+#include "../common/dsl.h"
template <typename T>
VARIANTS(T, float, half, bfloat)
diff --git a/crates/uzu/src/backends/metal/kernel/audio/audio.metal b/crates/uzu/src/backends/metal/kernel/audio/audio.metal
index aff2451b3..e95cb27f0 100644
--- a/crates/uzu/src/backends/metal/kernel/audio/audio.metal
+++ b/crates/uzu/src/backends/metal/kernel/audio/audio.metal
@@ -1,5 +1,5 @@
#include <metal_stdlib>
-#include "../definitions.metal"
+#include "../common/dsl.h"
using namespace metal;
diff --git a/crates/uzu/src/backends/metal/kernel/common/defines.h b/crates/uzu/src/backends/metal/kernel/common/defines.h
index 4ea67c43f..97e2fcfd3 100644
--- a/crates/uzu/src/backends/metal/kernel/common/defines.h
+++ b/crates/uzu/src/backends/metal/kernel/common/defines.h
@@ -1,22 +1,17 @@
#pragma once
-#if defined __METAL__
-#define MTL_CONST constant
-#else
-#define MTL_CONST
-#endif
+#define METAL_CONST static constant constexpr
+#define METAL_PRAGMA_UNROLL _Pragma("clang loop unroll(full)")
+#define METAL_SIMD_SIZE 32
+METAL_FUNC int pow2(int n) { return 1 << n; }
-static MTL_CONST constexpr int MAX_REDUCE_SPECIALIZED_DIMS = 4;
-static MTL_CONST constexpr int REDUCE_N_READS = 4;
-static MTL_CONST constexpr int REDUCE_N_WRITES = 4;
-static MTL_CONST constexpr int SOFTMAX_N_READS = 4;
-static MTL_CONST constexpr int RMS_N_READS = 4;
-static MTL_CONST constexpr int RMS_LOOPED_LIMIT = 4096;
+METAL_CONST int MAX_REDUCE_SPECIALIZED_DIMS = 4;
+METAL_CONST int REDUCE_N_READS = 4;
+METAL_CONST int REDUCE_N_WRITES = 4;
+METAL_CONST int SOFTMAX_N_READS = 4;
+METAL_CONST int RMS_N_READS = 4;
+METAL_CONST int RMS_LOOPED_LIMIT = 4096;
-// Instantiate a templated kernel.
-// Extra args are used as template parameters:
-// e.g. instantiate_kernel(binary_int, binary, a, b) ->
-// [[host_name(binary_int)]] [kernel] binary<a, b>
#define instantiate_kernel(name, func, ...) \
template [[host_name( \
name \
diff --git a/crates/uzu/src/backends/metal/kernel/common/dsl.h b/crates/uzu/src/backends/metal/kernel/common/dsl.h
new file mode 100644
index 000000000..bae47519e
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/common/dsl.h
@@ -0,0 +1,66 @@
+#pragma once
+
+#include <metal_stdlib>
+using namespace metal;
+
+#if defined(__cpp_if_constexpr)
+#define IF_CONSTEXPR(cond) if constexpr (cond)
+#else
+#define IF_CONSTEXPR(cond) if (cond)
+#endif
+
+#ifdef DSL_ANALYZE
+#define DSL_META(...) [[clang::annotate("", __VA_ARGS__)]]
+#else
+#define DSL_META(...)
+#endif
+
+#define DSL_STR(X) #X
+#define DSL_XSTR(X) DSL_STR(X)
+
+#define VARIANTS(TYPENAME, ...) \
+ DSL_META("dsl.variants", #TYPENAME, #__VA_ARGS__)
+#define PUBLIC DSL_META("dsl.public")
+#define KERNEL(NAME) DSL_META("dsl.kernel") void NAME
+
+#define SPECIALIZE DSL_META("dsl.specialize")
+#define OPTIONAL(EXPR) DSL_META("dsl.optional", DSL_XSTR(EXPR))
+
+#define AXIS(TDS, TPG) DSL_META("dsl.axis", DSL_XSTR(TDS), DSL_XSTR(TPG))
+#define GROUPS(EXPR) DSL_META("dsl.groups", DSL_XSTR(EXPR))
+#define THREADS(EXPR) DSL_META("dsl.threads", DSL_XSTR(EXPR))
+
+#define generateKernel( \
+ max_threads, \
+ functionName, \
+ scalarType, \
+ outerArgs, \
+ innerArgs \
+) \
+ [[max_total_threads_per_threadgroup(max_threads)]] \
+ kernel void functionName##_##scalarType outerArgs { \
+ functionName innerArgs; \
+ }
+
+#define generateKernels(max_threads, functionName) \
+ generateKernel( \
+ max_threads, \
+ functionName, \
+ float, \
+ outerArguments(float), \
+ innerArguments \
+ ); \
+ generateKernel( \
+ max_threads, \
+ functionName, \
+ bfloat, \
+ outerArguments(bfloat), \
+ innerArguments \
+ ); \
+ generateKernel( \
+ max_threads, \
+ functionName, \
+ half, \
+ outerArguments(half), \
+ innerArguments \
+ );
diff --git a/crates/uzu/src/backends/metal/kernel/common/tensor_view.h b/crates/uzu/src/backends/metal/kernel/common/tensor_view.h
new file mode 100644
index 000000000..ef5e977fd
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/common/tensor_view.h
@@ -0,0 +1,120 @@
+#pragma once
+
+#include <metal_stdlib>
+using namespace metal;
+
+template <typename T>
+struct TensorView2D {
+ device T* data;
+ uint dim0, dim1;
+ uint stride0, stride1;
+
+ TensorView2D(device T* ptr) : data(ptr) {}
+
+ TensorView2D(device T* ptr, int d0, int d1) : data(ptr), dim0(d0), dim1(d1) {
+ stride0 = d1;
+ stride1 = 1;
+ }
+
+ TensorView2D(device T* ptr, int d0, int d1, int s0, int s1)
+ : data(ptr), dim0(d0), dim1(d1), stride0(s0), stride1(s1) {}
+
+ thread TensorView2D& shaped(int d0, int d1) {
+ dim0 = d0;
+ dim1 = d1;
+ stride0 = d1;
+ stride1 = 1;
+ return *this;
+ }
+
+ device T& at(uint i, uint j) const { return data[i * stride0 + j * stride1]; }
+
+ device T& operator()(uint i, uint j) const { return at(i, j); }
+};
+
+template <typename T>
+struct TensorView3D {
+ device T* data;
+ uint dim0, dim1, dim2;
+ uint stride0, stride1, stride2;
+
+ TensorView3D(device T* ptr) : data(ptr) {}
+
+ TensorView3D(device T* ptr, int d0, int d1, int d2)
+ : data(ptr), dim0(d0), dim1(d1), dim2(d2) {
+ stride0 = d1 * d2;
+ stride1 = d2;
+ stride2 = 1;
+ }
+
+ TensorView3D(device T* ptr, int d0, int d1, int d2, int s0, int s1, int s2)
+ : data(ptr), dim0(d0), dim1(d1), dim2(d2), stride0(s0), stride1(s1),
+ stride2(s2) {}
+
+ thread TensorView3D& shaped(int d0, int d1, int d2) {
+ dim0 = d0;
+ dim1 = d1;
+ dim2 = d2;
+ stride0 = d1 * d2;
+ stride1 = d2;
+ stride2 = 1;
+ return *this;
+ }
+
+ device T& at(uint i, uint j, uint k) const {
+ return data[i * stride0 + j * stride1 + k * stride2];
+ }
+
+ device T& operator()(uint i, uint j, uint k) const { return at(i, j, k); }
+};
+
+template <typename T>
+struct TensorView4D {
+ device T* data;
+ uint dim0, dim1, dim2, dim3;
+ uint stride0, stride1, stride2, stride3;
+
+ TensorView4D(device T* ptr) : data(ptr) {}
+
+ TensorView4D(device T* ptr, int d0, int d1, int d2, int d3)
+ : data(ptr), dim0(d0), dim1(d1), dim2(d2), dim3(d3) {
+ stride0 = d1 * d2 * d3;
+ stride1 = d2 * d3;
+ stride2 = d3;
+ stride3 = 1;
+ }
+
+ TensorView4D(
+ device T* ptr,
+ int d0,
+ int d1,
+ int d2,
+ int d3,
+ int s0,
+ int s1,
+ int s2,
+ int s3
+ )
+ : data(ptr), dim0(d0), dim1(d1), dim2(d2), dim3(d3), stride0(s0),
+ stride1(s1), stride2(s2), stride3(s3) {}
+
+ thread TensorView4D& shaped(int d0, int d1, int d2, int d3) {
+ dim0 = d0;
+ dim1 = d1;
+ dim2 = d2;
+ dim3 = d3;
+ stride0 = d1 * d2 * d3;
+ stride1 = d2 * d3;
+ stride2 = d3;
+ stride3 = 1;
+ return *this;
+ }
+
+ device T& at(uint i, uint j, uint k, uint l) const {
+ return data[i * stride0 + j * stride1 + k * stride2 + l * stride3];
+ }
+
+ device T& operator()(uint i, uint j, uint k, uint l) const {
+ return at(i, j, k, l);
+ }
+};
diff --git a/crates/uzu/src/backends/metal/kernel/common/thread_context.h b/crates/uzu/src/backends/metal/kernel/common/thread_context.h
new file mode 100644
index 000000000..9a3f827a5
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/common/thread_context.h
@@ -0,0 +1,12 @@
+#pragma once
+
+#include <metal_stdlib>
+using namespace metal;
+
+struct ThreadContext {
+ uint simdgroup_index;
+ uint threadgroup_index;
+ uint simdgroup_size;
+ uint threadgroup_size;
+ uint simdgroups_per_threadgroup;
+};
diff --git a/crates/uzu/src/backends/metal/kernel/common/thread_functions.h b/crates/uzu/src/backends/metal/kernel/common/thread_functions.h
new file mode 100644
index 000000000..5185a9831
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/common/thread_functions.h
@@ -0,0 +1,54 @@
+#pragma once
+
+#include <metal_stdlib>
+using namespace metal;
+
+template <ushort LENGTH, typename T>
+static inline T thread_prefix_inclusive_sum(thread T (&values)[LENGTH]) {
+ for (ushort i = 1; i < LENGTH; i++) {
+ values[i] += values[i - 1];
+ }
+ return values[LENGTH - 1];
+}
+
+template <ushort LENGTH, typename T>
+static inline T thread_prefix_inclusive_sum(threadgroup T* values) {
+ for (ushort i = 1; i < LENGTH; i++) {
+ values[i] += values[i - 1];
+ }
+ return values[LENGTH - 1];
+}
+
+template <ushort LENGTH, typename T>
+static inline T thread_prefix_exclusive_sum(thread T (&values)[LENGTH]) {
+ T inclusive_prefix = thread_prefix_inclusive_sum<LENGTH>(values);
+ for (ushort i = LENGTH - 1; i > 0; i--) {
+ values[i] = values[i - 1];
+ }
+ values[0] = 0;
+ return inclusive_prefix;
+}
+
+template <ushort LENGTH, typename T>
+static inline T thread_prefix_exclusive_sum(threadgroup T* values) {
+ T inclusive_prefix = thread_prefix_inclusive_sum<LENGTH>(values);
+ for (ushort i = LENGTH - 1; i > 0; i--) {
+ values[i] = values[i - 1];
+ }
+ values[0] = 0;
+ return inclusive_prefix;
+}
+
+template <ushort LENGTH, typename T>
+static inline void thread_uniform_add(thread T (&values)[LENGTH], T uni) {
+ for (ushort i = 0; i < LENGTH; i++) {
+ values[i] += uni;
+ }
+}
+
+template <ushort LENGTH, typename T>
+static inline void thread_uniform_add(threadgroup T* values, T uni) {
+ for (ushort i = 0; i < LENGTH; i++) {
+ values[i] += uni;
+ }
+}
diff --git a/crates/uzu/src/backends/metal/kernel/common/threadgroup_reduce.h b/crates/uzu/src/backends/metal/kernel/common/threadgroup_reduce.h
new file mode 100644
index 000000000..2438e37a7
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/common/threadgroup_reduce.h
@@ -0,0 +1,229 @@
+#pragma once
+
+#include "thread_context.h"
+#include <metal_stdlib>
+using namespace metal;
+
+template <ushort BLOCK_SIZE, typename T>
+static T threadgroup_raking_prefix_exclusive_sum(
+ T value,
+ threadgroup T* shared,
+ const ushort lid
+) {
+ shared[lid] = value;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ if (lid < 32) {
+ const short values_per_thread = BLOCK_SIZE / 32;
+ const short first_index = lid * values_per_thread;
+ for (short i = first_index + 1; i < first_index + values_per_thread; i++) {
+ shared[i] += shared[i - 1];
+ }
+ T partial_sum = shared[first_index + values_per_thread - 1];
+ for (short i = first_index + values_per_thread - 1; i > first_index; i--) {
+ shared[i] = shared[i - 1];
+ }
+ shared[first_index] = 0;
+
+ T prefix = simd_prefix_exclusive_sum(partial_sum);
+
+ for (short i = first_index; i < first_index + values_per_thread; i++) {
+ shared[i] += prefix;
+ }
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ const T result = shared[lid];
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ return result;
+}
+
+template <ushort BLOCK_SIZE, typename T>
+static T threadgroup_raking_reduce_sum(
+ T value,
+ threadgroup T* shared,
+ const ushort lid
+) {
+ shared[lid] = value;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ if (lid < 32) {
+ const short values_per_thread = BLOCK_SIZE / 32;
+ const short first_index = lid * values_per_thread;
+
+ T thread_sum = shared[first_index];
+ for (short i = first_index + 1; i < first_index + values_per_thread; i++) {
+ thread_sum += shared[i];
+ }
+
+ T total_sum = simd_sum(thread_sum);
+
+ if (lid == 0) {
+ shared[0] = total_sum;
+ }
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ const T result = shared[0];
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ return result;
+}
+
+template <ushort BLOCK_SIZE, typename T>
+static T threadgroup_raking_reduce_max(
+ T value,
+ threadgroup T* shared,
+ const ushort lid
+) {
+ shared[lid] = value;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ if (lid < 32) {
+ const short values_per_thread = BLOCK_SIZE / 32;
+ const short first_index = lid * values_per_thread;
+
+ T thread_max = shared[first_index];
+ for (short i = first_index + 1; i < first_index + values_per_thread; i++) {
+ thread_max = max(thread_max, shared[i]);
+ }
+
+ T total_max = simd_max(thread_max);
+
+ if (lid == 0) {
+ shared[0] = total_max;
+ }
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ const T result = shared[0];
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ return result;
+}
+
+template <ushort BLOCK_SIZE, typename T>
+static T threadgroup_raking_reduce_min(
+ T value,
+ threadgroup T* shared,
+ const ushort lid
+) {
+ shared[lid] = value;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ if (lid < 32) {
+ const short values_per_thread = BLOCK_SIZE / 32;
+ const short first_index = lid * values_per_thread;
+
+ T thread_min = shared[first_index];
+ for (short i = first_index + 1; i < first_index + values_per_thread; i++) {
+ thread_min = min(thread_min, shared[i]);
+ }
+
+ T total_min = simd_min(thread_min);
+
+ if (lid == 0) {
+ shared[0] = total_min;
+ }
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ const T result = shared[0];
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ return result;
+}
+
+template <ushort BLOCK_SIZE, typename T>
+static T threadgroup_cooperative_reduce_sum(
+ T value,
+ threadgroup T* shared,
+ const ushort lid,
+ const thread ThreadContext& simd
+) {
+ T local_sum = simd_sum(value);
+
+ if (simd.simdgroup_index == 0) {
+ shared[simd.threadgroup_index] = local_sum;
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ T total_sum = T(0);
+ const ushort num_simd_groups =
+ (BLOCK_SIZE + simd.simdgroup_size - 1) / simd.simdgroup_size;
+ if (lid < num_simd_groups) {
+ total_sum = shared[lid];
+ }
+ total_sum = simd_sum(total_sum);
+
+ if (lid == 0) {
+ shared[0] = total_sum;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ const T result = shared[0];
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ return result;
+}
+
+template <ushort BLOCK_SIZE, typename T>
+static T threadgroup_cooperative_reduce_max(
+ T value,
+ threadgroup T* shared,
+ const ushort lid,
+ const thread ThreadContext& simd
+) {
+ T local_max = simd_max(value);
+
+ if (simd.simdgroup_index == 0) {
+ shared[simd.threadgroup_index] = local_max;
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ T total_max =
+ lid < (BLOCK_SIZE + simd.simdgroup_size - 1) / simd.simdgroup_size
+ ? shared[lid]
+ : T(-INFINITY);
+ total_max = simd_max(total_max);
+
+ if (lid == 0) {
+ shared[0] = total_max;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ const T result = shared[0];
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ return result;
+}
+
+template <ushort BLOCK_SIZE, typename T>
+static T threadgroup_cooperative_reduce_min(
+ T value,
+ threadgroup T* shared,
+ const ushort lid,
+ const thread ThreadContext& simd
+) {
+ T local_min = simd_min(value);
+
+ if (simd.simdgroup_index == 0) {
+ shared[simd.threadgroup_index] = local_min;
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ T total_min =
+ lid < (BLOCK_SIZE + simd.simdgroup_size - 1) / simd.simdgroup_size
+ ? shared[lid]
+ : T(INFINITY);
+ total_min = simd_min(total_min);
+
+ if (lid == 0) {
+ shared[0] = total_min;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ const T result = shared[0];
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ return result;
+}
diff --git a/crates/uzu/src/backends/metal/kernel/common/utils.h b/crates/uzu/src/backends/metal/kernel/common/utils.h
index c4e853f4c..71a185205 100644
--- a/crates/uzu/src/backends/metal/kernel/common/utils.h
+++ b/crates/uzu/src/backends/metal/kernel/common/utils.h
@@ -88,8 +88,6 @@ struct Limits<complex64_t> {
// Indexing utils
///////////////////////////////////////////////////////////////////////////////
-#define MTL_PRAGMA_UNROLL _Pragma("clang loop unroll(full)")
-
///////////////////////////////////////////////////////////////////////////////
// Single Array with generic dims
diff --git a/crates/uzu/src/backends/metal/kernel/definitions.metal b/crates/uzu/src/backends/metal/kernel/definitions.metal
deleted file mode 100644
index d6f5a7c8a..000000000
--- a/crates/uzu/src/backends/metal/kernel/definitions.metal
+++ /dev/null
@@ -1,536 +0,0 @@
-#ifndef definitions_metal
-#define definitions_metal
-
-#include <metal_stdlib>
-
-using namespace metal;
-
-// MARK: - Tensor Access Templates
-
-template <typename T>
-struct TensorView2D {
- device T* data;
- uint dim0, dim1;
- uint stride0, stride1;
-
- TensorView2D(device T* ptr) : data(ptr) {}
-
- TensorView2D(device T* ptr, int d0, int d1) : data(ptr), dim0(d0), dim1(d1) {
- stride0 = d1;
- stride1 = 1;
- }
-
- // Custom stride constructor
- TensorView2D(device T* ptr, int d0, int d1, int s0, int s1)
- : data(ptr), dim0(d0), dim1(d1), stride0(s0), stride1(s1) {}
-
- thread TensorView2D& shaped(int d0, int d1) {
- dim0 = d0;
- dim1 = d1;
- stride0 = d1;
- stride1 = 1;
- return *this;
- }
-
- device T& at(uint i, uint j) const { return data[i * stride0 + j * stride1]; }
-
- device T& operator()(uint i, uint j) const { return at(i, j); }
-};
-
-template <typename T>
-struct TensorView3D {
- device T* data;
- uint dim0, dim1, dim2;
- uint stride0, stride1, stride2;
-
- TensorView3D(device T* ptr) : data(ptr) {}
-
- TensorView3D(device T* ptr, int d0, int d1, int d2)
- : data(ptr), dim0(d0), dim1(d1), dim2(d2) {
- stride0 = d1 * d2;
- stride1 = d2;
- stride2 = 1;
- }
-
- // Custom stride constructor
- TensorView3D(device T* ptr, int d0, int d1, int d2, int s0, int s1, int s2)
- : data(ptr), dim0(d0), dim1(d1), dim2(d2), stride0(s0), stride1(s1),
- stride2(s2) {}
-
- thread TensorView3D& shaped(int d0, int d1, int d2) {
- dim0 = d0;
- dim1 = d1;
- dim2 = d2;
- stride0 = d1 * d2;
- stride1 = d2;
- stride2 = 1;
- return *this;
- }
-
- device T& at(uint i, uint j, uint k) const {
- return data[i * stride0 + j * stride1 + k * stride2];
- }
-
- device T& operator()(uint i, uint j, uint k) const { return at(i, j, k); }
-};
-
-template <typename T>
-struct TensorView4D {
- device T* data;
- uint dim0, dim1, dim2, dim3;
- uint stride0, stride1, stride2, stride3;
-
- TensorView4D(device T* ptr) : data(ptr) {}
-
- TensorView4D(device T* ptr, int d0, int d1, int d2, int d3)
- : data(ptr), dim0(d0), dim1(d1), dim2(d2), dim3(d3) {
- stride0 = d1 * d2 * d3;
- stride1 = d2 * d3;
- stride2 = d3;
- stride3 = 1;
- }
-
- // Custom stride constructor
- TensorView4D(
- device T* ptr,
- int d0,
- int d1,
- int d2,
- int d3,
- int s0,
- int s1,
- int s2,
- int s3
- )
- : data(ptr), dim0(d0), dim1(d1), dim2(d2), dim3(d3), stride0(s0),
- stride1(s1), stride2(s2), stride3(s3) {}
-
- thread TensorView4D& shaped(int d0, int d1, int d2, int d3) {
- dim0 = d0;
- dim1 = d1;
- dim2 = d2;
- dim3 = d3;
- stride0 = d1 * d2 * d3;
- stride1 = d2 * d3;
- stride2 = d3;
- stride3 = 1;
- return *this;
- }
-
- device T& at(uint i, uint j, uint k, uint l) const {
- return data[i * stride0 + j * stride1 + k * stride2 + l * stride3];
- }
-
- device T& operator()(uint i, uint j, uint k, uint l) const {
- return at(i, j, k, l);
- }
-};
-
-struct Simd {
- uint lane_idx;
- uint group_idx;
- uint group_size;
- uint groups_per_threadgroup;
-
- uint groups_count(const uint threads_count) const {
- return (threads_count + this->group_size - 1) / this->group_size;
- }
-};
-
-///////////////////////////////////////////////////////////////////////////////
-// MARK: - Thread Functions
-///////////////////////////////////////////////////////////////////////////////
-
-template <ushort LENGTH, typename T>
-static inline T thread_prefix_inclusive_sum(thread T (&values)[LENGTH]) {
- for (ushort i = 1; i < LENGTH; i++) {
- values[i] += values[i - 1];
- }
- return values[LENGTH - 1];
-}
-
-template <ushort LENGTH, typename T>
-static inline T thread_prefix_inclusive_sum(threadgroup T* values) {
- for (ushort i = 1; i < LENGTH; i++) {
- values[i] += values[i - 1];
- }
- return values[LENGTH - 1];
-}
-
-template <ushort LENGTH, typename T>
-static inline T thread_prefix_exclusive_sum(thread T (&values)[LENGTH]) {
- // do as an inclusive scan first
- T inclusive_prefix = thread_prefix_inclusive_sum<LENGTH>(values);
- // convert to an exclusive scan in the reverse direction
- for (ushort i = LENGTH - 1; i > 0; i--) {
- values[i] = values[i - 1];
- }
- values[0] = 0;
- return inclusive_prefix;
-}
-
-template <ushort LENGTH, typename T>
-static inline T thread_prefix_exclusive_sum(threadgroup T* values) {
- // do as an inclusive scan first
- T inclusive_prefix = thread_prefix_inclusive_sum<LENGTH>(values);
- // convert to an exclusive scan in the reverse direction
- for (ushort i = LENGTH - 1; i > 0; i--) {
- values[i] = values[i - 1];
- }
- values[0] = 0;
- return inclusive_prefix;
-}
-
-template <ushort LENGTH, typename T>
-static inline void thread_uniform_add(thread T (&values)[LENGTH], T uni) {
- for (ushort i = 0; i < LENGTH; i++) {
- values[i] += uni;
- }
-}
-
-template <ushort LENGTH, typename T>
-static inline void thread_uniform_add(threadgroup T* values, T uni) {
- for (ushort i = 0; i < LENGTH; i++) {
- values[i] += uni;
- }
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// MARK: - Threadgroup Functions
-///////////////////////////////////////////////////////////////////////////////
-
-//------------------------------------------------------------------------------------------------//
-// Raking threadgroup scan
-template <ushort BLOCK_SIZE, typename T>
-static T threadgroup_raking_prefix_exclusive_sum(
- T value,
- threadgroup T* shared,
- const ushort lid
-) {
- // load values into shared memory
- shared[lid] = value;
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // only the first 32 threads form the rake
- if (lid < 32) {
- // scan by thread in shared mem
- const short values_per_thread = BLOCK_SIZE / 32;
- const short first_index = lid * values_per_thread;
- for (short i = first_index + 1; i < first_index + values_per_thread; i++) {
- shared[i] += shared[i - 1];
- }
- T partial_sum = shared[first_index + values_per_thread - 1];
- for (short i = first_index + values_per_thread - 1; i > first_index; i--) {
- shared[i] = shared[i - 1];
- }
- shared[first_index] = 0;
-
- // scan the partial sums
- T prefix = simd_prefix_exclusive_sum(partial_sum);
-
- // add back the prefix
- for (short i = first_index; i < first_index + values_per_thread; i++) {
- shared[i] += prefix;
- }
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- const T result = shared[lid];
- threadgroup_barrier(mem_flags::mem_threadgroup);
- return result;
-}
-
-//------------------------------------------------------------------------------------------------//
-// Raking threadgroup sum reduction
-template <ushort BLOCK_SIZE, typename T>
-static T threadgroup_raking_reduce_sum(
- T value,
- threadgroup T* shared,
- const ushort lid
-) {
- // load values into shared memory
- shared[lid] = value;
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // only the first 32 threads form the rake
- if (lid < 32) {
- // reduce by thread in shared mem
- const short values_per_thread = BLOCK_SIZE / 32;
- const short first_index = lid * values_per_thread;
-
- T thread_sum = shared[first_index];
- for (short i = first_index + 1; i < first_index + values_per_thread; i++) {
- thread_sum += shared[i];
- }
-
- // reduce the partial sums using SIMD
- T total_sum = simd_sum(thread_sum);
-
- // broadcast result back to shared memory
- if (lid == 0) {
- shared[0] = total_sum;
- }
- }
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- const T result = shared[0];
- threadgroup_barrier(mem_flags::mem_threadgroup);
- return result;
-}
-
-//------------------------------------------------------------------------------------------------//
-// Raking threadgroup max reduction
-template <ushort BLOCK_SIZE, typename T>
-static T threadgroup_raking_reduce_max(
- T value,
- threadgroup T* shared,
- const ushort lid
-) {
- // load values into shared memory
- shared[lid] = value;
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // only the first 32 threads form the rake
- if (lid < 32) {
- // reduce by thread in shared mem
- const short values_per_thread = BLOCK_SIZE / 32;
- const short first_index = lid * values_per_thread;
-
- T thread_max = shared[first_index];
- for (short i = first_index + 1; i < first_index + values_per_thread; i++) {
- thread_max = max(thread_max, shared[i]);
- }
-
- // reduce the partial maxes using SIMD
- T total_max = simd_max(thread_max);
-
- // broadcast result back to shared memory
- if (lid == 0) {
- shared[0] = total_max;
- }
- }
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- const T result = shared[0];
- threadgroup_barrier(mem_flags::mem_threadgroup);
- return result;
-}
-
-//------------------------------------------------------------------------------------------------//
-// Raking threadgroup min reduction
-template <ushort BLOCK_SIZE, typename T>
-static T threadgroup_raking_reduce_min(
- T value,
- threadgroup T* shared,
- const ushort lid
-) {
- // load values into shared memory
- shared[lid] = value;
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // only the first 32 threads form the rake
- if (lid < 32) {
- // reduce by thread in shared mem
- const short values_per_thread = BLOCK_SIZE / 32;
- const short first_index = lid * values_per_thread;
-
- T thread_min = shared[first_index];
- for (short i = first_index + 1; i < first_index + values_per_thread; i++) {
- thread_min = min(thread_min, shared[i]);
- }
-
- // reduce the partial mins using SIMD
- T total_min = simd_min(thread_min);
-
- // broadcast result back to shared memory
- if (lid == 0) {
- shared[0] = total_min;
- }
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- const T result = shared[0];
- threadgroup_barrier(mem_flags::mem_threadgroup);
- return result;
-}
-
-//------------------------------------------------------------------------------------------------//
-// Cooperative threadgroup sum reduction (2-level hierarchical)
-template <ushort BLOCK_SIZE, typename T>
-static T threadgroup_cooperative_reduce_sum(
- T value,
- threadgroup T* shared,
- const ushort lid,
- const thread Simd& simd
-) {
- // Reduce within simdgroup
- T local_sum = simd_sum(value);
-
- // First thread in each simdgroup writes to shared memory
- if (simd.lane_idx == 0) {
- shared[simd.group_idx] = local_sum;
- }
-
- // Synchronize across the threadgroup
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // Reduce across simdgroups
- T total_sum = T(0);
- const ushort num_simd_groups = simd.groups_count(BLOCK_SIZE);
- if (lid < num_simd_groups) {
- total_sum = shared[lid];
- }
- total_sum = simd_sum(total_sum);
-
- // Broadcast the result to all threads
- if (lid == 0) {
- shared[0] = total_sum;
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- const T result = shared[0];
- threadgroup_barrier(mem_flags::mem_threadgroup);
- return result;
-}
-
-//------------------------------------------------------------------------------------------------//
-// Cooperative threadgroup max reduction (2-level hierarchical)
-template <ushort BLOCK_SIZE, typename T>
-static T threadgroup_cooperative_reduce_max(
- T value,
- threadgroup T* shared,
- const ushort lid,
- const thread Simd& simd
-) {
- // Reduce within simdgroup
- T local_max = simd_max(value);
-
- // First thread in each simdgroup writes to shared memory
- if (simd.lane_idx == 0) {
- shared[simd.group_idx] = local_max;
- }
-
- // Synchronize across the threadgroup
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // Reduce across simdgroups
- T total_max =
- lid < simd.groups_count(BLOCK_SIZE) ? shared[lid] : T(-INFINITY);
- total_max = simd_max(total_max);
-
- // Broadcast the result to all threads
- if (lid == 0) {
- shared[0] = total_max;
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- const T result = shared[0];
- threadgroup_barrier(mem_flags::mem_threadgroup);
- return result;
-}
-
-//------------------------------------------------------------------------------------------------//
-// Cooperative threadgroup min reduction (2-level hierarchical)
-template <ushort BLOCK_SIZE, typename T>
-static T threadgroup_cooperative_reduce_min(
- T value,
- threadgroup T* shared,
- const ushort lid,
- const thread Simd& simd
-) {
- // Reduce within simdgroup
- T local_min = simd_min(value);
-
- // First thread in each simdgroup writes to shared memory
- if (simd.lane_idx == 0) {
- shared[simd.group_idx] = local_min;
- }
-
- // Synchronize across the threadgroup
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // Reduce across simdgroups
- T total_min = lid < simd.groups_count(BLOCK_SIZE) ? shared[lid] : T(INFINITY);
- total_min = simd_min(total_min);
-
- // Broadcast the result to all threads
- if (lid == 0) {
- shared[0] = total_min;
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- const T result = shared[0];
- threadgroup_barrier(mem_flags::mem_threadgroup);
- return result;
-}
-
-// warning: constexpr if is a C++17 extension [-Wc++17-extensions]
-#if defined(__cpp_if_constexpr)
-#define IF_CONSTEXPR(cond) if constexpr (cond)
-#else
-#define IF_CONSTEXPR(cond) if (cond)
-#endif
-
-// MARK: - DSL Annotation Helpers
-
-#ifdef DSL_ANALYZE
-#define DSL_META(...) [[clang::annotate("", __VA_ARGS__)]]
-#else
-#define DSL_META(...)
-#endif
-
-#define DSL_STR(X) #X
-#define DSL_XSTR(X) DSL_STR(X)
-
-#define VARIANTS(TYPENAME, ...) \
- DSL_META("dsl.variants", #TYPENAME, #__VA_ARGS__)
-#define PUBLIC DSL_META("dsl.public")
-#define KERNEL(NAME) DSL_META("dsl.kernel") void NAME
-
-#define SPECIALIZE DSL_META("dsl.specialize")
-#define OPTIONAL(EXPR) DSL_META("dsl.optional", DSL_XSTR(EXPR))
-
-#define AXIS(TDS, TPG) DSL_META("dsl.axis", DSL_XSTR(TDS), DSL_XSTR(TPG))
-#define GROUPS(EXPR) DSL_META("dsl.groups", DSL_XSTR(EXPR))
-#define THREADS(EXPR) DSL_META("dsl.threads", DSL_XSTR(EXPR))
-
-// MARK: - Generate Template Kernels
-
-#define generateKernel( \
- max_threads, \
- functionName, \
- scalarType, \
- outerArgs, \
- innerArgs \
-) \
- [[max_total_threads_per_threadgroup(max_threads)]] \
- kernel void functionName##_##scalarType outerArgs { \
- functionName innerArgs; \
- }
-
-#define generateKernels(max_threads, functionName) \
- generateKernel( \
- max_threads, \
- functionName, \
- float, \
- outerArguments(float), \
- innerArguments \
- ); \
- generateKernel( \
- max_threads, \
- functionName, \
- bfloat, \
- outerArguments(bfloat), \
- innerArguments \
- ); \
- generateKernel( \
- max_threads, \
- functionName, \
- half, \
- outerArguments(half), \
- innerArguments \
- );
-
-#endif /* definitions_metal */
diff --git a/crates/uzu/src/backends/metal/kernel/embedding/full_precision_embedding.metal b/crates/uzu/src/backends/metal/kernel/embedding/full_precision_embedding.metal
index acc6e397e..c45f05e24 100644
--- a/crates/uzu/src/backends/metal/kernel/embedding/full_precision_embedding.metal
+++ b/crates/uzu/src/backends/metal/kernel/embedding/full_precision_embedding.metal
@@ -1,5 +1,5 @@
#include <metal_stdlib>
-#include "../definitions.metal"
+#include "../common/dsl.h"
#define BLOCK_SIZE 256
diff --git a/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal b/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal
index 4ee6a69a7..bc7881a3b 100644
--- a/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal
+++ b/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal
@@ -1,5 +1,5 @@
#include <metal_stdlib>
-#include "../definitions.metal"
+#include "../common/dsl.h"
enum QuantizationMode : uint {
QUANT_UINT4 = 0,
diff --git a/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal b/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal
index c4c379179..0d4fa0dc4 100644
--- a/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal
+++ b/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal
@@ -1,5 +1,5 @@
#include <metal_stdlib>
-#include "../definitions.metal"
+#include "../common/dsl.h"
#include "kv_cache_update.h"
diff --git a/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal b/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal
index 2c5ead11e..c6e54cee0 100644
--- a/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal
+++ b/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal
@@ -1,8 +1,10 @@
#include <metal_stdlib>
-#include "../definitions.metal"
+#include "../common/defines.h"
+#include "../common/dsl.h"
+#include "../common/thread_context.h"
+#include "../common/threadgroup_reduce.h"
#define BLOCK_SIZE 1024
-#define SIMD_SIZE 32
#define GRAIN_SIZE 4
// LayerNorm: subtract mean, then normalize (for BERT-style models)
@@ -18,7 +20,7 @@ void layer_norm_core(
threadgroup AccumT* shared_variance,
uint thread_in_row,
bool full_layer,
- const thread Simd& simd
+ const thread ThreadContext& simd
) {
AccumT partial_sum = static_cast<AccumT>(0.0f);
@@ -128,9 +130,9 @@ PUBLIC KERNEL(LayerNorm) (
constant float& scale_offset,
constant uint& full_layer,
const bool in_place SPECIALIZE,
- threadgroup ACC shared_mean[SIMD_SIZE],
- threadgroup ACC shared_variance[SIMD_SIZE],
- const Simd simd,
+ threadgroup ACC shared_mean[METAL_SIMD_SIZE],
+ threadgroup ACC shared_variance[METAL_SIMD_SIZE],
+ const ThreadContext simd,
uint batch_idx GROUPS(batch_size),
uint thread_in_row THREADS(BLOCK_SIZE)
) {
diff --git a/crates/uzu/src/backends/metal/kernel/mask_update/mask_update.metal b/crates/uzu/src/backends/metal/kernel/mask_update/mask_update.metal
index b9f61e282..06a9d52f2 100644
--- a/crates/uzu/src/backends/metal/kernel/mask_update/mask_update.metal
+++ b/crates/uzu/src/backends/metal/kernel/mask_update/mask_update.metal
@@ -1,5 +1,5 @@
#include <metal_stdlib>
-#include "../definitions.metal"
+#include "../common/dsl.h"
template <typename T>
VARIANTS(T, float, half, bfloat)
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/defines.h b/crates/uzu/src/backends/metal/kernel/matmul/common/defines.h
index 07ca0a072..4e4401177 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/common/defines.h
+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/defines.h
@@ -1,47 +1,43 @@
#pragma once
+#include "../../common/defines.h"
#include <metal_simdgroup>
#include <metal_simdgroup_matrix>
#include <metal_stdlib>
-#define UZU_MTL_CONST static constant constexpr
-#define UZU_PRAGMA_UNROLL _Pragma("clang loop unroll(full)")
-#define SIMD_SIZE 32
-
using namespace metal;
namespace uzu {
namespace matmul {
-// Pointer element type extraction
template <typename T>
-struct pointer_element_t_impl {
+struct PointerElementTypeImpl {
using type = T;
};
+
template <typename T>
-struct pointer_element_t_impl<device T*> {
+struct PointerElementTypeImpl<device T*> {
using type = T;
};
+
template <typename T>
-struct pointer_element_t_impl<const device T*> {
+struct PointerElementTypeImpl<const device T*> {
using type = T;
};
+
template <typename T>
-struct pointer_element_t_impl<threadgroup T*> {
+struct PointerElementTypeImpl<threadgroup T*> {
using type = T;
};
+
template <typename T>
-using pointer_element_t = typename pointer_element_t_impl<T>::type;
+using PointerElementType = typename PointerElementTypeImpl<T>::type;
-// Accumulator type helper
template <typename T>
-struct AccumHelper {
- typedef float accum_type;
+struct AccumulatorHelper {
+ typedef float AccumulatorType;
};
-// Transform operations for epilogue
-// Note: apply(x) is static for default store_result usage
-// apply(x, c) is non-static for epilogue operations with C matrix
template <typename OutT, typename InT>
struct TransformNone {
TransformNone(float = 1.0f, float = 0.0f) {}
@@ -61,11 +57,12 @@ struct TransformAdd {
};
template <typename OutT, typename InT>
-struct TransformAxpby {
+struct TransformScaleAccumulate {
const float alpha;
const float beta;
- TransformAxpby(float alpha_, float beta_) : alpha(alpha_), beta(beta_) {}
+ TransformScaleAccumulate(float alpha, float beta)
+ : alpha(alpha), beta(beta) {}
static METAL_FUNC OutT apply(InT x) { return static_cast<OutT>(x); }
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/gemm.h b/crates/uzu/src/backends/metal/kernel/matmul/common/gemm.h
index c90a562ac..8cee3d832 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/common/gemm.h
+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/gemm.h
@@ -1,8 +1,8 @@
#pragma once
#include "loader.h"
-#include "mma.h"
-#include "matmul.h"
+#include "threadgroup_tile.h"
+#include "../../../generated/matmul.h"
using namespace metal;
@@ -17,104 +17,111 @@ template <bool M_aligned, bool N_aligned, bool K_aligned>
struct LoopAlignment {};
///////////////////////////////////////////////////////////////////////////////
-// GEMM Kernel
+// ThreadgroupGemm Kernel
///////////////////////////////////////////////////////////////////////////////
template <
typename T,
typename U,
- int BM,
- int BN,
- int BK,
- int WM,
- int WN,
+ int BLOCK_ROWS,
+ int BLOCK_COLS,
+ int BLOCK_DEPTH,
+ int SIMDGROUPS_PER_ROW,
+ int SIMDGROUPS_PER_COLUMN,
bool transpose_a,
bool transpose_b,
bool MN_aligned,
bool K_aligned,
typename AccumType = float,
typename Epilogue = TransformNone<U, AccumType>>
-struct GEMMKernel {
- UZU_MTL_CONST short tgp_padding_a = 16 / sizeof(T);
- UZU_MTL_CONST short tgp_padding_b = 16 / sizeof(T);
- UZU_MTL_CONST short tgp_mem_size_a =
- transpose_a ? BK * (BM + tgp_padding_a) : BM * (BK + tgp_padding_a);
- UZU_MTL_CONST short tgp_mem_size_b =
- transpose_b ? BN * (BK + tgp_padding_b) : BK * (BN + tgp_padding_b);
- UZU_MTL_CONST short tgp_mem_size = tgp_mem_size_a + tgp_mem_size_b;
-
- UZU_MTL_CONST short tgp_size = WM * WN * 32;
-
- using loader_a_t = BlockLoader<
+struct ThreadgroupGemm {
+ METAL_CONST short THREADGROUP_PADDING_A = 16 / sizeof(T);
+ METAL_CONST short THREADGROUP_PADDING_B = 16 / sizeof(T);
+ METAL_CONST short THREADGROUP_MEMORY_SIZE_A =
+ transpose_a ? BLOCK_DEPTH * (BLOCK_ROWS + THREADGROUP_PADDING_A)
+ : BLOCK_ROWS * (BLOCK_DEPTH + THREADGROUP_PADDING_A);
+ METAL_CONST short THREADGROUP_MEMORY_SIZE_B =
+ transpose_b ? BLOCK_COLS * (BLOCK_DEPTH + THREADGROUP_PADDING_B)
+ : BLOCK_DEPTH * (BLOCK_COLS + THREADGROUP_PADDING_B);
+ METAL_CONST short THREADGROUP_MEMORY_SIZE =
+ THREADGROUP_MEMORY_SIZE_A + THREADGROUP_MEMORY_SIZE_B;
+
+ METAL_CONST short THREADGROUP_SIZE =
+ SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * 32;
+
+ using LoaderAType = ThreadgroupLoader<
T,
- transpose_a ? BK : BM,
- transpose_a ? BM : BK,
- transpose_a ? BM + tgp_padding_a : BK + tgp_padding_a,
+ transpose_a ? BLOCK_DEPTH : BLOCK_ROWS,
+ transpose_a ? BLOCK_ROWS : BLOCK_DEPTH,
+ transpose_a ? BLOCK_ROWS + THREADGROUP_PADDING_A
+ : BLOCK_DEPTH + THREADGROUP_PADDING_A,
!transpose_a,
- tgp_size>;
- using loader_b_t = BlockLoader<
+ THREADGROUP_SIZE>;
+ using LoaderBType = ThreadgroupLoader<
T,
- transpose_b ? BN : BK,
- transpose_b ? BK : BN,
- transpose_b ? BK + tgp_padding_b : BN + tgp_padding_b,
+ transpose_b ? BLOCK_COLS : BLOCK_DEPTH,
+ transpose_b ? BLOCK_DEPTH : BLOCK_COLS,
+ transpose_b ? BLOCK_DEPTH + THREADGROUP_PADDING_B
+ : BLOCK_COLS + THREADGROUP_PADDING_B,
transpose_b,
- tgp_size>;
- using mma_t = BlockMMA<
+ THREADGROUP_SIZE>;
+ using ThreadgroupTileType = ThreadgroupTile<
T,
U,
- BM,
- BN,
- BK,
- WM,
- WN,
+ BLOCK_ROWS,
+ BLOCK_COLS,
+ BLOCK_DEPTH,
+ SIMDGROUPS_PER_ROW,
+ SIMDGROUPS_PER_COLUMN,
transpose_a,
transpose_b,
- transpose_a ? BM + tgp_padding_a : BK + tgp_padding_a,
- transpose_b ? BK + tgp_padding_b : BN + tgp_padding_b,
+ transpose_a ? BLOCK_ROWS + THREADGROUP_PADDING_A
+ : BLOCK_DEPTH + THREADGROUP_PADDING_A,
+ transpose_b ? BLOCK_DEPTH + THREADGROUP_PADDING_B
+ : BLOCK_COLS + THREADGROUP_PADDING_B,
AccumType,
Epilogue>;
- /* Main kernel function */
template <bool M_aligned, bool N_aligned, bool K_aligned_>
static METAL_FUNC void gemm_loop(
- threadgroup T* As,
- threadgroup T* Bs,
+ threadgroup T* a_shared,
+ threadgroup T* b_shared,
const int gemm_k_iterations,
- thread loader_a_t& loader_a,
- thread loader_b_t& loader_b,
- thread mma_t& mma_op,
- thread const short& tgp_bm,
- thread const short& tgp_bn,
- thread const short& lbk,
- LoopAlignment<M_aligned, N_aligned, K_aligned_> l = {}
+ thread LoaderAType& loader_a,
+ thread LoaderBType& loader_b,
+ thread ThreadgroupTileType& threadgroup_tile,
+ thread const short& threadgroup_block_rows,
+ thread const short& threadgroup_block_cols,
+ thread const short& leftover_block_depth,
+ LoopAlignment<M_aligned, N_aligned, K_aligned_> alignment = {}
) {
- // Appease the compiler
- (void)l;
+ (void)alignment;
- short2 tile_dims_A = transpose_a ? short2(tgp_bm, BK) : short2(BK, tgp_bm);
- short2 tile_dims_B = transpose_b ? short2(BK, tgp_bn) : short2(tgp_bn, BK);
+ short2 tile_dimensions_a =
+ transpose_a ? short2(threadgroup_block_rows, BLOCK_DEPTH)
+ : short2(BLOCK_DEPTH, threadgroup_block_rows);
+ short2 tile_dimensions_b =
+ transpose_b ? short2(BLOCK_DEPTH, threadgroup_block_cols)
+ : short2(threadgroup_block_cols, BLOCK_DEPTH);
for (int k = 0; k < gemm_k_iterations; k++) {
threadgroup_barrier(mem_flags::mem_threadgroup);
- // Load elements into threadgroup
if (M_aligned) {
loader_a.load_unsafe();
} else {
- loader_a.load_safe(tile_dims_A);
+ loader_a.load_safe(tile_dimensions_a);
}
if (N_aligned) {
loader_b.load_unsafe();
} else {
- loader_b.load_safe(tile_dims_B);
+ loader_b.load_safe(tile_dimensions_b);
}
threadgroup_barrier(mem_flags::mem_threadgroup);
- // Multiply and accumulate threadgroup elements
- mma_op.mma(As, Bs);
+ threadgroup_tile.multiply_accumulate(a_shared, b_shared);
- // Prepare for next iteration
loader_a.next();
loader_b.next();
}
@@ -122,173 +129,200 @@ struct GEMMKernel {
if (!K_aligned_) {
threadgroup_barrier(mem_flags::mem_threadgroup);
- short2 tile_dims_A_last =
- transpose_a ? short2(tgp_bm, lbk) : short2(lbk, tgp_bm);
- short2 tile_dims_B_last =
- transpose_b ? short2(lbk, tgp_bn) : short2(tgp_bn, lbk);
+ short2 last_tile_dimensions_a =
+ transpose_a ? short2(threadgroup_block_rows, leftover_block_depth)
+ : short2(leftover_block_depth, threadgroup_block_rows);
+ short2 last_tile_dimensions_b =
+ transpose_b ? short2(leftover_block_depth, threadgroup_block_cols)
+ : short2(threadgroup_block_cols, leftover_block_depth);
- loader_a.load_safe(tile_dims_A_last);
- loader_b.load_safe(tile_dims_B_last);
+ loader_a.load_safe(last_tile_dimensions_a);
+ loader_b.load_safe(last_tile_dimensions_b);
threadgroup_barrier(mem_flags::mem_threadgroup);
- mma_op.mma(As, Bs);
+ threadgroup_tile.multiply_accumulate(a_shared, b_shared);
}
}
- /* Main kernel function */
static METAL_FUNC void run(
- const device T* A,
- const device T* B,
- device U* D,
- const constant GEMMParams* params,
- threadgroup T* As,
- threadgroup T* Bs,
+ const device T* a,
+ const device T* b,
+ device U* d,
+ const constant GemmParams* params,
+ threadgroup T* a_shared,
+ threadgroup T* b_shared,
uint simd_lane_id [[thread_index_in_simdgroup]],
uint simd_group_id [[simdgroup_index_in_threadgroup]],
uint2 tid [[threadgroup_position_in_grid]],
uint3 lid [[thread_position_in_threadgroup]]
) {
- // Pacifying compiler
(void)lid;
- const int tid_y = ((tid.y) << params->swizzle_log) +
- ((tid.x) & ((1 << params->swizzle_log) - 1));
- const int tid_x = (tid.x) >> params->swizzle_log;
+ const int swizzle_stride = pow2(params->swizzle_log);
+ const int swizzled_row_id =
+ tid.y * swizzle_stride + (tid.x % swizzle_stride);
+ const int swizzled_col_id = tid.x / swizzle_stride;
- if (params->tiles_n <= tid_x || params->tiles_m <= tid_y) {
+ if (params->threadgroups_per_row <= swizzled_col_id ||
+ params->threadgroups_per_column <= swizzled_row_id) {
return;
}
threadgroup_barrier(mem_flags::mem_none);
- // Find block in A, B, C
- const int c_row = tid_y * BM;
- const int c_col = tid_x * BN;
- const size_t c_row_long = size_t(c_row);
- const size_t c_col_long = size_t(c_col);
-
- A += transpose_a ? c_row_long : c_row_long * params->lda;
- B += transpose_b ? c_col_long * params->ldb : c_col_long;
- D += c_row_long * params->ldd + c_col_long;
-
- // Prepare threadgroup loading operations
- thread loader_a_t loader_a(A, params->lda, As, simd_group_id, simd_lane_id);
- thread loader_b_t loader_b(B, params->ldb, Bs, simd_group_id, simd_lane_id);
-
- // Prepare threadgroup mma operation
- thread mma_t mma_op(simd_group_id, simd_lane_id);
-
- int gemm_k_iterations = params->gemm_k_iterations_aligned;
+ const int output_row = swizzled_row_id * BLOCK_ROWS;
+ const int output_col = swizzled_col_id * BLOCK_COLS;
+ const size_t output_row_long = size_t(output_row);
+ const size_t output_col_long = size_t(output_col);
+
+ a += transpose_a ? output_row_long
+ : output_row_long * params->leading_dimension_a;
+ b += transpose_b ? output_col_long * params->leading_dimension_b
+ : output_col_long;
+ d += output_row_long * params->leading_dimension_d + output_col_long;
+
+ thread LoaderAType loader_a(
+ a,
+ params->leading_dimension_a,
+ a_shared,
+ simd_group_id,
+ simd_lane_id
+ );
+ thread LoaderBType loader_b(
+ b,
+ params->leading_dimension_b,
+ b_shared,
+ simd_group_id,
+ simd_lane_id
+ );
+
+ thread ThreadgroupTileType threadgroup_tile(simd_group_id, simd_lane_id);
+
+ int gemm_k_iterations = params->aligned_inner_iterations;
///////////////////////////////////////////////////////////////////////////
// MNK aligned loop
if (MN_aligned) {
for (int k = 0; k < gemm_k_iterations; k++) {
threadgroup_barrier(mem_flags::mem_threadgroup);
- // Load elements into threadgroup
loader_a.load_unsafe();
loader_b.load_unsafe();
threadgroup_barrier(mem_flags::mem_threadgroup);
- // Multiply and accumulate threadgroup elements
- mma_op.mma(As, Bs);
+ threadgroup_tile.multiply_accumulate(a_shared, b_shared);
- // Prepare for next iteration
loader_a.next();
loader_b.next();
}
threadgroup_barrier(mem_flags::mem_none);
- // Loop tail
if (!K_aligned) {
- int lbk = params->K - params->gemm_k_iterations_aligned * BK;
- short2 tile_dims_A = transpose_a ? short2(BM, lbk) : short2(lbk, BM);
- short2 tile_dims_B = transpose_b ? short2(lbk, BN) : short2(BN, lbk);
-
- loader_a.load_safe(tile_dims_A);
- loader_b.load_safe(tile_dims_B);
+ int leftover_block_depth =
+ params->K - params->aligned_inner_iterations * BLOCK_DEPTH;
+ short2 tile_dimensions_a =
+ transpose_a ? short2(BLOCK_ROWS, leftover_block_depth)
+ : short2(leftover_block_depth, BLOCK_ROWS);
+ short2 tile_dimensions_b =
+ transpose_b ? short2(leftover_block_depth, BLOCK_COLS)
+ : short2(BLOCK_COLS, leftover_block_depth);
+
+ loader_a.load_safe(tile_dimensions_a);
+ loader_b.load_safe(tile_dimensions_b);
threadgroup_barrier(mem_flags::mem_threadgroup);
- mma_op.mma(As, Bs);
+ threadgroup_tile.multiply_accumulate(a_shared, b_shared);
}
- // Store results to device memory
- mma_op.store_result(D, params->ldd);
+ threadgroup_tile.store_result(d, params->leading_dimension_d);
return;
}
///////////////////////////////////////////////////////////////////////////
// MN unaligned loop
- else { // Loop over K - unaligned case
- short tgp_bm = min(BM, params->M - c_row);
- short tgp_bn = min(BN, params->N - c_col);
- short leftover_bk = params->K - params->gemm_k_iterations_aligned * BK;
-
- if (tgp_bm == BM && tgp_bn == BN) {
+ else {
+ short threadgroup_block_rows = min(BLOCK_ROWS, params->M - output_row);
+ short threadgroup_block_cols = min(BLOCK_COLS, params->N - output_col);
+ short leftover_block_depth =
+ params->K - params->aligned_inner_iterations * BLOCK_DEPTH;
+
+ if (threadgroup_block_rows == BLOCK_ROWS &&
+ threadgroup_block_cols == BLOCK_COLS) {
gemm_loop<true, true, K_aligned>(
- As,
- Bs,
+ a_shared,
+ b_shared,
gemm_k_iterations,
loader_a,
loader_b,
- mma_op,
- tgp_bm,
- tgp_bn,
- leftover_bk
+ threadgroup_tile,
+ threadgroup_block_rows,
+ threadgroup_block_cols,
+ leftover_block_depth
);
- mma_op.store_result(D, params->ldd);
+ threadgroup_tile.store_result(d, params->leading_dimension_d);
return;
- } else if (tgp_bn == BN) {
+ } else if (threadgroup_block_cols == BLOCK_COLS) {
gemm_loop<false, true, K_aligned>(
- As,
- Bs,
+ a_shared,
+ b_shared,
gemm_k_iterations,
loader_a,
loader_b,
- mma_op,
- tgp_bm,
- tgp_bn,
- leftover_bk
+ threadgroup_tile,
+ threadgroup_block_rows,
+ threadgroup_block_cols,
+ leftover_block_depth
);
- mma_op.store_result_safe(D, params->ldd, short2(tgp_bn, tgp_bm));
+ threadgroup_tile.store_result_safe(
+ d,
+ params->leading_dimension_d,
+ short2(threadgroup_block_cols, threadgroup_block_rows)
+ );
return;
- } else if (tgp_bm == BM) {
+ } else if (threadgroup_block_rows == BLOCK_ROWS) {
gemm_loop<true, false, K_aligned>(
- As,
- Bs,
+ a_shared,
+ b_shared,
gemm_k_iterations,
loader_a,
loader_b,
- mma_op,
- tgp_bm,
- tgp_bn,
- leftover_bk
+ threadgroup_tile,
+ threadgroup_block_rows,
+ threadgroup_block_cols,
+ leftover_block_depth
);
- mma_op.store_result_safe(D, params->ldd, short2(tgp_bn, tgp_bm));
+ threadgroup_tile.store_result_safe(
+ d,
+ params->leading_dimension_d,
+ short2(threadgroup_block_cols, threadgroup_block_rows)
+ );
return;
} else {
gemm_loop<false, false, K_aligned>(
- As,
- Bs,
+ a_shared,
+ b_shared,
gemm_k_iterations,
loader_a,
loader_b,
- mma_op,
- tgp_bm,
- tgp_bn,
- leftover_bk
+ threadgroup_tile,
+ threadgroup_block_rows,
+ threadgroup_block_cols,
+ leftover_block_depth
);
- mma_op.store_result_safe(D, params->ldd, short2(tgp_bn, tgp_bm));
+ threadgroup_tile.store_result_safe(
+ d,
+ params->leading_dimension_d,
+ short2(threadgroup_block_cols, threadgroup_block_rows)
+ );
return;
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/loader.h b/crates/uzu/src/backends/metal/kernel/matmul/common/loader.h
index 885d536a9..33e45d2c8 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/common/loader.h
+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/loader.h
@@ -8,125 +8,127 @@ namespace uzu {
namespace matmul {
///////////////////////////////////////////////////////////////////////////////
-// Block Loader - loads tiles from device memory to threadgroup memory
+// Threadgroup Loader - loads tiles from device memory to threadgroup memory
///////////////////////////////////////////////////////////////////////////////
template <
typename T,
- short BROWS,
- short BCOLS,
- short dst_ld,
- short reduction_dim,
- short tgp_size,
- short alignment = 1,
- short n_reads = (BCOLS * BROWS) / (tgp_size),
- short TCOLS = BCOLS / n_reads,
- short TROWS = tgp_size / TCOLS>
-struct BlockLoader {
- UZU_MTL_CONST short n_rows = (BROWS + TROWS - 1) / TROWS;
- UZU_MTL_CONST short vec_size = n_reads;
-
- // Leading dimension for src
- const int src_ld;
+ short BLOCK_ROWS,
+ short BLOCK_COLS,
+ short DESTINATION_LEADING_DIMENSION,
+ short REDUCTION_DIMENSION,
+ short THREADGROUP_SIZE,
+ short ALIGNMENT = 1,
+ short READS_PER_THREAD = (BLOCK_COLS * BLOCK_ROWS) / (THREADGROUP_SIZE),
+ short THREAD_COLS = BLOCK_COLS / READS_PER_THREAD,
+ short THREAD_ROWS = THREADGROUP_SIZE / THREAD_COLS>
+struct ThreadgroupLoader {
+ METAL_CONST short ROW_ITERATIONS =
+ (BLOCK_ROWS + THREAD_ROWS - 1) / THREAD_ROWS;
+ METAL_CONST short VECTOR_SIZE = READS_PER_THREAD;
+
+ const int source_leading_dimension;
const int tile_stride;
- // Thread location indices
- const short thread_idx;
- const short bi;
- const short bj;
+ const short thread_index;
+ const short block_row_index;
+ const short block_col_index;
- // threadgroup and device memory
- threadgroup T* dst;
- const device T* src;
+ threadgroup T* destination;
+ const device T* source;
- struct alignas(alignment * sizeof(T)) ReadVector {
- uint8_t v[sizeof(T) * vec_size];
+ struct alignas(ALIGNMENT * sizeof(T)) ReadVector {
+ uint8_t bytes[sizeof(T) * VECTOR_SIZE];
};
- /* Constructor */
- METAL_FUNC BlockLoader(
- const device T* src_,
- const int src_ld_,
- threadgroup T* dst_,
+ METAL_FUNC ThreadgroupLoader(
+ const device T* source_ptr,
+ const int source_leading_dim,
+ threadgroup T* destination_ptr,
ushort simd_group_id [[simdgroup_index_in_threadgroup]],
ushort simd_lane_id [[thread_index_in_simdgroup]]
)
- : src_ld(src_ld_), tile_stride(reduction_dim ? BCOLS : BROWS * src_ld),
- thread_idx(simd_group_id * 32 + simd_lane_id), bi(thread_idx / TCOLS),
- bj(vec_size * (thread_idx % TCOLS)), dst(dst_ + bi * dst_ld + bj),
- src(src_ + bi * src_ld + bj) {}
+ : source_leading_dimension(source_leading_dim),
+ tile_stride(
+ REDUCTION_DIMENSION ? BLOCK_COLS : BLOCK_ROWS * source_leading_dim
+ ),
+ thread_index(simd_group_id * 32 + simd_lane_id),
+ block_row_index(thread_index / THREAD_COLS),
+ block_col_index(VECTOR_SIZE * (thread_index % THREAD_COLS)),
+ destination(
+ destination_ptr + block_row_index * DESTINATION_LEADING_DIMENSION +
+ block_col_index
+ ),
+ source(
+ source_ptr + block_row_index * source_leading_dim + block_col_index
+ ) {}
- /* Apply operation to threadgroup without bound checking */
template <typename UnaryOp>
- METAL_FUNC void apply_inplace_op(thread const UnaryOp& op) const {
- UZU_PRAGMA_UNROLL
- for (short i = 0; i < BROWS; i += TROWS) {
- UZU_PRAGMA_UNROLL
- for (short j = 0; j < vec_size; j++) {
- dst[i * dst_ld + j] = op.apply(dst[i * dst_ld + j]);
+ METAL_FUNC void apply_inplace_op(thread const UnaryOp& operation) const {
+ METAL_PRAGMA_UNROLL
+ for (short i = 0; i < BLOCK_ROWS; i += THREAD_ROWS) {
+ METAL_PRAGMA_UNROLL
+ for (short j = 0; j < VECTOR_SIZE; j++) {
+ destination[i * DESTINATION_LEADING_DIMENSION + j] =
+ operation.apply(destination[i * DESTINATION_LEADING_DIMENSION + j]);
}
}
}
- /* Load from device memory into threadgroup memory - without bound checking */
METAL_FUNC void load_unsafe() const {
- UZU_PRAGMA_UNROLL
- for (short i = 0; i < BROWS; i += TROWS) {
- *((threadgroup ReadVector*)(&dst[i * dst_ld])) =
- *((const device ReadVector*)(&src[i * src_ld]));
+ METAL_PRAGMA_UNROLL
+ for (short i = 0; i < BLOCK_ROWS; i += THREAD_ROWS) {
+ *((threadgroup ReadVector*)(&destination
+ [i * DESTINATION_LEADING_DIMENSION])) =
+ *((const device ReadVector*)(&source[i * source_leading_dimension]));
}
}
- /* Load from device memory into threadgroup memory - with bound checking */
- METAL_FUNC void load_safe(short2 src_tile_dim) const {
- src_tile_dim = src_tile_dim - short2(bj, bi);
-
- // Skip loading if thread has no valid reads
- if (src_tile_dim.x <= 0 || src_tile_dim.y <= 0) {
- UZU_PRAGMA_UNROLL
- for (short i = 0; i < BROWS; i += TROWS) {
- UZU_PRAGMA_UNROLL
- for (short j = 0; j < vec_size; j++) {
- dst[i * dst_ld + j] = T(0);
+ METAL_FUNC void load_safe(short2 source_tile_dimensions) const {
+ source_tile_dimensions =
+ source_tile_dimensions - short2(block_col_index, block_row_index);
+
+ if (source_tile_dimensions.x <= 0 || source_tile_dimensions.y <= 0) {
+ METAL_PRAGMA_UNROLL
+ for (short i = 0; i < BLOCK_ROWS; i += THREAD_ROWS) {
+ METAL_PRAGMA_UNROLL
+ for (short j = 0; j < VECTOR_SIZE; j++) {
+ destination[i * DESTINATION_LEADING_DIMENSION + j] = T(0);
}
}
return;
}
- // Use fast thread memory for bound checks
- bool tmp_idx[vec_size];
- T tmp_val[vec_size];
+ bool valid_mask[VECTOR_SIZE];
+ T loaded_values[VECTOR_SIZE];
- UZU_PRAGMA_UNROLL
- for (short i = 0; i < BROWS; i += TROWS) {
- // Make sure tmp_idx only contains valid indices
- UZU_PRAGMA_UNROLL
- for (short j = 0; j < vec_size; j++) {
- tmp_idx[j] = (i < src_tile_dim.y) && (j < src_tile_dim.x);
+ METAL_PRAGMA_UNROLL
+ for (short i = 0; i < BLOCK_ROWS; i += THREAD_ROWS) {
+ METAL_PRAGMA_UNROLL
+ for (short j = 0; j < VECTOR_SIZE; j++) {
+ valid_mask[j] =
+ (i < source_tile_dimensions.y) && (j < source_tile_dimensions.x);
}
- // Read valid indices into tmp_val
- UZU_PRAGMA_UNROLL
- for (short j = 0; j < vec_size; j++) {
- tmp_val[j] = src[(tmp_idx[j] ? i * src_ld + j : 0)];
+ METAL_PRAGMA_UNROLL
+ for (short j = 0; j < VECTOR_SIZE; j++) {
+ loaded_values[j] =
+ source[(valid_mask[j] ? i * source_leading_dimension + j : 0)];
}
- // Zero out unneeded values
- UZU_PRAGMA_UNROLL
- for (short j = 0; j < vec_size; j++) {
- tmp_val[j] = tmp_idx[j] ? tmp_val[j] : T(0);
+ METAL_PRAGMA_UNROLL
+ for (short j = 0; j < VECTOR_SIZE; j++) {
+ loaded_values[j] = valid_mask[j] ? loaded_values[j] : T(0);
}
- // Copy values to threadgroup memory
- UZU_PRAGMA_UNROLL
- for (short j = 0; j < vec_size; j++) {
- dst[i * dst_ld + j] = tmp_val[j];
+ METAL_PRAGMA_UNROLL
+ for (short j = 0; j < VECTOR_SIZE; j++) {
+ destination[i * DESTINATION_LEADING_DIMENSION + j] = loaded_values[j];
}
}
}
- /* Iteration helper */
- METAL_FUNC void next() { src += tile_stride; }
+ METAL_FUNC void next() { source += tile_stride; }
};
} // namespace matmul
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/matmul.h b/crates/uzu/src/backends/metal/kernel/matmul/common/matmul.h
deleted file mode 100644
index 7386d512f..000000000
--- a/crates/uzu/src/backends/metal/kernel/matmul/common/matmul.h
+++ /dev/null
@@ -1,9 +0,0 @@
-#pragma once
-
-#include "steel/gemm/params.h"
-
-namespace uzu {
-namespace matmul {
-using GEMMParams = steel::GEMMParams;
-} // namespace matmul
-} // namespace uzu
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/mma.h b/crates/uzu/src/backends/metal/kernel/matmul/common/mma.h
deleted file mode 100644
index 741ef4394..000000000
--- a/crates/uzu/src/backends/metal/kernel/matmul/common/mma.h
+++ /dev/null
@@ -1,524 +0,0 @@
-#pragma once
-
-#include "defines.h"
-
-using namespace metal;
-
-namespace uzu {
-namespace matmul {
-
-///////////////////////////////////////////////////////////////////////////////
-// MMA Fragment - 8x8 simdgroup matrix fragment
-///////////////////////////////////////////////////////////////////////////////
-
-template <typename T, int kFragRows_, int kFragCols_>
-struct BaseMMAFrag {
- static_assert(
- kFragRows_ == 8,
- "Only 8 x 8 fragment matrices are currently supported"
- );
- static_assert(
- kFragCols_ == 8,
- "Only 8 x 8 fragment matrices are currently supported"
- );
-};
-
-template <typename T>
-struct BaseMMAFrag<T, 8, 8> {
- UZU_MTL_CONST int kFragRows = 8;
- UZU_MTL_CONST int kFragCols = 8;
- UZU_MTL_CONST int kElemsPerFrag = (kFragRows * kFragCols) / 32;
- UZU_MTL_CONST int kElemRows = 1;
- UZU_MTL_CONST int kElemCols = 2;
-
- static_assert(
- kElemRows * kElemCols == kElemsPerFrag,
- "MMAFrag shape is not consistent with MMAFrag size"
- );
-
- typedef metal::simdgroup_matrix<T, kFragRows, kFragCols> mat_type;
- typedef metal::vec<T, kElemsPerFrag> frag_type;
-
- METAL_FUNC static constexpr short2 get_coord(
- ushort simd_lane_id [[thread_index_in_simdgroup]]
- ) {
- const short qid = simd_lane_id / 4;
- const short fm = (qid & 4) + ((simd_lane_id / 2) % 4);
- const short fn = (qid & 2) * 2 + (simd_lane_id % 2) * 2;
- return short2{fn, fm};
- }
-
- template <typename SrcPtrType, typename StrX, typename StrY>
- METAL_FUNC static constexpr void load(
- thread frag_type& dst,
- SrcPtrType src,
- StrX str_x,
- StrY str_y
- ) {
- UZU_PRAGMA_UNROLL
- for (short i = 0; i < kElemRows; i++) {
- UZU_PRAGMA_UNROLL
- for (short j = 0; j < kElemCols; j++) {
- dst[i * kElemCols + j] = static_cast<T>(src[i * str_x + j * str_y]);
- }
- }
- }
-
- template <
- typename SrcPtrType,
- typename StrX,
- typename StrY,
- typename LimX,
- typename LimY,
- typename OffX,
- typename OffY>
- METAL_FUNC static constexpr void load_safe(
- thread frag_type& dst,
- SrcPtrType src,
- StrX str_x,
- StrY str_y,
- LimX lim_x,
- LimY lim_y,
- OffX off_x = 0,
- OffY off_y = 0
- ) {
- UZU_PRAGMA_UNROLL
- for (short i = 0; i < kElemRows; i++) {
- UZU_PRAGMA_UNROLL
- for (short j = 0; j < kElemCols; j++) {
- if ((off_x + i) < lim_x && (off_y + j) < lim_y) {
- dst[i * kElemCols + j] =
- static_cast<T>(src[(off_x + i) * str_x + (off_y + j) * str_y]);
- } else {
- dst[i * kElemCols + j] = T(0);
- }
- }
- }
- }
-
- template <typename DstPtrType, typename StrX, typename StrY>
- METAL_FUNC static constexpr void store(
- const thread frag_type& src,
- DstPtrType dst,
- StrX str_x,
- StrY str_y
- ) {
- using U = pointer_element_t<DstPtrType>;
-
- UZU_PRAGMA_UNROLL
- for (short i = 0; i < kElemRows; i++) {
- UZU_PRAGMA_UNROLL
- for (short j = 0; j < kElemCols; j++) {
- dst[i * str_x + j * str_y] = static_cast<U>(src[i * kElemCols + j]);
- }
- }
- }
-
- template <
- typename DstPtrType,
- typename StrX,
- typename StrY,
- typename LimX,
- typename LimY,
- typename OffX,
- typename OffY>
- METAL_FUNC static constexpr void store_safe(
- const thread frag_type& src,
- DstPtrType dst,
- StrX str_x,
- StrY str_y,
- LimX lim_x,
- LimY lim_y,
- OffX off_x = 0,
- OffY off_y = 0
- ) {
- using U = pointer_element_t<DstPtrType>;
-
- UZU_PRAGMA_UNROLL
- for (short i = 0; i < kElemRows; i++) {
- UZU_PRAGMA_UNROLL
- for (short j = 0; j < kElemCols; j++) {
- if ((off_x + i) < lim_x && (off_y + j) < lim_y) {
- dst[(off_x + i) * str_x + (off_y + j) * str_y] =
- static_cast<U>(src[i * kElemCols + j]);
- }
- }
- }
- }
-
- METAL_FUNC static constexpr void mma(
- thread frag_type& D,
- thread frag_type& A,
- thread frag_type& B,
- thread frag_type& C
- ) {
- mat_type D_mat;
- mat_type A_mat;
- mat_type B_mat;
- mat_type C_mat;
-
- reinterpret_cast<thread frag_type&>(A_mat.thread_elements()) = A;
- reinterpret_cast<thread frag_type&>(B_mat.thread_elements()) = B;
- reinterpret_cast<thread frag_type&>(C_mat.thread_elements()) = C;
-
- mma(D_mat, A_mat, B_mat, C_mat);
-
- D = reinterpret_cast<thread frag_type&>(D_mat.thread_elements());
- }
-
- METAL_FUNC static constexpr void mma(
- thread mat_type& D,
- thread mat_type& A,
- thread mat_type& B,
- thread mat_type& C
- ) {
- simdgroup_multiply_accumulate(D, A, B, C);
- }
-};
-
-///////////////////////////////////////////////////////////////////////////////
-// MMA Tile - collection of MMA fragments
-///////////////////////////////////////////////////////////////////////////////
-
-template <
- typename T,
- int kTileRows_,
- int kTileCols_,
- class MMAFrag_ = BaseMMAFrag<T, 8, 8>>
-struct MMATile {
- using MMAFrag_t = MMAFrag_;
- using elem_type = T;
- UZU_MTL_CONST int kFragRows = MMAFrag_t::kFragRows;
- UZU_MTL_CONST int kFragCols = MMAFrag_t::kFragCols;
- UZU_MTL_CONST int kElemsPerFrag = MMAFrag_t::kElemsPerFrag;
-
- UZU_MTL_CONST int kTileRows = kTileRows_;
- UZU_MTL_CONST int kTileCols = kTileCols_;
-
- UZU_MTL_CONST int kRows = kTileRows * kFragRows;
- UZU_MTL_CONST int kCols = kTileCols * kFragCols;
-
- UZU_MTL_CONST int kNumFrags = kTileRows * kTileCols;
- UZU_MTL_CONST int kElemsPerTile = kNumFrags * kElemsPerFrag;
-
- typedef typename MMAFrag_t::mat_type mat_type;
- typedef typename MMAFrag_t::frag_type frag_type;
-
- frag_type val_frags[kNumFrags] = {frag_type(0)};
-
- METAL_FUNC MMATile() thread {}
-
- METAL_FUNC constexpr void clear() {
- UZU_PRAGMA_UNROLL
- for (short i = 0; i < kNumFrags; ++i) {
- val_frags[i] = frag_type(0);
- }
- }
-
- METAL_FUNC constexpr thread frag_type& frag_at(const short i, const short j) {
- return val_frags[i * kTileCols + j];
- }
-
- METAL_FUNC constexpr const thread frag_type& frag_at(
- const short i,
- const short j
- ) const {
- return val_frags[i * kTileCols + j];
- }
-
- METAL_FUNC thread elem_type* elems() {
- return reinterpret_cast<thread elem_type*>(val_frags);
- }
-
- METAL_FUNC const thread elem_type* elems() const {
- return reinterpret_cast<const thread elem_type*>(val_frags);
- }
-
- template <typename U, int w_x, int w_y, int str_x, int str_y>
- METAL_FUNC void load(const threadgroup U* src) {
- UZU_PRAGMA_UNROLL
- for (short i = 0; i < kTileRows; ++i) {
- UZU_PRAGMA_UNROLL
- for (short j = 0; j < kTileCols; ++j) {
- MMAFrag_t::load(
- frag_at(i, j),
- &(
- src[(i * kFragRows) * w_x * str_x +
- (j * kFragCols) * w_y * str_y]
- ),
- str_x,
- str_y
- );
- }
- }
- }
-
- template <typename U, int w_x, int w_y>
- METAL_FUNC void store(device U* dst, const int ld) const {
- UZU_PRAGMA_UNROLL
- for (short i = 0; i < kTileRows; ++i) {
- UZU_PRAGMA_UNROLL
- for (short j = 0; j < kTileCols; ++j) {
- MMAFrag_t::store(
- frag_at(i, j),
- &(dst[(i * kFragRows) * w_x * ld + (j * kFragCols) * w_y]),
- ld,
- 1
- );
- }
- }
- }
-
- template <typename U, int w_x, int w_y>
- METAL_FUNC void store_safe(
- device U* dst,
- const int ld,
- const short2 dst_tile_dims
- ) const {
- UZU_PRAGMA_UNROLL
- for (int i = 0; i < kTileRows; ++i) {
- UZU_PRAGMA_UNROLL
- for (int j = 0; j < kTileCols; ++j) {
- MMAFrag_t::store_safe(
- frag_at(i, j),
- dst,
- ld,
- 1,
- dst_tile_dims.y,
- dst_tile_dims.x,
- (i * kFragRows) * w_x,
- (j * kFragCols) * w_y
- );
- }
- }
- }
-};
-
-///////////////////////////////////////////////////////////////////////////////
-// Tile matrix multiply-accumulate
-///////////////////////////////////////////////////////////////////////////////
-
-template <typename T, typename U, int M, int N, int K>
-METAL_FUNC void tile_matmad(
- thread MMATile<T, M, N>& D,
- thread MMATile<U, M, K>& A,
- thread MMATile<U, K, N>& B,
- thread MMATile<T, M, N>& C
-) {
- UZU_PRAGMA_UNROLL
- for (short m = 0; m < M; ++m) {
- UZU_PRAGMA_UNROLL
- for (short n = 0; n < N; ++n) {
- short n_serp = (m % 2) ? (N - 1 - n) : n;
- UZU_PRAGMA_UNROLL
- for (short k = 0; k < K; ++k) {
- MMATile<T, M, N>::MMAFrag_t::mma(
- D.frag_at(m, n_serp),
- A.frag_at(m, k),
- B.frag_at(k, n_serp),
- C.frag_at(m, n_serp)
- );
- }
- }
- }
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// Block MMA - manages the GEMM computation for a threadgroup
-///////////////////////////////////////////////////////////////////////////////
-
-template <
- typename T,
- typename U,
- int BM,
- int BN,
- int BK,
- int WM,
- int WN,
- bool transpose_a,
- bool transpose_b,
- short lda_tgp,
- short ldb_tgp,
- typename AccumType = float,
- typename Epilogue = TransformNone<U, AccumType>>
-struct BlockMMA {
- // MMAFrag size
- UZU_MTL_CONST short kFragSize = 8;
- using MMAFrag_acc_t = BaseMMAFrag<AccumType, kFragSize, kFragSize>;
-
- // Warp tile simdgroup matrix strides along M
- UZU_MTL_CONST short TM_stride = kFragSize * WM;
- // Warp tile simdgroup matrix strides along N
- UZU_MTL_CONST short TN_stride = kFragSize * WN;
-
- // Warp tile size along M
- UZU_MTL_CONST short TM = BM / (kFragSize * WM);
- // Warp tile size along N
- UZU_MTL_CONST short TN = BN / (kFragSize * WN);
-
- // Threadgroup A strides
- UZU_MTL_CONST short A_str_m = transpose_a ? 1 : lda_tgp; // M
- UZU_MTL_CONST short A_str_k = transpose_a ? lda_tgp : 1; // K
-
- // Threadgroup B strides
- UZU_MTL_CONST short B_str_k = transpose_b ? 1 : ldb_tgp; // K
- UZU_MTL_CONST short B_str_n = transpose_b ? ldb_tgp : 1; // N
-
- // Threadgroup strides along K
- UZU_MTL_CONST short tile_stride_a = kFragSize * A_str_k;
- UZU_MTL_CONST short tile_stride_b = kFragSize * B_str_k;
-
- // Simdgroup matrices
- MMATile<AccumType, TM, 1, MMAFrag_acc_t> Atile;
- MMATile<AccumType, 1, TN, MMAFrag_acc_t> Btile;
- MMATile<AccumType, TM, TN, MMAFrag_acc_t> Ctile;
-
- // Offsets within threadgroup
- short sm;
- short sn;
-
- short As_offset;
- short Bs_offset;
-
- /* Constructor */
- METAL_FUNC BlockMMA(
- ushort simd_group_id [[simdgroup_index_in_threadgroup]],
- ushort simd_lane_id [[thread_index_in_simdgroup]]
- ) {
- // Determine thread position in simdgroup matrix
- short tm = kFragSize * (simd_group_id / WN);
- short tn = kFragSize * (simd_group_id % WN);
-
- short2 simd_coord = MMAFrag_acc_t::get_coord(simd_lane_id);
- sm = simd_coord.y;
- sn = simd_coord.x;
-
- // Determine thread and simdgroup offset
- As_offset = (tm + sm) * A_str_m + (sn)*A_str_k; // M, K
- Bs_offset = (sm)*B_str_k + (tn + sn) * B_str_n; // K, N
-
- sm += tm;
- sn += tn;
- }
-
- /* (BM, BK) X (BK, BN) multiply accumulate function */
- METAL_FUNC void mma(const threadgroup T* As, const threadgroup T* Bs) {
- // Adjust for simdgroup and thread location
- As += As_offset;
- Bs += Bs_offset;
-
- // Iterate over BK in blocks of kFragSize
- UZU_PRAGMA_UNROLL
- for (short kk = 0; kk < BK; kk += kFragSize) {
- simdgroup_barrier(mem_flags::mem_none);
-
- Atile.template load<T, WM, 1, A_str_m, A_str_k>(As);
-
- simdgroup_barrier(mem_flags::mem_none);
-
- Btile.template load<T, 1, WN, B_str_k, B_str_n>(Bs);
-
- simdgroup_barrier(mem_flags::mem_none);
-
- tile_matmad(Ctile, Atile, Btile, Ctile);
-
- // Progress to next simdgroup tile
- As += tile_stride_a;
- Bs += tile_stride_b;
- }
- }
-
- /* Store results from simdgroup_matrix results into device memory */
- METAL_FUNC void store_result(device U* D, const int ldd) {
- // Apply epilogue
- UZU_PRAGMA_UNROLL
- for (short i = 0; i < decltype(Ctile)::kElemsPerTile; i++) {
- Ctile.elems()[i] = Epilogue::apply(Ctile.elems()[i]);
- }
-
- // Adjust for simdgroup and thread location
- D += sm * ldd + sn;
-
- Ctile.template store<U, WM, WN>(D, ldd);
- }
-
- METAL_FUNC void store_result_safe(
- device U* D,
- const int ldd,
- short2 dst_tile_dims
- ) {
- // Apply epilogue
- UZU_PRAGMA_UNROLL
- for (short i = 0; i < decltype(Ctile)::kElemsPerTile; i++) {
- Ctile.elems()[i] = Epilogue::apply(Ctile.elems()[i]);
- }
-
- // Adjust for simdgroup and thread location
- D += sm * ldd + sn;
- dst_tile_dims -= short2(sn, sm);
-
- if (dst_tile_dims.x <= 0 || dst_tile_dims.y <= 0)
- return;
-
- Ctile.template store_safe<U, WM, WN>(D, ldd, dst_tile_dims);
- }
-
- /* Apply epilogue with C matrix */
- template <typename EpilogueOp>
- METAL_FUNC void apply_epilogue(
- const device U* C,
- const int ldc,
- const int fdc,
- thread const EpilogueOp& epilogue_op
- ) {
- const device U* C_ptr = C + sm * ldc + sn * fdc;
-
- UZU_PRAGMA_UNROLL
- for (short i = 0; i < TM; i++) {
- UZU_PRAGMA_UNROLL
- for (short j = 0; j < TN; j++) {
- thread auto& frag = Ctile.frag_at(i, j);
- UZU_PRAGMA_UNROLL
- for (short k = 0; k < decltype(Ctile)::kElemsPerFrag; k++) {
- short row_offset = (i * kFragSize * WM);
- short col_offset = (j * kFragSize * WN);
- // Get the C element (accounting for fragment layout)
- U c_val = C_ptr[row_offset * ldc + col_offset * fdc + k];
- frag[k] = epilogue_op.apply(frag[k], static_cast<AccumType>(c_val));
- }
- }
- }
- }
-
- template <typename EpilogueOp>
- METAL_FUNC void apply_epilogue_safe(
- const device U* C,
- const int ldc,
- const int fdc,
- short2 tile_dims,
- thread const EpilogueOp& epilogue_op
- ) {
- const device U* C_ptr = C + sm * ldc + sn * fdc;
- tile_dims -= short2(sn, sm);
-
- UZU_PRAGMA_UNROLL
- for (short i = 0; i < TM; i++) {
- UZU_PRAGMA_UNROLL
- for (short j = 0; j < TN; j++) {
- thread auto& frag = Ctile.frag_at(i, j);
- short row_offset = (i * kFragSize * WM);
- short col_offset = (j * kFragSize * WN);
- UZU_PRAGMA_UNROLL
- for (short k = 0; k < decltype(Ctile)::kElemsPerFrag; k++) {
- if (row_offset < tile_dims.y && col_offset + k < tile_dims.x) {
- U c_val = C_ptr[row_offset * ldc + col_offset * fdc + k];
- frag[k] = epilogue_op.apply(frag[k], static_cast<AccumType>(c_val));
- }
- }
- }
- }
- }
-};
-
-} // namespace matmul
-} // namespace uzu
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment.h b/crates/uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment.h
new file mode 100644
index 000000000..5c8dafae6
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment.h
@@ -0,0 +1,184 @@
+#pragma once
+
+#include "simdgroup_multiply_accumulate.h"
+
+using namespace metal;
+
+namespace uzu {
+namespace matmul {
+
+///////////////////////////////////////////////////////////////////////////////
+// SimdgroupFragment - thread's portion of simdgroup tile data
+///////////////////////////////////////////////////////////////////////////////
+
+template <
+ typename T,
+ int GRID_ROWS_,
+ int GRID_COLS_,
+ class SimdgroupMultiplyAccumulateOps_ =
+ SimdgroupMultiplyAccumulate<T, 8, 8>>
+struct SimdgroupFragment {
+ using SimdgroupMultiplyAccumulateOpsType = SimdgroupMultiplyAccumulateOps_;
+ using ElementType = T;
+ METAL_CONST int MULTIPLY_ACCUMULATE_ROWS =
+ SimdgroupMultiplyAccumulateOpsType::ROWS;
+ METAL_CONST int MULTIPLY_ACCUMULATE_COLS =
+ SimdgroupMultiplyAccumulateOpsType::COLS;
+ METAL_CONST int ELEMENTS_PER_MULTIPLY_ACCUMULATE =
+ SimdgroupMultiplyAccumulateOpsType::THREAD_ELEMENT_COUNT;
+
+ METAL_CONST int GRID_ROWS = GRID_ROWS_;
+ METAL_CONST int GRID_COLS = GRID_COLS_;
+
+ METAL_CONST int TOTAL_ROWS = GRID_ROWS * MULTIPLY_ACCUMULATE_ROWS;
+ METAL_CONST int TOTAL_COLS = GRID_COLS * MULTIPLY_ACCUMULATE_COLS;
+
+ METAL_CONST int MULTIPLY_ACCUMULATE_COUNT = GRID_ROWS * GRID_COLS;
+ METAL_CONST int ELEMENTS_PER_FRAGMENT =
+ MULTIPLY_ACCUMULATE_COUNT * ELEMENTS_PER_MULTIPLY_ACCUMULATE;
+
+ typedef typename SimdgroupMultiplyAccumulateOpsType::SimdgroupMatrixType
+ SimdgroupMatrixType;
+ typedef typename SimdgroupMultiplyAccumulateOpsType::ThreadDataType
+ ThreadDataType;
+
+ ThreadDataType multiply_accumulate_data[MULTIPLY_ACCUMULATE_COUNT] = {
+ ThreadDataType(0)
+ };
+
+ METAL_FUNC SimdgroupFragment() thread {}
+
+ METAL_FUNC constexpr void clear() {
+ METAL_PRAGMA_UNROLL
+ for (short i = 0; i < MULTIPLY_ACCUMULATE_COUNT; ++i) {
+ multiply_accumulate_data[i] = ThreadDataType(0);
+ }
+ }
+
+ METAL_FUNC constexpr thread ThreadDataType& multiply_accumulate_at(
+ const short i,
+ const short j
+ ) {
+ return multiply_accumulate_data[i * GRID_COLS + j];
+ }
+
+ METAL_FUNC constexpr const thread ThreadDataType& multiply_accumulate_at(
+ const short i,
+ const short j
+ ) const {
+ return multiply_accumulate_data[i * GRID_COLS + j];
+ }
+
+ METAL_FUNC thread ElementType* elements() {
+ return reinterpret_cast<thread ElementType*>(multiply_accumulate_data);
+ }
+
+ METAL_FUNC const thread ElementType* elements() const {
+ return reinterpret_cast<const thread ElementType*>(
+ multiply_accumulate_data
+ );
+ }
+
+ template <
+ typename U,
+ int SIMDGROUP_STRIDE_X,
+ int SIMDGROUP_STRIDE_Y,
+ int STRIDE_X,
+ int STRIDE_Y>
+ METAL_FUNC void load(const threadgroup U* src) {
+ METAL_PRAGMA_UNROLL
+ for (short i = 0; i < GRID_ROWS; ++i) {
+ METAL_PRAGMA_UNROLL
+ for (short j = 0; j < GRID_COLS; ++j) {
+ SimdgroupMultiplyAccumulateOpsType::load(
+ multiply_accumulate_at(i, j),
+ &(
+ src[(i * MULTIPLY_ACCUMULATE_ROWS) * SIMDGROUP_STRIDE_X *
+ STRIDE_X +
+ (j * MULTIPLY_ACCUMULATE_COLS) * SIMDGROUP_STRIDE_Y *
+ STRIDE_Y]
+ ),
+ STRIDE_X,
+ STRIDE_Y
+ );
+ }
+ }
+ }
+
+ template <typename U, int SIMDGROUP_STRIDE_X, int SIMDGROUP_STRIDE_Y>
+ METAL_FUNC void store(device U* dst, const int leading_dimension) const {
+ METAL_PRAGMA_UNROLL
+ for (short i = 0; i < GRID_ROWS; ++i) {
+ METAL_PRAGMA_UNROLL
+ for (short j = 0; j < GRID_COLS; ++j) {
+ SimdgroupMultiplyAccumulateOpsType::store(
+ multiply_accumulate_at(i, j),
+ &(
+ dst[(i * MULTIPLY_ACCUMULATE_ROWS) * SIMDGROUP_STRIDE_X *
+ leading_dimension +
+ (j * MULTIPLY_ACCUMULATE_COLS) * SIMDGROUP_STRIDE_Y]
+ ),
+ leading_dimension,
+ 1
+ );
+ }
+ }
+ }
+
+ template <typename U, int SIMDGROUP_STRIDE_X, int SIMDGROUP_STRIDE_Y>
+ METAL_FUNC void store_safe(
+ device U* dst,
+ const int leading_dimension,
+ const short2 destination_tile_dimensions
+ ) const {
+ METAL_PRAGMA_UNROLL
+ for (int i = 0; i < GRID_ROWS; ++i) {
+ METAL_PRAGMA_UNROLL
+ for (int j = 0; j < GRID_COLS; ++j) {
+ SimdgroupMultiplyAccumulateOpsType::store_safe(
+ multiply_accumulate_at(i, j),
+ dst,
+ leading_dimension,
+ 1,
+ destination_tile_dimensions.y,
+ destination_tile_dimensions.x,
+ (i * MULTIPLY_ACCUMULATE_ROWS) * SIMDGROUP_STRIDE_X,
+ (j * MULTIPLY_ACCUMULATE_COLS) * SIMDGROUP_STRIDE_Y
+ );
+ }
+ }
+ }
+};
+
+///////////////////////////////////////////////////////////////////////////////
+// Tile multiply-accumulate
+///////////////////////////////////////////////////////////////////////////////
+
+template <typename T, typename U, int M, int N, int K>
+METAL_FUNC void tile_multiply_accumulate(
+ thread SimdgroupFragment<T, M, N>& D,
+ thread SimdgroupFragment<U, M, K>& A,
+ thread SimdgroupFragment<U, K, N>& B,
+ thread SimdgroupFragment<T, M, N>& C
+) {
+ METAL_PRAGMA_UNROLL
+ for (short m = 0; m < M; ++m) {
+ METAL_PRAGMA_UNROLL
+ for (short n = 0; n < N; ++n) {
+ short column_serpentine = (m % 2) ? (N - 1 - n) : n;
+ METAL_PRAGMA_UNROLL
+ for (short k = 0; k < K; ++k) {
+ SimdgroupFragment<T, M, N>::SimdgroupMultiplyAccumulateOpsType::
+ multiply_accumulate(
+ D.multiply_accumulate_at(m, column_serpentine),
+ A.multiply_accumulate_at(m, k),
+ B.multiply_accumulate_at(k, column_serpentine),
+ C.multiply_accumulate_at(m, column_serpentine)
+ );
+ }
+ }
+ }
+}
+
+} // namespace matmul
+} // namespace uzu
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/simdgroup_multiply_accumulate.h b/crates/uzu/src/backends/metal/kernel/matmul/common/simdgroup_multiply_accumulate.h
new file mode 100644
index 000000000..6c4ddb93e
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/simdgroup_multiply_accumulate.h
@@ -0,0 +1,183 @@
+#pragma once
+
+#include "defines.h"
+
+using namespace metal;
+
+namespace uzu {
+namespace matmul {
+
+///////////////////////////////////////////////////////////////////////////////
+// SimdgroupMultiplyAccumulate - stateless traits/ops for 8x8 simdgroup_matrix
+///////////////////////////////////////////////////////////////////////////////
+
+template <typename T, int ROWS_, int COLS_>
+struct SimdgroupMultiplyAccumulate {
+ static_assert(
+ ROWS_ == 8,
+ "Only 8 x 8 fragment matrices are currently supported"
+ );
+ static_assert(
+ COLS_ == 8,
+ "Only 8 x 8 fragment matrices are currently supported"
+ );
+};
+
+template <typename T>
+struct SimdgroupMultiplyAccumulate<T, 8, 8> {
+ METAL_CONST int ROWS = 8;
+ METAL_CONST int COLS = 8;
+ METAL_CONST int THREAD_ELEMENT_COUNT = (ROWS * COLS) / 32;
+ METAL_CONST int THREAD_ELEMENT_ROWS = 1;
+ METAL_CONST int THREAD_ELEMENT_COLS = 2;
+
+ static_assert(
+ THREAD_ELEMENT_ROWS * THREAD_ELEMENT_COLS == THREAD_ELEMENT_COUNT,
+ "SimdgroupMultiplyAccumulate shape is not consistent with element count"
+ );
+
+ typedef metal::simdgroup_matrix<T, ROWS, COLS> SimdgroupMatrixType;
+ typedef metal::vec<T, THREAD_ELEMENT_COUNT> ThreadDataType;
+
+ METAL_FUNC static constexpr short2 get_lane_coordinates(
+ ushort simd_lane_id [[thread_index_in_simdgroup]]
+ ) {
+ const short quad_index = simd_lane_id / 4;
+ const short lane_row = (quad_index & 4) + ((simd_lane_id / 2) % 4);
+ const short lane_col = (quad_index & 2) * 2 + (simd_lane_id % 2) * 2;
+ return short2{lane_col, lane_row};
+ }
+
+ template <typename SrcPtrType, typename StrideX, typename StrideY>
+ METAL_FUNC static constexpr void load(
+ thread ThreadDataType& dst,
+ SrcPtrType src,
+ StrideX stride_x,
+ StrideY stride_y
+ ) {
+ METAL_PRAGMA_UNROLL
+ for (short i = 0; i < THREAD_ELEMENT_ROWS; i++) {
+ METAL_PRAGMA_UNROLL
+ for (short j = 0; j < THREAD_ELEMENT_COLS; j++) {
+ dst[i * THREAD_ELEMENT_COLS + j] =
+ static_cast<T>(src[i * stride_x + j * stride_y]);
+ }
+ }
+ }
+
+ template <
+ typename SrcPtrType,
+ typename StrideX,
+ typename StrideY,
+ typename LimitX,
+ typename LimitY,
+ typename OffsetX,
+ typename OffsetY>
+ METAL_FUNC static constexpr void load_safe(
+ thread ThreadDataType& dst,
+ SrcPtrType src,
+ StrideX stride_x,
+ StrideY stride_y,
+ LimitX limit_x,
+ LimitY limit_y,
+ OffsetX offset_x = 0,
+ OffsetY offset_y = 0
+ ) {
+ METAL_PRAGMA_UNROLL
+ for (short i = 0; i < THREAD_ELEMENT_ROWS; i++) {
+ METAL_PRAGMA_UNROLL
+ for (short j = 0; j < THREAD_ELEMENT_COLS; j++) {
+ if ((offset_x + i) < limit_x && (offset_y + j) < limit_y) {
+ dst[i * THREAD_ELEMENT_COLS + j] = static_cast<T>(
+ src[(offset_x + i) * stride_x + (offset_y + j) * stride_y]
+ );
+ } else {
+ dst[i * THREAD_ELEMENT_COLS + j] = T(0);
+ }
+ }
+ }
+ }
+
+ template <typename DstPtrType, typename StrideX, typename StrideY>
+ METAL_FUNC static constexpr void store(
+ const thread ThreadDataType& src,
+ DstPtrType dst,
+ StrideX stride_x,
+ StrideY stride_y
+ ) {
+ using U = PointerElementType<DstPtrType>;
+
+ METAL_PRAGMA_UNROLL
+ for (short i = 0; i < THREAD_ELEMENT_ROWS; i++) {
+ METAL_PRAGMA_UNROLL
+ for (short j = 0; j < THREAD_ELEMENT_COLS; j++) {
+ dst[i * stride_x + j * stride_y] =
+ static_cast<U>(src[i * THREAD_ELEMENT_COLS + j]);
+ }
+ }
+ }
+
+ template <
+ typename DstPtrType,
+ typename StrideX,
+ typename StrideY,
+ typename LimitX,
+ typename LimitY,
+ typename OffsetX,
+ typename OffsetY>
+ METAL_FUNC static constexpr void store_safe(
+ const thread ThreadDataType& src,
+ DstPtrType dst,
+ StrideX stride_x,
+ StrideY stride_y,
+ LimitX limit_x,
+ LimitY limit_y,
+ OffsetX offset_x = 0,
+ OffsetY offset_y = 0
+ ) {
+ using U = PointerElementType<DstPtrType>;
+
+ METAL_PRAGMA_UNROLL
+ for (short i = 0; i < THREAD_ELEMENT_ROWS; i++) {
+ METAL_PRAGMA_UNROLL
+ for (short j = 0; j < THREAD_ELEMENT_COLS; j++) {
+ if ((offset_x + i) < limit_x && (offset_y + j) < limit_y) {
+ dst[(offset_x + i) * stride_x + (offset_y + j) * stride_y] =
+ static_cast<U>(src[i * THREAD_ELEMENT_COLS + j]);
+ }
+ }
+ }
+ }
+
+ METAL_FUNC static constexpr void multiply_accumulate(
+ thread ThreadDataType& D,
+ thread ThreadDataType& A,
+ thread ThreadDataType& B,
+ thread ThreadDataType& C
+ ) {
+ SimdgroupMatrixType D_mat;
+ SimdgroupMatrixType A_mat;
+ SimdgroupMatrixType B_mat;
+ SimdgroupMatrixType C_mat;
+
+ reinterpret_cast<thread ThreadDataType&>(A_mat.thread_elements()) = A;
+ reinterpret_cast<thread ThreadDataType&>(B_mat.thread_elements()) = B;
+ reinterpret_cast<thread ThreadDataType&>(C_mat.thread_elements()) = C;
+
+ multiply_accumulate(D_mat, A_mat, B_mat, C_mat);
+
+ D = reinterpret_cast<thread ThreadDataType&>(D_mat.thread_elements());
+ }
+
+ METAL_FUNC static constexpr void multiply_accumulate(
+ thread SimdgroupMatrixType& D,
+ thread SimdgroupMatrixType& A,
+ thread SimdgroupMatrixType& B,
+ thread SimdgroupMatrixType& C
+ ) {
+ simdgroup_multiply_accumulate(D, A, B, C);
+ }
+};
+
+} // namespace matmul
+} // namespace uzu
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/params.h b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/params.h
deleted file mode 100644
index 9d1675443..000000000
--- a/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/params.h
+++ /dev/null
@@ -1,26 +0,0 @@
-
-#pragma once
-
-///////////////////////////////////////////////////////////////////////////////
-// GEMM param classes
-///////////////////////////////////////////////////////////////////////////////
-
-namespace steel {
-
-struct GEMMParams {
- const int M;
- const int N;
- const int K;
-
- const int lda;
- const int ldb;
- const int ldd;
-
- const int tiles_n;
- const int tiles_m;
-
- const int swizzle_log;
- const int gemm_k_iterations_aligned;
-};
-
-} // namespace steel
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h b/crates/uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h
new file mode 100644
index 000000000..c323dd73e
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h
@@ -0,0 +1,247 @@
+#pragma once
+
+#include "simdgroup_fragment.h"
+
+using namespace metal;
+
+namespace uzu {
+namespace matmul {
+
+///////////////////////////////////////////////////////////////////////////////
+// ThreadgroupTile - manages the GEMM computation for a threadgroup
+///////////////////////////////////////////////////////////////////////////////
+
+template <
+ typename T,
+ typename U,
+ int BLOCK_ROWS,
+ int BLOCK_COLS,
+ int BLOCK_DEPTH,
+ int SIMDGROUPS_PER_ROW,
+ int SIMDGROUPS_PER_COLUMN,
+ bool transpose_a,
+ bool transpose_b,
+ short THREADGROUP_LEADING_DIMENSION_A,
+ short THREADGROUP_LEADING_DIMENSION_B,
+ typename AccumType = float,
+ typename Epilogue = TransformNone<U, AccumType>>
+struct ThreadgroupTile {
+ METAL_CONST short SIMDGROUP_BLOCK_SIZE = 8;
+ using SimdgroupMultiplyAccumulateType = SimdgroupMultiplyAccumulate<
+ AccumType,
+ SIMDGROUP_BLOCK_SIZE,
+ SIMDGROUP_BLOCK_SIZE>;
+
+ METAL_CONST short TILE_ROW_STRIDE = SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW;
+ METAL_CONST short TILE_COL_STRIDE =
+ SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN;
+
+ METAL_CONST short TILE_ROWS =
+ BLOCK_ROWS / (SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW);
+ METAL_CONST short TILE_COLS =
+ BLOCK_COLS / (SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN);
+
+ METAL_CONST short A_STRIDE_ROW =
+ transpose_a ? 1 : THREADGROUP_LEADING_DIMENSION_A;
+ METAL_CONST short A_STRIDE_INNER =
+ transpose_a ? THREADGROUP_LEADING_DIMENSION_A : 1;
+
+ METAL_CONST short B_STRIDE_INNER =
+ transpose_b ? 1 : THREADGROUP_LEADING_DIMENSION_B;
+ METAL_CONST short B_STRIDE_COL =
+ transpose_b ? THREADGROUP_LEADING_DIMENSION_B : 1;
+
+ METAL_CONST short TILE_STRIDE_A = SIMDGROUP_BLOCK_SIZE * A_STRIDE_INNER;
+ METAL_CONST short TILE_STRIDE_B = SIMDGROUP_BLOCK_SIZE * B_STRIDE_INNER;
+
+ SimdgroupFragment<AccumType, TILE_ROWS, 1, SimdgroupMultiplyAccumulateType>
+ a_fragment;
+ SimdgroupFragment<AccumType, 1, TILE_COLS, SimdgroupMultiplyAccumulateType>
+ b_fragment;
+ SimdgroupFragment<
+ AccumType,
+ TILE_ROWS,
+ TILE_COLS,
+ SimdgroupMultiplyAccumulateType>
+ c_fragment;
+
+ short simdgroup_row_offset;
+ short simdgroup_col_offset;
+
+ short a_shared_offset;
+ short b_shared_offset;
+
+ METAL_FUNC ThreadgroupTile(
+ ushort simd_group_id [[simdgroup_index_in_threadgroup]],
+ ushort simd_lane_id [[thread_index_in_simdgroup]]
+ ) {
+ short tile_row_base =
+ SIMDGROUP_BLOCK_SIZE * (simd_group_id / SIMDGROUPS_PER_COLUMN);
+ short tile_col_base =
+ SIMDGROUP_BLOCK_SIZE * (simd_group_id % SIMDGROUPS_PER_COLUMN);
+
+ short2 simdgroup_coordinates =
+ SimdgroupMultiplyAccumulateType::get_lane_coordinates(simd_lane_id);
+ simdgroup_row_offset = simdgroup_coordinates.y;
+ simdgroup_col_offset = simdgroup_coordinates.x;
+
+ a_shared_offset = (tile_row_base + simdgroup_row_offset) * A_STRIDE_ROW +
+ (simdgroup_col_offset)*A_STRIDE_INNER;
+ b_shared_offset = (simdgroup_row_offset)*B_STRIDE_INNER +
+ (tile_col_base + simdgroup_col_offset) * B_STRIDE_COL;
+
+ simdgroup_row_offset += tile_row_base;
+ simdgroup_col_offset += tile_col_base;
+ }
+
+ METAL_FUNC void multiply_accumulate(
+ const threadgroup T* a_shared,
+ const threadgroup T* b_shared
+ ) {
+ a_shared += a_shared_offset;
+ b_shared += b_shared_offset;
+
+ METAL_PRAGMA_UNROLL
+ for (short k_block_index = 0; k_block_index < BLOCK_DEPTH;
+ k_block_index += SIMDGROUP_BLOCK_SIZE) {
+ simdgroup_barrier(mem_flags::mem_none);
+
+ a_fragment.template load<
+ T,
+ SIMDGROUPS_PER_ROW,
+ 1,
+ A_STRIDE_ROW,
+ A_STRIDE_INNER>(a_shared);
+
+ simdgroup_barrier(mem_flags::mem_none);
+
+ b_fragment.template load<
+ T,
+ 1,
+ SIMDGROUPS_PER_COLUMN,
+ B_STRIDE_INNER,
+ B_STRIDE_COL>(b_shared);
+
+ simdgroup_barrier(mem_flags::mem_none);
+
+ tile_multiply_accumulate(c_fragment, a_fragment, b_fragment, c_fragment);
+
+ a_shared += TILE_STRIDE_A;
+ b_shared += TILE_STRIDE_B;
+ }
+ }
+
+ METAL_FUNC void store_result(device U* D, const int leading_dimension_d) {
+ METAL_PRAGMA_UNROLL
+ for (short i = 0; i < decltype(c_fragment)::ELEMENTS_PER_FRAGMENT; i++) {
+ c_fragment.elements()[i] = Epilogue::apply(c_fragment.elements()[i]);
+ }
+
+ D += simdgroup_row_offset * leading_dimension_d + simdgroup_col_offset;
+
+ c_fragment.template store<U, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN>(
+ D,
+ leading_dimension_d
+ );
+ }
+
+ METAL_FUNC void store_result_safe(
+ device U* D,
+ const int leading_dimension_d,
+ short2 destination_tile_dimensions
+ ) {
+ METAL_PRAGMA_UNROLL
+ for (short i = 0; i < decltype(c_fragment)::ELEMENTS_PER_FRAGMENT; i++) {
+ c_fragment.elements()[i] = Epilogue::apply(c_fragment.elements()[i]);
+ }
+
+ D += simdgroup_row_offset * leading_dimension_d + simdgroup_col_offset;
+ destination_tile_dimensions -=
+ short2(simdgroup_col_offset, simdgroup_row_offset);
+
+ if (destination_tile_dimensions.x <= 0 ||
+ destination_tile_dimensions.y <= 0)
+ return;
+
+ c_fragment
+ .template store_safe<U, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN>(
+ D,
+ leading_dimension_d,
+ destination_tile_dimensions
+ );
+ }
+
+ template <typename EpilogueOp>
+ METAL_FUNC void apply_epilogue(
+ const device U* C,
+ const int leading_dimension_c,
+ const int column_stride_c,
+ thread const EpilogueOp& epilogue_operation
+ ) {
+ const device U* c_pointer = C + simdgroup_row_offset * leading_dimension_c +
+ simdgroup_col_offset * column_stride_c;
+
+ METAL_PRAGMA_UNROLL
+ for (short i = 0; i < TILE_ROWS; i++) {
+ METAL_PRAGMA_UNROLL
+ for (short j = 0; j < TILE_COLS; j++) {
+ thread auto& block_data = c_fragment.multiply_accumulate_at(i, j);
+ METAL_PRAGMA_UNROLL
+ for (short k = 0;
+ k < decltype(c_fragment)::ELEMENTS_PER_MULTIPLY_ACCUMULATE;
+ k++) {
+ short row_offset = (i * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW);
+ short col_offset = (j * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN);
+ U c_value = c_pointer
+ [row_offset * leading_dimension_c + col_offset * column_stride_c +
+ k];
+ block_data[k] = epilogue_operation.apply(
+ block_data[k],
+ static_cast<AccumType>(c_value)
+ );
+ }
+ }
+ }
+ }
+
+ template <typename EpilogueOp>
+ METAL_FUNC void apply_epilogue_safe(
+ const device U* C,
+ const int leading_dimension_c,
+ const int column_stride_c,
+ short2 tile_dimensions,
+ thread const EpilogueOp& epilogue_operation
+ ) {
+ const device U* c_pointer = C + simdgroup_row_offset * leading_dimension_c +
+ simdgroup_col_offset * column_stride_c;
+ tile_dimensions -= short2(simdgroup_col_offset, simdgroup_row_offset);
+
+ METAL_PRAGMA_UNROLL
+ for (short i = 0; i < TILE_ROWS; i++) {
+ METAL_PRAGMA_UNROLL
+ for (short j = 0; j < TILE_COLS; j++) {
+ thread auto& block_data = c_fragment.multiply_accumulate_at(i, j);
+ short row_offset = (i * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW);
+ short col_offset = (j * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN);
+ METAL_PRAGMA_UNROLL
+ for (short k = 0;
+ k < decltype(c_fragment)::ELEMENTS_PER_MULTIPLY_ACCUMULATE;
+ k++) {
+ if (row_offset < tile_dimensions.y &&
+ col_offset + k < tile_dimensions.x) {
+ U c_value = c_pointer
+ [row_offset * leading_dimension_c +
+ col_offset * column_stride_c + k];
+ block_data[k] = epilogue_operation.apply(
+ block_data[k],
+ static_cast<AccumType>(c_value)
+ );
+ }
+ }
+ }
+ }
+ }
+};
+
+} // namespace matmul
+} // namespace uzu
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.metal
index ade37da08..6608d1cb7 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.metal
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.metal
@@ -1,47 +1,48 @@
// clang-format off
#include "../../../common/utils.h"
-#include "../../../definitions.metal"
+#include "../../../common/dsl.h"
+#include "../../../common/thread_context.h"
#include "../../common/gemm.h"
using namespace uzu::matmul;
-template <typename T, int BM, int BN, int BK, int WM, int WN, bool MN_aligned, bool K_aligned>
+template <typename T, int BLOCK_ROWS, int BLOCK_COLS, int BLOCK_DEPTH, int SIMDGROUPS_PER_ROW, int SIMDGROUPS_PER_COLUMN, bool MN_aligned, bool K_aligned>
METAL_FUNC void gemm_dispatch(
const device T* a,
const device T* b,
device T* d,
- const constant GEMMParams* params,
- threadgroup T* As,
- threadgroup T* Bs,
+ const constant GemmParams* params,
+ threadgroup T* a_shared,
+ threadgroup T* b_shared,
uint simd_lane_id,
uint simd_group_id,
uint2 tid,
uint3 lid
) {
- using gemm_kernel = GEMMKernel<T, T, BM, BN, BK, WM, WN, false, true, MN_aligned, K_aligned, float>;
- gemm_kernel::run(a, b, d, params, As, Bs, simd_lane_id, simd_group_id, tid, lid);
+ using ThreadgroupGemmType = ThreadgroupGemm<T, T, BLOCK_ROWS, BLOCK_COLS, BLOCK_DEPTH, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN, false, true, MN_aligned, K_aligned, float>;
+ ThreadgroupGemmType::run(a, b, d, params, a_shared, b_shared, simd_lane_id, simd_group_id, tid, lid);
}
-#define GEMM_DISPATCH(T, BM, BN, BK, WM, WN) \
- if (block_rows == BM && block_cols == BN && block_depth == BK && \
- warps_per_row == WM && warps_per_col == WN) { \
+#define GEMM_DISPATCH(T, BLOCK_ROWS, BLOCK_COLS, BLOCK_DEPTH, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN) \
+ if (block_rows == BLOCK_ROWS && block_cols == BLOCK_COLS && block_depth == BLOCK_DEPTH && \
+ simdgroups_per_row == SIMDGROUPS_PER_ROW && simdgroups_per_column == SIMDGROUPS_PER_COLUMN) { \
if (align_m && align_n) { \
if (align_k) \
- gemm_dispatch<T, BM, BN, BK, WM, WN, true, true>(a, b, d, params, a_shared, b_shared, simd.lane_idx, simd.group_idx, uint2(group_x, group_y), uint3(thread_x, thread_y, thread_z)); \
+ gemm_dispatch<T, BLOCK_ROWS, BLOCK_COLS, BLOCK_DEPTH, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN, true, true>(a, b, d, params, a_shared, b_shared, simd.simdgroup_index, simd.threadgroup_index, uint2(group_x, group_y), uint3(thread_x, thread_y, thread_z)); \
else \
- gemm_dispatch<T, BM, BN, BK, WM, WN, true, false>(a, b, d, params, a_shared, b_shared, simd.lane_idx, simd.group_idx, uint2(group_x, group_y), uint3(thread_x, thread_y, thread_z)); \
+ gemm_dispatch<T, BLOCK_ROWS, BLOCK_COLS, BLOCK_DEPTH, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN, true, false>(a, b, d, params, a_shared, b_shared, simd.simdgroup_index, simd.threadgroup_index, uint2(group_x, group_y), uint3(thread_x, thread_y, thread_z)); \
} else { \
if (align_k) \
- gemm_dispatch<T, BM, BN, BK, WM, WN, false, true>(a, b, d, params, a_shared, b_shared, simd.lane_idx, simd.group_idx, uint2(group_x, group_y), uint3(thread_x, thread_y, thread_z)); \
+ gemm_dispatch<T, BLOCK_ROWS, BLOCK_COLS, BLOCK_DEPTH, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN, false, true>(a, b, d, params, a_shared, b_shared, simd.simdgroup_index, simd.threadgroup_index, uint2(group_x, group_y), uint3(thread_x, thread_y, thread_z)); \
else \
- gemm_dispatch<T, BM, BN, BK, WM, WN, false, false>(a, b, d, params, a_shared, b_shared, simd.lane_idx, simd.group_idx, uint2(group_x, group_y), uint3(thread_x, thread_y, thread_z)); \
+ gemm_dispatch<T, BLOCK_ROWS, BLOCK_COLS, BLOCK_DEPTH, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN, false, false>(a, b, d, params, a_shared, b_shared, simd.simdgroup_index, simd.threadgroup_index, uint2(group_x, group_y), uint3(thread_x, thread_y, thread_z)); \
} \
return; \
}
-#define GEMM_MAX_TGP_A 2560
-#define GEMM_MAX_TGP_B 1536
+#define GEMM_MAX_THREADGROUP_A 2560
+#define GEMM_MAX_THREADGROUP_B 1536
template <typename T>
VARIANTS(T, float, half, bfloat)
@@ -49,16 +50,16 @@ PUBLIC KERNEL(MatmulGemm)(
const device T* a,
const device T* b,
device T* d,
- const constant uzu::matmul::GEMMParams* params,
+ const constant uzu::matmul::GemmParams* params,
const constant uint& group_count_x,
const constant uint& group_count_y,
- threadgroup T a_shared[GEMM_MAX_TGP_A],
- threadgroup T b_shared[GEMM_MAX_TGP_B],
+ threadgroup T a_shared[GEMM_MAX_THREADGROUP_A],
+ threadgroup T b_shared[GEMM_MAX_THREADGROUP_B],
const uint block_rows SPECIALIZE,
const uint block_cols SPECIALIZE,
const uint block_depth SPECIALIZE,
- const uint warps_per_row SPECIALIZE,
- const uint warps_per_col SPECIALIZE,
+ const uint simdgroups_per_row SPECIALIZE,
+ const uint simdgroups_per_column SPECIALIZE,
const bool align_m SPECIALIZE,
const bool align_n SPECIALIZE,
const bool align_k SPECIALIZE,
@@ -67,7 +68,7 @@ PUBLIC KERNEL(MatmulGemm)(
const uint thread_x THREADS(32),
const uint thread_y THREADS(2),
const uint thread_z THREADS(2),
- const Simd simd
+ const ThreadContext simd
) {
GEMM_DISPATCH(T, 64, 64, 16, 2, 2)
GEMM_DISPATCH(T, 64, 64, 16, 1, 2)
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/shaders/gemv.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemv/shaders/gemv.metal
index b2fe633e5..ca4aeecf7 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv/shaders/gemv.metal
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv/shaders/gemv.metal
@@ -1,5 +1,6 @@
#include "../../../common/utils.h"
-#include "../../../definitions.metal"
+#include "../../../common/dsl.h"
+#include "../../../common/thread_context.h"
#include <metal_simdgroup>
@@ -8,7 +9,7 @@ using namespace metal;
// Upper bound for threadgroup memory (in floats).
// Max config: tg_simd_cols=16, output_rows_per_tg=16, thread_out_rows=4
// → 16 * (16 + 4) = 320. Unused when tg_simd_cols == 1.
-#define GEMV_MAX_TG_MEMORY 320
+#define GEMV_MAX_THREADGROUP_MEMORY 320
template <typename T>
VARIANTS(T, float, half, bfloat)
@@ -25,7 +26,7 @@ PUBLIC KERNEL(MatmulGemv)(
const constant int& output_source_stride,
const constant int& batch_rows,
const constant int& output_rows_per_threadgroup,
- threadgroup float threadgroup_memory[GEMV_MAX_TG_MEMORY],
+ threadgroup float threadgroup_memory[GEMV_MAX_THREADGROUP_MEMORY],
const uint tg_simd_rows SPECIALIZE,
const uint tg_simd_cols SPECIALIZE,
const uint sg_thread_rows SPECIALIZE,
@@ -38,14 +39,14 @@ PUBLIC KERNEL(MatmulGemv)(
const uint thread_index_x THREADS(32),
const uint thread_index_y THREADS(16),
const uint thread_index_z THREADS(1),
- const Simd simd
+ const ThreadContext simd
) {
if (output_rows_per_threadgroup <= 0) {
return;
}
// Simdgroup guard — kill excess simdgroups beyond what this tile uses
- if (simd.group_idx >= tg_simd_rows * tg_simd_cols) {
+ if (simd.threadgroup_index >= tg_simd_rows * tg_simd_cols) {
return;
}
@@ -65,21 +66,22 @@ PUBLIC KERNEL(MatmulGemv)(
thread float vector_coefficients[4];
const int thread_row_in_simdgroup =
- sg_thread_cols != 32 ? int(simd.lane_idx) / int(sg_thread_cols) : 0;
+ sg_thread_cols != 32 ? int(simd.simdgroup_index) / int(sg_thread_cols)
+ : 0;
const int thread_col_in_simdgroup =
- sg_thread_cols != 32 ? int(simd.lane_idx) % int(sg_thread_cols)
- : int(simd.lane_idx);
+ sg_thread_cols != 32 ? int(simd.simdgroup_index) % int(sg_thread_cols)
+ : int(simd.simdgroup_index);
const int simdgroup_column_index =
- tg_simd_cols != 1 ? int(simd.group_idx % tg_simd_cols) : 0;
+ tg_simd_cols != 1 ? int(simd.threadgroup_index % tg_simd_cols) : 0;
const int simdgroup_row_thread_base =
tg_simd_cols != 1
- ? int(sg_thread_rows) * int(simd.group_idx / tg_simd_cols)
- : int(sg_thread_rows) * int(simd.group_idx);
+ ? int(sg_thread_rows) * int(simd.threadgroup_index / tg_simd_cols)
+ : int(sg_thread_rows) * int(simd.threadgroup_index);
const int simdgroup_col_thread_base =
tg_simd_cols != 1
- ? int(sg_thread_cols) * int(simd.group_idx % tg_simd_cols)
+ ? int(sg_thread_cols) * int(simd.threadgroup_index % tg_simd_cols)
: 0;
int output_block_row_offset =
@@ -121,7 +123,7 @@ PUBLIC KERNEL(MatmulGemv)(
{
const device T* input_vector_row =
input_vector + batch_row * input_dimension;
- MTL_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (uint input_col_offset = 0; input_col_offset < thread_out_cols;
input_col_offset++) {
vector_coefficients[input_col_offset] = static_cast<float>(
@@ -132,11 +134,11 @@ PUBLIC KERNEL(MatmulGemv)(
// Per thread work loop
int matrix_row_offset = 0;
- MTL_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (uint output_row_offset = 0; output_row_offset < thread_out_rows;
output_row_offset++) {
// Load matrix row (unchecked)
- MTL_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (uint input_col_offset = 0; input_col_offset < thread_out_cols;
input_col_offset++) {
matrix_values[input_col_offset] = thread_matrix
@@ -144,7 +146,7 @@ PUBLIC KERNEL(MatmulGemv)(
}
// Accumulate results
- MTL_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (uint input_col_offset = 0; input_col_offset < thread_out_cols;
input_col_offset++) {
accumulated_values[output_row_offset] +=
@@ -165,7 +167,7 @@ PUBLIC KERNEL(MatmulGemv)(
input_vector + batch_row * input_dimension;
if (input_block_col_offset + int(thread_out_cols) <=
input_vector_length) {
- MTL_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (uint input_col_offset = 0; input_col_offset < thread_out_cols;
input_col_offset++) {
vector_coefficients[input_col_offset] = static_cast<float>(
@@ -173,7 +175,7 @@ PUBLIC KERNEL(MatmulGemv)(
);
}
} else {
- MTL_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (uint input_col_offset = 0; input_col_offset < thread_out_cols;
input_col_offset++) {
vector_coefficients[input_col_offset] =
@@ -189,13 +191,13 @@ PUBLIC KERNEL(MatmulGemv)(
}
// Per thread work loop
- MTL_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (uint output_row_offset = 0; output_row_offset < thread_out_rows;
output_row_offset++) {
// Load matrix row (checked)
if (input_block_col_offset + int(thread_out_cols) <=
input_vector_length) {
- MTL_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (uint input_col_offset = 0; input_col_offset < thread_out_cols;
input_col_offset++) {
matrix_values[input_col_offset] = thread_matrix
@@ -203,7 +205,7 @@ PUBLIC KERNEL(MatmulGemv)(
input_block_col_offset + input_col_offset];
}
} else {
- MTL_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (uint input_col_offset = 0; input_col_offset < thread_out_cols;
input_col_offset++) {
matrix_values[input_col_offset] =
@@ -217,7 +219,7 @@ PUBLIC KERNEL(MatmulGemv)(
}
// Accumulate results
- MTL_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (uint input_col_offset = 0; input_col_offset < thread_out_cols;
input_col_offset++) {
accumulated_values[output_row_offset] +=
@@ -228,13 +230,13 @@ PUBLIC KERNEL(MatmulGemv)(
}
// Simdgroup accumulations
- MTL_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (uint output_row_offset = 0; output_row_offset < thread_out_rows;
output_row_offset++) {
- MTL_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (ushort simd_shuffle_offset = (sg_thread_cols / 2);
simd_shuffle_offset >= 1;
- simd_shuffle_offset >>= 1) {
+ simd_shuffle_offset /= 2) {
accumulated_values[output_row_offset] += simd_shuffle_down(
accumulated_values[output_row_offset],
simd_shuffle_offset
@@ -253,7 +255,7 @@ PUBLIC KERNEL(MatmulGemv)(
simdgroup_column_index *
(computed_output_rows_per_tg + int(thread_out_rows)) +
output_block_row_offset;
- MTL_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (uint output_row_offset = 0; output_row_offset < thread_out_rows;
output_row_offset++) {
threadgroup_partial_accumulations[output_row_offset] =
@@ -268,7 +270,7 @@ PUBLIC KERNEL(MatmulGemv)(
for (uint reduction_simdgroup_col = 1;
reduction_simdgroup_col < tg_simd_cols;
reduction_simdgroup_col++) {
- MTL_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (uint output_row_offset = 0; output_row_offset < thread_out_rows;
output_row_offset++) {
accumulated_values[output_row_offset] += base_partial
@@ -284,7 +286,7 @@ PUBLIC KERNEL(MatmulGemv)(
// Write outputs
if (simdgroup_col_thread_base == 0 && thread_col_in_simdgroup == 0) {
device T* output_row_values = output_vector + batch_row * output_dimension;
- MTL_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (uint output_row_offset = 0; output_row_offset < thread_out_rows;
output_row_offset++) {
if (apply_output_scale_and_accumulate) {
diff --git a/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal b/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal
index 0e2215a54..05b5f58e1 100644
--- a/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal
+++ b/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal
@@ -1,6 +1,6 @@
#include <metal_stdlib>
#include "../activation/activation.h"
-#include "../definitions.metal"
+#include "../common/dsl.h"
template <typename T>
VARIANTS(T, float, half, bfloat)
diff --git a/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal b/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal
index d86888590..1281a2bd8 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal
@@ -1,7 +1,9 @@
#include <metal_stdlib>
#include <metal_atomic>
-#include "../definitions.metal"
+#include "../common/dsl.h"
+#include "../common/thread_context.h"
+#include "../common/threadgroup_reduce.h"
#define BLOCK_SIZE 128
#define TILE_E 512
@@ -23,7 +25,7 @@ PUBLIC KERNEL(MoeCountsOffsetsFused)(
threadgroup uint
counts_shared[BLOCK_SIZE], // Cache counts in threadgroup memory
threadgroup uint& carry,
- const Simd simd,
+ const ThreadContext simd,
const uint lid THREADS(128)
) {
if (e_input == 0) {
diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal
index 2607983e5..70320f217 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal
@@ -1,6 +1,7 @@
#include <metal_stdlib>
#include <metal_simdgroup>
-#include "../definitions.metal"
+#include "../common/dsl.h"
+#include "../common/thread_context.h"
#include "moe_commons.h"
// ============================================================================
@@ -26,7 +27,7 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassA)(
constant float& up_clip_min,
constant float& up_clip_max,
const uint gating_sel SPECIALIZE, // 0=GELU, 1=SiLU, 2=SwiGLU, 3=GEGLU
- const Simd simd,
+ const ThreadContext simd,
const uint h_block_idx GROUPS(d_ff.div_ceil(4)),
const uint k_slot GROUPS(k),
const uint tid THREADS(128)
@@ -36,7 +37,7 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassA)(
return;
const uint expert_u = uint(expert_id);
- const uint h_idx = h_block_idx * 4 + simd.group_idx;
+ const uint h_idx = h_block_idx * 4 + simd.threadgroup_index;
if (h_idx >= d_ff)
return;
@@ -54,7 +55,7 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassA)(
const uint vec_iters = d_model / 128;
for (uint i = 0; i < vec_iters; ++i) {
- uint base_idx = i * 128 + simd.lane_idx * 4;
+ uint base_idx = i * 128 + simd.simdgroup_index * 4;
device const T* x_vec2 = reinterpret_cast<device const T*>(x + base_idx);
device const T* w_up_vec2 =
@@ -74,7 +75,7 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassA)(
}
}
- uint leftover_start = vec_iters * 128 + simd.lane_idx;
+ uint leftover_start = vec_iters * 128 + simd.simdgroup_index;
for (uint idx = leftover_start; idx < d_model; idx += 32) {
float xv = float(x[idx]);
acc_up += xv * float(w_up_row[idx]);
@@ -88,7 +89,7 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassA)(
acc_gate = simd_sum(acc_gate);
}
- if (simd.lane_idx == 0) {
+ if (simd.simdgroup_index == 0) {
float up_val = clamp(
acc_up + float(biases[bias_base + h_idx]),
up_clip_min,
@@ -132,11 +133,11 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassB)(
constant uint& d_model,
constant uint& d_ff,
constant uint& k_input,
- const Simd simd,
+ const ThreadContext simd,
const uint d_block GROUPS(d_model.div_ceil(8)),
const uint tid THREADS(256)
) {
- const uint my_col = d_block * 8 + simd.group_idx;
+ const uint my_col = d_block * 8 + simd.threadgroup_index;
if (my_col >= d_model)
return;
@@ -158,7 +159,7 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassB)(
// Vectorized reduction
for (uint i = 0; i < vec_iters; ++i) {
- uint base_idx = i * 128 + simd.lane_idx * 4;
+ uint base_idx = i * 128 + simd.simdgroup_index * 4;
float4 h_vec =
*reinterpret_cast<device const float4*>(hidden_ptr + base_idx);
@@ -171,19 +172,20 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassB)(
}
// Remainder
- for (uint idx = vec_iters * 128 + simd.lane_idx; idx < d_ff; idx += 32) {
+ for (uint idx = vec_iters * 128 + simd.simdgroup_index; idx < d_ff;
+ idx += 32) {
acc += hidden_ptr[idx] * float(w2_ptr[idx]);
}
// Simdgroup reduction and accumulate
float result = simd_sum(acc);
- if (simd.lane_idx == 0) {
+ if (simd.simdgroup_index == 0) {
result += float(biases[(ulong)expert_u * (ulong)d_model + (ulong)my_col]);
final_acc += prob * result;
}
}
- if (simd.lane_idx == 0) {
+ if (simd.simdgroup_index == 0) {
y[my_col] = T(final_acc);
}
}
\ No newline at end of file
diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal
index c7400665f..deab8e40c 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal
@@ -1,6 +1,7 @@
#include <metal_stdlib>
#include <metal_simdgroup>
-#include "../definitions.metal"
+#include "../common/dsl.h"
+#include "../common/thread_context.h"
#include "moe_commons.h"
using namespace metal;
@@ -26,7 +27,7 @@ PUBLIC KERNEL(MoeExpertsDecodePassA)(
constant float& silu_alpha,
device const uint* tile_map,
const uint gating_sel SPECIALIZE,
- const Simd simd,
+ const ThreadContext simd,
const uint tile_idx GROUPS(INDIRECT),
const uint tid THREADS(128)
) {
@@ -43,7 +44,7 @@ PUBLIC KERNEL(MoeExpertsDecodePassA)(
return;
// Each simdgroup outputs one hidden element
- const uint h_idx = h_block_idx * 4 + simd.group_idx;
+ const uint h_idx = h_block_idx * 4 + simd.threadgroup_index;
if (h_idx >= d_ff)
return;
@@ -65,7 +66,7 @@ PUBLIC KERNEL(MoeExpertsDecodePassA)(
const uint vec_iters = d_model / 128;
for (uint i = 0; i < vec_iters; ++i) {
- uint base_idx = i * 128 + simd.lane_idx * 4;
+ uint base_idx = i * 128 + simd.simdgroup_index * 4;
device const T* x_vec = reinterpret_cast<device const T*>(x_ptr + base_idx);
device const T* w_up_vec =
@@ -86,7 +87,7 @@ PUBLIC KERNEL(MoeExpertsDecodePassA)(
}
// Handle leftover elements
- uint leftover_start = vec_iters * 128 + simd.lane_idx;
+ uint leftover_start = vec_iters * 128 + simd.simdgroup_index;
for (uint idx = leftover_start; idx < d_model; idx += 32) {
float xv = float(x_ptr[idx]);
acc_up += xv * float(w_up_row[idx]);
@@ -102,7 +103,7 @@ PUBLIC KERNEL(MoeExpertsDecodePassA)(
}
// Lane 0 applies activation and writes result
- if (simd.lane_idx == 0) {
+ if (simd.simdgroup_index == 0) {
float up_val = acc_up + float(up_biases[bias_base + h_idx]);
up_val = clamp(up_val, up_clip_min, up_clip_max);
@@ -141,7 +142,7 @@ PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(
constant uint& d_model,
constant uint& d_ff,
constant uint& e,
- const Simd simd,
+ const ThreadContext simd,
const uint tgpig_x GROUPS(d_model.div_ceil(SIMDGROUPS_PER_TG)),
const uint tgpig_y GROUPS(total_rows),
const uint tid THREADS(256)
@@ -149,7 +150,7 @@ PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(
const uint row_idx = tgpig_y;
// Each simdgroup computes one output column
- const uint my_col = tgpig_x * SIMDGROUPS_PER_TG + simd.group_idx;
+ const uint my_col = tgpig_x * SIMDGROUPS_PER_TG + simd.threadgroup_index;
if (my_col >= d_model)
return;
@@ -170,7 +171,7 @@ PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(
const uint k_vec_iters = k_iters / 8;
for (uint iter = 0; iter < k_vec_iters; ++iter) {
- const uint k_base = iter * (8 * THREADS_PER_SIMD) + simd.lane_idx;
+ const uint k_base = iter * (8 * THREADS_PER_SIMD) + simd.simdgroup_index;
// hidden: stride-32 per lane, already f32 from Pass A
const AccumT h0 = hidden[hidden_base + k_base + 0 * THREADS_PER_SIMD];
@@ -216,7 +217,7 @@ PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(
// Handle remaining full iterations
for (uint iter = k_vec_iters * 8; iter < k_iters; ++iter) {
- const uint k = iter * THREADS_PER_SIMD + simd.lane_idx;
+ const uint k = iter * THREADS_PER_SIMD + simd.simdgroup_index;
acc =
fma(AccumT(hidden[hidden_base + k]),
AccumT(w2_all[w2_col_base + k]),
@@ -225,8 +226,8 @@ PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(
// Handle leftover elements (d_ff % 32)
const uint leftover_start = k_iters * THREADS_PER_SIMD;
- if (leftover_start + simd.lane_idx < d_ff) {
- const uint k = leftover_start + simd.lane_idx;
+ if (leftover_start + simd.simdgroup_index < d_ff) {
+ const uint k = leftover_start + simd.simdgroup_index;
acc =
fma(AccumT(hidden[hidden_base + k]),
AccumT(w2_all[w2_col_base + k]),
@@ -237,7 +238,7 @@ PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(
AccumT result = simd_sum(acc);
// Lane 0 writes result
- if (simd.lane_idx == 0) {
+ if (simd.simdgroup_index == 0) {
const ulong bias_idx = (ulong)expert_idx * (ulong)d_model + (ulong)my_col;
result += AccumT(down_biases[bias_idx]);
diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal
index 92f9ae0c8..f91b44dbc 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal
@@ -1,5 +1,6 @@
#include <metal_stdlib>
-#include "../definitions.metal"
+#include "../common/dsl.h"
+#include "../common/thread_context.h"
#include "../quant_matmul/mma.h"
#include "moe_commons.h"
@@ -59,7 +60,7 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(
const uint n_tile_idx GROUPS(INDIRECT),
const uint row_tile_idx GROUPS(INDIRECT),
const uint local_tid_x THREADS(128),
- const Simd simd
+ const ThreadContext simd
) {
const uint base = row_tile_idx * 3u;
const uint expert_idx = tile_map[base + 0u];
@@ -105,15 +106,15 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(
// simdgroup tile mapping
const uint sg_col_count = Bn / SgBn;
- const uint row_sg = simd.group_idx / sg_col_count;
- const uint col_sg = simd.group_idx % sg_col_count;
+ const uint row_sg = simd.threadgroup_index / sg_col_count;
+ const uint col_sg = simd.threadgroup_index % sg_col_count;
const uint row_sg_off = row_sg * SgBm;
const uint col_sg_off = col_sg * SgBn;
// Guard against misconfigured TG sizes
constexpr uint SG_TILE = 8; // simdgroup fragment dimension (8x8)
constexpr uint SG_EXPECTED = (Bm / SG_TILE) * (Bn / SG_TILE);
- if (simd.group_idx >= SG_EXPECTED)
+ if (simd.threadgroup_index >= SG_EXPECTED)
return;
// partial accumulators (8x8 tiles per simdgroup)
@@ -144,7 +145,7 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(
constexpr uint total_vecs = Bm * vec_cols; // 16*8 = 128
const uint iters = ceil_div(total_vecs, threads_total);
- UZU_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (uint t = 0; t < iters; ++t) {
const uint i = t * threads_total + lin;
if (i < total_vecs) {
@@ -158,7 +159,7 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(
// Safe to read - at least first element is valid
const ulong base =
x_base + (ulong)r * (ulong)d_model + (ulong)(k_off + col_base);
- UZU_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (uint j = 0; j < vec_size; j++) {
dst[j] = (col_base + j < valid_k)
? static_cast<T>(float(x_perm[base + j]))
@@ -166,7 +167,7 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(
}
} else {
// Row out of bounds or entire vec8 chunk is out of K bounds
- UZU_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (uint j = 0; j < vec_size; j++) {
dst[j] = static_cast<T>(0.0f);
}
@@ -183,7 +184,7 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(
constexpr uint total_vecs = Bn * vec_cols; // 32*8 = 256
const uint iters = ceil_div(total_vecs, threads_total);
- UZU_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (uint t = 0; t < iters; ++t) {
const uint i = t * threads_total + lin;
if (i < total_vecs) {
@@ -203,7 +204,7 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(
const ulong gate_base = w13_expert_base +
(ulong)(d_ff + gcol) * (ulong)d_model +
(ulong)(k_off + col_base);
- UZU_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (uint j = 0; j < vec_size; j++) {
up_dst[j] = (col_base + j < valid_k)
? static_cast<T>(float(w13_all[up_base + j]))
@@ -216,7 +217,7 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(
}
} else {
// Row out of bounds or entire vec8 chunk is out of K bounds
- UZU_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (uint j = 0; j < vec_size; j++) {
up_dst[j] = static_cast<T>(0.0f);
if (gating_sel > 1u)
@@ -230,15 +231,15 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(
threadgroup_barrier(mem_flags::mem_threadgroup);
// ---- MMA across the BK tile (in simdgroup-sized chunks) ----
- UZU_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (uint kk = 0; kk < Bk; kk += SG_TILE) {
- UZU_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (uint m_sub = 0; m_sub < SgBm; m_sub += SG_TILE) {
const uint r_idx = m_sub / SG_TILE;
metal::simdgroup_matrix<T, 8, 8> lhs_frag;
simdgroup_load(lhs_frag, Xs, X_LD, ulong2(kk, row_sg_off + m_sub));
- UZU_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (uint n_sub = 0; n_sub < SgBn; n_sub += SG_TILE) {
const uint c_idx = n_sub / SG_TILE;
const uint tile = r_idx * (SgBn / SG_TILE) + c_idx;
@@ -294,7 +295,7 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(
threadgroup_barrier(mem_flags::mem_threadgroup);
// ---- Epilogue: activation/gating → hidden_out (f32) ----
- const uint sg_lane_start = simd.group_idx * 32;
+ const uint sg_lane_start = simd.threadgroup_index * 32;
const uint sg_lane_end = sg_lane_start + 32;
if (lin < sg_lane_start || lin >= sg_lane_end)
return;
@@ -305,10 +306,10 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(
const uint lane_row = (lane_qid & 4u) + ((sg_lin >> 1) & 3u);
const uint lane_col_base = ((lane_qid & 2u) << 1) + ((sg_lin & 1u) << 1);
- UZU_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (uint n_sub = 0; n_sub < SgBn; n_sub += SG_TILE) {
const uint c_idx = n_sub / SG_TILE;
- UZU_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (uint m_sub = 0; m_sub < SgBm; m_sub += SG_TILE) {
const uint r_idx = m_sub / SG_TILE;
const uint tile = r_idx * (SgBn / SG_TILE) + c_idx;
@@ -403,7 +404,7 @@ PUBLIC KERNEL(MoeExpertsPrefillPassB)(
threadgroup float Hs[PASSB_BM * (PASSB_BK + PASSB_TG_PAD)],
threadgroup T Wk[PASSB_BN * (PASSB_BK + PASSB_TG_PAD)],
threadgroup float bias_tile[PASSB_BN],
- const Simd simd,
+ const ThreadContext simd,
const uint n_tile_idx GROUPS(INDIRECT),
const uint row_tile_idx GROUPS(INDIRECT),
const uint lin THREADS(128)
@@ -445,20 +446,20 @@ PUBLIC KERNEL(MoeExpertsPrefillPassB)(
const ulong bias_base = (ulong)expert_idx * (ulong)d_model;
// 2×2 simdgroup layout for 16×64 output (each sg handles 8×32)
- const uint row_sg = simd.group_idx / 2; // 0-1 → rows
- const uint col_sg = simd.group_idx % 2; // 0-1 → cols
+ const uint row_sg = simd.threadgroup_index / 2; // 0-1 → rows
+ const uint col_sg = simd.threadgroup_index % 2; // 0-1 → cols
const uint row_sg_off = row_sg * SgBm;
const uint col_sg_off = col_sg * SgBn;
// Guard against excessive simdgroups
constexpr uint SG_EXPECTED = (PASSB_BM / SgBm) * (PASSB_BN / SgBn);
- if (simd.group_idx >= SG_EXPECTED)
+ if (simd.threadgroup_index >= SG_EXPECTED)
return;
// 4 accumulators per simdgroup: 1×4 layout of 8×8 tiles
constexpr uint TEMP = (SgBm / SG_TILE) * (SgBn / SG_TILE); // 1×4 = 4
metal::simdgroup_float8x8 Out[TEMP];
- UZU_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (uint i = 0; i < TEMP; ++i) {
Out[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);
}
@@ -488,12 +489,12 @@ PUBLIC KERNEL(MoeExpertsPrefillPassB)(
// Safe to read - at least first element is valid
device const float* my_src =
hidden + h_base + (ulong)h_bi * (ulong)d_ff + (ulong)(k_off + h_bj);
- UZU_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (uint j = 0; j < H_VEC; j++) {
my_dst[j] = (h_bj + j < valid_k) ? my_src[j] : 0.0f;
}
} else {
- UZU_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (uint j = 0; j < H_VEC; j++) {
my_dst[j] = 0.0f;
}
@@ -512,13 +513,13 @@ PUBLIC KERNEL(MoeExpertsPrefillPassB)(
device const T* my_src = w2_all + w2_expert_base +
(ulong)gcol * (ulong)d_ff +
(ulong)(k_off + w_bj);
- UZU_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (uint j = 0; j < W_VEC; j++) {
my_dst[j] = (w_bj + j < valid_k) ? static_cast<T>(float(my_src[j]))
: static_cast<T>(0.0f);
}
} else {
- UZU_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (uint j = 0; j < W_VEC; j++) {
my_dst[j] = static_cast<T>(0.0f);
}
@@ -529,7 +530,7 @@ PUBLIC KERNEL(MoeExpertsPrefillPassB)(
threadgroup_barrier(mem_flags::mem_threadgroup);
// ---- Serpentine MMA: 1×4 tiles per simdgroup ----
- UZU_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (uint kk = 0; kk < Bk; kk += SG_TILE) {
// Load 1 LHS fragment
metal::simdgroup_float8x8 lhs;
@@ -560,17 +561,17 @@ PUBLIC KERNEL(MoeExpertsPrefillPassB)(
threadgroup_barrier(mem_flags::mem_threadgroup);
// ---- Writeout with fragment extraction ----
- const uint lane_qid = simd.lane_idx >> 2;
- const uint lane_row = (lane_qid & 4u) + ((simd.lane_idx >> 1) & 3u);
+ const uint lane_qid = simd.simdgroup_index >> 2;
+ const uint lane_row = (lane_qid & 4u) + ((simd.simdgroup_index >> 1) & 3u);
const uint lane_col_base =
- ((lane_qid & 2u) << 1) + ((simd.lane_idx & 1u) << 1);
+ ((lane_qid & 2u) << 1) + ((simd.simdgroup_index & 1u) << 1);
const uint local_row = row_sg_off + lane_row;
if (local_row >= m_rows)
return;
const ulong out_row = seg_start + row_tg_off + local_row;
- UZU_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (uint ni = 0; ni < 4; ni++) { // 4 col tiles per simdgroup
const auto accum = Out[ni].thread_elements();
const uint local_col = col_sg_off + ni * 8 + lane_col_base;
diff --git a/crates/uzu/src/backends/metal/kernel/moe/finalize.metal b/crates/uzu/src/backends/metal/kernel/moe/finalize.metal
index 435eb592a..f94928258 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/finalize.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/finalize.metal
@@ -1,5 +1,5 @@
#include <metal_stdlib>
-#include "../definitions.metal"
+#include "../common/dsl.h"
#define BM 32
#define BN 64
diff --git a/crates/uzu/src/backends/metal/kernel/moe/gather.metal b/crates/uzu/src/backends/metal/kernel/moe/gather.metal
index 74f724df0..98365a997 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/gather.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/gather.metal
@@ -1,5 +1,5 @@
#include <metal_stdlib>
-#include "../definitions.metal"
+#include "../common/dsl.h"
// Tiling config for 2D gather
#define ROWS_PER_TG 8
diff --git a/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal b/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal
index 3c0a29155..ad16c6b40 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal
@@ -1,6 +1,8 @@
#include <metal_stdlib>
#include <metal_simdgroup>
-#include "../definitions.metal"
+#include "../common/dsl.h"
+#include "../common/thread_context.h"
+#include "../common/threadgroup_reduce.h"
using namespace metal;
constant uint THREADS_PER_TG = 256; // 8 simdgroups
@@ -28,7 +30,7 @@ PUBLIC KERNEL(MoeRouterTopK)(
threadgroup uint reduce_tmp_u[THREADS_PER_TG],
threadgroup uint shared_best_idx[1],
threadgroup float shared_best_val[1],
- const Simd simd,
+ const ThreadContext simd,
const uint tgpig_x GROUPS(1),
const uint token_idx GROUPS(t),
const uint lid THREADS(256)
@@ -50,11 +52,12 @@ PUBLIC KERNEL(MoeRouterTopK)(
}
threadgroup_barrier(mem_flags::mem_threadgroup);
- for (uint row = simd.group_idx; row < e; row += simd.groups_per_threadgroup) {
+ for (uint row = simd.threadgroup_index; row < e;
+ row += simd.simdgroups_per_threadgroup) {
const device ScalarT* w_vec = weight + (ulong)row * (ulong)vecs * 4;
float4 accum4 = float4(0.0f);
- for (uint c = simd.lane_idx; c < vecs; c += 32u) {
+ for (uint c = simd.simdgroup_index; c < vecs; c += 32u) {
const float4 wv = float4(
w_vec[c * 4 + 0],
w_vec[c * 4 + 1],
diff --git a/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal b/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal
index b32d13f4f..6cc4b4701 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal
@@ -1,6 +1,7 @@
#include <metal_stdlib>
#include <metal_atomic>
-#include "../definitions.metal"
+#include "../common/dsl.h"
+#include "../common/thread_context.h"
using namespace metal;
#define BLOCK_SIZE 256
@@ -29,7 +30,7 @@ inline void moe_scatter_buckets_impl(
uint3 tgpig,
threadgroup _atomic<uint>* sg_counts,
threadgroup uint* sg_base,
- const thread Simd& simd
+ const thread ThreadContext& simd
) {
if (E == 0 || T == 0 || K == 0)
return;
@@ -64,7 +65,7 @@ inline void moe_scatter_buckets_impl(
if (ue >= e0 && ue < e0 + tile_e) {
const uint te = ue - e0;
atomic_fetch_add_explicit(
- &sg_counts[simd.group_idx * TILE_E + te],
+ &sg_counts[simd.threadgroup_index * TILE_E + te],
1u,
memory_order_relaxed
);
@@ -102,8 +103,8 @@ inline void moe_scatter_buckets_impl(
threadgroup_barrier(mem_flags::mem_threadgroup);
// Phase 3: deterministic write with 32-lane sequencing, no device atomics
- for (uint step = 0; step < simd.group_size; ++step) {
- if (simd.lane_idx == step) {
+ for (uint step = 0; step < simd.simdgroup_size; ++step) {
+ if (simd.simdgroup_index == step) {
const uint t = t_start + lid;
if (t < t_end) {
const uint base = t * K;
@@ -116,7 +117,7 @@ inline void moe_scatter_buckets_impl(
const uint tile_id = e0 / TILE_E;
const uint base_idx_block =
(block_id * num_tiles + tile_id) * TILE_E + te;
- const uint idx_sg_te = simd.group_idx * TILE_E + te;
+ const uint idx_sg_te = simd.threadgroup_index * TILE_E + te;
const uint local = atomic_fetch_add_explicit(
&sg_counts[idx_sg_te],
1u,
@@ -201,7 +202,7 @@ PUBLIC KERNEL(MoeScatterBuckets)(
constant uint& num_tiles,
threadgroup _atomic<uint> sg_counts[NUM_SG * TILE_E],
threadgroup uint sg_base[NUM_SG * TILE_E],
- const Simd simd,
+ const ThreadContext simd,
const uint tgpig_x GROUPS(num_blocks),
const uint lid THREADS(256)
) {
@@ -250,7 +251,7 @@ PUBLIC KERNEL(MoeScatterBucketsMap)(
device int* tok2row,
threadgroup _atomic<uint> sg_counts[NUM_SG * TILE_E],
threadgroup uint sg_base[NUM_SG * TILE_E],
- const Simd simd,
+ const ThreadContext simd,
const uint tgpig_x GROUPS(num_blocks),
const uint lid THREADS(256)
) {
diff --git a/crates/uzu/src/backends/metal/kernel/moe/tiles_map.metal b/crates/uzu/src/backends/metal/kernel/moe/tiles_map.metal
index 05a4cd5a4..a6489393a 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/tiles_map.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/tiles_map.metal
@@ -1,6 +1,6 @@
#include <metal_stdlib>
#include <metal_atomic>
-#include "../definitions.metal"
+#include "../common/dsl.h"
#define BM 16
diff --git a/crates/uzu/src/backends/metal/kernel/moe/tiles_pass_a.metal b/crates/uzu/src/backends/metal/kernel/moe/tiles_pass_a.metal
index f0de182c1..899c6c168 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/tiles_pass_a.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/tiles_pass_a.metal
@@ -1,5 +1,5 @@
#include <metal_stdlib>
-#include "../definitions.metal"
+#include "../common/dsl.h"
// === Helper kernels for indirect dispatch of Pass A ===
diff --git a/crates/uzu/src/backends/metal/kernel/pooling/pooling.metal b/crates/uzu/src/backends/metal/kernel/pooling/pooling.metal
index 185ae7d16..ba4c7a47f 100644
--- a/crates/uzu/src/backends/metal/kernel/pooling/pooling.metal
+++ b/crates/uzu/src/backends/metal/kernel/pooling/pooling.metal
@@ -1,5 +1,5 @@
#include <metal_stdlib>
-#include "../definitions.metal"
+#include "../common/dsl.h"
// CLS pooling: Extract first token [batch, seq_len, hidden_dim] → [batch,
// hidden_dim]
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/mma.h b/crates/uzu/src/backends/metal/kernel/quant_matmul/mma.h
index 56000c057..162e1ba18 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/mma.h
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/mma.h
@@ -1,13 +1,10 @@
#pragma once
+#include "../common/defines.h"
#include <metal_simdgroup>
#include <metal_simdgroup_matrix>
#include <metal_stdlib>
-#define UZU_MTL_CONST static constant constexpr
-#define UZU_PRAGMA_UNROLL _Pragma("clang loop unroll(full)")
-#define SIMD_SIZE 32
-
using namespace metal;
namespace matmul_utils {
@@ -47,11 +44,11 @@ struct BaseMMAFrag {
template <typename T>
struct BaseMMAFrag<T, 8, 8> {
- UZU_MTL_CONST int kFragRows = 8;
- UZU_MTL_CONST int kFragCols = 8;
- UZU_MTL_CONST int kElemsPerFrag = (kFragRows * kFragCols) / 32;
- UZU_MTL_CONST int kElemRows = 1;
- UZU_MTL_CONST int kElemCols = 2;
+ METAL_CONST int kFragRows = 8;
+ METAL_CONST int kFragCols = 8;
+ METAL_CONST int kElemsPerFrag = (kFragRows * kFragCols) / 32;
+ METAL_CONST int kElemRows = 1;
+ METAL_CONST int kElemCols = 2;
static_assert(
kElemRows * kElemCols == kElemsPerFrag,
@@ -77,9 +74,9 @@ struct BaseMMAFrag<T, 8, 8> {
StrX str_x,
StrY str_y
) {
- UZU_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (short i = 0; i < kElemRows; i++) {
- UZU_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (short j = 0; j < kElemCols; j++) {
dst[i * kElemCols + j] = static_cast<T>(src[i * str_x + j * str_y]);
}
@@ -95,9 +92,9 @@ struct BaseMMAFrag<T, 8, 8> {
) {
using U = pointer_element_t<DstPtrType>;
- UZU_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (short i = 0; i < kElemRows; i++) {
- UZU_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (short j = 0; j < kElemCols; j++) {
dst[i * str_x + j * str_y] = static_cast<U>(src[i * kElemCols + j]);
}
@@ -124,9 +121,9 @@ struct BaseMMAFrag<T, 8, 8> {
) {
using U = pointer_element_t<DstPtrType>;
- UZU_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (short i = 0; i < kElemRows; i++) {
- UZU_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (short j = 0; j < kElemCols; j++) {
if ((off_x + i) < lim_x && (off_y + j) < lim_y) {
dst[(off_x + i) * str_x + (off_y + j) * str_y] =
@@ -174,18 +171,18 @@ template <
struct MMATile {
using MMAFrag_t = MMAFrag_;
using elem_type = T;
- UZU_MTL_CONST int kFragRows = MMAFrag_t::kFragRows;
- UZU_MTL_CONST int kFragCols = MMAFrag_t::kFragCols;
- UZU_MTL_CONST int kElemsPerFrag = MMAFrag_t::kElemsPerFrag;
+ METAL_CONST int kFragRows = MMAFrag_t::kFragRows;
+ METAL_CONST int kFragCols = MMAFrag_t::kFragCols;
+ METAL_CONST int kElemsPerFrag = MMAFrag_t::kElemsPerFrag;
- UZU_MTL_CONST int kTileRows = kTileRows_;
- UZU_MTL_CONST int kTileCols = kTileCols_;
+ METAL_CONST int kTileRows = kTileRows_;
+ METAL_CONST int kTileCols = kTileCols_;
- UZU_MTL_CONST int kRows = kTileRows * kFragRows;
- UZU_MTL_CONST int kCols = kTileCols * kFragCols;
+ METAL_CONST int kRows = kTileRows * kFragRows;
+ METAL_CONST int kCols = kTileCols * kFragCols;
- UZU_MTL_CONST int kNumFrags = kTileRows * kTileCols;
- UZU_MTL_CONST int kElemsPerTile = kNumFrags * kElemsPerFrag;
+ METAL_CONST int kNumFrags = kTileRows * kTileCols;
+ METAL_CONST int kElemsPerTile = kNumFrags * kElemsPerFrag;
typedef typename MMAFrag_t::mat_type mat_type;
typedef typename MMAFrag_t::frag_type frag_type;
@@ -195,7 +192,7 @@ struct MMATile {
METAL_FUNC MMATile() thread {}
METAL_FUNC constexpr void clear() {
- UZU_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (short i = 0; i < kNumFrags; ++i) {
val_frags[i] = frag_type(0);
}
@@ -222,9 +219,9 @@ struct MMATile {
template <typename U, int w_x, int w_y, int str_x, int str_y>
METAL_FUNC void load(const threadgroup U* src) {
- UZU_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (short i = 0; i < kTileRows; ++i) {
- UZU_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (short j = 0; j < kTileCols; ++j) {
MMAFrag_t::load(
frag_at(i, j),
@@ -241,9 +238,9 @@ struct MMATile {
template <typename U, int w_x, int w_y>
METAL_FUNC void store(device U* dst, const int ld) const {
- UZU_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (short i = 0; i < kTileRows; ++i) {
- UZU_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (short j = 0; j < kTileCols; ++j) {
MMAFrag_t::store(
frag_at(i, j),
@@ -258,9 +255,9 @@ struct MMATile {
// Store results into threadgroup memory (same layout as device variant)
template <typename U, int w_x, int w_y>
METAL_FUNC void store_tg(threadgroup U* dst, const int ld) const {
- UZU_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (short i = 0; i < kTileRows; ++i) {
- UZU_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (short j = 0; j < kTileCols; ++j) {
MMAFrag_t::store(
frag_at(i, j),
@@ -278,9 +275,9 @@ struct MMATile {
const int ld,
const short2 dst_tile_dims
) const {
- UZU_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (int i = 0; i < kTileRows; ++i) {
- UZU_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (int j = 0; j < kTileCols; ++j) {
MMAFrag_t::store_safe(
frag_at(i, j),
@@ -304,12 +301,12 @@ METAL_FUNC void tile_matmad(
thread MMATile<U, K, N>& B,
thread MMATile<T, M, N>& C
) {
- UZU_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (short m = 0; m < M; ++m) {
- UZU_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (short n = 0; n < N; ++n) {
short n_serp = (m % 2) ? (N - 1 - n) : n;
- UZU_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (short k = 0; k < K; ++k) {
MMATile<T, M, N>::MMAFrag_t::mma(
D.frag_at(m, n_serp),
@@ -338,30 +335,30 @@ template <
typename Epilogue = TransformNone<U, AccumType>>
struct BlockMMA {
// MMAFrag size
- UZU_MTL_CONST short kFragSize = 8;
+ METAL_CONST short kFragSize = 8;
using MMAFrag_acc_t = BaseMMAFrag<AccumType, kFragSize, kFragSize>;
// Warp tile simdgroup matrix strides along M
- UZU_MTL_CONST short TM_stride = kFragSize * WM;
+ METAL_CONST short TM_stride = kFragSize * WM;
// Warp tile simdgroup matrix strides along M
- UZU_MTL_CONST short TN_stride = kFragSize * WN;
+ METAL_CONST short TN_stride = kFragSize * WN;
// Warp tile size along M
- UZU_MTL_CONST short TM = BM / (kFragSize * WM);
+ METAL_CONST short TM = BM / (kFragSize * WM);
// Warp tile size along N
- UZU_MTL_CONST short TN = BN / (kFragSize * WN);
+ METAL_CONST short TN = BN / (kFragSize * WN);
// Threadgroup A strides
- UZU_MTL_CONST short A_str_m = transpose_a ? 1 : lda_tgp; // M
- UZU_MTL_CONST short A_str_k = transpose_a ? lda_tgp : 1; // K
+ METAL_CONST short A_str_m = transpose_a ? 1 : lda_tgp; // M
+ METAL_CONST short A_str_k = transpose_a ? lda_tgp : 1; // K
// Threadgroup B strides
- UZU_MTL_CONST short B_str_k = transpose_b ? 1 : ldb_tgp; // K
- UZU_MTL_CONST short B_str_n = transpose_b ? ldb_tgp : 1; // N
+ METAL_CONST short B_str_k = transpose_b ? 1 : ldb_tgp; // K
+ METAL_CONST short B_str_n = transpose_b ? ldb_tgp : 1; // N
// Threadgroup strides along K
- UZU_MTL_CONST short tile_stride_a = kFragSize * A_str_k;
- UZU_MTL_CONST short tile_stride_b = kFragSize * B_str_k;
+ METAL_CONST short tile_stride_a = kFragSize * A_str_k;
+ METAL_CONST short tile_stride_b = kFragSize * B_str_k;
// Simdgroup matrices
MMATile<AccumType, TM, 1, MMAFrag_acc_t> Atile;
@@ -403,7 +400,7 @@ struct BlockMMA {
Bs += Bs_offset;
// Iterate over BK in blocks of kFragSize
- UZU_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (short kk = 0; kk < BK; kk += kFragSize) {
simdgroup_barrier(mem_flags::mem_none);
@@ -426,7 +423,7 @@ struct BlockMMA {
/* Store results from simdgroup_matrix results into device memory */
METAL_FUNC void store_result(device U* D, const int ldd) {
// Apply epilogue
- UZU_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (short i = 0; i < decltype(Ctile)::kElemsPerTile; i++) {
Ctile.elems()[i] = Epilogue::apply(Ctile.elems()[i]);
}
@@ -440,7 +437,7 @@ struct BlockMMA {
/* Store results into threadgroup memory */
METAL_FUNC void store_result_tg(threadgroup U* D, const int ldd) {
// Apply epilogue
- UZU_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (short i = 0; i < decltype(Ctile)::kElemsPerTile; i++) {
Ctile.elems()[i] = Epilogue::apply(Ctile.elems()[i]);
}
@@ -457,7 +454,7 @@ struct BlockMMA {
short2 dst_tile_dims
) {
// Apply epilogue
- UZU_PRAGMA_UNROLL
+ METAL_PRAGMA_UNROLL
for (short i = 0; i < decltype(Ctile)::kElemsPerTile; i++) {
Ctile.elems()[i] = Epilogue::apply(Ctile.elems()[i]);
}
@@ -483,7 +480,7 @@ template <
int align_M = 1,
int align_K = 4>
struct BlockLoader {
- UZU_MTL_CONST short n_reads = (BM * BK < tgp_size) ? 1 : (BM * BK) / tgp_size;
+ METAL_CONST short n_reads = (BM * BK < tgp_size) ? 1 : (BM * BK) / tgp_size;
const int src_ld;
const short thread_idx;
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm.metal
index f020cc70b..f16fba9bb 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm.metal
@@ -1,5 +1,5 @@
#include <metal_stdlib>
-#include "../definitions.metal"
+#include "../common/dsl.h"
#include "quant_matmul.metal"
template <typename T, int GROUP_SIZE, int BITS>
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal
index 7f26b135c..c324e936f 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal
@@ -1,5 +1,5 @@
#include <metal_stdlib>
-#include "../definitions.metal"
+#include "../common/dsl.h"
#include "quant_matmul.metal"
template <typename T, int GROUP_SIZE, int BITS>
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metal
index 012359d6b..61d6fc85f 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metal
@@ -1,5 +1,5 @@
#include <metal_stdlib>
-#include "../definitions.metal"
+#include "../common/dsl.h"
#include "quant_matmul.metal"
template <typename T, int GROUP_SIZE, int BITS>
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal
index ea76f8f7c..6bcfc03b9 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal
@@ -1,5 +1,5 @@
#include <metal_stdlib>
-#include "../definitions.metal"
+#include "../common/dsl.h"
#include "quant_matmul.metal"
template <typename T, int GROUP_SIZE, int BITS>
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal
index 756cb624a..4b3120dee 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal
@@ -1,5 +1,5 @@
#include <metal_stdlib>
-#include "../definitions.metal"
+#include "../common/dsl.h"
#include "quant_matmul.metal"
template <typename T, int GROUP_SIZE, int BITS>
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal
index ed923674d..59657ecff 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal
@@ -287,12 +287,12 @@ struct QuantizedBlockLoaderMlx {
);
static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");
- UZU_MTL_CONST short pack_factor = get_pack_factor<bits, 8>();
- UZU_MTL_CONST short bytes_per_pack = get_bytes_per_pack<bits>();
- UZU_MTL_CONST short BCOLS_PACKED = BCOLS / pack_factor;
- UZU_MTL_CONST short n_reads =
+ METAL_CONST short pack_factor = get_pack_factor<bits, 8>();
+ METAL_CONST short bytes_per_pack = get_bytes_per_pack<bits>();
+ METAL_CONST short BCOLS_PACKED = BCOLS / pack_factor;
+ METAL_CONST short n_reads =
(BCOLS_PACKED * BROWS < tgp_size) ? 1 : (BCOLS_PACKED * BROWS) / tgp_size;
- UZU_MTL_CONST short group_steps = group_size / BCOLS;
+ METAL_CONST short group_steps = group_size / BCOLS;
const int src_ld;
const int tile_stride;
@@ -422,12 +422,12 @@ struct QuantizedBlockLoaderZp {
);
static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");
- UZU_MTL_CONST short pack_factor = get_pack_factor<bits, 8>();
- UZU_MTL_CONST short bytes_per_pack = get_bytes_per_pack<bits>();
- UZU_MTL_CONST short BCOLS_PACKED = BCOLS / pack_factor;
- UZU_MTL_CONST short n_reads =
+ METAL_CONST short pack_factor = get_pack_factor<bits, 8>();
+ METAL_CONST short bytes_per_pack = get_bytes_per_pack<bits>();
+ METAL_CONST short BCOLS_PACKED = BCOLS / pack_factor;
+ METAL_CONST short n_reads =
(BCOLS_PACKED * BROWS < tgp_size) ? 1 : (BCOLS_PACKED * BROWS) / tgp_size;
- UZU_MTL_CONST short group_steps = group_size / BCOLS;
+ METAL_CONST short group_steps = group_size / BCOLS;
const int src_ld;
const int groups_per_row;
@@ -744,8 +744,8 @@ void qmm_impl(
uint simd_gid [[simdgroup_index_in_threadgroup]],
uint simd_lid [[thread_index_in_simdgroup]]
) {
- static_assert(BK >= 32, "BK should be larger than SIMD_SIZE");
- static_assert(BK % 32 == 0, "BK should be divisible by SIMD_SIZE");
+ static_assert(BK >= 32, "BK should be larger than METAL_SIMD_SIZE");
+ static_assert(BK % 32 == 0, "BK should be divisible by METAL_SIMD_SIZE");
(void)lid;
@@ -953,8 +953,8 @@ void qmm_transposed_impl(
uint simd_gid [[simdgroup_index_in_threadgroup]],
uint simd_lid [[thread_index_in_simdgroup]]
) {
- static_assert(BK >= 32, "BK should be larger than SIMD_SIZE");
- static_assert(BK % 32 == 0, "BK should be divisible by SIMD_SIZE");
+ static_assert(BK >= 32, "BK should be larger than METAL_SIMD_SIZE");
+ static_assert(BK % 32 == 0, "BK should be divisible by METAL_SIMD_SIZE");
constexpr int WM = 2;
constexpr int WN = 2;
@@ -1377,7 +1377,7 @@ void qmv_fast_impl(
constexpr int pack_factor = get_pack_factor<bits, 32>();
constexpr int bytes_per_pack = get_bytes_per_pack<bits, 32>();
constexpr int values_per_thread = pack_factor * packs_per_thread;
- constexpr int block_size = values_per_thread * SIMD_SIZE;
+ constexpr int block_size = values_per_thread * METAL_SIMD_SIZE;
constexpr int scale_step_per_thread = group_size / values_per_thread;
const device uint8_t* ws = (const device uint8_t*)w;
typedef float U;
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qvm.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qvm.metal
index e12ac5063..3c389f961 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qvm.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qvm.metal
@@ -1,5 +1,5 @@
#include <metal_stdlib>
-#include "../definitions.metal"
+#include "../common/dsl.h"
#include "quant_matmul.metal"
template <typename T, int GROUP_SIZE, int BITS>
diff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/qk_norm.metal b/crates/uzu/src/backends/metal/kernel/rms_norm/qk_norm.metal
index 10cb61900..c9d2ae75c 100644
--- a/crates/uzu/src/backends/metal/kernel/rms_norm/qk_norm.metal
+++ b/crates/uzu/src/backends/metal/kernel/rms_norm/qk_norm.metal
@@ -1,9 +1,8 @@
#include <metal_stdlib>
-#include "../definitions.metal"
+#include "../common/defines.h"
+#include "../common/dsl.h"
using namespace metal;
-
-#define SIMD_SIZE 32
#define GRAIN_SIZE 4
// QK norm: normalize per-head vectors (small head_dim) efficiently.
@@ -31,7 +30,7 @@ PUBLIC KERNEL(QKNorm)(
constant bool& full_layer,
const uint batch_idx GROUPS(batch_size),
const uint head_idx GROUPS(head_count),
- const uint lane_id THREADS(SIMD_SIZE),
+ const uint lane_id THREADS(METAL_SIMD_SIZE),
const bool in_place SPECIALIZE
) {
if (in_place) {
@@ -59,7 +58,7 @@ PUBLIC KERNEL(QKNorm)(
// Sum of squares: each lane processes GRAIN_SIZE elements per iteration.
for (uint base_i = lane_id * GRAIN_SIZE; base_i < element_count;
- base_i += SIMD_SIZE * GRAIN_SIZE) {
+ base_i += METAL_SIMD_SIZE * GRAIN_SIZE) {
AccumT vals[GRAIN_SIZE];
for (uint j = 0; j < GRAIN_SIZE; ++j) {
uint i = base_i + j;
@@ -80,7 +79,7 @@ PUBLIC KERNEL(QKNorm)(
// Normalize + scale.
for (uint base_i = lane_id * GRAIN_SIZE; base_i < element_count;
- base_i += SIMD_SIZE * GRAIN_SIZE) {
+ base_i += METAL_SIMD_SIZE * GRAIN_SIZE) {
AccumT vals[GRAIN_SIZE];
for (uint j = 0; j < GRAIN_SIZE; ++j) {
uint i = base_i + j;
diff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
index f846ad0d2..eb0d04ac4 100644
--- a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
+++ b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
@@ -1,10 +1,12 @@
#include <metal_stdlib>
-#include "../definitions.metal"
+#include "../common/defines.h"
+#include "../common/dsl.h"
+#include "../common/thread_context.h"
+#include "../common/threadgroup_reduce.h"
using namespace metal;
#define BLOCK_SIZE 1024
-#define SIMD_SIZE 32
#define GRAIN_SIZE 4
template <typename InputT, typename ScaleT, typename OutputT, typename AccumT>
@@ -22,8 +24,8 @@ PUBLIC KERNEL(RMSNorm)(
constant float& scale_offset,
constant bool& full_layer,
const bool in_place SPECIALIZE,
- threadgroup AccumT shared_sum[SIMD_SIZE],
- const Simd simd,
+ threadgroup AccumT shared_sum[METAL_SIMD_SIZE],
+ const ThreadContext simd,
const uint batch_idx GROUPS(batch_size),
const uint thread_in_row THREADS(1024)
) {
diff --git a/crates/uzu/src/backends/metal/kernel/rope/rope.metal b/crates/uzu/src/backends/metal/kernel/rope/rope.metal
index bb4cc6217..5f6274e14 100644
--- a/crates/uzu/src/backends/metal/kernel/rope/rope.metal
+++ b/crates/uzu/src/backends/metal/kernel/rope/rope.metal
@@ -1,5 +1,6 @@
#include <metal_stdlib>
-#include "../definitions.metal"
+#include "../common/dsl.h"
+#include "../common/tensor_view.h"
template <typename T>
inline T applyRopeTransform(
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal b/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal
index 89d317024..86939cf03 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal
@@ -1,6 +1,7 @@
#include <metal_stdlib>
#include <metal_atomic>
-#include "../definitions.metal"
+#include "../common/dsl.h"
+#include "../common/thread_context.h"
#include "argmax.h"
@@ -26,7 +27,7 @@ static ArgmaxPair threadgroup_cooperative_argmax(
ArgmaxPair value,
threadgroup ArgmaxPair* shared,
const ushort lid,
- const thread Simd& simd
+ const thread ThreadContext& simd
) {
// Reduce within simdgroup using manual shuffle operations
ArgmaxPair local_result = value;
@@ -39,8 +40,8 @@ static ArgmaxPair threadgroup_cooperative_argmax(
}
// First thread in each simdgroup writes to shared memory
- if (simd.lane_idx == 0) {
- shared[simd.group_idx] = local_result;
+ if (simd.simdgroup_index == 0) {
+ shared[simd.threadgroup_index] = local_result;
}
// Synchronize across the threadgroup
@@ -202,7 +203,7 @@ PUBLIC KERNEL(ArgmaxMain)(
constant uint& batch_size,
constant uint& vocab_size,
threadgroup ArgmaxPair shared[BLOCK_SIZE],
- const Simd simd,
+ const ThreadContext simd,
uint batch_idx GROUPS(batch_size),
uint vocab_group_idx GROUPS(vocab_size.div_ceil(BLOCK_SIZE * GRAIN_SIZE)),
ushort local_id THREADS(BLOCK_SIZE)
@@ -246,7 +247,7 @@ PUBLIC KERNEL(ArgmaxFinal)(
constant uint& batch_size,
constant uint& vocab_size,
threadgroup ArgmaxPair shared[BLOCK_SIZE],
- const Simd simd,
+ const ThreadContext simd,
uint batch_idx GROUPS(batch_size),
ushort local_id THREADS(BLOCK_SIZE)
) {
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/bitmask.metal b/crates/uzu/src/backends/metal/kernel/sampling/bitmask.metal
index 010fa690b..3dd8b2011 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/bitmask.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/bitmask.metal
@@ -1,5 +1,5 @@
#include <metal_stdlib>
-#include "../definitions.metal"
+#include "../common/dsl.h"
#define BLOCK_SIZE 1024
#define GRAIN_SIZE 64
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metal b/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metal
index 349e49cbe..6fb5ce481 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metal
@@ -1,5 +1,5 @@
#include <metal_stdlib>
-#include "../definitions.metal"
+#include "../common/dsl.h"
#include "../rng.metal"
#define BLOCK_SIZE 1024
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/min_p.metal b/crates/uzu/src/backends/metal/kernel/sampling/min_p.metal
index 327195b5a..121a1cca3 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/min_p.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/min_p.metal
@@ -1,5 +1,7 @@
#include <metal_stdlib>
-#include "../definitions.metal"
+#include "../common/dsl.h"
+#include "../common/thread_context.h"
+#include "../common/threadgroup_reduce.h"
#define BLOCK_SIZE 1024
#define BLOCK_SIZE_IN_SIMDS (BLOCK_SIZE / 32)
@@ -13,7 +15,7 @@ PUBLIC KERNEL(MinP) (
constant uint& batch_size,
constant uint& vocab_size,
constant float& min_p,
- const Simd simd,
+ const ThreadContext simd,
uint batch_idx GROUPS(batch_size),
uint thread_idx THREADS(BLOCK_SIZE),
const bool in_place SPECIALIZE
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/temperature.metal b/crates/uzu/src/backends/metal/kernel/sampling/temperature.metal
index cba9ff056..73c7bd3e4 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/temperature.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/temperature.metal
@@ -1,5 +1,5 @@
#include <metal_stdlib>
-#include "../definitions.metal"
+#include "../common/dsl.h"
#define BLOCK_SIZE 1024
#define GRAIN_SIZE 64
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal b/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal
index 5113ee31c..d921385b2 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal
@@ -1,5 +1,7 @@
#include <metal_stdlib>
-#include "../definitions.metal"
+#include "../common/dsl.h"
+#include "../common/thread_context.h"
+#include "../common/threadgroup_reduce.h"
#define BLOCK_SIZE 1024
#define BLOCK_SIZE_IN_SIMDS (BLOCK_SIZE / 32)
@@ -14,7 +16,7 @@ PUBLIC KERNEL(TopK)(
constant uint& batch_size,
constant uint& vocab_size,
constant uint& top_k,
- const Simd simd,
+ const ThreadContext simd,
uint batch_idx GROUPS(batch_size),
uint thread_idx THREADS(BLOCK_SIZE),
const bool in_place SPECIALIZE
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal b/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal
index f94ddb033..cd4f01774 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal
@@ -1,5 +1,7 @@
#include <metal_stdlib>
-#include "../definitions.metal"
+#include "../common/dsl.h"
+#include "../common/thread_context.h"
+#include "../common/threadgroup_reduce.h"
#define BLOCK_SIZE 1024
#define BLOCK_SIZE_IN_SIMDS (BLOCK_SIZE / 32)
@@ -15,7 +17,7 @@ PUBLIC KERNEL(TopP) (
constant uint& batch_size,
constant uint& vocab_size,
constant float& top_p,
- const Simd simd,
+ const ThreadContext simd,
uint batch_idx GROUPS(batch_size),
uint thread_idx THREADS(BLOCK_SIZE),
const bool in_place SPECIALIZE
diff --git a/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal b/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal
index d4088a527..7b5d757a6 100644
--- a/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal
+++ b/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal
@@ -1,5 +1,5 @@
#include <metal_stdlib>
-#include "../definitions.metal"
+#include "../common/dsl.h"
template <typename T>
VARIANTS(T, float, half, bfloat)
diff --git a/crates/uzu/src/backends/metal/kernel/sigmoid/sigmoid.metal b/crates/uzu/src/backends/metal/kernel/sigmoid/sigmoid.metal
index 69d5c400a..4ddc4c8b7 100644
--- a/crates/uzu/src/backends/metal/kernel/sigmoid/sigmoid.metal
+++ b/crates/uzu/src/backends/metal/kernel/sigmoid/sigmoid.metal
@@ -1,5 +1,5 @@
#include <metal_stdlib>
-#include "../definitions.metal"
+#include "../common/dsl.h"
template <typename T>
VARIANTS(T, float, half, bfloat)
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/conv1d.metal b/crates/uzu/src/backends/metal/kernel/ssm/conv1d.metal
index 1ff690c80..0c48c0d01 100644
--- a/crates/uzu/src/backends/metal/kernel/ssm/conv1d.metal
+++ b/crates/uzu/src/backends/metal/kernel/ssm/conv1d.metal
@@ -1,5 +1,5 @@
#include <metal_stdlib>
-#include "../definitions.metal"
+#include "../common/dsl.h"
#include "ssm_common.h"
template <typename T>
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/split_inproj.metal b/crates/uzu/src/backends/metal/kernel/ssm/split_inproj.metal
index b6c6709ad..fb1b85ebd 100644
--- a/crates/uzu/src/backends/metal/kernel/ssm/split_inproj.metal
+++ b/crates/uzu/src/backends/metal/kernel/ssm/split_inproj.metal
@@ -1,5 +1,5 @@
#include <metal_stdlib>
-#include "../definitions.metal"
+#include "../common/dsl.h"
template <typename T>
VARIANTS(T, float, half, bfloat)
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal b/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal
index db7910523..f84606b78 100644
--- a/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal
+++ b/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal
@@ -1,6 +1,7 @@
#include <metal_stdlib>
#include <metal_simdgroup>
-#include "../definitions.metal"
+#include "../common/dsl.h"
+#include "../common/thread_context.h"
#include "ssm_common.h"
using namespace metal;
@@ -27,7 +28,7 @@ PUBLIC KERNEL(SSDPrefill64)(
constant const uint* state_strides,
constant const uint& num_heads,
constant const uint& head_dim,
- const Simd simd,
+ const ThreadContext simd,
const uint pair_idx GROUPS(num_heads * head_dim),
const uint lane_idx THREADS(32)
) {
@@ -55,7 +56,7 @@ PUBLIC KERNEL(SSDPrefill64)(
const float d_scalar = float(d[h_idx]);
const uint idx0 = lane_idx;
- const uint idx1 = lane_idx + simd.group_size;
+ const uint idx1 = lane_idx + simd.simdgroup_size;
float state0 = 0.0f;
float state1 = 0.0f;
@@ -125,7 +126,7 @@ PUBLIC KERNEL(SSDPrefill)(
constant const uint* state_strides,
constant const uint& num_heads,
constant const uint& head_dim,
- const Simd simd,
+ const ThreadContext simd,
const uint pair_idx GROUPS(num_heads * head_dim),
const uint lane_idx THREADS(32)
) {
@@ -153,9 +154,9 @@ PUBLIC KERNEL(SSDPrefill)(
const uint cb_group_base = group_idx * cb_group_stride;
const float d_scalar = float(d[h_idx]);
- const int max_chunks = SSM_PREFILL_MAX_STATE / simd.group_size;
+ const int max_chunks = SSM_PREFILL_MAX_STATE / simd.simdgroup_size;
const int chunk_count =
- (state_dim + int(simd.group_size) - 1) / int(simd.group_size);
+ (state_dim + int(simd.simdgroup_size) - 1) / int(simd.simdgroup_size);
if (state_dim <= 0 || chunk_count > max_chunks) {
return;
}
@@ -164,7 +165,7 @@ PUBLIC KERNEL(SSDPrefill)(
#pragma unroll
for (int chunk = 0; chunk < chunk_count; ++chunk) {
- const int idx = chunk * int(simd.group_size) + int(lane_idx);
+ const int idx = chunk * int(simd.simdgroup_size) + int(lane_idx);
const uint state_idx = state_base + idx * state_inner_stride;
lane_states[chunk] = float(state[state_idx]);
}
@@ -182,7 +183,7 @@ PUBLIC KERNEL(SSDPrefill)(
float contrib_sum = 0.0f;
#pragma unroll
for (int chunk = 0; chunk < chunk_count; ++chunk) {
- const int idx = chunk * int(simd.group_size) + int(lane_idx);
+ const int idx = chunk * int(simd.simdgroup_size) + int(lane_idx);
const uint cb_idx =
cb_group_base + uint(idx) * cb_state_stride + token * cb_token_stride;
const float new_state =
@@ -199,7 +200,7 @@ PUBLIC KERNEL(SSDPrefill)(
#pragma unroll
for (int chunk = 0; chunk < chunk_count; ++chunk) {
- const int idx = chunk * int(simd.group_size) + int(lane_idx);
+ const int idx = chunk * int(simd.simdgroup_size) + int(lane_idx);
const uint state_idx = state_base + uint(idx) * state_inner_stride;
state[state_idx] = static_cast<T>(lane_states[chunk]);
}
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal b/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal
index 33857f7fb..8ddf31f79 100644
--- a/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal
+++ b/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal
@@ -1,5 +1,5 @@
#include <metal_stdlib>
-#include "../definitions.metal"
+#include "../common/dsl.h"
#include "ssm_common.h"
using namespace metal;
diff --git a/crates/uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal b/crates/uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal
index 48e6dcb6f..184350e6f 100644
--- a/crates/uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal
+++ b/crates/uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal
@@ -1,5 +1,5 @@
#include <metal_stdlib>
-#include "../definitions.metal"
+#include "../common/dsl.h"
template <typename T>
VARIANTS(T, float, half, bfloat)
diff --git a/crates/uzu/src/backends/metal/kernel/tensor_add_swap/tensor_add_swap.metal b/crates/uzu/src/backends/metal/kernel/tensor_add_swap/tensor_add_swap.metal
index 3524e2e77..82a033113 100644
--- a/crates/uzu/src/backends/metal/kernel/tensor_add_swap/tensor_add_swap.metal
+++ b/crates/uzu/src/backends/metal/kernel/tensor_add_swap/tensor_add_swap.metal
@@ -1,5 +1,5 @@
#include <metal_stdlib>
-#include "../definitions.metal"
+#include "../common/dsl.h"
template <typename T>
VARIANTS(T, float, half, bfloat)
diff --git a/crates/uzu/src/backends/metal/kernel/tensor_copy/tensor_copy.metal b/crates/uzu/src/backends/metal/kernel/tensor_copy/tensor_copy.metal
index d619ed30b..052a48efe 100644
--- a/crates/uzu/src/backends/metal/kernel/tensor_copy/tensor_copy.metal
+++ b/crates/uzu/src/backends/metal/kernel/tensor_copy/tensor_copy.metal
@@ -1,5 +1,5 @@
#include <metal_stdlib>
-#include "../definitions.metal"
+#include "../common/dsl.h"
template <typename T>
VARIANTS(T, float, half, bfloat)
diff --git a/crates/uzu/src/backends/metal/kernel/token_copy/token_copy.metal b/crates/uzu/src/backends/metal/kernel/token_copy/token_copy.metal
index f4ebbf08f..e5cc942f6 100644
--- a/crates/uzu/src/backends/metal/kernel/token_copy/token_copy.metal
+++ b/crates/uzu/src/backends/metal/kernel/token_copy/token_copy.metal
@@ -1,5 +1,5 @@
#include <metal_stdlib>
-#include "../definitions.metal"
+#include "../common/dsl.h"
/// Copies a single sampled token (u32) to the token_ids buffer (u64).
/// Used in async pipeline to pass sampled token to next forward pass.
diff --git a/crates/uzu/tests/kernel/matmul/gemv_test.rs b/crates/uzu/tests/kernel/matmul/gemv_test.rs
index 9660f02b9..2381dfe8f 100644
--- a/crates/uzu/tests/kernel/matmul/gemv_test.rs
+++ b/crates/uzu/tests/kernel/matmul/gemv_test.rs
@@ -78,9 +78,9 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
batch: m,
input_dim: k,
output_dim: n,
- lda: k,
- ldb: k,
- ldd: n,
+ leading_dimension_a: k,
+ leading_dimension_b: k,
+ leading_dimension_d: n,
transpose_b: true,
};
diff --git a/crates/uzu/tests/kernel/matmul/steel_gemm_test.rs b/crates/uzu/tests/kernel/matmul/steel_gemm_test.rs
index 8a539aebe..70c4d305f 100644
--- a/crates/uzu/tests/kernel/matmul/steel_gemm_test.rs
+++ b/crates/uzu/tests/kernel/matmul/steel_gemm_test.rs
@@ -12,7 +12,7 @@ use uzu::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
Context,
- gpu_types::GEMMParams,
+ gpu_types::GemmParams,
kernel::matmul::{
GridSize, MatmulArguments,
gemm::{GemmDispatchDescriptor, GemmKernel, GemmSpecialization},
@@ -81,28 +81,28 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
..base
});
- let tiles_n = (n + config.block_cols - 1) / config.block_cols;
- let tiles_m = (m + config.block_rows - 1) / config.block_rows;
+ let threadgroups_per_row = (n + config.block_cols - 1) / config.block_cols;
+ let threadgroups_per_column = (m + config.block_rows - 1) / config.block_rows;
- let params = GEMMParams {
+ let params = GemmParams {
M: m,
N: n,
K: k,
- lda: k,
- ldb: k,
- ldd: n,
- tiles_n,
- tiles_m,
+ leading_dimension_a: k,
+ leading_dimension_b: k,
+ leading_dimension_d: n,
+ threadgroups_per_row,
+ threadgroups_per_column,
swizzle_log: 0,
- gemm_k_iterations_aligned: k / config.block_depth,
+ aligned_inner_iterations: k / config.block_depth,
};
let descriptor = GemmDispatchDescriptor {
specialization: config,
params,
threadgroups: GridSize {
- x: tiles_n as usize,
- y: tiles_m as usize,
+ x: threadgroups_per_row as usize,
+ y: threadgroups_per_column as usize,
z: 1,
},
};
@@ -126,9 +126,9 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
batch: m,
input_dim: k,
output_dim: n,
- lda: k,
- ldb: k,
- ldd: n,
+ leading_dimension_a: k,
+ leading_dimension_b: k,
+ leading_dimension_d: n,
transpose_b: true,
};
kernel.encode(&context, &mut arguments, &descriptor, &mut command_buffer).expect("Failed to encode");
diff --git a/crates/uzu/tests/matmul_correctness_test.rs b/crates/uzu/tests/matmul_correctness_test.rs
index 3d687081c..802873345 100644
--- a/crates/uzu/tests/matmul_correctness_test.rs
+++ b/crates/uzu/tests/matmul_correctness_test.rs
@@ -38,7 +38,7 @@ fn run_metal_matmul(
.new_buffer(m * n * core::mem::size_of::<bf16>(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let ldb = if transpose_b {
+ let leading_dim_b = if transpose_b {
k
} else {
n
@@ -56,9 +56,9 @@ fn run_metal_matmul(
batch: m as i32,
input_dim: k as i32,
output_dim: n as i32,
- lda: k as i32,
- ldb: ldb as i32,
- ldd: n as i32,
+ leading_dimension_a: k as i32,
+ leading_dimension_b: leading_dim_b as i32,
+ leading_dimension_d: n as i32,
transpose_b,
};
let descriptor =
diff --git a/crates/uzu/tests/matmul_perf_test/bench.rs b/crates/uzu/tests/matmul_perf_test/bench.rs
index 783e3f324..460f3ed83 100644
--- a/crates/uzu/tests/matmul_perf_test/bench.rs
+++ b/crates/uzu/tests/matmul_perf_test/bench.rs
@@ -46,9 +46,9 @@ pub fn make_arguments<'a>(
batch: shape.batch as i32,
input_dim: shape.input_dim as i32,
output_dim: shape.output_dim as i32,
- lda: shape.input_dim as i32,
- ldb: shape.input_dim as i32,
- ldd: shape.output_dim as i32,
+ leading_dimension_a: shape.input_dim as i32,
+ leading_dimension_b: shape.input_dim as i32,
+ leading_dimension_d: shape.output_dim as i32,
transpose_b: true,
}
}