A high-performance inference engine for AI models
shaders refactoring
eugenec7b328fb4bb8
diff --git a/crates/uzu/build/metal/ast.rs b/crates/uzu/build/metal/ast.rsindex 5d7e49614..8bbc95f8b 100644--- a/crates/uzu/build/metal/ast.rs+++ b/crates/uzu/build/metal/ast.rs@@ -119,7 +119,7 @@ pub enum MetalArgumentType {Axis(Box<str>, Box<str>),Groups(MetalGroupsType),Threads(Box<str>),- Simd,+ ThreadContext,}#[derive(Debug, Serialize, Deserialize)]@@ -251,8 +251,8 @@ impl MetalArgument {},_ => bail!("unknown annotation: {annotation_key}"),}- } else if self.c_type.as_ref() == "Simd" || self.c_type.as_ref() == "const Simd" {- Ok(MetalArgumentType::Simd)+ } else if self.c_type.as_ref() == "ThreadContext" || self.c_type.as_ref() == "const ThreadContext" {+ Ok(MetalArgumentType::ThreadContext)} else if self.c_type.contains("device") && self.c_type.contains('*') && !self.c_type.contains('&') {Ok(MetalArgumentType::Buffer(if self.c_type.contains("const") {MetalBufferAccess::Read@@ -354,8 +354,8 @@ impl MetalKernelInfo {self.arguments.iter().any(|a| matches!(a.argument_type(), Ok(MetalArgumentType::Threads(_))))}- pub fn has_simd(&self) -> bool {- self.arguments.iter().any(|a| matches!(a.argument_type(), Ok(MetalArgumentType::Simd)))+ pub fn has_thread_context(&self) -> bool {+ self.arguments.iter().any(|a| matches!(a.argument_type(), Ok(MetalArgumentType::ThreadContext)))}pub fn to_kernel(&self) -> Option<Kernel> {diff --git a/crates/uzu/build/metal/wrapper.rs b/crates/uzu/build/metal/wrapper.rsindex b02c373c4..6e63c71df 100644--- a/crates/uzu/build/metal/wrapper.rs+++ b/crates/uzu/build/metal/wrapper.rs@@ -170,7 +170,7 @@ fn kernel_wrappers(wrapper_arguments.push("uint3 __dsl_thread_idx [[thread_position_in_threadgroup]]".into());}- if kernel.has_simd() {+ if kernel.has_thread_context() {wrapper_arguments.push("uint __dsl_simd_lane_idx [[thread_index_in_simdgroup]]".into());wrapper_arguments.push("uint __dsl_simd_group_idx [[simdgroup_index_in_threadgroup]]".into());wrapper_arguments.push("uint __dsl_simd_group_size [[threads_per_simdgroup]]".into());@@ -210,7 +210,7 @@ fn kernel_wrappers(MetalArgumentType::Threads(_) => {format!("__dsl_thread_idx.{}", thread_axis_letters.next().unwrap())},- MetalArgumentType::Simd => "Simd { .lane_idx = __dsl_simd_lane_idx, .group_idx = __dsl_simd_group_idx, .group_size = __dsl_simd_group_size, .groups_per_threadgroup = __dsl_simd_group_per_threadgroup }".into(),+ MetalArgumentType::ThreadContext => "ThreadContext { .simdgroup_index = __dsl_simd_lane_idx, .threadgroup_index = __dsl_simd_group_idx, .simdgroup_size = __dsl_simd_group_size, .threadgroup_size = __dsl_simd_group_size * __dsl_simd_group_per_threadgroup, .simdgroups_per_threadgroup = __dsl_simd_group_per_threadgroup }".into(),}).collect::<Vec<_>>().join(", ")diff --git a/crates/uzu/src/backends/common/gpu_types/matmul.rs b/crates/uzu/src/backends/common/gpu_types/matmul.rsindex d54a635a0..96713f917 100644--- a/crates/uzu/src/backends/common/gpu_types/matmul.rs+++ b/crates/uzu/src/backends/common/gpu_types/matmul.rs@@ -1,22 +1,16 @@-//! GEMM kernel parameter structs.-//!-//! Field names use uppercase for matrix dimensions (M, N, K) to match-//! standard BLAS/GEMM convention and generated shader headers.-#![allow(non_snake_case)]-/// Main GEMM parameters passed to backend kernels as constant buffer.#[repr(C)]#[derive(Debug, Default, Copy, Clone)]-pub struct GEMMParams {+pub struct GemmParams {pub M: i32,pub N: i32,pub K: i32,- pub lda: i32,- pub ldb: i32,- pub ldd: i32,- pub tiles_n: i32,- pub tiles_m: i32,+ pub leading_dimension_a: i32,+ pub leading_dimension_b: i32,+ pub leading_dimension_d: i32,+ pub threadgroups_per_column: i32,+ pub threadgroups_per_row: i32,pub swizzle_log: i32,- pub gemm_k_iterations_aligned: i32,+ pub aligned_inner_iterations: i32,}diff --git a/crates/uzu/src/backends/common/kernel/matmul/full_precision.rs b/crates/uzu/src/backends/common/kernel/matmul/full_precision.rsindex b22a6aaaf..5cf2ad669 100644--- a/crates/uzu/src/backends/common/kernel/matmul/full_precision.rs+++ b/crates/uzu/src/backends/common/kernel/matmul/full_precision.rs@@ -62,9 +62,9 @@ impl<B: Backend<Kernels: MatmulKernels<FullPrecisionMatmulKernel = Self>>> FullPbatch: arguments.batch as i32,input_dim: arguments.input_dim as i32,output_dim: arguments.output_dim as i32,- lda: arguments.input_dim as i32,- ldb: arguments.input_dim as i32,- ldd: arguments.output_dim as i32,+ leading_dimension_a: arguments.input_dim as i32,+ leading_dimension_b: arguments.input_dim as i32,+ leading_dimension_d: arguments.output_dim as i32,transpose_b: true,};diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemm/dispatch_descriptor.rs b/crates/uzu/src/backends/common/kernel/matmul/gemm/dispatch_descriptor.rsindex a4cbbad71..21ce06389 100644--- a/crates/uzu/src/backends/common/kernel/matmul/gemm/dispatch_descriptor.rs+++ b/crates/uzu/src/backends/common/kernel/matmul/gemm/dispatch_descriptor.rs@@ -3,7 +3,7 @@ use crate::{DataType,backends::common::{Backend,- gpu_types::GEMMParams,+ gpu_types::GemmParams,kernel::matmul::{MatmulArguments, MatmulError},},};@@ -11,7 +11,7 @@ use crate::{#[derive(Debug, Clone)]pub struct GemmDispatchDescriptor {pub specialization: GemmSpecialization,- pub params: GEMMParams,+ pub params: GemmParams,pub threadgroups: GridSize,}@@ -31,25 +31,25 @@ impl GemmDispatchDescriptor {let n = arguments.output_dim;let k = arguments.input_dim;- let tiles_n = (n + config.block_cols - 1) / config.block_cols;- let tiles_m = (m + config.block_rows - 1) / config.block_rows;+ let threadgroups_per_row = (n + config.block_cols - 1) / config.block_cols;+ let threadgroups_per_column = (m + config.block_rows - 1) / config.block_rows;let swizzle_log = config.swizzle_log2;- let tile_swizzle = 1 << swizzle_log;- let tm_swizzled = (tiles_m + tile_swizzle - 1) / tile_swizzle;- let tn_swizzled = tiles_n * tile_swizzle;+ let swizzle_stride = 1_i32 << swizzle_log;+ let tm_swizzled = (threadgroups_per_column + swizzle_stride - 1) / swizzle_stride;+ let tn_swizzled = threadgroups_per_row * swizzle_stride;- let params = GEMMParams {+ let params = GemmParams {M: m,N: n,K: k,- lda: arguments.lda,- ldb: arguments.ldb,- ldd: arguments.ldd,- tiles_n,- tiles_m,+ leading_dimension_a: arguments.leading_dimension_a,+ leading_dimension_b: arguments.leading_dimension_b,+ leading_dimension_d: arguments.leading_dimension_d,+ threadgroups_per_row,+ threadgroups_per_column,swizzle_log,- gemm_k_iterations_aligned: k / config.block_depth,+ aligned_inner_iterations: k / config.block_depth,};let threadgroups = GridSize {diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemm/kernel.rs b/crates/uzu/src/backends/common/kernel/matmul/gemm/kernel.rsindex 7e6d673ff..a1502a1bd 100644--- a/crates/uzu/src/backends/common/kernel/matmul/gemm/kernel.rs+++ b/crates/uzu/src/backends/common/kernel/matmul/gemm/kernel.rs@@ -50,8 +50,8 @@ impl<B: Backend> GemmKernel<B> {config.block_rows as u32,config.block_cols as u32,config.block_depth as u32,- config.warps_per_row as u32,- config.warps_per_col as u32,+ config.simdgroups_per_row as u32,+ config.simdgroups_per_column as u32,config.align_m,config.align_n,config.align_k,diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemm/specialization.rs b/crates/uzu/src/backends/common/kernel/matmul/gemm/specialization.rsindex da1cd1ed4..42e60d822 100644--- a/crates/uzu/src/backends/common/kernel/matmul/gemm/specialization.rs+++ b/crates/uzu/src/backends/common/kernel/matmul/gemm/specialization.rs@@ -8,8 +8,8 @@ pub struct GemmSpecialization {pub block_rows: i32,pub block_cols: i32,pub block_depth: i32,- pub warps_per_row: u64,- pub warps_per_col: u64,+ pub simdgroups_per_row: u64,+ pub simdgroups_per_column: u64,pub swizzle_log2: i32,pub align_m: bool,pub align_n: bool,@@ -24,8 +24,8 @@ impl GemmSpecialization {block_rows: 64,block_cols: 32,block_depth: 32,- warps_per_row: 2,- warps_per_col: 2,+ simdgroups_per_row: 2,+ simdgroups_per_column: 2,swizzle_log2: 0,align_m: false,align_n: true,@@ -35,8 +35,8 @@ impl GemmSpecialization {block_rows: 64,block_cols: 32,block_depth: 32,- warps_per_row: 2,- warps_per_col: 2,+ simdgroups_per_row: 2,+ simdgroups_per_column: 2,swizzle_log2: 0,align_m: true,align_n: true,@@ -46,8 +46,8 @@ impl GemmSpecialization {block_rows: 64,block_cols: 64,block_depth: 16,- warps_per_row: 2,- warps_per_col: 2,+ simdgroups_per_row: 2,+ simdgroups_per_column: 2,swizzle_log2: 0,align_m: false,align_n: true,@@ -57,8 +57,8 @@ impl GemmSpecialization {block_rows: 64,block_cols: 64,block_depth: 16,- warps_per_row: 2,- warps_per_col: 2,+ simdgroups_per_row: 2,+ simdgroups_per_column: 2,swizzle_log2: 0,align_m: true,align_n: false,@@ -68,8 +68,8 @@ impl GemmSpecialization {block_rows: 64,block_cols: 64,block_depth: 16,- warps_per_row: 2,- warps_per_col: 2,+ simdgroups_per_row: 2,+ simdgroups_per_column: 2,swizzle_log2: 0,align_m: true,align_n: true,@@ -79,8 +79,8 @@ impl GemmSpecialization {block_rows: 64,block_cols: 64,block_depth: 16,- warps_per_row: 1,- warps_per_col: 2,+ simdgroups_per_row: 1,+ simdgroups_per_column: 2,swizzle_log2: 0,align_m: true,align_n: true,@@ -92,8 +92,8 @@ impl GemmSpecialization {block_rows: 64,block_cols: 64,block_depth: 16,- warps_per_row: 2,- warps_per_col: 2,+ simdgroups_per_row: 2,+ simdgroups_per_column: 2,swizzle_log2: 0,align_m: true,align_n: true,@@ -103,8 +103,8 @@ impl GemmSpecialization {block_rows: 64,block_cols: 64,block_depth: 16,- warps_per_row: 2,- warps_per_col: 2,+ simdgroups_per_row: 2,+ simdgroups_per_column: 2,swizzle_log2: 0,align_m: false,align_n: true,@@ -116,8 +116,8 @@ impl GemmSpecialization {block_rows: 32,block_cols: 64,block_depth: 16,- warps_per_row: 2,- warps_per_col: 2,+ simdgroups_per_row: 2,+ simdgroups_per_column: 2,swizzle_log2: 0,align_m: false,align_n: true,@@ -127,8 +127,8 @@ impl GemmSpecialization {block_rows: 32,block_cols: 64,block_depth: 16,- warps_per_row: 2,- warps_per_col: 2,+ simdgroups_per_row: 2,+ simdgroups_per_column: 2,swizzle_log2: 0,align_m: true,align_n: true,@@ -148,7 +148,7 @@ impl GemmSpecialization {let is_float32 = matches!(data_type, DataType::F32);let prefer_half_or_tf32 = !is_float32 || B::Context::tf32_enabled();- let (block_rows, block_cols, block_depth, warps_per_row, warps_per_col, swizzle_log2) =+ let (block_rows, block_cols, block_depth, simdgroups_per_row, simdgroups_per_column, swizzle_log2) =if context.is_high_performance() && overall_work_elements >= (1_i64 << 20) {if prefer_half_or_tf32 {if 2 * std::cmp::max(arguments.batch, arguments.output_dim) > arguments.input_dim {@@ -183,8 +183,8 @@ impl GemmSpecialization {block_rows,block_cols,block_depth,- warps_per_row,- warps_per_col,+ simdgroups_per_row,+ simdgroups_per_column,swizzle_log2,align_m: (m % block_rows) == 0,align_n: (n % block_cols) == 0,diff --git a/crates/uzu/src/backends/common/kernel/matmul/gemv/dispatch_descriptor.rs b/crates/uzu/src/backends/common/kernel/matmul/gemv/dispatch_descriptor.rsindex 77e77df26..e45ac57bf 100644--- a/crates/uzu/src/backends/common/kernel/matmul/gemv/dispatch_descriptor.rs+++ b/crates/uzu/src/backends/common/kernel/matmul/gemv/dispatch_descriptor.rs@@ -95,9 +95,9 @@ impl GemvDispatchDescriptor {let input_dimension = arguments.input_dim;let matrix_leading_dim = if matrix_is_rhs {- arguments.ldb+ arguments.leading_dimension_b} else {- arguments.lda+ arguments.leading_dimension_a};let batch_rows = arguments.batch;diff --git a/crates/uzu/src/backends/common/kernel/matmul/matmul_arguments.rs b/crates/uzu/src/backends/common/kernel/matmul/matmul_arguments.rsindex 4981baff7..a432f4e0b 100644--- a/crates/uzu/src/backends/common/kernel/matmul/matmul_arguments.rs+++ b/crates/uzu/src/backends/common/kernel/matmul/matmul_arguments.rs@@ -14,8 +14,8 @@ pub struct MatmulArguments<'a, B: Backend> {pub input_dim: i32,/// N dimension - output_dim (cols of B, cols of D)pub output_dim: i32,- pub lda: i32,- pub ldb: i32,- pub ldd: i32,+ pub leading_dimension_a: i32,+ pub leading_dimension_b: i32,+ pub leading_dimension_d: i32,pub transpose_b: bool,}diff --git a/crates/uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs b/crates/uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rsindex f9043cb45..302d77480 100644--- a/crates/uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs+++ b/crates/uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs@@ -15,19 +15,19 @@ unsafe fn matmul_gemm_compute_row<T: ArrayElement + Float>(row: usize,n: usize,k: usize,- lda: usize,- ldb: usize,- ldd: usize,+ leading_dimension_a: usize,+ leading_dimension_b: usize,+ leading_dimension_d: usize,) {unsafe {for col in 0..n {let mut acc: f32 = 0.0;for i in 0..k {- let a_val = *a.0.add(row * lda + i);- let b_val = *b.0.add(col * ldb + i);+ let a_val = *a.0.add(row * leading_dimension_a + i);+ let b_val = *b.0.add(col * leading_dimension_b + i);acc = acc + a_val.to_f32().unwrap() * b_val.to_f32().unwrap();}- *d.0.add(row * ldd + col) = T::from(acc).unwrap();+ *d.0.add(row * leading_dimension_d + col) = T::from(acc).unwrap();}}}@@ -39,9 +39,9 @@ fn matmul_gemm_impl<T: ArrayElement + Float>(m: usize,n: usize,k: usize,- lda: usize,- ldb: usize,- ldd: usize,+ leading_dimension_a: usize,+ leading_dimension_b: usize,+ leading_dimension_d: usize,) {let num_threads = std::thread::available_parallelism().map(|n| n.get()).unwrap_or(1);let a = SendPtr(a);@@ -65,7 +65,17 @@ fn matmul_gemm_impl<T: ArrayElement + Float>(s.spawn(move || unsafe {for row in start..end {- matmul_gemm_compute_row(a, b, d, row, n, k, lda, ldb, ldd);+ matmul_gemm_compute_row(+ a,+ b,+ d,+ row,+ n,+ k,+ leading_dimension_a,+ leading_dimension_b,+ leading_dimension_d,+ );}});@@ -81,7 +91,7 @@ pub fn matmul_gemm<T: ArrayElement + Float>(a: *const T,b: *const T,d: *mut T,- params: &[crate::backends::common::gpu_types::matmul::GEMMParams],+ params: &[crate::backends::common::gpu_types::matmul::GemmParams],#[allow(unused)] group_count_x: u32,#[allow(unused)] group_count_y: u32,#[allow(unused)]@@ -95,10 +105,10 @@ pub fn matmul_gemm<T: ArrayElement + Float>(block_depth: u32,#[allow(unused)]#[specialize]- warps_per_row: u32,+ simdgroups_per_row: u32,#[allow(unused)]#[specialize]- warps_per_col: u32,+ simdgroups_per_column: u32,#[allow(unused)]#[specialize]align_m: bool,@@ -118,9 +128,9 @@ pub fn matmul_gemm<T: ArrayElement + Float>(p.M as usize,p.N as usize,p.K as usize,- p.lda as usize,- p.ldb as usize,- p.ldd as usize,+ p.leading_dimension_a as usize,+ p.leading_dimension_b as usize,+ p.leading_dimension_d as usize,);}}diff --git a/crates/uzu/src/backends/metal/generated/matmul.h b/crates/uzu/src/backends/metal/generated/matmul.hindex 2f5fe3580..86d45c6ec 100644--- a/crates/uzu/src/backends/metal/generated/matmul.h+++ b/crates/uzu/src/backends/metal/generated/matmul.h@@ -14,19 +14,18 @@ namespace matmul {#include <stdint.h>#endif-/**Main GEMM parameters passed to backend kernels as constant buffer. */typedef struct {int32_t M;int32_t N;int32_t K;- int32_t lda;- int32_t ldb;- int32_t ldd;- int32_t tiles_n;- int32_t tiles_m;+ int32_t leading_dimension_a;+ int32_t leading_dimension_b;+ int32_t leading_dimension_d;+ int32_t threadgroups_per_column;+ int32_t threadgroups_per_row;int32_t swizzle_log;- int32_t gemm_k_iterations_aligned;-} GEMMParams;+ int32_t aligned_inner_iterations;+} GemmParams;#ifdef __METAL_VERSION__} // namespace matmuldiff --git a/crates/uzu/src/backends/metal/kernel/activation/activation.metal b/crates/uzu/src/backends/metal/kernel/activation/activation.metalindex 074a77e44..a815a2722 100644--- a/crates/uzu/src/backends/metal/kernel/activation/activation.metal+++ b/crates/uzu/src/backends/metal/kernel/activation/activation.metal@@ -1,5 +1,5 @@#include <metal_stdlib>-#include "../definitions.metal"+#include "../common/dsl.h"#include "activation.h"diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metalindex cba3a797b..fbe1a533b 100644--- a/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_gemm.metal@@ -1,7 +1,8 @@#include <metal_stdlib>-#include "../definitions.metal"+#include "../common/dsl.h"+#include "../common/thread_context.h"#include "../matmul/common/loader.h"-#include "../matmul/common/mma.h"+#include "../matmul/common/simdgroup_fragment.h"#include "attention.h"using namespace metal;@@ -17,8 +18,8 @@ struct TransformScale {template <typename T>METAL_FUNC T row_reduce_max(T v) {- // BaseMMAFrag::get_coord mapping groups lanes for a row as:- // {lane, lane^1, lane^8, (lane^1)^8}. Reduce in two steps.+ // SimdgroupMultiplyAccumulate::get_lane_coordinates mapping groups lanes for+ // a row as: {lane, lane^1, lane^8, (lane^1)^8}. Reduce in two steps.v = metal::max(v, simd_shuffle_xor(v, 1));v = metal::max(v, simd_shuffle_xor(v, 8));return v;@@ -31,9 +32,9 @@ METAL_FUNC T row_reduce_sum(T v) {return v;}-#define BQ 32-#define WM 4-#define WN 1+#define BLOCK_QUERY_ROWS 32+#define SIMDGROUPS_PER_ROW 4+#define SIMDGROUPS_PER_COLUMN 1template <typename T, uint BK, uint BD>VARIANTS(T, float, half, bfloat)@@ -55,17 +56,17 @@ PUBLIC KERNEL(AttentionGemm)(const bool do_causal SPECIALIZE,const bool has_mask SPECIALIZE,const bool has_sinks SPECIALIZE,- threadgroup T q_smem[BQ * (BD + 16 / sizeof(T))],+ threadgroup T q_smem[BLOCK_QUERY_ROWS * (BD + 16 / sizeof(T))],threadgroup T kv_smem[BK * (BD + 16 / sizeof(T))],- const Simd simd,- const uint tgid_x GROUPS(suffix_length.div_ceil(BQ)),+ const ThreadContext simd,+ const uint tgid_x GROUPS(suffix_length.div_ceil(BLOCK_QUERY_ROWS)),const uint tgid_y GROUPS(num_heads),const uint tgid_z GROUPS(1),const uint lid THREADS(128)) {// -------------------------------------------------------------------------// Pointer setup (all strides are in elements)- // tgid_x: query tile index (BQ rows)+ // tgid_x: query tile index (BLOCK_QUERY_ROWS rows)// tgid_y: query head index// tgid_z: batch index (currently 1 in uzu, but kept for completeness)const uint batch_idx = tgid_z;@@ -73,7 +74,7 @@ PUBLIC KERNEL(AttentionGemm)(const uint q_tile_idx = tgid_x;q += batch_idx * params.q_strides[0] + head_idx * params.q_strides[1] +- q_tile_idx * int64_t(BQ) * params.q_strides[2];+ q_tile_idx * int64_t(BLOCK_QUERY_ROWS) * params.q_strides[2];const int kv_head_idx = int(tgid_y) / params.gqa_factor;k += batch_idx * params.k_strides[0] +@@ -82,7 +83,7 @@ PUBLIC KERNEL(AttentionGemm)(int64_t(kv_head_idx) * params.v_strides[1];o += batch_idx * params.o_strides[0] + head_idx * params.o_strides[1] +- q_tile_idx * int64_t(BQ) * params.o_strides[2];+ q_tile_idx * int64_t(BLOCK_QUERY_ROWS) * params.o_strides[2];if (has_mask) {mask += batch_idx * mask_params.m_strides[0] +@@ -91,115 +92,162 @@ PUBLIC KERNEL(AttentionGemm)(// -------------------------------------------------------------------------// Threadgroup memory- constexpr short padQ = 16 / sizeof(T);- constexpr short padK = 16 / sizeof(T);- constexpr short padV = 16 / sizeof(T);+ constexpr short query_padding = 16 / sizeof(T);+ constexpr short key_padding = 16 / sizeof(T);+ constexpr short value_padding = 16 / sizeof(T);- constexpr short LDQ_tgp = BD + padQ;- constexpr short LDK_tgp = BD + padK; // K stored as [BK, BD] row-major- constexpr short LDV_tgp = BD + padV;+ constexpr short query_leading_dimension = BD + query_padding;+ constexpr short key_leading_dimension = BD + key_padding;+ constexpr short value_leading_dimension = BD + value_padding;- threadgroup T* Qs = q_smem;- threadgroup T* Ks = kv_smem;- threadgroup T* Vs = kv_smem;+ threadgroup T* query_shared = q_smem;+ threadgroup T* key_shared = kv_smem;+ threadgroup T* value_shared = kv_smem;//// -------------------------------------------------------------------------// Block loaders- using QBlockLoader = BlockLoader<+ using QueryLoader = ThreadgroupLoader<T,- /*BROWS=*/BQ,- /*BCOLS=*/BD,- /*dst_ld=*/LDQ_tgp,- /*reduction_dim=*/1,- /*tgp_size=*/WM * WN * 32>;+ BLOCK_QUERY_ROWS,+ BD,+ query_leading_dimension,+ 1,+ SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * 32>;- using KBlockLoader = BlockLoader<+ using KeyLoader = ThreadgroupLoader<T,- /*BROWS=*/BK,- /*BCOLS=*/BD,- /*dst_ld=*/LDK_tgp,- /*reduction_dim=*/0,- /*tgp_size=*/WM * WN * 32>;+ BK,+ BD,+ key_leading_dimension,+ 0,+ SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * 32>;- using VBlockLoader = BlockLoader<+ using ValueLoader = ThreadgroupLoader<T,- /*BROWS=*/BK,- /*BCOLS=*/BD,- /*dst_ld=*/LDV_tgp,- /*reduction_dim=*/0,- /*tgp_size=*/WM * WN * 32>;-- const int q_src_ld = int(params.q_strides[2]);- const int k_src_ld = int(params.k_strides[2]);- const int v_src_ld = int(params.v_strides[2]);-- thread QBlockLoader loader_q(q, q_src_ld, Qs, simd.group_idx, simd.lane_idx);- thread KBlockLoader loader_k(k, k_src_ld, Ks, simd.group_idx, simd.lane_idx);- thread VBlockLoader loader_v(v, v_src_ld, Vs, simd.group_idx, simd.lane_idx);+ BK,+ BD,+ value_leading_dimension,+ 0,+ SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * 32>;++ const int query_source_stride = int(params.q_strides[2]);+ const int key_source_stride = int(params.k_strides[2]);+ const int value_source_stride = int(params.v_strides[2]);++ thread QueryLoader query_loader(+ q,+ query_source_stride,+ query_shared,+ simd.threadgroup_index,+ simd.simdgroup_index+ );+ thread KeyLoader key_loader(+ k,+ key_source_stride,+ key_shared,+ simd.threadgroup_index,+ simd.simdgroup_index+ );+ thread ValueLoader value_loader(+ v,+ value_source_stride,+ value_shared,+ simd.threadgroup_index,+ simd.simdgroup_index+ );TransformScale<T> ts(static_cast<T>(params.scale * M_LOG2E_F));// -------------------------------------------------------------------------// MMA tiles- constexpr short kFragSize = 8;+ constexpr short SIMDGROUP_BLOCK_SIZE = 8;using AccumType = float;- using MMAFrag_acc_t = BaseMMAFrag<AccumType, kFragSize, kFragSize>;+ using SimdgroupMultiplyAccumulateType = SimdgroupMultiplyAccumulate<+ AccumType,+ SIMDGROUP_BLOCK_SIZE,+ SIMDGROUP_BLOCK_SIZE>;- constexpr int kNWarps = WM * WN;+ constexpr int SIMDGROUPS_PER_THREADGROUP =+ SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN;static_assert(- BQ >= (kNWarps * kFragSize) && BQ % (kNWarps * kFragSize) == 0,+ BLOCK_QUERY_ROWS >= (SIMDGROUPS_PER_THREADGROUP * SIMDGROUP_BLOCK_SIZE) &&+ BLOCK_QUERY_ROWS %+ (SIMDGROUPS_PER_THREADGROUP * SIMDGROUP_BLOCK_SIZE) ==+ 0,"Each simdgroup must host at least 1 simdgroup matrix along Q sequence.");- // Q seq frags per warp (we keep TQ == 1 for the 32-row block layout)- constexpr int TQ = BQ / (kNWarps * kFragSize);- // KV sequence frags- constexpr int TK = BK / kFragSize;- // HeadDim frags- constexpr int TD = BD / kFragSize;-- static_assert(TQ == 1, "Expected TQ == 1");-- MMATile<AccumType, TQ, 1, MMAFrag_acc_t> Qtile;- MMATile<AccumType, 1, TK, MMAFrag_acc_t> Ktile; // represents K^T slice- MMATile<AccumType, TQ, TK, MMAFrag_acc_t> Stile;- MMATile<AccumType, 1, 1, MMAFrag_acc_t> Vtile;- MMATile<AccumType, TQ, TD, MMAFrag_acc_t> Otile;-- Otile.clear();+ // Q sequence multiply-accumulate blocks per simdgroup (QUERY_GRID_ROWS == 1+ // for the 32-row block layout)+ constexpr int QUERY_GRID_ROWS =+ BLOCK_QUERY_ROWS / (SIMDGROUPS_PER_THREADGROUP * SIMDGROUP_BLOCK_SIZE);+ constexpr int KEY_GRID_COLS = BK / SIMDGROUP_BLOCK_SIZE;+ constexpr int HEAD_DIM_GRID_COLS = BD / SIMDGROUP_BLOCK_SIZE;++ static_assert(QUERY_GRID_ROWS == 1, "Expected QUERY_GRID_ROWS == 1");++ SimdgroupFragment<+ AccumType,+ QUERY_GRID_ROWS,+ 1,+ SimdgroupMultiplyAccumulateType>+ query_fragment;+ SimdgroupFragment<+ AccumType,+ 1,+ KEY_GRID_COLS,+ SimdgroupMultiplyAccumulateType>+ key_fragment;+ SimdgroupFragment<+ AccumType,+ QUERY_GRID_ROWS,+ KEY_GRID_COLS,+ SimdgroupMultiplyAccumulateType>+ score_fragment;+ SimdgroupFragment<AccumType, 1, 1, SimdgroupMultiplyAccumulateType>+ value_fragment;+ SimdgroupFragment<+ AccumType,+ QUERY_GRID_ROWS,+ HEAD_DIM_GRID_COLS,+ SimdgroupMultiplyAccumulateType>+ output_fragment;++ output_fragment.clear();// -------------------------------------------------------------------------// Lane coordinates and pointer offsets- const short2 simd_coord = MMAFrag_acc_t::get_coord(simd.lane_idx);- const short sm = simd_coord.y;- const short sn = simd_coord.x;+ const short2 lane_coordinates =+ SimdgroupMultiplyAccumulateType::get_lane_coordinates(+ simd.simdgroup_index+ );+ const short lane_row = lane_coordinates.y;+ const short lane_col = lane_coordinates.x;- const short tm = kFragSize * TQ * short(simd.group_idx);+ const short simdgroup_row_base =+ SIMDGROUP_BLOCK_SIZE * QUERY_GRID_ROWS * short(simd.threadgroup_index);- // Qs is row-major [BQ, BD]- const short Qs_offset = (tm + sm) * LDQ_tgp + sn;- constexpr short Qs_tile_stride = kFragSize;+ const short query_shared_offset =+ (simdgroup_row_base + lane_row) * query_leading_dimension + lane_col;+ constexpr short query_tile_stride = SIMDGROUP_BLOCK_SIZE;- // Ks is row-major [BK, BD] but we load K^T by swapping strides:- // B_str_k = 1, B_str_n = LDK_tgp => offset = sn * LDK_tgp + sm- const short Ks_offset = sn * LDK_tgp + sm;- constexpr short Ks_tile_stride =- kFragSize; // advance along head-dim (contiguous)+ const short key_shared_offset = lane_col * key_leading_dimension + lane_row;+ constexpr short key_tile_stride = SIMDGROUP_BLOCK_SIZE;- // Vs is row-major [BK, BD]- const short Vs_offset = sm * LDV_tgp + sn;+ const short value_shared_offset =+ lane_row * value_leading_dimension + lane_col;// -------------------------------------------------------------------------// Load Q block once (and apply scaling)threadgroup_barrier(mem_flags::mem_threadgroup);if (!align_q && int(tgid_x) == params.nq_aligned) {- loader_q.load_safe(short2(BD, params.q_rem));+ query_loader.load_safe(short2(BD, params.q_rem));} else {- loader_q.load_unsafe();+ query_loader.load_unsafe();}- loader_q.apply_inplace_op(ts);+ query_loader.apply_inplace_op(ts);// -------------------------------------------------------------------------// Streaming softmax state for this row (shared across lanes in a row)@@ -215,51 +263,57 @@ PUBLIC KERNEL(AttentionGemm)(// Determine K block loop limit (causal can early-stop)int kb_lim = params.nk;if (do_causal) {- const int q_max = (int(tgid_x) + 1) * BQ + params.q_off;+ const int q_max = (int(tgid_x) + 1) * BLOCK_QUERY_ROWS + params.q_off;kb_lim = (q_max + BK - 1) / BK;kb_lim = min(params.nk, kb_lim);}- const int q_rel = int(tgid_x) * BQ + int(tm) + int(sm); // [0, q_len)- const int q_abs = q_rel + params.q_off; // [0, k_len)+ const int q_rel = int(tgid_x) * BLOCK_QUERY_ROWS + int(simdgroup_row_base) ++ int(lane_row); // [0, q_len)+ const int q_abs = q_rel + params.q_off; // [0, k_len)// Loop over KV blocksfor (int kb = 0; kb < kb_lim; kb++) {// Load K blockthreadgroup_barrier(mem_flags::mem_threadgroup);if (!align_k && kb == params.nk_aligned) {- loader_k.load_safe(short2(BD, params.k_rem));+ key_loader.load_safe(short2(BD, params.k_rem));} else {- loader_k.load_unsafe();+ key_loader.load_unsafe();}// Compute S = Q @ K^T for this block- Stile.clear();+ score_fragment.clear();threadgroup_barrier(mem_flags::mem_threadgroup);- UZU_PRAGMA_UNROLL- for (short dd = 0; dd < TD; dd++) {+ METAL_PRAGMA_UNROLL+ for (short dd = 0; dd < HEAD_DIM_GRID_COLS; dd++) {simdgroup_barrier(mem_flags::mem_none);- Qtile.template load<T, 1, 1, LDQ_tgp, 1>(- &Qs[Qs_offset + dd * Qs_tile_stride]+ query_fragment.template load<T, 1, 1, query_leading_dimension, 1>(+ &query_shared[query_shared_offset + dd * query_tile_stride]);- Ktile.template load<T, 1, 1, 1, LDK_tgp>(- &Ks[Ks_offset + dd * Ks_tile_stride]+ key_fragment.template load<T, 1, 1, 1, key_leading_dimension>(+ &key_shared[key_shared_offset + dd * key_tile_stride]);simdgroup_barrier(mem_flags::mem_none);- tile_matmad(Stile, Qtile, Ktile, Stile);+ tile_multiply_accumulate(+ score_fragment,+ query_fragment,+ key_fragment,+ score_fragment+ );}// Mask out tail keys for the last (unaligned) K blockif (!align_k && kb == params.nk_aligned) {const int k_rem = params.k_rem;- UZU_PRAGMA_UNROLL- for (short j = 0; j < TK; j++) {- thread auto& frag = Stile.frag_at(0, j);- const int col0 = int(sn) + int(j) * kFragSize;+ METAL_PRAGMA_UNROLL+ for (short j = 0; j < KEY_GRID_COLS; j++) {+ thread auto& frag = score_fragment.multiply_accumulate_at(0, j);+ const int col0 = int(lane_col) + int(j) * SIMDGROUP_BLOCK_SIZE;if (col0 >= k_rem) {frag[0] = neg_inf;}@@ -271,13 +325,14 @@ PUBLIC KERNEL(AttentionGemm)(// Causal mask (only needed for the last few blocks near the diagonal)if (do_causal) {- const int tail_blocks = (BQ + BK - 1) / BK + int(!align_k);+ const int tail_blocks = (BLOCK_QUERY_ROWS + BK - 1) / BK + int(!align_k);const int tail_start = kb_lim - tail_blocks;if (kb >= tail_start) {- UZU_PRAGMA_UNROLL- for (short j = 0; j < TK; j++) {- thread auto& frag = Stile.frag_at(0, j);- const int col_base = kb * BK + int(sn) + int(j) * kFragSize;+ METAL_PRAGMA_UNROLL+ for (short j = 0; j < KEY_GRID_COLS; j++) {+ thread auto& frag = score_fragment.multiply_accumulate_at(0, j);+ const int col_base =+ kb * BK + int(lane_col) + int(j) * SIMDGROUP_BLOCK_SIZE;if (q_abs < col_base) {frag[0] = neg_inf;}@@ -293,10 +348,11 @@ PUBLIC KERNEL(AttentionGemm)(const int64_t row_stride = mask_params.m_strides[2];const int64_t row_base = int64_t(q_rel) * row_stride;- UZU_PRAGMA_UNROLL- for (short j = 0; j < TK; j++) {- thread auto& frag = Stile.frag_at(0, j);- const int col_base = kb * BK + int(sn) + int(j) * kFragSize;+ METAL_PRAGMA_UNROLL+ for (short j = 0; j < KEY_GRID_COLS; j++) {+ thread auto& frag = score_fragment.multiply_accumulate_at(0, j);+ const int col_base =+ kb * BK + int(lane_col) + int(j) * SIMDGROUP_BLOCK_SIZE;const int k0 = col_base;const int k1 = col_base + 1;@@ -317,9 +373,9 @@ PUBLIC KERNEL(AttentionGemm)(// Load V block (overwriting K in shared memory)threadgroup_barrier(mem_flags::mem_threadgroup);if (!align_k && kb == params.nk_aligned) {- loader_v.load_safe(short2(BD, params.k_rem));+ value_loader.load_safe(short2(BD, params.k_rem));} else {- loader_v.load_unsafe();+ value_loader.load_unsafe();}// -----------------------------------------------------------------------@@ -327,9 +383,9 @@ PUBLIC KERNEL(AttentionGemm)(// Row max for this blockAccumType block_max_local = -INFINITY;- UZU_PRAGMA_UNROLL- for (short j = 0; j < TK; j++) {- const thread auto& frag = Stile.frag_at(0, j);+ METAL_PRAGMA_UNROLL+ for (short j = 0; j < KEY_GRID_COLS; j++) {+ const thread auto& frag = score_fragment.multiply_accumulate_at(0, j);block_max_local = metal::max(block_max_local, frag[0]);block_max_local = metal::max(block_max_local, frag[1]);}@@ -344,9 +400,9 @@ PUBLIC KERNEL(AttentionGemm)(// exp2(S - new_max) and row sumAccumType block_sum_local = AccumType(0);- UZU_PRAGMA_UNROLL- for (short j = 0; j < TK; j++) {- thread auto& frag = Stile.frag_at(0, j);+ METAL_PRAGMA_UNROLL+ for (short j = 0; j < KEY_GRID_COLS; j++) {+ thread auto& frag = score_fragment.multiply_accumulate_at(0, j);frag[0] = fast::exp2(frag[0] - new_max);frag[1] = fast::exp2(frag[1] - new_max);block_sum_local += frag[0] + frag[1];@@ -355,50 +411,51 @@ PUBLIC KERNEL(AttentionGemm)(sum_score += block_sum;// Rescale output accumulator- UZU_PRAGMA_UNROLL- for (short id = 0; id < TD; id++) {- thread auto& frag = Otile.frag_at(0, id);+ METAL_PRAGMA_UNROLL+ for (short id = 0; id < HEAD_DIM_GRID_COLS; id++) {+ thread auto& frag = output_fragment.multiply_accumulate_at(0, id);frag[0] *= factor;frag[1] *= factor;}- // Accumulate output: Otile += Stile * Vblock+ // Accumulate output: output_fragment += score_fragment * Vblockthreadgroup_barrier(mem_flags::mem_threadgroup);- UZU_PRAGMA_UNROLL- for (short id = 0; id < TD; id++) {- UZU_PRAGMA_UNROLL- for (short ik = 0; ik < TK; ik++) {+ METAL_PRAGMA_UNROLL+ for (short id = 0; id < HEAD_DIM_GRID_COLS; id++) {+ METAL_PRAGMA_UNROLL+ for (short ik = 0; ik < KEY_GRID_COLS; ik++) {IF_CONSTEXPR(BD == 128) { simdgroup_barrier(mem_flags::mem_none); }- const short kk = ik * kFragSize;- const short dd = id * kFragSize;+ const short kk = ik * SIMDGROUP_BLOCK_SIZE;+ const short dd = id * SIMDGROUP_BLOCK_SIZE;- Vtile.template load<T, 1, 1, LDV_tgp, 1>(- &Vs[Vs_offset + kk * LDV_tgp + dd]+ value_fragment.template load<T, 1, 1, value_leading_dimension, 1>(+ &value_shared+ [value_shared_offset + kk * value_leading_dimension + dd]);IF_CONSTEXPR(BD == 128) { simdgroup_barrier(mem_flags::mem_none); }- MMAFrag_acc_t::mma(- Otile.frag_at(0, id),- Stile.frag_at(0, ik),- Vtile.frag_at(0, 0),- Otile.frag_at(0, id)+ SimdgroupMultiplyAccumulateType::multiply_accumulate(+ output_fragment.multiply_accumulate_at(0, id),+ score_fragment.multiply_accumulate_at(0, ik),+ value_fragment.multiply_accumulate_at(0, 0),+ output_fragment.multiply_accumulate_at(0, id));}}// Prepare for next iteration- loader_k.next();- loader_v.next();+ key_loader.next();+ value_loader.next();}// -------------------------------------------------------------------------// Normalize output by sum_score (avoid div-by-zero for masked-out rows)const AccumType inv_sum = AccumType(1) / sum_score;- UZU_PRAGMA_UNROLL- for (short id = 0; id < TD; id++) {- thread auto& frag = Otile.frag_at(0, id);+ METAL_PRAGMA_UNROLL+ for (short id = 0; id < HEAD_DIM_GRID_COLS; id++) {+ thread auto& frag = output_fragment.multiply_accumulate_at(0, id);frag[0] *= inv_sum;frag[1] *= inv_sum;}@@ -406,21 +463,23 @@ PUBLIC KERNEL(AttentionGemm)(threadgroup_barrier(mem_flags::mem_none);// Store results (O is row-major with row-stride params.o_strides[2])- o += int64_t(tm + sm) * params.o_strides[2] + int64_t(sn);+ o += int64_t(simdgroup_row_base + lane_row) * params.o_strides[2] ++ int64_t(lane_col);if (!align_q && int(tgid_x) == params.nq_aligned) {- const short2 dst_tile_dims = short2(BD - sn, params.q_rem - (tm + sm));+ const short2 dst_tile_dims =+ short2(BD - lane_col, params.q_rem - (simdgroup_row_base + lane_row));if (dst_tile_dims.x <= 0 || dst_tile_dims.y <= 0) {return;}- Otile.template store_safe<T, 1, 1>(+ output_fragment.template store_safe<T, 1, 1>(o,int(params.o_strides[2]),dst_tile_dims);} else {- Otile.template store<T, 1, 1>(o, int(params.o_strides[2]));+ output_fragment.template store<T, 1, 1>(o, int(params.o_strides[2]));}}diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metalindex 7b693dd5f..1a09cb6be 100644--- a/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_single_pass.metal@@ -1,6 +1,7 @@#include <metal_stdlib>#include <metal_simdgroup>-#include "../definitions.metal"+#include "../common/dsl.h"+#include "../common/thread_context.h"#define SEQUENCE_BLOCK_SIZE 32#define HEAD_BLOCK_SIZE 32@@ -34,7 +35,7 @@ PUBLIC KERNEL(AttentionSinglePass)(const bool has_mask SPECIALIZE,const bool has_sinks SPECIALIZE,const bool do_causal SPECIALIZE,- const Simd simd,+ const ThreadContext simd,const uint head_idx GROUPS(num_heads),const uint q_seq_idx GROUPS(suffix_length),const uint tid THREADS(1024)@@ -59,18 +60,22 @@ PUBLIC KERNEL(AttentionSinglePass)(const uint q_offset = query_transposed ? tpg.x * q_seq_idx + head_idx: head_idx * tpg.y + q_seq_idx;- queries += q_offset * HEAD_DIM + simd.lane_idx * qk_elements_per_thread;- keys += kv_head_idx * k_head_stride + simd.group_idx * k_seq_stride +- simd.lane_idx * qk_elements_per_thread;- values += kv_head_idx * v_head_stride + simd.group_idx * v_seq_stride +- simd.lane_idx * value_elements_per_thread;+ queries +=+ q_offset * HEAD_DIM + simd.simdgroup_index * qk_elements_per_thread;+ keys += kv_head_idx * k_head_stride + simd.threadgroup_index * k_seq_stride ++ simd.simdgroup_index * qk_elements_per_thread;+ values += kv_head_idx * v_head_stride ++ simd.threadgroup_index * v_seq_stride ++ simd.simdgroup_index * value_elements_per_thread;if (float_mask) {- fmask += head_idx * mask_head_stride + simd.group_idx * mask_kv_seq_stride ++ fmask += head_idx * mask_head_stride ++ simd.threadgroup_index * mask_kv_seq_stride +q_seq_idx * mask_q_seq_stride;}- out += o_offset * value_dim + simd.group_idx * value_elements_per_thread;+ out +=+ o_offset * value_dim + simd.threadgroup_index * value_elements_per_thread;// Read the query and 0 the output accumulatorfor (int i = 0; i < qk_elements_per_thread; i++) {@@ -82,7 +87,7 @@ PUBLIC KERNEL(AttentionSinglePass)(U max_score = -INFINITY;U sum_exp_score = 0;- if (has_sinks && simd.group_idx == 0) {+ if (has_sinks && simd.threadgroup_index == 0) {const int num_q_heads = static_cast<int>(tpg.x);int q_head_idx = head_idx % num_q_heads;max_score = static_cast<U>(sinks[q_head_idx]);@@ -90,7 +95,8 @@ PUBLIC KERNEL(AttentionSinglePass)(}// For each key- for (uint i = simd.group_idx; i < sequence_length; i += SEQUENCE_BLOCK_SIZE) {+ for (uint i = simd.threadgroup_index; i < sequence_length;+ i += SEQUENCE_BLOCK_SIZE) {bool use_key = true;if (do_causal) {use_key = i <= (sequence_length - tpg.y + q_seq_idx);@@ -135,31 +141,35 @@ PUBLIC KERNEL(AttentionSinglePass)(}// Each thread has a partial part of the output so we need to combine them.- if (simd.lane_idx == 0) {- shared_max_scores[simd.group_idx] = max_score;- shared_sum_exp_scores[simd.group_idx] = sum_exp_score;+ if (simd.simdgroup_index == 0) {+ shared_max_scores[simd.threadgroup_index] = max_score;+ shared_sum_exp_scores[simd.threadgroup_index] = sum_exp_score;}threadgroup_barrier(mem_flags::mem_threadgroup);- max_score = shared_max_scores[simd.lane_idx];+ max_score = shared_max_scores[simd.simdgroup_index];U new_max = simd_max(max_score);U factor = fast::exp(max_score - new_max);- sum_exp_score = simd_sum(shared_sum_exp_scores[simd.lane_idx] * factor);+ sum_exp_score =+ simd_sum(shared_sum_exp_scores[simd.simdgroup_index] * factor);// Now we need to aggregate all the outputsfor (int i = 0; i < value_elements_per_thread; i++) {- shared_outputs[simd.lane_idx * HEAD_BLOCK_SIZE + simd.group_idx] = o[i];+ shared_outputs+ [simd.simdgroup_index * HEAD_BLOCK_SIZE + simd.threadgroup_index] =+ o[i];threadgroup_barrier(mem_flags::mem_threadgroup);- o[i] =- simd_sum(- shared_outputs[simd.group_idx * HEAD_BLOCK_SIZE + simd.lane_idx] *- factor- ) /- sum_exp_score;+ o[i] = simd_sum(+ shared_outputs+ [simd.threadgroup_index * HEAD_BLOCK_SIZE ++ simd.simdgroup_index] *+ factor+ ) /+ sum_exp_score;threadgroup_barrier(mem_flags::mem_threadgroup);}// And write the output- if (simd.lane_idx == 0) {+ if (simd.simdgroup_index == 0) {for (int i = 0; i < value_elements_per_thread; i++) {out[i] = static_cast<T>(o[i]);}diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metalindex 9db9a7e86..a47849b78 100644--- a/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_two_pass.metal@@ -1,6 +1,7 @@#include <metal_stdlib>#include <metal_simdgroup>-#include "../definitions.metal"+#include "../common/dsl.h"+#include "../common/thread_context.h"#define HEAD_BLOCK_SIZE 32#define SEQUENCE_BLOCK_SIZE_1 8@@ -38,7 +39,7 @@ PUBLIC KERNEL(AttentionTwoPass1)(threadgroup float shared_max_scores[SEQUENCE_BLOCK_SIZE_1],threadgroup float shared_sum_exp_scores[SEQUENCE_BLOCK_SIZE_1],threadgroup float shared_outputs[SEQUENCE_BLOCK_SIZE_1 * HEAD_BLOCK_SIZE],- const Simd simd,+ const ThreadContext simd,const uint head_idx GROUPS(num_heads),const uint q_seq_idx GROUPS(suffix_length),const uint block_idx GROUPS(TOTAL_BLOCKS_COUNT),@@ -66,19 +67,21 @@ PUBLIC KERNEL(AttentionTwoPass1)(: head_idx * tpg.y + q_seq_idx; // Consistent with single-passconst uint kv_head_idx = head_idx / gqa_factor;- queries += q_offset * HEAD_DIM + simd.lane_idx * qk_elements_per_thread;+ queries +=+ q_offset * HEAD_DIM + simd.simdgroup_index * qk_elements_per_thread;keys += kv_head_idx * k_head_stride +- (block_idx * SEQUENCE_BLOCK_SIZE_1 + simd.group_idx) * k_seq_stride +- simd.lane_idx * qk_elements_per_thread;- values +=- kv_head_idx * v_head_stride +- (block_idx * SEQUENCE_BLOCK_SIZE_1 + simd.group_idx) * v_seq_stride +- simd.lane_idx * value_elements_per_thread;+ (block_idx * SEQUENCE_BLOCK_SIZE_1 + simd.threadgroup_index) *+ k_seq_stride ++ simd.simdgroup_index * qk_elements_per_thread;+ values += kv_head_idx * v_head_stride ++ (block_idx * SEQUENCE_BLOCK_SIZE_1 + simd.threadgroup_index) *+ v_seq_stride ++ simd.simdgroup_index * value_elements_per_thread;out += o_offset * TOTAL_BLOCKS_COUNT * value_dim + block_idx * value_dim +- simd.lane_idx * value_elements_per_thread;+ simd.simdgroup_index * value_elements_per_thread;if (float_mask) {fmask += head_idx * mask_head_stride +- (block_idx * SEQUENCE_BLOCK_SIZE_1 + simd.group_idx) *+ (block_idx * SEQUENCE_BLOCK_SIZE_1 + simd.threadgroup_index) *mask_kv_seq_stride +q_seq_idx * mask_q_seq_stride;}@@ -95,7 +98,7 @@ PUBLIC KERNEL(AttentionTwoPass1)(U max_score = -1e9;U sum_exp_score = 0;- if (has_sinks && block_idx == 0 && simd.group_idx == 0) {+ if (has_sinks && block_idx == 0 && simd.threadgroup_index == 0) {const uint num_q_heads = tpg.x;int q_head_idx = head_idx % num_q_heads;max_score = static_cast<U>(sinks[q_head_idx]);@@ -103,7 +106,7 @@ PUBLIC KERNEL(AttentionTwoPass1)(}// For each key- for (uint i = block_idx * SEQUENCE_BLOCK_SIZE_1 + simd.group_idx;+ for (uint i = block_idx * SEQUENCE_BLOCK_SIZE_1 + simd.threadgroup_index;i < sequence_length;i += TOTAL_BLOCKS_COUNT * SEQUENCE_BLOCK_SIZE_1) {bool use_key = true;@@ -150,38 +153,42 @@ PUBLIC KERNEL(AttentionTwoPass1)(}// Each thread has a partial part of the output so we need to combine them.- if (simd.lane_idx == 0) {- shared_max_scores[simd.group_idx] = max_score;- shared_sum_exp_scores[simd.group_idx] = sum_exp_score;+ if (simd.simdgroup_index == 0) {+ shared_max_scores[simd.threadgroup_index] = max_score;+ shared_sum_exp_scores[simd.threadgroup_index] = sum_exp_score;}threadgroup_barrier(mem_flags::mem_threadgroup);- max_score = (simd.lane_idx < SEQUENCE_BLOCK_SIZE_1)- ? shared_max_scores[simd.lane_idx]+ max_score = (simd.simdgroup_index < SEQUENCE_BLOCK_SIZE_1)+ ? shared_max_scores[simd.simdgroup_index]: -1e9;U new_max = simd_max(max_score);U factor = fast::exp(max_score - new_max);- sum_exp_score = (simd.lane_idx < SEQUENCE_BLOCK_SIZE_1)- ? shared_sum_exp_scores[simd.lane_idx]+ sum_exp_score = (simd.simdgroup_index < SEQUENCE_BLOCK_SIZE_1)+ ? shared_sum_exp_scores[simd.simdgroup_index]: 0;sum_exp_score = simd_sum(sum_exp_score * factor);// Write the sum and new max- if (simd.group_idx == 0) {+ if (simd.threadgroup_index == 0) {sums[0] = sum_exp_score;maxs[0] = new_max;}// Now we need to aggregate all the outputsfor (uint i = 0; i < value_elements_per_thread; i++) {- shared_outputs[simd.lane_idx * SEQUENCE_BLOCK_SIZE_1 + simd.group_idx] =- o[i] * fast::exp(shared_max_scores[simd.group_idx] - new_max);+ shared_outputs+ [simd.simdgroup_index * SEQUENCE_BLOCK_SIZE_1 ++ simd.threadgroup_index] =+ o[i] *+ fast::exp(shared_max_scores[simd.threadgroup_index] - new_max);threadgroup_barrier(mem_flags::mem_threadgroup);// And write the output- if (simd.group_idx == 0) {- U output = shared_outputs[simd.lane_idx * SEQUENCE_BLOCK_SIZE_1];+ if (simd.threadgroup_index == 0) {+ U output = shared_outputs[simd.simdgroup_index * SEQUENCE_BLOCK_SIZE_1];for (uint j = 1; j < SEQUENCE_BLOCK_SIZE_1; j++) {- output += shared_outputs[simd.lane_idx * SEQUENCE_BLOCK_SIZE_1 + j];+ output +=+ shared_outputs[simd.simdgroup_index * SEQUENCE_BLOCK_SIZE_1 + j];}out[i] = static_cast<T>(output);}@@ -200,7 +207,7 @@ PUBLIC KERNEL(AttentionTwoPass2)(const constant uint& num_heads,const constant uint& suffix_length,threadgroup float shared_outputs[SEQUENCE_BLOCK_SIZE_2 * HEAD_BLOCK_SIZE],- const Simd simd,+ const ThreadContext simd,const uint head_idx GROUPS(num_heads),const uint q_seq_idx GROUPS(suffix_length),const uint tid THREADS(1024)@@ -214,17 +221,19 @@ PUBLIC KERNEL(AttentionTwoPass2)(const uint o_offset = q_seq_idx * tpg.x + head_idx; // Our custom layoutpartials += o_offset * TOTAL_BLOCKS_COUNT * HEAD_DIM +- simd.group_idx * HEAD_DIM + simd.lane_idx * elements_per_thread;+ simd.threadgroup_index * HEAD_DIM ++ simd.simdgroup_index * elements_per_thread;sums += o_offset * TOTAL_BLOCKS_COUNT;maxs += o_offset * TOTAL_BLOCKS_COUNT;- out += o_offset * HEAD_DIM +- simd.group_idx * elements_per_thread; // Our custom output layout+ out +=+ o_offset * HEAD_DIM ++ simd.threadgroup_index * elements_per_thread; // Our custom output layout// First everybody reads the max and sum_exp- U max_score = maxs[simd.lane_idx];+ U max_score = maxs[simd.simdgroup_index];U new_max = simd_max(max_score);U factor = fast::exp(max_score - new_max);- U sum_exp_score = simd_sum(sums[simd.lane_idx] * factor);+ U sum_exp_score = simd_sum(sums[simd.simdgroup_index] * factor);// Now read the block into registers and then use shared memory to transpose// it@@ -232,19 +241,22 @@ PUBLIC KERNEL(AttentionTwoPass2)(o[i] = partials[i];}for (uint i = 0; i < elements_per_thread; i++) {- shared_outputs[simd.lane_idx * HEAD_BLOCK_SIZE + simd.group_idx] = o[i];+ shared_outputs+ [simd.simdgroup_index * HEAD_BLOCK_SIZE + simd.threadgroup_index] =+ o[i];threadgroup_barrier(mem_flags::mem_threadgroup);- o[i] =- simd_sum(- shared_outputs[simd.group_idx * HEAD_BLOCK_SIZE + simd.lane_idx] *- factor- ) /- sum_exp_score;+ o[i] = simd_sum(+ shared_outputs+ [simd.threadgroup_index * HEAD_BLOCK_SIZE ++ simd.simdgroup_index] *+ factor+ ) /+ sum_exp_score;threadgroup_barrier(mem_flags::mem_threadgroup);}// And write the output- if (simd.lane_idx == 0) {+ if (simd.simdgroup_index == 0) {for (uint i = 0; i < elements_per_thread; i++) {out[i] = static_cast<T>(o[i]);}diff --git a/crates/uzu/src/backends/metal/kernel/attention/attention_update_kv_cache.metal b/crates/uzu/src/backends/metal/kernel/attention/attention_update_kv_cache.metalindex e28f779b4..f27f2c3af 100644--- a/crates/uzu/src/backends/metal/kernel/attention/attention_update_kv_cache.metal+++ b/crates/uzu/src/backends/metal/kernel/attention/attention_update_kv_cache.metal@@ -1,5 +1,5 @@#include <metal_stdlib>-#include "../definitions.metal"+#include "../common/dsl.h"template <typename T>VARIANTS(T, float, half, bfloat)diff --git a/crates/uzu/src/backends/metal/kernel/audio/audio.metal b/crates/uzu/src/backends/metal/kernel/audio/audio.metalindex aff2451b3..e95cb27f0 100644--- a/crates/uzu/src/backends/metal/kernel/audio/audio.metal+++ b/crates/uzu/src/backends/metal/kernel/audio/audio.metal@@ -1,5 +1,5 @@#include <metal_stdlib>-#include "../definitions.metal"+#include "../common/dsl.h"using namespace metal;diff --git a/crates/uzu/src/backends/metal/kernel/common/defines.h b/crates/uzu/src/backends/metal/kernel/common/defines.hindex 4ea67c43f..97e2fcfd3 100644--- a/crates/uzu/src/backends/metal/kernel/common/defines.h+++ b/crates/uzu/src/backends/metal/kernel/common/defines.h@@ -1,22 +1,17 @@#pragma once-#if defined __METAL__-#define MTL_CONST constant-#else-#define MTL_CONST-#endif+#define METAL_CONST static constant constexpr+#define METAL_PRAGMA_UNROLL _Pragma("clang loop unroll(full)")+#define METAL_SIMD_SIZE 32+METAL_FUNC int pow2(int n) { return 1 << n; }-static MTL_CONST constexpr int MAX_REDUCE_SPECIALIZED_DIMS = 4;-static MTL_CONST constexpr int REDUCE_N_READS = 4;-static MTL_CONST constexpr int REDUCE_N_WRITES = 4;-static MTL_CONST constexpr int SOFTMAX_N_READS = 4;-static MTL_CONST constexpr int RMS_N_READS = 4;-static MTL_CONST constexpr int RMS_LOOPED_LIMIT = 4096;+METAL_CONST int MAX_REDUCE_SPECIALIZED_DIMS = 4;+METAL_CONST int REDUCE_N_READS = 4;+METAL_CONST int REDUCE_N_WRITES = 4;+METAL_CONST int SOFTMAX_N_READS = 4;+METAL_CONST int RMS_N_READS = 4;+METAL_CONST int RMS_LOOPED_LIMIT = 4096;-// Instantiate a templated kernel.-// Extra args are used as template parameters:-// e.g. instantiate_kernel(binary_int, binary, a, b) ->-// [[host_name(binary_int)]] [kernel] binary<a, b>#define instantiate_kernel(name, func, ...) \template [[host_name( \name \diff --git a/crates/uzu/src/backends/metal/kernel/common/dsl.h b/crates/uzu/src/backends/metal/kernel/common/dsl.hnew file mode 100644index 000000000..bae47519e--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/common/dsl.h@@ -0,0 +1,66 @@+#pragma once++#include <metal_stdlib>+using namespace metal;++#if defined(__cpp_if_constexpr)+#define IF_CONSTEXPR(cond) if constexpr (cond)+#else+#define IF_CONSTEXPR(cond) if (cond)+#endif++#ifdef DSL_ANALYZE+#define DSL_META(...) [[clang::annotate("", __VA_ARGS__)]]+#else+#define DSL_META(...)+#endif++#define DSL_STR(X) #X+#define DSL_XSTR(X) DSL_STR(X)++#define VARIANTS(TYPENAME, ...) \+ DSL_META("dsl.variants", #TYPENAME, #__VA_ARGS__)+#define PUBLIC DSL_META("dsl.public")+#define KERNEL(NAME) DSL_META("dsl.kernel") void NAME++#define SPECIALIZE DSL_META("dsl.specialize")+#define OPTIONAL(EXPR) DSL_META("dsl.optional", DSL_XSTR(EXPR))++#define AXIS(TDS, TPG) DSL_META("dsl.axis", DSL_XSTR(TDS), DSL_XSTR(TPG))+#define GROUPS(EXPR) DSL_META("dsl.groups", DSL_XSTR(EXPR))+#define THREADS(EXPR) DSL_META("dsl.threads", DSL_XSTR(EXPR))++#define generateKernel( \+ max_threads, \+ functionName, \+ scalarType, \+ outerArgs, \+ innerArgs \+) \+ [[max_total_threads_per_threadgroup(max_threads)]] \+ kernel void functionName##_##scalarType outerArgs { \+ functionName innerArgs; \+ }++#define generateKernels(max_threads, functionName) \+ generateKernel( \+ max_threads, \+ functionName, \+ float, \+ outerArguments(float), \+ innerArguments \+ ); \+ generateKernel( \+ max_threads, \+ functionName, \+ bfloat, \+ outerArguments(bfloat), \+ innerArguments \+ ); \+ generateKernel( \+ max_threads, \+ functionName, \+ half, \+ outerArguments(half), \+ innerArguments \+ );diff --git a/crates/uzu/src/backends/metal/kernel/common/tensor_view.h b/crates/uzu/src/backends/metal/kernel/common/tensor_view.hnew file mode 100644index 000000000..ef5e977fd--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/common/tensor_view.h@@ -0,0 +1,120 @@+#pragma once++#include <metal_stdlib>+using namespace metal;++template <typename T>+struct TensorView2D {+ device T* data;+ uint dim0, dim1;+ uint stride0, stride1;++ TensorView2D(device T* ptr) : data(ptr) {}++ TensorView2D(device T* ptr, int d0, int d1) : data(ptr), dim0(d0), dim1(d1) {+ stride0 = d1;+ stride1 = 1;+ }++ TensorView2D(device T* ptr, int d0, int d1, int s0, int s1)+ : data(ptr), dim0(d0), dim1(d1), stride0(s0), stride1(s1) {}++ thread TensorView2D& shaped(int d0, int d1) {+ dim0 = d0;+ dim1 = d1;+ stride0 = d1;+ stride1 = 1;+ return *this;+ }++ device T& at(uint i, uint j) const { return data[i * stride0 + j * stride1]; }++ device T& operator()(uint i, uint j) const { return at(i, j); }+};++template <typename T>+struct TensorView3D {+ device T* data;+ uint dim0, dim1, dim2;+ uint stride0, stride1, stride2;++ TensorView3D(device T* ptr) : data(ptr) {}++ TensorView3D(device T* ptr, int d0, int d1, int d2)+ : data(ptr), dim0(d0), dim1(d1), dim2(d2) {+ stride0 = d1 * d2;+ stride1 = d2;+ stride2 = 1;+ }++ TensorView3D(device T* ptr, int d0, int d1, int d2, int s0, int s1, int s2)+ : data(ptr), dim0(d0), dim1(d1), dim2(d2), stride0(s0), stride1(s1),+ stride2(s2) {}++ thread TensorView3D& shaped(int d0, int d1, int d2) {+ dim0 = d0;+ dim1 = d1;+ dim2 = d2;+ stride0 = d1 * d2;+ stride1 = d2;+ stride2 = 1;+ return *this;+ }++ device T& at(uint i, uint j, uint k) const {+ return data[i * stride0 + j * stride1 + k * stride2];+ }++ device T& operator()(uint i, uint j, uint k) const { return at(i, j, k); }+};++template <typename T>+struct TensorView4D {+ device T* data;+ uint dim0, dim1, dim2, dim3;+ uint stride0, stride1, stride2, stride3;++ TensorView4D(device T* ptr) : data(ptr) {}++ TensorView4D(device T* ptr, int d0, int d1, int d2, int d3)+ : data(ptr), dim0(d0), dim1(d1), dim2(d2), dim3(d3) {+ stride0 = d1 * d2 * d3;+ stride1 = d2 * d3;+ stride2 = d3;+ stride3 = 1;+ }++ TensorView4D(+ device T* ptr,+ int d0,+ int d1,+ int d2,+ int d3,+ int s0,+ int s1,+ int s2,+ int s3+ )+ : data(ptr), dim0(d0), dim1(d1), dim2(d2), dim3(d3), stride0(s0),+ stride1(s1), stride2(s2), stride3(s3) {}++ thread TensorView4D& shaped(int d0, int d1, int d2, int d3) {+ dim0 = d0;+ dim1 = d1;+ dim2 = d2;+ dim3 = d3;+ stride0 = d1 * d2 * d3;+ stride1 = d2 * d3;+ stride2 = d3;+ stride3 = 1;+ return *this;+ }++ device T& at(uint i, uint j, uint k, uint l) const {+ return data[i * stride0 + j * stride1 + k * stride2 + l * stride3];+ }++ device T& operator()(uint i, uint j, uint k, uint l) const {+ return at(i, j, k, l);+ }+};diff --git a/crates/uzu/src/backends/metal/kernel/common/thread_context.h b/crates/uzu/src/backends/metal/kernel/common/thread_context.hnew file mode 100644index 000000000..9a3f827a5--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/common/thread_context.h@@ -0,0 +1,12 @@+#pragma once++#include <metal_stdlib>+using namespace metal;++struct ThreadContext {+ uint simdgroup_index;+ uint threadgroup_index;+ uint simdgroup_size;+ uint threadgroup_size;+ uint simdgroups_per_threadgroup;+};diff --git a/crates/uzu/src/backends/metal/kernel/common/thread_functions.h b/crates/uzu/src/backends/metal/kernel/common/thread_functions.hnew file mode 100644index 000000000..5185a9831--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/common/thread_functions.h@@ -0,0 +1,54 @@+#pragma once++#include <metal_stdlib>+using namespace metal;++template <ushort LENGTH, typename T>+static inline T thread_prefix_inclusive_sum(thread T (&values)[LENGTH]) {+ for (ushort i = 1; i < LENGTH; i++) {+ values[i] += values[i - 1];+ }+ return values[LENGTH - 1];+}++template <ushort LENGTH, typename T>+static inline T thread_prefix_inclusive_sum(threadgroup T* values) {+ for (ushort i = 1; i < LENGTH; i++) {+ values[i] += values[i - 1];+ }+ return values[LENGTH - 1];+}++template <ushort LENGTH, typename T>+static inline T thread_prefix_exclusive_sum(thread T (&values)[LENGTH]) {+ T inclusive_prefix = thread_prefix_inclusive_sum<LENGTH>(values);+ for (ushort i = LENGTH - 1; i > 0; i--) {+ values[i] = values[i - 1];+ }+ values[0] = 0;+ return inclusive_prefix;+}++template <ushort LENGTH, typename T>+static inline T thread_prefix_exclusive_sum(threadgroup T* values) {+ T inclusive_prefix = thread_prefix_inclusive_sum<LENGTH>(values);+ for (ushort i = LENGTH - 1; i > 0; i--) {+ values[i] = values[i - 1];+ }+ values[0] = 0;+ return inclusive_prefix;+}++template <ushort LENGTH, typename T>+static inline void thread_uniform_add(thread T (&values)[LENGTH], T uni) {+ for (ushort i = 0; i < LENGTH; i++) {+ values[i] += uni;+ }+}++template <ushort LENGTH, typename T>+static inline void thread_uniform_add(threadgroup T* values, T uni) {+ for (ushort i = 0; i < LENGTH; i++) {+ values[i] += uni;+ }+}diff --git a/crates/uzu/src/backends/metal/kernel/common/threadgroup_reduce.h b/crates/uzu/src/backends/metal/kernel/common/threadgroup_reduce.hnew file mode 100644index 000000000..2438e37a7--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/common/threadgroup_reduce.h@@ -0,0 +1,229 @@+#pragma once++#include "thread_context.h"+#include <metal_stdlib>+using namespace metal;++template <ushort BLOCK_SIZE, typename T>+static T threadgroup_raking_prefix_exclusive_sum(+ T value,+ threadgroup T* shared,+ const ushort lid+) {+ shared[lid] = value;+ threadgroup_barrier(mem_flags::mem_threadgroup);++ if (lid < 32) {+ const short values_per_thread = BLOCK_SIZE / 32;+ const short first_index = lid * values_per_thread;+ for (short i = first_index + 1; i < first_index + values_per_thread; i++) {+ shared[i] += shared[i - 1];+ }+ T partial_sum = shared[first_index + values_per_thread - 1];+ for (short i = first_index + values_per_thread - 1; i > first_index; i--) {+ shared[i] = shared[i - 1];+ }+ shared[first_index] = 0;++ T prefix = simd_prefix_exclusive_sum(partial_sum);++ for (short i = first_index; i < first_index + values_per_thread; i++) {+ shared[i] += prefix;+ }+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);++ const T result = shared[lid];+ threadgroup_barrier(mem_flags::mem_threadgroup);+ return result;+}++template <ushort BLOCK_SIZE, typename T>+static T threadgroup_raking_reduce_sum(+ T value,+ threadgroup T* shared,+ const ushort lid+) {+ shared[lid] = value;+ threadgroup_barrier(mem_flags::mem_threadgroup);++ if (lid < 32) {+ const short values_per_thread = BLOCK_SIZE / 32;+ const short first_index = lid * values_per_thread;++ T thread_sum = shared[first_index];+ for (short i = first_index + 1; i < first_index + values_per_thread; i++) {+ thread_sum += shared[i];+ }++ T total_sum = simd_sum(thread_sum);++ if (lid == 0) {+ shared[0] = total_sum;+ }+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ const T result = shared[0];+ threadgroup_barrier(mem_flags::mem_threadgroup);+ return result;+}++template <ushort BLOCK_SIZE, typename T>+static T threadgroup_raking_reduce_max(+ T value,+ threadgroup T* shared,+ const ushort lid+) {+ shared[lid] = value;+ threadgroup_barrier(mem_flags::mem_threadgroup);++ if (lid < 32) {+ const short values_per_thread = BLOCK_SIZE / 32;+ const short first_index = lid * values_per_thread;++ T thread_max = shared[first_index];+ for (short i = first_index + 1; i < first_index + values_per_thread; i++) {+ thread_max = max(thread_max, shared[i]);+ }++ T total_max = simd_max(thread_max);++ if (lid == 0) {+ shared[0] = total_max;+ }+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ const T result = shared[0];+ threadgroup_barrier(mem_flags::mem_threadgroup);+ return result;+}++template <ushort BLOCK_SIZE, typename T>+static T threadgroup_raking_reduce_min(+ T value,+ threadgroup T* shared,+ const ushort lid+) {+ shared[lid] = value;+ threadgroup_barrier(mem_flags::mem_threadgroup);++ if (lid < 32) {+ const short values_per_thread = BLOCK_SIZE / 32;+ const short first_index = lid * values_per_thread;++ T thread_min = shared[first_index];+ for (short i = first_index + 1; i < first_index + values_per_thread; i++) {+ thread_min = min(thread_min, shared[i]);+ }++ T total_min = simd_min(thread_min);++ if (lid == 0) {+ shared[0] = total_min;+ }+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);++ const T result = shared[0];+ threadgroup_barrier(mem_flags::mem_threadgroup);+ return result;+}++template <ushort BLOCK_SIZE, typename T>+static T threadgroup_cooperative_reduce_sum(+ T value,+ threadgroup T* shared,+ const ushort lid,+ const thread ThreadContext& simd+) {+ T local_sum = simd_sum(value);++ if (simd.simdgroup_index == 0) {+ shared[simd.threadgroup_index] = local_sum;+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ T total_sum = T(0);+ const ushort num_simd_groups =+ (BLOCK_SIZE + simd.simdgroup_size - 1) / simd.simdgroup_size;+ if (lid < num_simd_groups) {+ total_sum = shared[lid];+ }+ total_sum = simd_sum(total_sum);++ if (lid == 0) {+ shared[0] = total_sum;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);++ const T result = shared[0];+ threadgroup_barrier(mem_flags::mem_threadgroup);+ return result;+}++template <ushort BLOCK_SIZE, typename T>+static T threadgroup_cooperative_reduce_max(+ T value,+ threadgroup T* shared,+ const ushort lid,+ const thread ThreadContext& simd+) {+ T local_max = simd_max(value);++ if (simd.simdgroup_index == 0) {+ shared[simd.threadgroup_index] = local_max;+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ T total_max =+ lid < (BLOCK_SIZE + simd.simdgroup_size - 1) / simd.simdgroup_size+ ? shared[lid]+ : T(-INFINITY);+ total_max = simd_max(total_max);++ if (lid == 0) {+ shared[0] = total_max;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);++ const T result = shared[0];+ threadgroup_barrier(mem_flags::mem_threadgroup);+ return result;+}++template <ushort BLOCK_SIZE, typename T>+static T threadgroup_cooperative_reduce_min(+ T value,+ threadgroup T* shared,+ const ushort lid,+ const thread ThreadContext& simd+) {+ T local_min = simd_min(value);++ if (simd.simdgroup_index == 0) {+ shared[simd.threadgroup_index] = local_min;+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ T total_min =+ lid < (BLOCK_SIZE + simd.simdgroup_size - 1) / simd.simdgroup_size+ ? shared[lid]+ : T(INFINITY);+ total_min = simd_min(total_min);++ if (lid == 0) {+ shared[0] = total_min;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);++ const T result = shared[0];+ threadgroup_barrier(mem_flags::mem_threadgroup);+ return result;+}diff --git a/crates/uzu/src/backends/metal/kernel/common/utils.h b/crates/uzu/src/backends/metal/kernel/common/utils.hindex c4e853f4c..71a185205 100644--- a/crates/uzu/src/backends/metal/kernel/common/utils.h+++ b/crates/uzu/src/backends/metal/kernel/common/utils.h@@ -88,8 +88,6 @@ struct Limits<complex64_t> {// Indexing utils///////////////////////////////////////////////////////////////////////////////-#define MTL_PRAGMA_UNROLL _Pragma("clang loop unroll(full)")-///////////////////////////////////////////////////////////////////////////////// Single Array with generic dimsdiff --git a/crates/uzu/src/backends/metal/kernel/definitions.metal b/crates/uzu/src/backends/metal/kernel/definitions.metaldeleted file mode 100644index d6f5a7c8a..000000000--- a/crates/uzu/src/backends/metal/kernel/definitions.metal+++ /dev/null@@ -1,536 +0,0 @@-#ifndef definitions_metal-#define definitions_metal--#include <metal_stdlib>--using namespace metal;--// MARK: - Tensor Access Templates--template <typename T>-struct TensorView2D {- device T* data;- uint dim0, dim1;- uint stride0, stride1;-- TensorView2D(device T* ptr) : data(ptr) {}-- TensorView2D(device T* ptr, int d0, int d1) : data(ptr), dim0(d0), dim1(d1) {- stride0 = d1;- stride1 = 1;- }-- // Custom stride constructor- TensorView2D(device T* ptr, int d0, int d1, int s0, int s1)- : data(ptr), dim0(d0), dim1(d1), stride0(s0), stride1(s1) {}-- thread TensorView2D& shaped(int d0, int d1) {- dim0 = d0;- dim1 = d1;- stride0 = d1;- stride1 = 1;- return *this;- }-- device T& at(uint i, uint j) const { return data[i * stride0 + j * stride1]; }-- device T& operator()(uint i, uint j) const { return at(i, j); }-};--template <typename T>-struct TensorView3D {- device T* data;- uint dim0, dim1, dim2;- uint stride0, stride1, stride2;-- TensorView3D(device T* ptr) : data(ptr) {}-- TensorView3D(device T* ptr, int d0, int d1, int d2)- : data(ptr), dim0(d0), dim1(d1), dim2(d2) {- stride0 = d1 * d2;- stride1 = d2;- stride2 = 1;- }-- // Custom stride constructor- TensorView3D(device T* ptr, int d0, int d1, int d2, int s0, int s1, int s2)- : data(ptr), dim0(d0), dim1(d1), dim2(d2), stride0(s0), stride1(s1),- stride2(s2) {}-- thread TensorView3D& shaped(int d0, int d1, int d2) {- dim0 = d0;- dim1 = d1;- dim2 = d2;- stride0 = d1 * d2;- stride1 = d2;- stride2 = 1;- return *this;- }-- device T& at(uint i, uint j, uint k) const {- return data[i * stride0 + j * stride1 + k * stride2];- }-- device T& operator()(uint i, uint j, uint k) const { return at(i, j, k); }-};--template <typename T>-struct TensorView4D {- device T* data;- uint dim0, dim1, dim2, dim3;- uint stride0, stride1, stride2, stride3;-- TensorView4D(device T* ptr) : data(ptr) {}-- TensorView4D(device T* ptr, int d0, int d1, int d2, int d3)- : data(ptr), dim0(d0), dim1(d1), dim2(d2), dim3(d3) {- stride0 = d1 * d2 * d3;- stride1 = d2 * d3;- stride2 = d3;- stride3 = 1;- }-- // Custom stride constructor- TensorView4D(- device T* ptr,- int d0,- int d1,- int d2,- int d3,- int s0,- int s1,- int s2,- int s3- )- : data(ptr), dim0(d0), dim1(d1), dim2(d2), dim3(d3), stride0(s0),- stride1(s1), stride2(s2), stride3(s3) {}-- thread TensorView4D& shaped(int d0, int d1, int d2, int d3) {- dim0 = d0;- dim1 = d1;- dim2 = d2;- dim3 = d3;- stride0 = d1 * d2 * d3;- stride1 = d2 * d3;- stride2 = d3;- stride3 = 1;- return *this;- }-- device T& at(uint i, uint j, uint k, uint l) const {- return data[i * stride0 + j * stride1 + k * stride2 + l * stride3];- }-- device T& operator()(uint i, uint j, uint k, uint l) const {- return at(i, j, k, l);- }-};--struct Simd {- uint lane_idx;- uint group_idx;- uint group_size;- uint groups_per_threadgroup;-- uint groups_count(const uint threads_count) const {- return (threads_count + this->group_size - 1) / this->group_size;- }-};--///////////////////////////////////////////////////////////////////////////////-// MARK: - Thread Functions-///////////////////////////////////////////////////////////////////////////////--template <ushort LENGTH, typename T>-static inline T thread_prefix_inclusive_sum(thread T (&values)[LENGTH]) {- for (ushort i = 1; i < LENGTH; i++) {- values[i] += values[i - 1];- }- return values[LENGTH - 1];-}--template <ushort LENGTH, typename T>-static inline T thread_prefix_inclusive_sum(threadgroup T* values) {- for (ushort i = 1; i < LENGTH; i++) {- values[i] += values[i - 1];- }- return values[LENGTH - 1];-}--template <ushort LENGTH, typename T>-static inline T thread_prefix_exclusive_sum(thread T (&values)[LENGTH]) {- // do as an inclusive scan first- T inclusive_prefix = thread_prefix_inclusive_sum<LENGTH>(values);- // convert to an exclusive scan in the reverse direction- for (ushort i = LENGTH - 1; i > 0; i--) {- values[i] = values[i - 1];- }- values[0] = 0;- return inclusive_prefix;-}--template <ushort LENGTH, typename T>-static inline T thread_prefix_exclusive_sum(threadgroup T* values) {- // do as an inclusive scan first- T inclusive_prefix = thread_prefix_inclusive_sum<LENGTH>(values);- // convert to an exclusive scan in the reverse direction- for (ushort i = LENGTH - 1; i > 0; i--) {- values[i] = values[i - 1];- }- values[0] = 0;- return inclusive_prefix;-}--template <ushort LENGTH, typename T>-static inline void thread_uniform_add(thread T (&values)[LENGTH], T uni) {- for (ushort i = 0; i < LENGTH; i++) {- values[i] += uni;- }-}--template <ushort LENGTH, typename T>-static inline void thread_uniform_add(threadgroup T* values, T uni) {- for (ushort i = 0; i < LENGTH; i++) {- values[i] += uni;- }-}--///////////////////////////////////////////////////////////////////////////////-// MARK: - Threadgroup Functions-///////////////////////////////////////////////////////////////////////////////--//------------------------------------------------------------------------------------------------//-// Raking threadgroup scan-template <ushort BLOCK_SIZE, typename T>-static T threadgroup_raking_prefix_exclusive_sum(- T value,- threadgroup T* shared,- const ushort lid-) {- // load values into shared memory- shared[lid] = value;- threadgroup_barrier(mem_flags::mem_threadgroup);-- // only the first 32 threads form the rake- if (lid < 32) {- // scan by thread in shared mem- const short values_per_thread = BLOCK_SIZE / 32;- const short first_index = lid * values_per_thread;- for (short i = first_index + 1; i < first_index + values_per_thread; i++) {- shared[i] += shared[i - 1];- }- T partial_sum = shared[first_index + values_per_thread - 1];- for (short i = first_index + values_per_thread - 1; i > first_index; i--) {- shared[i] = shared[i - 1];- }- shared[first_index] = 0;-- // scan the partial sums- T prefix = simd_prefix_exclusive_sum(partial_sum);-- // add back the prefix- for (short i = first_index; i < first_index + values_per_thread; i++) {- shared[i] += prefix;- }- }- threadgroup_barrier(mem_flags::mem_threadgroup);-- const T result = shared[lid];- threadgroup_barrier(mem_flags::mem_threadgroup);- return result;-}--//------------------------------------------------------------------------------------------------//-// Raking threadgroup sum reduction-template <ushort BLOCK_SIZE, typename T>-static T threadgroup_raking_reduce_sum(- T value,- threadgroup T* shared,- const ushort lid-) {- // load values into shared memory- shared[lid] = value;- threadgroup_barrier(mem_flags::mem_threadgroup);-- // only the first 32 threads form the rake- if (lid < 32) {- // reduce by thread in shared mem- const short values_per_thread = BLOCK_SIZE / 32;- const short first_index = lid * values_per_thread;-- T thread_sum = shared[first_index];- for (short i = first_index + 1; i < first_index + values_per_thread; i++) {- thread_sum += shared[i];- }-- // reduce the partial sums using SIMD- T total_sum = simd_sum(thread_sum);-- // broadcast result back to shared memory- if (lid == 0) {- shared[0] = total_sum;- }- }-- threadgroup_barrier(mem_flags::mem_threadgroup);-- const T result = shared[0];- threadgroup_barrier(mem_flags::mem_threadgroup);- return result;-}--//------------------------------------------------------------------------------------------------//-// Raking threadgroup max reduction-template <ushort BLOCK_SIZE, typename T>-static T threadgroup_raking_reduce_max(- T value,- threadgroup T* shared,- const ushort lid-) {- // load values into shared memory- shared[lid] = value;- threadgroup_barrier(mem_flags::mem_threadgroup);-- // only the first 32 threads form the rake- if (lid < 32) {- // reduce by thread in shared mem- const short values_per_thread = BLOCK_SIZE / 32;- const short first_index = lid * values_per_thread;-- T thread_max = shared[first_index];- for (short i = first_index + 1; i < first_index + values_per_thread; i++) {- thread_max = max(thread_max, shared[i]);- }-- // reduce the partial maxes using SIMD- T total_max = simd_max(thread_max);-- // broadcast result back to shared memory- if (lid == 0) {- shared[0] = total_max;- }- }-- threadgroup_barrier(mem_flags::mem_threadgroup);-- const T result = shared[0];- threadgroup_barrier(mem_flags::mem_threadgroup);- return result;-}--//------------------------------------------------------------------------------------------------//-// Raking threadgroup min reduction-template <ushort BLOCK_SIZE, typename T>-static T threadgroup_raking_reduce_min(- T value,- threadgroup T* shared,- const ushort lid-) {- // load values into shared memory- shared[lid] = value;- threadgroup_barrier(mem_flags::mem_threadgroup);-- // only the first 32 threads form the rake- if (lid < 32) {- // reduce by thread in shared mem- const short values_per_thread = BLOCK_SIZE / 32;- const short first_index = lid * values_per_thread;-- T thread_min = shared[first_index];- for (short i = first_index + 1; i < first_index + values_per_thread; i++) {- thread_min = min(thread_min, shared[i]);- }-- // reduce the partial mins using SIMD- T total_min = simd_min(thread_min);-- // broadcast result back to shared memory- if (lid == 0) {- shared[0] = total_min;- }- }- threadgroup_barrier(mem_flags::mem_threadgroup);-- const T result = shared[0];- threadgroup_barrier(mem_flags::mem_threadgroup);- return result;-}--//------------------------------------------------------------------------------------------------//-// Cooperative threadgroup sum reduction (2-level hierarchical)-template <ushort BLOCK_SIZE, typename T>-static T threadgroup_cooperative_reduce_sum(- T value,- threadgroup T* shared,- const ushort lid,- const thread Simd& simd-) {- // Reduce within simdgroup- T local_sum = simd_sum(value);-- // First thread in each simdgroup writes to shared memory- if (simd.lane_idx == 0) {- shared[simd.group_idx] = local_sum;- }-- // Synchronize across the threadgroup- threadgroup_barrier(mem_flags::mem_threadgroup);-- // Reduce across simdgroups- T total_sum = T(0);- const ushort num_simd_groups = simd.groups_count(BLOCK_SIZE);- if (lid < num_simd_groups) {- total_sum = shared[lid];- }- total_sum = simd_sum(total_sum);-- // Broadcast the result to all threads- if (lid == 0) {- shared[0] = total_sum;- }- threadgroup_barrier(mem_flags::mem_threadgroup);-- const T result = shared[0];- threadgroup_barrier(mem_flags::mem_threadgroup);- return result;-}--//------------------------------------------------------------------------------------------------//-// Cooperative threadgroup max reduction (2-level hierarchical)-template <ushort BLOCK_SIZE, typename T>-static T threadgroup_cooperative_reduce_max(- T value,- threadgroup T* shared,- const ushort lid,- const thread Simd& simd-) {- // Reduce within simdgroup- T local_max = simd_max(value);-- // First thread in each simdgroup writes to shared memory- if (simd.lane_idx == 0) {- shared[simd.group_idx] = local_max;- }-- // Synchronize across the threadgroup- threadgroup_barrier(mem_flags::mem_threadgroup);-- // Reduce across simdgroups- T total_max =- lid < simd.groups_count(BLOCK_SIZE) ? shared[lid] : T(-INFINITY);- total_max = simd_max(total_max);-- // Broadcast the result to all threads- if (lid == 0) {- shared[0] = total_max;- }- threadgroup_barrier(mem_flags::mem_threadgroup);-- const T result = shared[0];- threadgroup_barrier(mem_flags::mem_threadgroup);- return result;-}--//------------------------------------------------------------------------------------------------//-// Cooperative threadgroup min reduction (2-level hierarchical)-template <ushort BLOCK_SIZE, typename T>-static T threadgroup_cooperative_reduce_min(- T value,- threadgroup T* shared,- const ushort lid,- const thread Simd& simd-) {- // Reduce within simdgroup- T local_min = simd_min(value);-- // First thread in each simdgroup writes to shared memory- if (simd.lane_idx == 0) {- shared[simd.group_idx] = local_min;- }-- // Synchronize across the threadgroup- threadgroup_barrier(mem_flags::mem_threadgroup);-- // Reduce across simdgroups- T total_min = lid < simd.groups_count(BLOCK_SIZE) ? shared[lid] : T(INFINITY);- total_min = simd_min(total_min);-- // Broadcast the result to all threads- if (lid == 0) {- shared[0] = total_min;- }- threadgroup_barrier(mem_flags::mem_threadgroup);-- const T result = shared[0];- threadgroup_barrier(mem_flags::mem_threadgroup);- return result;-}--// warning: constexpr if is a C++17 extension [-Wc++17-extensions]-#if defined(__cpp_if_constexpr)-#define IF_CONSTEXPR(cond) if constexpr (cond)-#else-#define IF_CONSTEXPR(cond) if (cond)-#endif--// MARK: - DSL Annotation Helpers--#ifdef DSL_ANALYZE-#define DSL_META(...) [[clang::annotate("", __VA_ARGS__)]]-#else-#define DSL_META(...)-#endif--#define DSL_STR(X) #X-#define DSL_XSTR(X) DSL_STR(X)--#define VARIANTS(TYPENAME, ...) \- DSL_META("dsl.variants", #TYPENAME, #__VA_ARGS__)-#define PUBLIC DSL_META("dsl.public")-#define KERNEL(NAME) DSL_META("dsl.kernel") void NAME--#define SPECIALIZE DSL_META("dsl.specialize")-#define OPTIONAL(EXPR) DSL_META("dsl.optional", DSL_XSTR(EXPR))--#define AXIS(TDS, TPG) DSL_META("dsl.axis", DSL_XSTR(TDS), DSL_XSTR(TPG))-#define GROUPS(EXPR) DSL_META("dsl.groups", DSL_XSTR(EXPR))-#define THREADS(EXPR) DSL_META("dsl.threads", DSL_XSTR(EXPR))--// MARK: - Generate Template Kernels--#define generateKernel( \- max_threads, \- functionName, \- scalarType, \- outerArgs, \- innerArgs \-) \- [[max_total_threads_per_threadgroup(max_threads)]] \- kernel void functionName##_##scalarType outerArgs { \- functionName innerArgs; \- }--#define generateKernels(max_threads, functionName) \- generateKernel( \- max_threads, \- functionName, \- float, \- outerArguments(float), \- innerArguments \- ); \- generateKernel( \- max_threads, \- functionName, \- bfloat, \- outerArguments(bfloat), \- innerArguments \- ); \- generateKernel( \- max_threads, \- functionName, \- half, \- outerArguments(half), \- innerArguments \- );--#endif /* definitions_metal */diff --git a/crates/uzu/src/backends/metal/kernel/embedding/full_precision_embedding.metal b/crates/uzu/src/backends/metal/kernel/embedding/full_precision_embedding.metalindex acc6e397e..c45f05e24 100644--- a/crates/uzu/src/backends/metal/kernel/embedding/full_precision_embedding.metal+++ b/crates/uzu/src/backends/metal/kernel/embedding/full_precision_embedding.metal@@ -1,5 +1,5 @@#include <metal_stdlib>-#include "../definitions.metal"+#include "../common/dsl.h"#define BLOCK_SIZE 256diff --git a/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal b/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metalindex 4ee6a69a7..bc7881a3b 100644--- a/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal+++ b/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal@@ -1,5 +1,5 @@#include <metal_stdlib>-#include "../definitions.metal"+#include "../common/dsl.h"enum QuantizationMode : uint {QUANT_UINT4 = 0,diff --git a/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal b/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metalindex c4c379179..0d4fa0dc4 100644--- a/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal+++ b/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal@@ -1,5 +1,5 @@#include <metal_stdlib>-#include "../definitions.metal"+#include "../common/dsl.h"#include "kv_cache_update.h"diff --git a/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal b/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metalindex 2c5ead11e..c6e54cee0 100644--- a/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal+++ b/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal@@ -1,8 +1,10 @@#include <metal_stdlib>-#include "../definitions.metal"+#include "../common/defines.h"+#include "../common/dsl.h"+#include "../common/thread_context.h"+#include "../common/threadgroup_reduce.h"#define BLOCK_SIZE 1024-#define SIMD_SIZE 32#define GRAIN_SIZE 4// LayerNorm: subtract mean, then normalize (for BERT-style models)@@ -18,7 +20,7 @@ void layer_norm_core(threadgroup AccumT* shared_variance,uint thread_in_row,bool full_layer,- const thread Simd& simd+ const thread ThreadContext& simd) {AccumT partial_sum = static_cast<AccumT>(0.0f);@@ -128,9 +130,9 @@ PUBLIC KERNEL(LayerNorm) (constant float& scale_offset,constant uint& full_layer,const bool in_place SPECIALIZE,- threadgroup ACC shared_mean[SIMD_SIZE],- threadgroup ACC shared_variance[SIMD_SIZE],- const Simd simd,+ threadgroup ACC shared_mean[METAL_SIMD_SIZE],+ threadgroup ACC shared_variance[METAL_SIMD_SIZE],+ const ThreadContext simd,uint batch_idx GROUPS(batch_size),uint thread_in_row THREADS(BLOCK_SIZE)) {diff --git a/crates/uzu/src/backends/metal/kernel/mask_update/mask_update.metal b/crates/uzu/src/backends/metal/kernel/mask_update/mask_update.metalindex b9f61e282..06a9d52f2 100644--- a/crates/uzu/src/backends/metal/kernel/mask_update/mask_update.metal+++ b/crates/uzu/src/backends/metal/kernel/mask_update/mask_update.metal@@ -1,5 +1,5 @@#include <metal_stdlib>-#include "../definitions.metal"+#include "../common/dsl.h"template <typename T>VARIANTS(T, float, half, bfloat)diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/defines.h b/crates/uzu/src/backends/metal/kernel/matmul/common/defines.hindex 07ca0a072..4e4401177 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/common/defines.h+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/defines.h@@ -1,47 +1,43 @@#pragma once+#include "../../common/defines.h"#include <metal_simdgroup>#include <metal_simdgroup_matrix>#include <metal_stdlib>-#define UZU_MTL_CONST static constant constexpr-#define UZU_PRAGMA_UNROLL _Pragma("clang loop unroll(full)")-#define SIMD_SIZE 32-using namespace metal;namespace uzu {namespace matmul {-// Pointer element type extractiontemplate <typename T>-struct pointer_element_t_impl {+struct PointerElementTypeImpl {using type = T;};+template <typename T>-struct pointer_element_t_impl<device T*> {+struct PointerElementTypeImpl<device T*> {using type = T;};+template <typename T>-struct pointer_element_t_impl<const device T*> {+struct PointerElementTypeImpl<const device T*> {using type = T;};+template <typename T>-struct pointer_element_t_impl<threadgroup T*> {+struct PointerElementTypeImpl<threadgroup T*> {using type = T;};+template <typename T>-using pointer_element_t = typename pointer_element_t_impl<T>::type;+using PointerElementType = typename PointerElementTypeImpl<T>::type;-// Accumulator type helpertemplate <typename T>-struct AccumHelper {- typedef float accum_type;+struct AccumulatorHelper {+ typedef float AccumulatorType;};-// Transform operations for epilogue-// Note: apply(x) is static for default store_result usage-// apply(x, c) is non-static for epilogue operations with C matrixtemplate <typename OutT, typename InT>struct TransformNone {TransformNone(float = 1.0f, float = 0.0f) {}@@ -61,11 +57,12 @@ struct TransformAdd {};template <typename OutT, typename InT>-struct TransformAxpby {+struct TransformScaleAccumulate {const float alpha;const float beta;- TransformAxpby(float alpha_, float beta_) : alpha(alpha_), beta(beta_) {}+ TransformScaleAccumulate(float alpha, float beta)+ : alpha(alpha), beta(beta) {}static METAL_FUNC OutT apply(InT x) { return static_cast<OutT>(x); }diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/gemm.h b/crates/uzu/src/backends/metal/kernel/matmul/common/gemm.hindex c90a562ac..8cee3d832 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/common/gemm.h+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/gemm.h@@ -1,8 +1,8 @@#pragma once#include "loader.h"-#include "mma.h"-#include "matmul.h"+#include "threadgroup_tile.h"+#include "../../../generated/matmul.h"using namespace metal;@@ -17,104 +17,111 @@ template <bool M_aligned, bool N_aligned, bool K_aligned>struct LoopAlignment {};///////////////////////////////////////////////////////////////////////////////-// GEMM Kernel+// ThreadgroupGemm Kernel///////////////////////////////////////////////////////////////////////////////template <typename T,typename U,- int BM,- int BN,- int BK,- int WM,- int WN,+ int BLOCK_ROWS,+ int BLOCK_COLS,+ int BLOCK_DEPTH,+ int SIMDGROUPS_PER_ROW,+ int SIMDGROUPS_PER_COLUMN,bool transpose_a,bool transpose_b,bool MN_aligned,bool K_aligned,typename AccumType = float,typename Epilogue = TransformNone<U, AccumType>>-struct GEMMKernel {- UZU_MTL_CONST short tgp_padding_a = 16 / sizeof(T);- UZU_MTL_CONST short tgp_padding_b = 16 / sizeof(T);- UZU_MTL_CONST short tgp_mem_size_a =- transpose_a ? BK * (BM + tgp_padding_a) : BM * (BK + tgp_padding_a);- UZU_MTL_CONST short tgp_mem_size_b =- transpose_b ? BN * (BK + tgp_padding_b) : BK * (BN + tgp_padding_b);- UZU_MTL_CONST short tgp_mem_size = tgp_mem_size_a + tgp_mem_size_b;-- UZU_MTL_CONST short tgp_size = WM * WN * 32;-- using loader_a_t = BlockLoader<+struct ThreadgroupGemm {+ METAL_CONST short THREADGROUP_PADDING_A = 16 / sizeof(T);+ METAL_CONST short THREADGROUP_PADDING_B = 16 / sizeof(T);+ METAL_CONST short THREADGROUP_MEMORY_SIZE_A =+ transpose_a ? BLOCK_DEPTH * (BLOCK_ROWS + THREADGROUP_PADDING_A)+ : BLOCK_ROWS * (BLOCK_DEPTH + THREADGROUP_PADDING_A);+ METAL_CONST short THREADGROUP_MEMORY_SIZE_B =+ transpose_b ? BLOCK_COLS * (BLOCK_DEPTH + THREADGROUP_PADDING_B)+ : BLOCK_DEPTH * (BLOCK_COLS + THREADGROUP_PADDING_B);+ METAL_CONST short THREADGROUP_MEMORY_SIZE =+ THREADGROUP_MEMORY_SIZE_A + THREADGROUP_MEMORY_SIZE_B;++ METAL_CONST short THREADGROUP_SIZE =+ SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * 32;++ using LoaderAType = ThreadgroupLoader<T,- transpose_a ? BK : BM,- transpose_a ? BM : BK,- transpose_a ? BM + tgp_padding_a : BK + tgp_padding_a,+ transpose_a ? BLOCK_DEPTH : BLOCK_ROWS,+ transpose_a ? BLOCK_ROWS : BLOCK_DEPTH,+ transpose_a ? BLOCK_ROWS + THREADGROUP_PADDING_A+ : BLOCK_DEPTH + THREADGROUP_PADDING_A,!transpose_a,- tgp_size>;- using loader_b_t = BlockLoader<+ THREADGROUP_SIZE>;+ using LoaderBType = ThreadgroupLoader<T,- transpose_b ? BN : BK,- transpose_b ? BK : BN,- transpose_b ? BK + tgp_padding_b : BN + tgp_padding_b,+ transpose_b ? BLOCK_COLS : BLOCK_DEPTH,+ transpose_b ? BLOCK_DEPTH : BLOCK_COLS,+ transpose_b ? BLOCK_DEPTH + THREADGROUP_PADDING_B+ : BLOCK_COLS + THREADGROUP_PADDING_B,transpose_b,- tgp_size>;- using mma_t = BlockMMA<+ THREADGROUP_SIZE>;+ using ThreadgroupTileType = ThreadgroupTile<T,U,- BM,- BN,- BK,- WM,- WN,+ BLOCK_ROWS,+ BLOCK_COLS,+ BLOCK_DEPTH,+ SIMDGROUPS_PER_ROW,+ SIMDGROUPS_PER_COLUMN,transpose_a,transpose_b,- transpose_a ? BM + tgp_padding_a : BK + tgp_padding_a,- transpose_b ? BK + tgp_padding_b : BN + tgp_padding_b,+ transpose_a ? BLOCK_ROWS + THREADGROUP_PADDING_A+ : BLOCK_DEPTH + THREADGROUP_PADDING_A,+ transpose_b ? BLOCK_DEPTH + THREADGROUP_PADDING_B+ : BLOCK_COLS + THREADGROUP_PADDING_B,AccumType,Epilogue>;- /* Main kernel function */template <bool M_aligned, bool N_aligned, bool K_aligned_>static METAL_FUNC void gemm_loop(- threadgroup T* As,- threadgroup T* Bs,+ threadgroup T* a_shared,+ threadgroup T* b_shared,const int gemm_k_iterations,- thread loader_a_t& loader_a,- thread loader_b_t& loader_b,- thread mma_t& mma_op,- thread const short& tgp_bm,- thread const short& tgp_bn,- thread const short& lbk,- LoopAlignment<M_aligned, N_aligned, K_aligned_> l = {}+ thread LoaderAType& loader_a,+ thread LoaderBType& loader_b,+ thread ThreadgroupTileType& threadgroup_tile,+ thread const short& threadgroup_block_rows,+ thread const short& threadgroup_block_cols,+ thread const short& leftover_block_depth,+ LoopAlignment<M_aligned, N_aligned, K_aligned_> alignment = {}) {- // Appease the compiler- (void)l;+ (void)alignment;- short2 tile_dims_A = transpose_a ? short2(tgp_bm, BK) : short2(BK, tgp_bm);- short2 tile_dims_B = transpose_b ? short2(BK, tgp_bn) : short2(tgp_bn, BK);+ short2 tile_dimensions_a =+ transpose_a ? short2(threadgroup_block_rows, BLOCK_DEPTH)+ : short2(BLOCK_DEPTH, threadgroup_block_rows);+ short2 tile_dimensions_b =+ transpose_b ? short2(BLOCK_DEPTH, threadgroup_block_cols)+ : short2(threadgroup_block_cols, BLOCK_DEPTH);for (int k = 0; k < gemm_k_iterations; k++) {threadgroup_barrier(mem_flags::mem_threadgroup);- // Load elements into threadgroupif (M_aligned) {loader_a.load_unsafe();} else {- loader_a.load_safe(tile_dims_A);+ loader_a.load_safe(tile_dimensions_a);}if (N_aligned) {loader_b.load_unsafe();} else {- loader_b.load_safe(tile_dims_B);+ loader_b.load_safe(tile_dimensions_b);}threadgroup_barrier(mem_flags::mem_threadgroup);- // Multiply and accumulate threadgroup elements- mma_op.mma(As, Bs);+ threadgroup_tile.multiply_accumulate(a_shared, b_shared);- // Prepare for next iterationloader_a.next();loader_b.next();}@@ -122,173 +129,200 @@ struct GEMMKernel {if (!K_aligned_) {threadgroup_barrier(mem_flags::mem_threadgroup);- short2 tile_dims_A_last =- transpose_a ? short2(tgp_bm, lbk) : short2(lbk, tgp_bm);- short2 tile_dims_B_last =- transpose_b ? short2(lbk, tgp_bn) : short2(tgp_bn, lbk);+ short2 last_tile_dimensions_a =+ transpose_a ? short2(threadgroup_block_rows, leftover_block_depth)+ : short2(leftover_block_depth, threadgroup_block_rows);+ short2 last_tile_dimensions_b =+ transpose_b ? short2(leftover_block_depth, threadgroup_block_cols)+ : short2(threadgroup_block_cols, leftover_block_depth);- loader_a.load_safe(tile_dims_A_last);- loader_b.load_safe(tile_dims_B_last);+ loader_a.load_safe(last_tile_dimensions_a);+ loader_b.load_safe(last_tile_dimensions_b);threadgroup_barrier(mem_flags::mem_threadgroup);- mma_op.mma(As, Bs);+ threadgroup_tile.multiply_accumulate(a_shared, b_shared);}}- /* Main kernel function */static METAL_FUNC void run(- const device T* A,- const device T* B,- device U* D,- const constant GEMMParams* params,- threadgroup T* As,- threadgroup T* Bs,+ const device T* a,+ const device T* b,+ device U* d,+ const constant GemmParams* params,+ threadgroup T* a_shared,+ threadgroup T* b_shared,uint simd_lane_id [[thread_index_in_simdgroup]],uint simd_group_id [[simdgroup_index_in_threadgroup]],uint2 tid [[threadgroup_position_in_grid]],uint3 lid [[thread_position_in_threadgroup]]) {- // Pacifying compiler(void)lid;- const int tid_y = ((tid.y) << params->swizzle_log) +- ((tid.x) & ((1 << params->swizzle_log) - 1));- const int tid_x = (tid.x) >> params->swizzle_log;+ const int swizzle_stride = pow2(params->swizzle_log);+ const int swizzled_row_id =+ tid.y * swizzle_stride + (tid.x % swizzle_stride);+ const int swizzled_col_id = tid.x / swizzle_stride;- if (params->tiles_n <= tid_x || params->tiles_m <= tid_y) {+ if (params->threadgroups_per_row <= swizzled_col_id ||+ params->threadgroups_per_column <= swizzled_row_id) {return;}threadgroup_barrier(mem_flags::mem_none);- // Find block in A, B, C- const int c_row = tid_y * BM;- const int c_col = tid_x * BN;- const size_t c_row_long = size_t(c_row);- const size_t c_col_long = size_t(c_col);-- A += transpose_a ? c_row_long : c_row_long * params->lda;- B += transpose_b ? c_col_long * params->ldb : c_col_long;- D += c_row_long * params->ldd + c_col_long;-- // Prepare threadgroup loading operations- thread loader_a_t loader_a(A, params->lda, As, simd_group_id, simd_lane_id);- thread loader_b_t loader_b(B, params->ldb, Bs, simd_group_id, simd_lane_id);-- // Prepare threadgroup mma operation- thread mma_t mma_op(simd_group_id, simd_lane_id);-- int gemm_k_iterations = params->gemm_k_iterations_aligned;+ const int output_row = swizzled_row_id * BLOCK_ROWS;+ const int output_col = swizzled_col_id * BLOCK_COLS;+ const size_t output_row_long = size_t(output_row);+ const size_t output_col_long = size_t(output_col);++ a += transpose_a ? output_row_long+ : output_row_long * params->leading_dimension_a;+ b += transpose_b ? output_col_long * params->leading_dimension_b+ : output_col_long;+ d += output_row_long * params->leading_dimension_d + output_col_long;++ thread LoaderAType loader_a(+ a,+ params->leading_dimension_a,+ a_shared,+ simd_group_id,+ simd_lane_id+ );+ thread LoaderBType loader_b(+ b,+ params->leading_dimension_b,+ b_shared,+ simd_group_id,+ simd_lane_id+ );++ thread ThreadgroupTileType threadgroup_tile(simd_group_id, simd_lane_id);++ int gemm_k_iterations = params->aligned_inner_iterations;///////////////////////////////////////////////////////////////////////////// MNK aligned loopif (MN_aligned) {for (int k = 0; k < gemm_k_iterations; k++) {threadgroup_barrier(mem_flags::mem_threadgroup);- // Load elements into threadgrouploader_a.load_unsafe();loader_b.load_unsafe();threadgroup_barrier(mem_flags::mem_threadgroup);- // Multiply and accumulate threadgroup elements- mma_op.mma(As, Bs);+ threadgroup_tile.multiply_accumulate(a_shared, b_shared);- // Prepare for next iterationloader_a.next();loader_b.next();}threadgroup_barrier(mem_flags::mem_none);- // Loop tailif (!K_aligned) {- int lbk = params->K - params->gemm_k_iterations_aligned * BK;- short2 tile_dims_A = transpose_a ? short2(BM, lbk) : short2(lbk, BM);- short2 tile_dims_B = transpose_b ? short2(lbk, BN) : short2(BN, lbk);-- loader_a.load_safe(tile_dims_A);- loader_b.load_safe(tile_dims_B);+ int leftover_block_depth =+ params->K - params->aligned_inner_iterations * BLOCK_DEPTH;+ short2 tile_dimensions_a =+ transpose_a ? short2(BLOCK_ROWS, leftover_block_depth)+ : short2(leftover_block_depth, BLOCK_ROWS);+ short2 tile_dimensions_b =+ transpose_b ? short2(leftover_block_depth, BLOCK_COLS)+ : short2(BLOCK_COLS, leftover_block_depth);++ loader_a.load_safe(tile_dimensions_a);+ loader_b.load_safe(tile_dimensions_b);threadgroup_barrier(mem_flags::mem_threadgroup);- mma_op.mma(As, Bs);+ threadgroup_tile.multiply_accumulate(a_shared, b_shared);}- // Store results to device memory- mma_op.store_result(D, params->ldd);+ threadgroup_tile.store_result(d, params->leading_dimension_d);return;}///////////////////////////////////////////////////////////////////////////// MN unaligned loop- else { // Loop over K - unaligned case- short tgp_bm = min(BM, params->M - c_row);- short tgp_bn = min(BN, params->N - c_col);- short leftover_bk = params->K - params->gemm_k_iterations_aligned * BK;-- if (tgp_bm == BM && tgp_bn == BN) {+ else {+ short threadgroup_block_rows = min(BLOCK_ROWS, params->M - output_row);+ short threadgroup_block_cols = min(BLOCK_COLS, params->N - output_col);+ short leftover_block_depth =+ params->K - params->aligned_inner_iterations * BLOCK_DEPTH;++ if (threadgroup_block_rows == BLOCK_ROWS &&+ threadgroup_block_cols == BLOCK_COLS) {gemm_loop<true, true, K_aligned>(- As,- Bs,+ a_shared,+ b_shared,gemm_k_iterations,loader_a,loader_b,- mma_op,- tgp_bm,- tgp_bn,- leftover_bk+ threadgroup_tile,+ threadgroup_block_rows,+ threadgroup_block_cols,+ leftover_block_depth);- mma_op.store_result(D, params->ldd);+ threadgroup_tile.store_result(d, params->leading_dimension_d);return;- } else if (tgp_bn == BN) {+ } else if (threadgroup_block_cols == BLOCK_COLS) {gemm_loop<false, true, K_aligned>(- As,- Bs,+ a_shared,+ b_shared,gemm_k_iterations,loader_a,loader_b,- mma_op,- tgp_bm,- tgp_bn,- leftover_bk+ threadgroup_tile,+ threadgroup_block_rows,+ threadgroup_block_cols,+ leftover_block_depth);- mma_op.store_result_safe(D, params->ldd, short2(tgp_bn, tgp_bm));+ threadgroup_tile.store_result_safe(+ d,+ params->leading_dimension_d,+ short2(threadgroup_block_cols, threadgroup_block_rows)+ );return;- } else if (tgp_bm == BM) {+ } else if (threadgroup_block_rows == BLOCK_ROWS) {gemm_loop<true, false, K_aligned>(- As,- Bs,+ a_shared,+ b_shared,gemm_k_iterations,loader_a,loader_b,- mma_op,- tgp_bm,- tgp_bn,- leftover_bk+ threadgroup_tile,+ threadgroup_block_rows,+ threadgroup_block_cols,+ leftover_block_depth);- mma_op.store_result_safe(D, params->ldd, short2(tgp_bn, tgp_bm));+ threadgroup_tile.store_result_safe(+ d,+ params->leading_dimension_d,+ short2(threadgroup_block_cols, threadgroup_block_rows)+ );return;} else {gemm_loop<false, false, K_aligned>(- As,- Bs,+ a_shared,+ b_shared,gemm_k_iterations,loader_a,loader_b,- mma_op,- tgp_bm,- tgp_bn,- leftover_bk+ threadgroup_tile,+ threadgroup_block_rows,+ threadgroup_block_cols,+ leftover_block_depth);- mma_op.store_result_safe(D, params->ldd, short2(tgp_bn, tgp_bm));+ threadgroup_tile.store_result_safe(+ d,+ params->leading_dimension_d,+ short2(threadgroup_block_cols, threadgroup_block_rows)+ );return;}}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/loader.h b/crates/uzu/src/backends/metal/kernel/matmul/common/loader.hindex 885d536a9..33e45d2c8 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/common/loader.h+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/loader.h@@ -8,125 +8,127 @@ namespace uzu {namespace matmul {///////////////////////////////////////////////////////////////////////////////-// Block Loader - loads tiles from device memory to threadgroup memory+// Threadgroup Loader - loads tiles from device memory to threadgroup memory///////////////////////////////////////////////////////////////////////////////template <typename T,- short BROWS,- short BCOLS,- short dst_ld,- short reduction_dim,- short tgp_size,- short alignment = 1,- short n_reads = (BCOLS * BROWS) / (tgp_size),- short TCOLS = BCOLS / n_reads,- short TROWS = tgp_size / TCOLS>-struct BlockLoader {- UZU_MTL_CONST short n_rows = (BROWS + TROWS - 1) / TROWS;- UZU_MTL_CONST short vec_size = n_reads;-- // Leading dimension for src- const int src_ld;+ short BLOCK_ROWS,+ short BLOCK_COLS,+ short DESTINATION_LEADING_DIMENSION,+ short REDUCTION_DIMENSION,+ short THREADGROUP_SIZE,+ short ALIGNMENT = 1,+ short READS_PER_THREAD = (BLOCK_COLS * BLOCK_ROWS) / (THREADGROUP_SIZE),+ short THREAD_COLS = BLOCK_COLS / READS_PER_THREAD,+ short THREAD_ROWS = THREADGROUP_SIZE / THREAD_COLS>+struct ThreadgroupLoader {+ METAL_CONST short ROW_ITERATIONS =+ (BLOCK_ROWS + THREAD_ROWS - 1) / THREAD_ROWS;+ METAL_CONST short VECTOR_SIZE = READS_PER_THREAD;++ const int source_leading_dimension;const int tile_stride;- // Thread location indices- const short thread_idx;- const short bi;- const short bj;+ const short thread_index;+ const short block_row_index;+ const short block_col_index;- // threadgroup and device memory- threadgroup T* dst;- const device T* src;+ threadgroup T* destination;+ const device T* source;- struct alignas(alignment * sizeof(T)) ReadVector {- uint8_t v[sizeof(T) * vec_size];+ struct alignas(ALIGNMENT * sizeof(T)) ReadVector {+ uint8_t bytes[sizeof(T) * VECTOR_SIZE];};- /* Constructor */- METAL_FUNC BlockLoader(- const device T* src_,- const int src_ld_,- threadgroup T* dst_,+ METAL_FUNC ThreadgroupLoader(+ const device T* source_ptr,+ const int source_leading_dim,+ threadgroup T* destination_ptr,ushort simd_group_id [[simdgroup_index_in_threadgroup]],ushort simd_lane_id [[thread_index_in_simdgroup]])- : src_ld(src_ld_), tile_stride(reduction_dim ? BCOLS : BROWS * src_ld),- thread_idx(simd_group_id * 32 + simd_lane_id), bi(thread_idx / TCOLS),- bj(vec_size * (thread_idx % TCOLS)), dst(dst_ + bi * dst_ld + bj),- src(src_ + bi * src_ld + bj) {}+ : source_leading_dimension(source_leading_dim),+ tile_stride(+ REDUCTION_DIMENSION ? BLOCK_COLS : BLOCK_ROWS * source_leading_dim+ ),+ thread_index(simd_group_id * 32 + simd_lane_id),+ block_row_index(thread_index / THREAD_COLS),+ block_col_index(VECTOR_SIZE * (thread_index % THREAD_COLS)),+ destination(+ destination_ptr + block_row_index * DESTINATION_LEADING_DIMENSION ++ block_col_index+ ),+ source(+ source_ptr + block_row_index * source_leading_dim + block_col_index+ ) {}- /* Apply operation to threadgroup without bound checking */template <typename UnaryOp>- METAL_FUNC void apply_inplace_op(thread const UnaryOp& op) const {- UZU_PRAGMA_UNROLL- for (short i = 0; i < BROWS; i += TROWS) {- UZU_PRAGMA_UNROLL- for (short j = 0; j < vec_size; j++) {- dst[i * dst_ld + j] = op.apply(dst[i * dst_ld + j]);+ METAL_FUNC void apply_inplace_op(thread const UnaryOp& operation) const {+ METAL_PRAGMA_UNROLL+ for (short i = 0; i < BLOCK_ROWS; i += THREAD_ROWS) {+ METAL_PRAGMA_UNROLL+ for (short j = 0; j < VECTOR_SIZE; j++) {+ destination[i * DESTINATION_LEADING_DIMENSION + j] =+ operation.apply(destination[i * DESTINATION_LEADING_DIMENSION + j]);}}}- /* Load from device memory into threadgroup memory - without bound checking */METAL_FUNC void load_unsafe() const {- UZU_PRAGMA_UNROLL- for (short i = 0; i < BROWS; i += TROWS) {- *((threadgroup ReadVector*)(&dst[i * dst_ld])) =- *((const device ReadVector*)(&src[i * src_ld]));+ METAL_PRAGMA_UNROLL+ for (short i = 0; i < BLOCK_ROWS; i += THREAD_ROWS) {+ *((threadgroup ReadVector*)(&destination+ [i * DESTINATION_LEADING_DIMENSION])) =+ *((const device ReadVector*)(&source[i * source_leading_dimension]));}}- /* Load from device memory into threadgroup memory - with bound checking */- METAL_FUNC void load_safe(short2 src_tile_dim) const {- src_tile_dim = src_tile_dim - short2(bj, bi);-- // Skip loading if thread has no valid reads- if (src_tile_dim.x <= 0 || src_tile_dim.y <= 0) {- UZU_PRAGMA_UNROLL- for (short i = 0; i < BROWS; i += TROWS) {- UZU_PRAGMA_UNROLL- for (short j = 0; j < vec_size; j++) {- dst[i * dst_ld + j] = T(0);+ METAL_FUNC void load_safe(short2 source_tile_dimensions) const {+ source_tile_dimensions =+ source_tile_dimensions - short2(block_col_index, block_row_index);++ if (source_tile_dimensions.x <= 0 || source_tile_dimensions.y <= 0) {+ METAL_PRAGMA_UNROLL+ for (short i = 0; i < BLOCK_ROWS; i += THREAD_ROWS) {+ METAL_PRAGMA_UNROLL+ for (short j = 0; j < VECTOR_SIZE; j++) {+ destination[i * DESTINATION_LEADING_DIMENSION + j] = T(0);}}return;}- // Use fast thread memory for bound checks- bool tmp_idx[vec_size];- T tmp_val[vec_size];+ bool valid_mask[VECTOR_SIZE];+ T loaded_values[VECTOR_SIZE];- UZU_PRAGMA_UNROLL- for (short i = 0; i < BROWS; i += TROWS) {- // Make sure tmp_idx only contains valid indices- UZU_PRAGMA_UNROLL- for (short j = 0; j < vec_size; j++) {- tmp_idx[j] = (i < src_tile_dim.y) && (j < src_tile_dim.x);+ METAL_PRAGMA_UNROLL+ for (short i = 0; i < BLOCK_ROWS; i += THREAD_ROWS) {+ METAL_PRAGMA_UNROLL+ for (short j = 0; j < VECTOR_SIZE; j++) {+ valid_mask[j] =+ (i < source_tile_dimensions.y) && (j < source_tile_dimensions.x);}- // Read valid indices into tmp_val- UZU_PRAGMA_UNROLL- for (short j = 0; j < vec_size; j++) {- tmp_val[j] = src[(tmp_idx[j] ? i * src_ld + j : 0)];+ METAL_PRAGMA_UNROLL+ for (short j = 0; j < VECTOR_SIZE; j++) {+ loaded_values[j] =+ source[(valid_mask[j] ? i * source_leading_dimension + j : 0)];}- // Zero out unneeded values- UZU_PRAGMA_UNROLL- for (short j = 0; j < vec_size; j++) {- tmp_val[j] = tmp_idx[j] ? tmp_val[j] : T(0);+ METAL_PRAGMA_UNROLL+ for (short j = 0; j < VECTOR_SIZE; j++) {+ loaded_values[j] = valid_mask[j] ? loaded_values[j] : T(0);}- // Copy values to threadgroup memory- UZU_PRAGMA_UNROLL- for (short j = 0; j < vec_size; j++) {- dst[i * dst_ld + j] = tmp_val[j];+ METAL_PRAGMA_UNROLL+ for (short j = 0; j < VECTOR_SIZE; j++) {+ destination[i * DESTINATION_LEADING_DIMENSION + j] = loaded_values[j];}}}- /* Iteration helper */- METAL_FUNC void next() { src += tile_stride; }+ METAL_FUNC void next() { source += tile_stride; }};} // namespace matmuldiff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/matmul.h b/crates/uzu/src/backends/metal/kernel/matmul/common/matmul.hdeleted file mode 100644index 7386d512f..000000000--- a/crates/uzu/src/backends/metal/kernel/matmul/common/matmul.h+++ /dev/null@@ -1,9 +0,0 @@-#pragma once--#include "steel/gemm/params.h"--namespace uzu {-namespace matmul {-using GEMMParams = steel::GEMMParams;-} // namespace matmul-} // namespace uzudiff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/mma.h b/crates/uzu/src/backends/metal/kernel/matmul/common/mma.hdeleted file mode 100644index 741ef4394..000000000--- a/crates/uzu/src/backends/metal/kernel/matmul/common/mma.h+++ /dev/null@@ -1,524 +0,0 @@-#pragma once--#include "defines.h"--using namespace metal;--namespace uzu {-namespace matmul {--///////////////////////////////////////////////////////////////////////////////-// MMA Fragment - 8x8 simdgroup matrix fragment-///////////////////////////////////////////////////////////////////////////////--template <typename T, int kFragRows_, int kFragCols_>-struct BaseMMAFrag {- static_assert(- kFragRows_ == 8,- "Only 8 x 8 fragment matrices are currently supported"- );- static_assert(- kFragCols_ == 8,- "Only 8 x 8 fragment matrices are currently supported"- );-};--template <typename T>-struct BaseMMAFrag<T, 8, 8> {- UZU_MTL_CONST int kFragRows = 8;- UZU_MTL_CONST int kFragCols = 8;- UZU_MTL_CONST int kElemsPerFrag = (kFragRows * kFragCols) / 32;- UZU_MTL_CONST int kElemRows = 1;- UZU_MTL_CONST int kElemCols = 2;-- static_assert(- kElemRows * kElemCols == kElemsPerFrag,- "MMAFrag shape is not consistent with MMAFrag size"- );-- typedef metal::simdgroup_matrix<T, kFragRows, kFragCols> mat_type;- typedef metal::vec<T, kElemsPerFrag> frag_type;-- METAL_FUNC static constexpr short2 get_coord(- ushort simd_lane_id [[thread_index_in_simdgroup]]- ) {- const short qid = simd_lane_id / 4;- const short fm = (qid & 4) + ((simd_lane_id / 2) % 4);- const short fn = (qid & 2) * 2 + (simd_lane_id % 2) * 2;- return short2{fn, fm};- }-- template <typename SrcPtrType, typename StrX, typename StrY>- METAL_FUNC static constexpr void load(- thread frag_type& dst,- SrcPtrType src,- StrX str_x,- StrY str_y- ) {- UZU_PRAGMA_UNROLL- for (short i = 0; i < kElemRows; i++) {- UZU_PRAGMA_UNROLL- for (short j = 0; j < kElemCols; j++) {- dst[i * kElemCols + j] = static_cast<T>(src[i * str_x + j * str_y]);- }- }- }-- template <- typename SrcPtrType,- typename StrX,- typename StrY,- typename LimX,- typename LimY,- typename OffX,- typename OffY>- METAL_FUNC static constexpr void load_safe(- thread frag_type& dst,- SrcPtrType src,- StrX str_x,- StrY str_y,- LimX lim_x,- LimY lim_y,- OffX off_x = 0,- OffY off_y = 0- ) {- UZU_PRAGMA_UNROLL- for (short i = 0; i < kElemRows; i++) {- UZU_PRAGMA_UNROLL- for (short j = 0; j < kElemCols; j++) {- if ((off_x + i) < lim_x && (off_y + j) < lim_y) {- dst[i * kElemCols + j] =- static_cast<T>(src[(off_x + i) * str_x + (off_y + j) * str_y]);- } else {- dst[i * kElemCols + j] = T(0);- }- }- }- }-- template <typename DstPtrType, typename StrX, typename StrY>- METAL_FUNC static constexpr void store(- const thread frag_type& src,- DstPtrType dst,- StrX str_x,- StrY str_y- ) {- using U = pointer_element_t<DstPtrType>;-- UZU_PRAGMA_UNROLL- for (short i = 0; i < kElemRows; i++) {- UZU_PRAGMA_UNROLL- for (short j = 0; j < kElemCols; j++) {- dst[i * str_x + j * str_y] = static_cast<U>(src[i * kElemCols + j]);- }- }- }-- template <- typename DstPtrType,- typename StrX,- typename StrY,- typename LimX,- typename LimY,- typename OffX,- typename OffY>- METAL_FUNC static constexpr void store_safe(- const thread frag_type& src,- DstPtrType dst,- StrX str_x,- StrY str_y,- LimX lim_x,- LimY lim_y,- OffX off_x = 0,- OffY off_y = 0- ) {- using U = pointer_element_t<DstPtrType>;-- UZU_PRAGMA_UNROLL- for (short i = 0; i < kElemRows; i++) {- UZU_PRAGMA_UNROLL- for (short j = 0; j < kElemCols; j++) {- if ((off_x + i) < lim_x && (off_y + j) < lim_y) {- dst[(off_x + i) * str_x + (off_y + j) * str_y] =- static_cast<U>(src[i * kElemCols + j]);- }- }- }- }-- METAL_FUNC static constexpr void mma(- thread frag_type& D,- thread frag_type& A,- thread frag_type& B,- thread frag_type& C- ) {- mat_type D_mat;- mat_type A_mat;- mat_type B_mat;- mat_type C_mat;-- reinterpret_cast<thread frag_type&>(A_mat.thread_elements()) = A;- reinterpret_cast<thread frag_type&>(B_mat.thread_elements()) = B;- reinterpret_cast<thread frag_type&>(C_mat.thread_elements()) = C;-- mma(D_mat, A_mat, B_mat, C_mat);-- D = reinterpret_cast<thread frag_type&>(D_mat.thread_elements());- }-- METAL_FUNC static constexpr void mma(- thread mat_type& D,- thread mat_type& A,- thread mat_type& B,- thread mat_type& C- ) {- simdgroup_multiply_accumulate(D, A, B, C);- }-};--///////////////////////////////////////////////////////////////////////////////-// MMA Tile - collection of MMA fragments-///////////////////////////////////////////////////////////////////////////////--template <- typename T,- int kTileRows_,- int kTileCols_,- class MMAFrag_ = BaseMMAFrag<T, 8, 8>>-struct MMATile {- using MMAFrag_t = MMAFrag_;- using elem_type = T;- UZU_MTL_CONST int kFragRows = MMAFrag_t::kFragRows;- UZU_MTL_CONST int kFragCols = MMAFrag_t::kFragCols;- UZU_MTL_CONST int kElemsPerFrag = MMAFrag_t::kElemsPerFrag;-- UZU_MTL_CONST int kTileRows = kTileRows_;- UZU_MTL_CONST int kTileCols = kTileCols_;-- UZU_MTL_CONST int kRows = kTileRows * kFragRows;- UZU_MTL_CONST int kCols = kTileCols * kFragCols;-- UZU_MTL_CONST int kNumFrags = kTileRows * kTileCols;- UZU_MTL_CONST int kElemsPerTile = kNumFrags * kElemsPerFrag;-- typedef typename MMAFrag_t::mat_type mat_type;- typedef typename MMAFrag_t::frag_type frag_type;-- frag_type val_frags[kNumFrags] = {frag_type(0)};-- METAL_FUNC MMATile() thread {}-- METAL_FUNC constexpr void clear() {- UZU_PRAGMA_UNROLL- for (short i = 0; i < kNumFrags; ++i) {- val_frags[i] = frag_type(0);- }- }-- METAL_FUNC constexpr thread frag_type& frag_at(const short i, const short j) {- return val_frags[i * kTileCols + j];- }-- METAL_FUNC constexpr const thread frag_type& frag_at(- const short i,- const short j- ) const {- return val_frags[i * kTileCols + j];- }-- METAL_FUNC thread elem_type* elems() {- return reinterpret_cast<thread elem_type*>(val_frags);- }-- METAL_FUNC const thread elem_type* elems() const {- return reinterpret_cast<const thread elem_type*>(val_frags);- }-- template <typename U, int w_x, int w_y, int str_x, int str_y>- METAL_FUNC void load(const threadgroup U* src) {- UZU_PRAGMA_UNROLL- for (short i = 0; i < kTileRows; ++i) {- UZU_PRAGMA_UNROLL- for (short j = 0; j < kTileCols; ++j) {- MMAFrag_t::load(- frag_at(i, j),- &(- src[(i * kFragRows) * w_x * str_x +- (j * kFragCols) * w_y * str_y]- ),- str_x,- str_y- );- }- }- }-- template <typename U, int w_x, int w_y>- METAL_FUNC void store(device U* dst, const int ld) const {- UZU_PRAGMA_UNROLL- for (short i = 0; i < kTileRows; ++i) {- UZU_PRAGMA_UNROLL- for (short j = 0; j < kTileCols; ++j) {- MMAFrag_t::store(- frag_at(i, j),- &(dst[(i * kFragRows) * w_x * ld + (j * kFragCols) * w_y]),- ld,- 1- );- }- }- }-- template <typename U, int w_x, int w_y>- METAL_FUNC void store_safe(- device U* dst,- const int ld,- const short2 dst_tile_dims- ) const {- UZU_PRAGMA_UNROLL- for (int i = 0; i < kTileRows; ++i) {- UZU_PRAGMA_UNROLL- for (int j = 0; j < kTileCols; ++j) {- MMAFrag_t::store_safe(- frag_at(i, j),- dst,- ld,- 1,- dst_tile_dims.y,- dst_tile_dims.x,- (i * kFragRows) * w_x,- (j * kFragCols) * w_y- );- }- }- }-};--///////////////////////////////////////////////////////////////////////////////-// Tile matrix multiply-accumulate-///////////////////////////////////////////////////////////////////////////////--template <typename T, typename U, int M, int N, int K>-METAL_FUNC void tile_matmad(- thread MMATile<T, M, N>& D,- thread MMATile<U, M, K>& A,- thread MMATile<U, K, N>& B,- thread MMATile<T, M, N>& C-) {- UZU_PRAGMA_UNROLL- for (short m = 0; m < M; ++m) {- UZU_PRAGMA_UNROLL- for (short n = 0; n < N; ++n) {- short n_serp = (m % 2) ? (N - 1 - n) : n;- UZU_PRAGMA_UNROLL- for (short k = 0; k < K; ++k) {- MMATile<T, M, N>::MMAFrag_t::mma(- D.frag_at(m, n_serp),- A.frag_at(m, k),- B.frag_at(k, n_serp),- C.frag_at(m, n_serp)- );- }- }- }-}--///////////////////////////////////////////////////////////////////////////////-// Block MMA - manages the GEMM computation for a threadgroup-///////////////////////////////////////////////////////////////////////////////--template <- typename T,- typename U,- int BM,- int BN,- int BK,- int WM,- int WN,- bool transpose_a,- bool transpose_b,- short lda_tgp,- short ldb_tgp,- typename AccumType = float,- typename Epilogue = TransformNone<U, AccumType>>-struct BlockMMA {- // MMAFrag size- UZU_MTL_CONST short kFragSize = 8;- using MMAFrag_acc_t = BaseMMAFrag<AccumType, kFragSize, kFragSize>;-- // Warp tile simdgroup matrix strides along M- UZU_MTL_CONST short TM_stride = kFragSize * WM;- // Warp tile simdgroup matrix strides along N- UZU_MTL_CONST short TN_stride = kFragSize * WN;-- // Warp tile size along M- UZU_MTL_CONST short TM = BM / (kFragSize * WM);- // Warp tile size along N- UZU_MTL_CONST short TN = BN / (kFragSize * WN);-- // Threadgroup A strides- UZU_MTL_CONST short A_str_m = transpose_a ? 1 : lda_tgp; // M- UZU_MTL_CONST short A_str_k = transpose_a ? lda_tgp : 1; // K-- // Threadgroup B strides- UZU_MTL_CONST short B_str_k = transpose_b ? 1 : ldb_tgp; // K- UZU_MTL_CONST short B_str_n = transpose_b ? ldb_tgp : 1; // N-- // Threadgroup strides along K- UZU_MTL_CONST short tile_stride_a = kFragSize * A_str_k;- UZU_MTL_CONST short tile_stride_b = kFragSize * B_str_k;-- // Simdgroup matrices- MMATile<AccumType, TM, 1, MMAFrag_acc_t> Atile;- MMATile<AccumType, 1, TN, MMAFrag_acc_t> Btile;- MMATile<AccumType, TM, TN, MMAFrag_acc_t> Ctile;-- // Offsets within threadgroup- short sm;- short sn;-- short As_offset;- short Bs_offset;-- /* Constructor */- METAL_FUNC BlockMMA(- ushort simd_group_id [[simdgroup_index_in_threadgroup]],- ushort simd_lane_id [[thread_index_in_simdgroup]]- ) {- // Determine thread position in simdgroup matrix- short tm = kFragSize * (simd_group_id / WN);- short tn = kFragSize * (simd_group_id % WN);-- short2 simd_coord = MMAFrag_acc_t::get_coord(simd_lane_id);- sm = simd_coord.y;- sn = simd_coord.x;-- // Determine thread and simdgroup offset- As_offset = (tm + sm) * A_str_m + (sn)*A_str_k; // M, K- Bs_offset = (sm)*B_str_k + (tn + sn) * B_str_n; // K, N-- sm += tm;- sn += tn;- }-- /* (BM, BK) X (BK, BN) multiply accumulate function */- METAL_FUNC void mma(const threadgroup T* As, const threadgroup T* Bs) {- // Adjust for simdgroup and thread location- As += As_offset;- Bs += Bs_offset;-- // Iterate over BK in blocks of kFragSize- UZU_PRAGMA_UNROLL- for (short kk = 0; kk < BK; kk += kFragSize) {- simdgroup_barrier(mem_flags::mem_none);-- Atile.template load<T, WM, 1, A_str_m, A_str_k>(As);-- simdgroup_barrier(mem_flags::mem_none);-- Btile.template load<T, 1, WN, B_str_k, B_str_n>(Bs);-- simdgroup_barrier(mem_flags::mem_none);-- tile_matmad(Ctile, Atile, Btile, Ctile);-- // Progress to next simdgroup tile- As += tile_stride_a;- Bs += tile_stride_b;- }- }-- /* Store results from simdgroup_matrix results into device memory */- METAL_FUNC void store_result(device U* D, const int ldd) {- // Apply epilogue- UZU_PRAGMA_UNROLL- for (short i = 0; i < decltype(Ctile)::kElemsPerTile; i++) {- Ctile.elems()[i] = Epilogue::apply(Ctile.elems()[i]);- }-- // Adjust for simdgroup and thread location- D += sm * ldd + sn;-- Ctile.template store<U, WM, WN>(D, ldd);- }-- METAL_FUNC void store_result_safe(- device U* D,- const int ldd,- short2 dst_tile_dims- ) {- // Apply epilogue- UZU_PRAGMA_UNROLL- for (short i = 0; i < decltype(Ctile)::kElemsPerTile; i++) {- Ctile.elems()[i] = Epilogue::apply(Ctile.elems()[i]);- }-- // Adjust for simdgroup and thread location- D += sm * ldd + sn;- dst_tile_dims -= short2(sn, sm);-- if (dst_tile_dims.x <= 0 || dst_tile_dims.y <= 0)- return;-- Ctile.template store_safe<U, WM, WN>(D, ldd, dst_tile_dims);- }-- /* Apply epilogue with C matrix */- template <typename EpilogueOp>- METAL_FUNC void apply_epilogue(- const device U* C,- const int ldc,- const int fdc,- thread const EpilogueOp& epilogue_op- ) {- const device U* C_ptr = C + sm * ldc + sn * fdc;-- UZU_PRAGMA_UNROLL- for (short i = 0; i < TM; i++) {- UZU_PRAGMA_UNROLL- for (short j = 0; j < TN; j++) {- thread auto& frag = Ctile.frag_at(i, j);- UZU_PRAGMA_UNROLL- for (short k = 0; k < decltype(Ctile)::kElemsPerFrag; k++) {- short row_offset = (i * kFragSize * WM);- short col_offset = (j * kFragSize * WN);- // Get the C element (accounting for fragment layout)- U c_val = C_ptr[row_offset * ldc + col_offset * fdc + k];- frag[k] = epilogue_op.apply(frag[k], static_cast<AccumType>(c_val));- }- }- }- }-- template <typename EpilogueOp>- METAL_FUNC void apply_epilogue_safe(- const device U* C,- const int ldc,- const int fdc,- short2 tile_dims,- thread const EpilogueOp& epilogue_op- ) {- const device U* C_ptr = C + sm * ldc + sn * fdc;- tile_dims -= short2(sn, sm);-- UZU_PRAGMA_UNROLL- for (short i = 0; i < TM; i++) {- UZU_PRAGMA_UNROLL- for (short j = 0; j < TN; j++) {- thread auto& frag = Ctile.frag_at(i, j);- short row_offset = (i * kFragSize * WM);- short col_offset = (j * kFragSize * WN);- UZU_PRAGMA_UNROLL- for (short k = 0; k < decltype(Ctile)::kElemsPerFrag; k++) {- if (row_offset < tile_dims.y && col_offset + k < tile_dims.x) {- U c_val = C_ptr[row_offset * ldc + col_offset * fdc + k];- frag[k] = epilogue_op.apply(frag[k], static_cast<AccumType>(c_val));- }- }- }- }- }-};--} // namespace matmul-} // namespace uzudiff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment.h b/crates/uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment.hnew file mode 100644index 000000000..5c8dafae6--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment.h@@ -0,0 +1,184 @@+#pragma once++#include "simdgroup_multiply_accumulate.h"++using namespace metal;++namespace uzu {+namespace matmul {++///////////////////////////////////////////////////////////////////////////////+// SimdgroupFragment - thread's portion of simdgroup tile data+///////////////////////////////////////////////////////////////////////////////++template <+ typename T,+ int GRID_ROWS_,+ int GRID_COLS_,+ class SimdgroupMultiplyAccumulateOps_ =+ SimdgroupMultiplyAccumulate<T, 8, 8>>+struct SimdgroupFragment {+ using SimdgroupMultiplyAccumulateOpsType = SimdgroupMultiplyAccumulateOps_;+ using ElementType = T;+ METAL_CONST int MULTIPLY_ACCUMULATE_ROWS =+ SimdgroupMultiplyAccumulateOpsType::ROWS;+ METAL_CONST int MULTIPLY_ACCUMULATE_COLS =+ SimdgroupMultiplyAccumulateOpsType::COLS;+ METAL_CONST int ELEMENTS_PER_MULTIPLY_ACCUMULATE =+ SimdgroupMultiplyAccumulateOpsType::THREAD_ELEMENT_COUNT;++ METAL_CONST int GRID_ROWS = GRID_ROWS_;+ METAL_CONST int GRID_COLS = GRID_COLS_;++ METAL_CONST int TOTAL_ROWS = GRID_ROWS * MULTIPLY_ACCUMULATE_ROWS;+ METAL_CONST int TOTAL_COLS = GRID_COLS * MULTIPLY_ACCUMULATE_COLS;++ METAL_CONST int MULTIPLY_ACCUMULATE_COUNT = GRID_ROWS * GRID_COLS;+ METAL_CONST int ELEMENTS_PER_FRAGMENT =+ MULTIPLY_ACCUMULATE_COUNT * ELEMENTS_PER_MULTIPLY_ACCUMULATE;++ typedef typename SimdgroupMultiplyAccumulateOpsType::SimdgroupMatrixType+ SimdgroupMatrixType;+ typedef typename SimdgroupMultiplyAccumulateOpsType::ThreadDataType+ ThreadDataType;++ ThreadDataType multiply_accumulate_data[MULTIPLY_ACCUMULATE_COUNT] = {+ ThreadDataType(0)+ };++ METAL_FUNC SimdgroupFragment() thread {}++ METAL_FUNC constexpr void clear() {+ METAL_PRAGMA_UNROLL+ for (short i = 0; i < MULTIPLY_ACCUMULATE_COUNT; ++i) {+ multiply_accumulate_data[i] = ThreadDataType(0);+ }+ }++ METAL_FUNC constexpr thread ThreadDataType& multiply_accumulate_at(+ const short i,+ const short j+ ) {+ return multiply_accumulate_data[i * GRID_COLS + j];+ }++ METAL_FUNC constexpr const thread ThreadDataType& multiply_accumulate_at(+ const short i,+ const short j+ ) const {+ return multiply_accumulate_data[i * GRID_COLS + j];+ }++ METAL_FUNC thread ElementType* elements() {+ return reinterpret_cast<thread ElementType*>(multiply_accumulate_data);+ }++ METAL_FUNC const thread ElementType* elements() const {+ return reinterpret_cast<const thread ElementType*>(+ multiply_accumulate_data+ );+ }++ template <+ typename U,+ int SIMDGROUP_STRIDE_X,+ int SIMDGROUP_STRIDE_Y,+ int STRIDE_X,+ int STRIDE_Y>+ METAL_FUNC void load(const threadgroup U* src) {+ METAL_PRAGMA_UNROLL+ for (short i = 0; i < GRID_ROWS; ++i) {+ METAL_PRAGMA_UNROLL+ for (short j = 0; j < GRID_COLS; ++j) {+ SimdgroupMultiplyAccumulateOpsType::load(+ multiply_accumulate_at(i, j),+ &(+ src[(i * MULTIPLY_ACCUMULATE_ROWS) * SIMDGROUP_STRIDE_X *+ STRIDE_X ++ (j * MULTIPLY_ACCUMULATE_COLS) * SIMDGROUP_STRIDE_Y *+ STRIDE_Y]+ ),+ STRIDE_X,+ STRIDE_Y+ );+ }+ }+ }++ template <typename U, int SIMDGROUP_STRIDE_X, int SIMDGROUP_STRIDE_Y>+ METAL_FUNC void store(device U* dst, const int leading_dimension) const {+ METAL_PRAGMA_UNROLL+ for (short i = 0; i < GRID_ROWS; ++i) {+ METAL_PRAGMA_UNROLL+ for (short j = 0; j < GRID_COLS; ++j) {+ SimdgroupMultiplyAccumulateOpsType::store(+ multiply_accumulate_at(i, j),+ &(+ dst[(i * MULTIPLY_ACCUMULATE_ROWS) * SIMDGROUP_STRIDE_X *+ leading_dimension ++ (j * MULTIPLY_ACCUMULATE_COLS) * SIMDGROUP_STRIDE_Y]+ ),+ leading_dimension,+ 1+ );+ }+ }+ }++ template <typename U, int SIMDGROUP_STRIDE_X, int SIMDGROUP_STRIDE_Y>+ METAL_FUNC void store_safe(+ device U* dst,+ const int leading_dimension,+ const short2 destination_tile_dimensions+ ) const {+ METAL_PRAGMA_UNROLL+ for (int i = 0; i < GRID_ROWS; ++i) {+ METAL_PRAGMA_UNROLL+ for (int j = 0; j < GRID_COLS; ++j) {+ SimdgroupMultiplyAccumulateOpsType::store_safe(+ multiply_accumulate_at(i, j),+ dst,+ leading_dimension,+ 1,+ destination_tile_dimensions.y,+ destination_tile_dimensions.x,+ (i * MULTIPLY_ACCUMULATE_ROWS) * SIMDGROUP_STRIDE_X,+ (j * MULTIPLY_ACCUMULATE_COLS) * SIMDGROUP_STRIDE_Y+ );+ }+ }+ }+};++///////////////////////////////////////////////////////////////////////////////+// Tile multiply-accumulate+///////////////////////////////////////////////////////////////////////////////++template <typename T, typename U, int M, int N, int K>+METAL_FUNC void tile_multiply_accumulate(+ thread SimdgroupFragment<T, M, N>& D,+ thread SimdgroupFragment<U, M, K>& A,+ thread SimdgroupFragment<U, K, N>& B,+ thread SimdgroupFragment<T, M, N>& C+) {+ METAL_PRAGMA_UNROLL+ for (short m = 0; m < M; ++m) {+ METAL_PRAGMA_UNROLL+ for (short n = 0; n < N; ++n) {+ short column_serpentine = (m % 2) ? (N - 1 - n) : n;+ METAL_PRAGMA_UNROLL+ for (short k = 0; k < K; ++k) {+ SimdgroupFragment<T, M, N>::SimdgroupMultiplyAccumulateOpsType::+ multiply_accumulate(+ D.multiply_accumulate_at(m, column_serpentine),+ A.multiply_accumulate_at(m, k),+ B.multiply_accumulate_at(k, column_serpentine),+ C.multiply_accumulate_at(m, column_serpentine)+ );+ }+ }+ }+}++} // namespace matmul+} // namespace uzudiff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/simdgroup_multiply_accumulate.h b/crates/uzu/src/backends/metal/kernel/matmul/common/simdgroup_multiply_accumulate.hnew file mode 100644index 000000000..6c4ddb93e--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/simdgroup_multiply_accumulate.h@@ -0,0 +1,183 @@+#pragma once++#include "defines.h"++using namespace metal;++namespace uzu {+namespace matmul {++///////////////////////////////////////////////////////////////////////////////+// SimdgroupMultiplyAccumulate - stateless traits/ops for 8x8 simdgroup_matrix+///////////////////////////////////////////////////////////////////////////////++template <typename T, int ROWS_, int COLS_>+struct SimdgroupMultiplyAccumulate {+ static_assert(+ ROWS_ == 8,+ "Only 8 x 8 fragment matrices are currently supported"+ );+ static_assert(+ COLS_ == 8,+ "Only 8 x 8 fragment matrices are currently supported"+ );+};++template <typename T>+struct SimdgroupMultiplyAccumulate<T, 8, 8> {+ METAL_CONST int ROWS = 8;+ METAL_CONST int COLS = 8;+ METAL_CONST int THREAD_ELEMENT_COUNT = (ROWS * COLS) / 32;+ METAL_CONST int THREAD_ELEMENT_ROWS = 1;+ METAL_CONST int THREAD_ELEMENT_COLS = 2;++ static_assert(+ THREAD_ELEMENT_ROWS * THREAD_ELEMENT_COLS == THREAD_ELEMENT_COUNT,+ "SimdgroupMultiplyAccumulate shape is not consistent with element count"+ );++ typedef metal::simdgroup_matrix<T, ROWS, COLS> SimdgroupMatrixType;+ typedef metal::vec<T, THREAD_ELEMENT_COUNT> ThreadDataType;++ METAL_FUNC static constexpr short2 get_lane_coordinates(+ ushort simd_lane_id [[thread_index_in_simdgroup]]+ ) {+ const short quad_index = simd_lane_id / 4;+ const short lane_row = (quad_index & 4) + ((simd_lane_id / 2) % 4);+ const short lane_col = (quad_index & 2) * 2 + (simd_lane_id % 2) * 2;+ return short2{lane_col, lane_row};+ }++ template <typename SrcPtrType, typename StrideX, typename StrideY>+ METAL_FUNC static constexpr void load(+ thread ThreadDataType& dst,+ SrcPtrType src,+ StrideX stride_x,+ StrideY stride_y+ ) {+ METAL_PRAGMA_UNROLL+ for (short i = 0; i < THREAD_ELEMENT_ROWS; i++) {+ METAL_PRAGMA_UNROLL+ for (short j = 0; j < THREAD_ELEMENT_COLS; j++) {+ dst[i * THREAD_ELEMENT_COLS + j] =+ static_cast<T>(src[i * stride_x + j * stride_y]);+ }+ }+ }++ template <+ typename SrcPtrType,+ typename StrideX,+ typename StrideY,+ typename LimitX,+ typename LimitY,+ typename OffsetX,+ typename OffsetY>+ METAL_FUNC static constexpr void load_safe(+ thread ThreadDataType& dst,+ SrcPtrType src,+ StrideX stride_x,+ StrideY stride_y,+ LimitX limit_x,+ LimitY limit_y,+ OffsetX offset_x = 0,+ OffsetY offset_y = 0+ ) {+ METAL_PRAGMA_UNROLL+ for (short i = 0; i < THREAD_ELEMENT_ROWS; i++) {+ METAL_PRAGMA_UNROLL+ for (short j = 0; j < THREAD_ELEMENT_COLS; j++) {+ if ((offset_x + i) < limit_x && (offset_y + j) < limit_y) {+ dst[i * THREAD_ELEMENT_COLS + j] = static_cast<T>(+ src[(offset_x + i) * stride_x + (offset_y + j) * stride_y]+ );+ } else {+ dst[i * THREAD_ELEMENT_COLS + j] = T(0);+ }+ }+ }+ }++ template <typename DstPtrType, typename StrideX, typename StrideY>+ METAL_FUNC static constexpr void store(+ const thread ThreadDataType& src,+ DstPtrType dst,+ StrideX stride_x,+ StrideY stride_y+ ) {+ using U = PointerElementType<DstPtrType>;++ METAL_PRAGMA_UNROLL+ for (short i = 0; i < THREAD_ELEMENT_ROWS; i++) {+ METAL_PRAGMA_UNROLL+ for (short j = 0; j < THREAD_ELEMENT_COLS; j++) {+ dst[i * stride_x + j * stride_y] =+ static_cast<U>(src[i * THREAD_ELEMENT_COLS + j]);+ }+ }+ }++ template <+ typename DstPtrType,+ typename StrideX,+ typename StrideY,+ typename LimitX,+ typename LimitY,+ typename OffsetX,+ typename OffsetY>+ METAL_FUNC static constexpr void store_safe(+ const thread ThreadDataType& src,+ DstPtrType dst,+ StrideX stride_x,+ StrideY stride_y,+ LimitX limit_x,+ LimitY limit_y,+ OffsetX offset_x = 0,+ OffsetY offset_y = 0+ ) {+ using U = PointerElementType<DstPtrType>;++ METAL_PRAGMA_UNROLL+ for (short i = 0; i < THREAD_ELEMENT_ROWS; i++) {+ METAL_PRAGMA_UNROLL+ for (short j = 0; j < THREAD_ELEMENT_COLS; j++) {+ if ((offset_x + i) < limit_x && (offset_y + j) < limit_y) {+ dst[(offset_x + i) * stride_x + (offset_y + j) * stride_y] =+ static_cast<U>(src[i * THREAD_ELEMENT_COLS + j]);+ }+ }+ }+ }++ METAL_FUNC static constexpr void multiply_accumulate(+ thread ThreadDataType& D,+ thread ThreadDataType& A,+ thread ThreadDataType& B,+ thread ThreadDataType& C+ ) {+ SimdgroupMatrixType D_mat;+ SimdgroupMatrixType A_mat;+ SimdgroupMatrixType B_mat;+ SimdgroupMatrixType C_mat;++ reinterpret_cast<thread ThreadDataType&>(A_mat.thread_elements()) = A;+ reinterpret_cast<thread ThreadDataType&>(B_mat.thread_elements()) = B;+ reinterpret_cast<thread ThreadDataType&>(C_mat.thread_elements()) = C;++ multiply_accumulate(D_mat, A_mat, B_mat, C_mat);++ D = reinterpret_cast<thread ThreadDataType&>(D_mat.thread_elements());+ }++ METAL_FUNC static constexpr void multiply_accumulate(+ thread SimdgroupMatrixType& D,+ thread SimdgroupMatrixType& A,+ thread SimdgroupMatrixType& B,+ thread SimdgroupMatrixType& C+ ) {+ simdgroup_multiply_accumulate(D, A, B, C);+ }+};++} // namespace matmul+} // namespace uzudiff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/params.h b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/params.hdeleted file mode 100644index 9d1675443..000000000--- a/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/params.h+++ /dev/null@@ -1,26 +0,0 @@--#pragma once--///////////////////////////////////////////////////////////////////////////////-// GEMM param classes-///////////////////////////////////////////////////////////////////////////////--namespace steel {--struct GEMMParams {- const int M;- const int N;- const int K;-- const int lda;- const int ldb;- const int ldd;-- const int tiles_n;- const int tiles_m;-- const int swizzle_log;- const int gemm_k_iterations_aligned;-};--} // namespace steeldiff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h b/crates/uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.hnew file mode 100644index 000000000..c323dd73e--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h@@ -0,0 +1,247 @@+#pragma once++#include "simdgroup_fragment.h"++using namespace metal;++namespace uzu {+namespace matmul {++///////////////////////////////////////////////////////////////////////////////+// ThreadgroupTile - manages the GEMM computation for a threadgroup+///////////////////////////////////////////////////////////////////////////////++template <+ typename T,+ typename U,+ int BLOCK_ROWS,+ int BLOCK_COLS,+ int BLOCK_DEPTH,+ int SIMDGROUPS_PER_ROW,+ int SIMDGROUPS_PER_COLUMN,+ bool transpose_a,+ bool transpose_b,+ short THREADGROUP_LEADING_DIMENSION_A,+ short THREADGROUP_LEADING_DIMENSION_B,+ typename AccumType = float,+ typename Epilogue = TransformNone<U, AccumType>>+struct ThreadgroupTile {+ METAL_CONST short SIMDGROUP_BLOCK_SIZE = 8;+ using SimdgroupMultiplyAccumulateType = SimdgroupMultiplyAccumulate<+ AccumType,+ SIMDGROUP_BLOCK_SIZE,+ SIMDGROUP_BLOCK_SIZE>;++ METAL_CONST short TILE_ROW_STRIDE = SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW;+ METAL_CONST short TILE_COL_STRIDE =+ SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN;++ METAL_CONST short TILE_ROWS =+ BLOCK_ROWS / (SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW);+ METAL_CONST short TILE_COLS =+ BLOCK_COLS / (SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN);++ METAL_CONST short A_STRIDE_ROW =+ transpose_a ? 1 : THREADGROUP_LEADING_DIMENSION_A;+ METAL_CONST short A_STRIDE_INNER =+ transpose_a ? THREADGROUP_LEADING_DIMENSION_A : 1;++ METAL_CONST short B_STRIDE_INNER =+ transpose_b ? 1 : THREADGROUP_LEADING_DIMENSION_B;+ METAL_CONST short B_STRIDE_COL =+ transpose_b ? THREADGROUP_LEADING_DIMENSION_B : 1;++ METAL_CONST short TILE_STRIDE_A = SIMDGROUP_BLOCK_SIZE * A_STRIDE_INNER;+ METAL_CONST short TILE_STRIDE_B = SIMDGROUP_BLOCK_SIZE * B_STRIDE_INNER;++ SimdgroupFragment<AccumType, TILE_ROWS, 1, SimdgroupMultiplyAccumulateType>+ a_fragment;+ SimdgroupFragment<AccumType, 1, TILE_COLS, SimdgroupMultiplyAccumulateType>+ b_fragment;+ SimdgroupFragment<+ AccumType,+ TILE_ROWS,+ TILE_COLS,+ SimdgroupMultiplyAccumulateType>+ c_fragment;++ short simdgroup_row_offset;+ short simdgroup_col_offset;++ short a_shared_offset;+ short b_shared_offset;++ METAL_FUNC ThreadgroupTile(+ ushort simd_group_id [[simdgroup_index_in_threadgroup]],+ ushort simd_lane_id [[thread_index_in_simdgroup]]+ ) {+ short tile_row_base =+ SIMDGROUP_BLOCK_SIZE * (simd_group_id / SIMDGROUPS_PER_COLUMN);+ short tile_col_base =+ SIMDGROUP_BLOCK_SIZE * (simd_group_id % SIMDGROUPS_PER_COLUMN);++ short2 simdgroup_coordinates =+ SimdgroupMultiplyAccumulateType::get_lane_coordinates(simd_lane_id);+ simdgroup_row_offset = simdgroup_coordinates.y;+ simdgroup_col_offset = simdgroup_coordinates.x;++ a_shared_offset = (tile_row_base + simdgroup_row_offset) * A_STRIDE_ROW ++ (simdgroup_col_offset)*A_STRIDE_INNER;+ b_shared_offset = (simdgroup_row_offset)*B_STRIDE_INNER ++ (tile_col_base + simdgroup_col_offset) * B_STRIDE_COL;++ simdgroup_row_offset += tile_row_base;+ simdgroup_col_offset += tile_col_base;+ }++ METAL_FUNC void multiply_accumulate(+ const threadgroup T* a_shared,+ const threadgroup T* b_shared+ ) {+ a_shared += a_shared_offset;+ b_shared += b_shared_offset;++ METAL_PRAGMA_UNROLL+ for (short k_block_index = 0; k_block_index < BLOCK_DEPTH;+ k_block_index += SIMDGROUP_BLOCK_SIZE) {+ simdgroup_barrier(mem_flags::mem_none);++ a_fragment.template load<+ T,+ SIMDGROUPS_PER_ROW,+ 1,+ A_STRIDE_ROW,+ A_STRIDE_INNER>(a_shared);++ simdgroup_barrier(mem_flags::mem_none);++ b_fragment.template load<+ T,+ 1,+ SIMDGROUPS_PER_COLUMN,+ B_STRIDE_INNER,+ B_STRIDE_COL>(b_shared);++ simdgroup_barrier(mem_flags::mem_none);++ tile_multiply_accumulate(c_fragment, a_fragment, b_fragment, c_fragment);++ a_shared += TILE_STRIDE_A;+ b_shared += TILE_STRIDE_B;+ }+ }++ METAL_FUNC void store_result(device U* D, const int leading_dimension_d) {+ METAL_PRAGMA_UNROLL+ for (short i = 0; i < decltype(c_fragment)::ELEMENTS_PER_FRAGMENT; i++) {+ c_fragment.elements()[i] = Epilogue::apply(c_fragment.elements()[i]);+ }++ D += simdgroup_row_offset * leading_dimension_d + simdgroup_col_offset;++ c_fragment.template store<U, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN>(+ D,+ leading_dimension_d+ );+ }++ METAL_FUNC void store_result_safe(+ device U* D,+ const int leading_dimension_d,+ short2 destination_tile_dimensions+ ) {+ METAL_PRAGMA_UNROLL+ for (short i = 0; i < decltype(c_fragment)::ELEMENTS_PER_FRAGMENT; i++) {+ c_fragment.elements()[i] = Epilogue::apply(c_fragment.elements()[i]);+ }++ D += simdgroup_row_offset * leading_dimension_d + simdgroup_col_offset;+ destination_tile_dimensions -=+ short2(simdgroup_col_offset, simdgroup_row_offset);++ if (destination_tile_dimensions.x <= 0 ||+ destination_tile_dimensions.y <= 0)+ return;++ c_fragment+ .template store_safe<U, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN>(+ D,+ leading_dimension_d,+ destination_tile_dimensions+ );+ }++ template <typename EpilogueOp>+ METAL_FUNC void apply_epilogue(+ const device U* C,+ const int leading_dimension_c,+ const int column_stride_c,+ thread const EpilogueOp& epilogue_operation+ ) {+ const device U* c_pointer = C + simdgroup_row_offset * leading_dimension_c ++ simdgroup_col_offset * column_stride_c;++ METAL_PRAGMA_UNROLL+ for (short i = 0; i < TILE_ROWS; i++) {+ METAL_PRAGMA_UNROLL+ for (short j = 0; j < TILE_COLS; j++) {+ thread auto& block_data = c_fragment.multiply_accumulate_at(i, j);+ METAL_PRAGMA_UNROLL+ for (short k = 0;+ k < decltype(c_fragment)::ELEMENTS_PER_MULTIPLY_ACCUMULATE;+ k++) {+ short row_offset = (i * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW);+ short col_offset = (j * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN);+ U c_value = c_pointer+ [row_offset * leading_dimension_c + col_offset * column_stride_c ++ k];+ block_data[k] = epilogue_operation.apply(+ block_data[k],+ static_cast<AccumType>(c_value)+ );+ }+ }+ }+ }++ template <typename EpilogueOp>+ METAL_FUNC void apply_epilogue_safe(+ const device U* C,+ const int leading_dimension_c,+ const int column_stride_c,+ short2 tile_dimensions,+ thread const EpilogueOp& epilogue_operation+ ) {+ const device U* c_pointer = C + simdgroup_row_offset * leading_dimension_c ++ simdgroup_col_offset * column_stride_c;+ tile_dimensions -= short2(simdgroup_col_offset, simdgroup_row_offset);++ METAL_PRAGMA_UNROLL+ for (short i = 0; i < TILE_ROWS; i++) {+ METAL_PRAGMA_UNROLL+ for (short j = 0; j < TILE_COLS; j++) {+ thread auto& block_data = c_fragment.multiply_accumulate_at(i, j);+ short row_offset = (i * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW);+ short col_offset = (j * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN);+ METAL_PRAGMA_UNROLL+ for (short k = 0;+ k < decltype(c_fragment)::ELEMENTS_PER_MULTIPLY_ACCUMULATE;+ k++) {+ if (row_offset < tile_dimensions.y &&+ col_offset + k < tile_dimensions.x) {+ U c_value = c_pointer+ [row_offset * leading_dimension_c ++ col_offset * column_stride_c + k];+ block_data[k] = epilogue_operation.apply(+ block_data[k],+ static_cast<AccumType>(c_value)+ );+ }+ }+ }+ }+ }+};++} // namespace matmul+} // namespace uzudiff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.metalindex ade37da08..6608d1cb7 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.metal+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.metal@@ -1,47 +1,48 @@// clang-format off#include "../../../common/utils.h"-#include "../../../definitions.metal"+#include "../../../common/dsl.h"+#include "../../../common/thread_context.h"#include "../../common/gemm.h"using namespace uzu::matmul;-template <typename T, int BM, int BN, int BK, int WM, int WN, bool MN_aligned, bool K_aligned>+template <typename T, int BLOCK_ROWS, int BLOCK_COLS, int BLOCK_DEPTH, int SIMDGROUPS_PER_ROW, int SIMDGROUPS_PER_COLUMN, bool MN_aligned, bool K_aligned>METAL_FUNC void gemm_dispatch(const device T* a,const device T* b,device T* d,- const constant GEMMParams* params,- threadgroup T* As,- threadgroup T* Bs,+ const constant GemmParams* params,+ threadgroup T* a_shared,+ threadgroup T* b_shared,uint simd_lane_id,uint simd_group_id,uint2 tid,uint3 lid) {- using gemm_kernel = GEMMKernel<T, T, BM, BN, BK, WM, WN, false, true, MN_aligned, K_aligned, float>;- gemm_kernel::run(a, b, d, params, As, Bs, simd_lane_id, simd_group_id, tid, lid);+ using ThreadgroupGemmType = ThreadgroupGemm<T, T, BLOCK_ROWS, BLOCK_COLS, BLOCK_DEPTH, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN, false, true, MN_aligned, K_aligned, float>;+ ThreadgroupGemmType::run(a, b, d, params, a_shared, b_shared, simd_lane_id, simd_group_id, tid, lid);}-#define GEMM_DISPATCH(T, BM, BN, BK, WM, WN) \- if (block_rows == BM && block_cols == BN && block_depth == BK && \- warps_per_row == WM && warps_per_col == WN) { \+#define GEMM_DISPATCH(T, BLOCK_ROWS, BLOCK_COLS, BLOCK_DEPTH, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN) \+ if (block_rows == BLOCK_ROWS && block_cols == BLOCK_COLS && block_depth == BLOCK_DEPTH && \+ simdgroups_per_row == SIMDGROUPS_PER_ROW && simdgroups_per_column == SIMDGROUPS_PER_COLUMN) { \if (align_m && align_n) { \if (align_k) \- gemm_dispatch<T, BM, BN, BK, WM, WN, true, true>(a, b, d, params, a_shared, b_shared, simd.lane_idx, simd.group_idx, uint2(group_x, group_y), uint3(thread_x, thread_y, thread_z)); \+ gemm_dispatch<T, BLOCK_ROWS, BLOCK_COLS, BLOCK_DEPTH, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN, true, true>(a, b, d, params, a_shared, b_shared, simd.simdgroup_index, simd.threadgroup_index, uint2(group_x, group_y), uint3(thread_x, thread_y, thread_z)); \else \- gemm_dispatch<T, BM, BN, BK, WM, WN, true, false>(a, b, d, params, a_shared, b_shared, simd.lane_idx, simd.group_idx, uint2(group_x, group_y), uint3(thread_x, thread_y, thread_z)); \+ gemm_dispatch<T, BLOCK_ROWS, BLOCK_COLS, BLOCK_DEPTH, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN, true, false>(a, b, d, params, a_shared, b_shared, simd.simdgroup_index, simd.threadgroup_index, uint2(group_x, group_y), uint3(thread_x, thread_y, thread_z)); \} else { \if (align_k) \- gemm_dispatch<T, BM, BN, BK, WM, WN, false, true>(a, b, d, params, a_shared, b_shared, simd.lane_idx, simd.group_idx, uint2(group_x, group_y), uint3(thread_x, thread_y, thread_z)); \+ gemm_dispatch<T, BLOCK_ROWS, BLOCK_COLS, BLOCK_DEPTH, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN, false, true>(a, b, d, params, a_shared, b_shared, simd.simdgroup_index, simd.threadgroup_index, uint2(group_x, group_y), uint3(thread_x, thread_y, thread_z)); \else \- gemm_dispatch<T, BM, BN, BK, WM, WN, false, false>(a, b, d, params, a_shared, b_shared, simd.lane_idx, simd.group_idx, uint2(group_x, group_y), uint3(thread_x, thread_y, thread_z)); \+ gemm_dispatch<T, BLOCK_ROWS, BLOCK_COLS, BLOCK_DEPTH, SIMDGROUPS_PER_ROW, SIMDGROUPS_PER_COLUMN, false, false>(a, b, d, params, a_shared, b_shared, simd.simdgroup_index, simd.threadgroup_index, uint2(group_x, group_y), uint3(thread_x, thread_y, thread_z)); \} \return; \}-#define GEMM_MAX_TGP_A 2560-#define GEMM_MAX_TGP_B 1536+#define GEMM_MAX_THREADGROUP_A 2560+#define GEMM_MAX_THREADGROUP_B 1536template <typename T>VARIANTS(T, float, half, bfloat)@@ -49,16 +50,16 @@ PUBLIC KERNEL(MatmulGemm)(const device T* a,const device T* b,device T* d,- const constant uzu::matmul::GEMMParams* params,+ const constant uzu::matmul::GemmParams* params,const constant uint& group_count_x,const constant uint& group_count_y,- threadgroup T a_shared[GEMM_MAX_TGP_A],- threadgroup T b_shared[GEMM_MAX_TGP_B],+ threadgroup T a_shared[GEMM_MAX_THREADGROUP_A],+ threadgroup T b_shared[GEMM_MAX_THREADGROUP_B],const uint block_rows SPECIALIZE,const uint block_cols SPECIALIZE,const uint block_depth SPECIALIZE,- const uint warps_per_row SPECIALIZE,- const uint warps_per_col SPECIALIZE,+ const uint simdgroups_per_row SPECIALIZE,+ const uint simdgroups_per_column SPECIALIZE,const bool align_m SPECIALIZE,const bool align_n SPECIALIZE,const bool align_k SPECIALIZE,@@ -67,7 +68,7 @@ PUBLIC KERNEL(MatmulGemm)(const uint thread_x THREADS(32),const uint thread_y THREADS(2),const uint thread_z THREADS(2),- const Simd simd+ const ThreadContext simd) {GEMM_DISPATCH(T, 64, 64, 16, 2, 2)GEMM_DISPATCH(T, 64, 64, 16, 1, 2)diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/shaders/gemv.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemv/shaders/gemv.metalindex b2fe633e5..ca4aeecf7 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv/shaders/gemv.metal+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv/shaders/gemv.metal@@ -1,5 +1,6 @@#include "../../../common/utils.h"-#include "../../../definitions.metal"+#include "../../../common/dsl.h"+#include "../../../common/thread_context.h"#include <metal_simdgroup>@@ -8,7 +9,7 @@ using namespace metal;// Upper bound for threadgroup memory (in floats).// Max config: tg_simd_cols=16, output_rows_per_tg=16, thread_out_rows=4// → 16 * (16 + 4) = 320. Unused when tg_simd_cols == 1.-#define GEMV_MAX_TG_MEMORY 320+#define GEMV_MAX_THREADGROUP_MEMORY 320template <typename T>VARIANTS(T, float, half, bfloat)@@ -25,7 +26,7 @@ PUBLIC KERNEL(MatmulGemv)(const constant int& output_source_stride,const constant int& batch_rows,const constant int& output_rows_per_threadgroup,- threadgroup float threadgroup_memory[GEMV_MAX_TG_MEMORY],+ threadgroup float threadgroup_memory[GEMV_MAX_THREADGROUP_MEMORY],const uint tg_simd_rows SPECIALIZE,const uint tg_simd_cols SPECIALIZE,const uint sg_thread_rows SPECIALIZE,@@ -38,14 +39,14 @@ PUBLIC KERNEL(MatmulGemv)(const uint thread_index_x THREADS(32),const uint thread_index_y THREADS(16),const uint thread_index_z THREADS(1),- const Simd simd+ const ThreadContext simd) {if (output_rows_per_threadgroup <= 0) {return;}// Simdgroup guard — kill excess simdgroups beyond what this tile uses- if (simd.group_idx >= tg_simd_rows * tg_simd_cols) {+ if (simd.threadgroup_index >= tg_simd_rows * tg_simd_cols) {return;}@@ -65,21 +66,22 @@ PUBLIC KERNEL(MatmulGemv)(thread float vector_coefficients[4];const int thread_row_in_simdgroup =- sg_thread_cols != 32 ? int(simd.lane_idx) / int(sg_thread_cols) : 0;+ sg_thread_cols != 32 ? int(simd.simdgroup_index) / int(sg_thread_cols)+ : 0;const int thread_col_in_simdgroup =- sg_thread_cols != 32 ? int(simd.lane_idx) % int(sg_thread_cols)- : int(simd.lane_idx);+ sg_thread_cols != 32 ? int(simd.simdgroup_index) % int(sg_thread_cols)+ : int(simd.simdgroup_index);const int simdgroup_column_index =- tg_simd_cols != 1 ? int(simd.group_idx % tg_simd_cols) : 0;+ tg_simd_cols != 1 ? int(simd.threadgroup_index % tg_simd_cols) : 0;const int simdgroup_row_thread_base =tg_simd_cols != 1- ? int(sg_thread_rows) * int(simd.group_idx / tg_simd_cols)- : int(sg_thread_rows) * int(simd.group_idx);+ ? int(sg_thread_rows) * int(simd.threadgroup_index / tg_simd_cols)+ : int(sg_thread_rows) * int(simd.threadgroup_index);const int simdgroup_col_thread_base =tg_simd_cols != 1- ? int(sg_thread_cols) * int(simd.group_idx % tg_simd_cols)+ ? int(sg_thread_cols) * int(simd.threadgroup_index % tg_simd_cols): 0;int output_block_row_offset =@@ -121,7 +123,7 @@ PUBLIC KERNEL(MatmulGemv)({const device T* input_vector_row =input_vector + batch_row * input_dimension;- MTL_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (uint input_col_offset = 0; input_col_offset < thread_out_cols;input_col_offset++) {vector_coefficients[input_col_offset] = static_cast<float>(@@ -132,11 +134,11 @@ PUBLIC KERNEL(MatmulGemv)(// Per thread work loopint matrix_row_offset = 0;- MTL_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (uint output_row_offset = 0; output_row_offset < thread_out_rows;output_row_offset++) {// Load matrix row (unchecked)- MTL_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (uint input_col_offset = 0; input_col_offset < thread_out_cols;input_col_offset++) {matrix_values[input_col_offset] = thread_matrix@@ -144,7 +146,7 @@ PUBLIC KERNEL(MatmulGemv)(}// Accumulate results- MTL_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (uint input_col_offset = 0; input_col_offset < thread_out_cols;input_col_offset++) {accumulated_values[output_row_offset] +=@@ -165,7 +167,7 @@ PUBLIC KERNEL(MatmulGemv)(input_vector + batch_row * input_dimension;if (input_block_col_offset + int(thread_out_cols) <=input_vector_length) {- MTL_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (uint input_col_offset = 0; input_col_offset < thread_out_cols;input_col_offset++) {vector_coefficients[input_col_offset] = static_cast<float>(@@ -173,7 +175,7 @@ PUBLIC KERNEL(MatmulGemv)();}} else {- MTL_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (uint input_col_offset = 0; input_col_offset < thread_out_cols;input_col_offset++) {vector_coefficients[input_col_offset] =@@ -189,13 +191,13 @@ PUBLIC KERNEL(MatmulGemv)(}// Per thread work loop- MTL_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (uint output_row_offset = 0; output_row_offset < thread_out_rows;output_row_offset++) {// Load matrix row (checked)if (input_block_col_offset + int(thread_out_cols) <=input_vector_length) {- MTL_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (uint input_col_offset = 0; input_col_offset < thread_out_cols;input_col_offset++) {matrix_values[input_col_offset] = thread_matrix@@ -203,7 +205,7 @@ PUBLIC KERNEL(MatmulGemv)(input_block_col_offset + input_col_offset];}} else {- MTL_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (uint input_col_offset = 0; input_col_offset < thread_out_cols;input_col_offset++) {matrix_values[input_col_offset] =@@ -217,7 +219,7 @@ PUBLIC KERNEL(MatmulGemv)(}// Accumulate results- MTL_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (uint input_col_offset = 0; input_col_offset < thread_out_cols;input_col_offset++) {accumulated_values[output_row_offset] +=@@ -228,13 +230,13 @@ PUBLIC KERNEL(MatmulGemv)(}// Simdgroup accumulations- MTL_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (uint output_row_offset = 0; output_row_offset < thread_out_rows;output_row_offset++) {- MTL_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (ushort simd_shuffle_offset = (sg_thread_cols / 2);simd_shuffle_offset >= 1;- simd_shuffle_offset >>= 1) {+ simd_shuffle_offset /= 2) {accumulated_values[output_row_offset] += simd_shuffle_down(accumulated_values[output_row_offset],simd_shuffle_offset@@ -253,7 +255,7 @@ PUBLIC KERNEL(MatmulGemv)(simdgroup_column_index *(computed_output_rows_per_tg + int(thread_out_rows)) +output_block_row_offset;- MTL_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (uint output_row_offset = 0; output_row_offset < thread_out_rows;output_row_offset++) {threadgroup_partial_accumulations[output_row_offset] =@@ -268,7 +270,7 @@ PUBLIC KERNEL(MatmulGemv)(for (uint reduction_simdgroup_col = 1;reduction_simdgroup_col < tg_simd_cols;reduction_simdgroup_col++) {- MTL_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (uint output_row_offset = 0; output_row_offset < thread_out_rows;output_row_offset++) {accumulated_values[output_row_offset] += base_partial@@ -284,7 +286,7 @@ PUBLIC KERNEL(MatmulGemv)(// Write outputsif (simdgroup_col_thread_base == 0 && thread_col_in_simdgroup == 0) {device T* output_row_values = output_vector + batch_row * output_dimension;- MTL_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (uint output_row_offset = 0; output_row_offset < thread_out_rows;output_row_offset++) {if (apply_output_scale_and_accumulate) {diff --git a/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal b/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metalindex 0e2215a54..05b5f58e1 100644--- a/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal+++ b/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal@@ -1,6 +1,6 @@#include <metal_stdlib>#include "../activation/activation.h"-#include "../definitions.metal"+#include "../common/dsl.h"template <typename T>VARIANTS(T, float, half, bfloat)diff --git a/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal b/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metalindex d86888590..1281a2bd8 100644--- a/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal@@ -1,7 +1,9 @@#include <metal_stdlib>#include <metal_atomic>-#include "../definitions.metal"+#include "../common/dsl.h"+#include "../common/thread_context.h"+#include "../common/threadgroup_reduce.h"#define BLOCK_SIZE 128#define TILE_E 512@@ -23,7 +25,7 @@ PUBLIC KERNEL(MoeCountsOffsetsFused)(threadgroup uintcounts_shared[BLOCK_SIZE], // Cache counts in threadgroup memorythreadgroup uint& carry,- const Simd simd,+ const ThreadContext simd,const uint lid THREADS(128)) {if (e_input == 0) {diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_single.metalindex 2607983e5..70320f217 100644--- a/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal@@ -1,6 +1,7 @@#include <metal_stdlib>#include <metal_simdgroup>-#include "../definitions.metal"+#include "../common/dsl.h"+#include "../common/thread_context.h"#include "moe_commons.h"// ============================================================================@@ -26,7 +27,7 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassA)(constant float& up_clip_min,constant float& up_clip_max,const uint gating_sel SPECIALIZE, // 0=GELU, 1=SiLU, 2=SwiGLU, 3=GEGLU- const Simd simd,+ const ThreadContext simd,const uint h_block_idx GROUPS(d_ff.div_ceil(4)),const uint k_slot GROUPS(k),const uint tid THREADS(128)@@ -36,7 +37,7 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassA)(return;const uint expert_u = uint(expert_id);- const uint h_idx = h_block_idx * 4 + simd.group_idx;+ const uint h_idx = h_block_idx * 4 + simd.threadgroup_index;if (h_idx >= d_ff)return;@@ -54,7 +55,7 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassA)(const uint vec_iters = d_model / 128;for (uint i = 0; i < vec_iters; ++i) {- uint base_idx = i * 128 + simd.lane_idx * 4;+ uint base_idx = i * 128 + simd.simdgroup_index * 4;device const T* x_vec2 = reinterpret_cast<device const T*>(x + base_idx);device const T* w_up_vec2 =@@ -74,7 +75,7 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassA)(}}- uint leftover_start = vec_iters * 128 + simd.lane_idx;+ uint leftover_start = vec_iters * 128 + simd.simdgroup_index;for (uint idx = leftover_start; idx < d_model; idx += 32) {float xv = float(x[idx]);acc_up += xv * float(w_up_row[idx]);@@ -88,7 +89,7 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassA)(acc_gate = simd_sum(acc_gate);}- if (simd.lane_idx == 0) {+ if (simd.simdgroup_index == 0) {float up_val = clamp(acc_up + float(biases[bias_base + h_idx]),up_clip_min,@@ -132,11 +133,11 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassB)(constant uint& d_model,constant uint& d_ff,constant uint& k_input,- const Simd simd,+ const ThreadContext simd,const uint d_block GROUPS(d_model.div_ceil(8)),const uint tid THREADS(256)) {- const uint my_col = d_block * 8 + simd.group_idx;+ const uint my_col = d_block * 8 + simd.threadgroup_index;if (my_col >= d_model)return;@@ -158,7 +159,7 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassB)(// Vectorized reductionfor (uint i = 0; i < vec_iters; ++i) {- uint base_idx = i * 128 + simd.lane_idx * 4;+ uint base_idx = i * 128 + simd.simdgroup_index * 4;float4 h_vec =*reinterpret_cast<device const float4*>(hidden_ptr + base_idx);@@ -171,19 +172,20 @@ PUBLIC KERNEL(MoeExpertsDecodeSinglePassB)(}// Remainder- for (uint idx = vec_iters * 128 + simd.lane_idx; idx < d_ff; idx += 32) {+ for (uint idx = vec_iters * 128 + simd.simdgroup_index; idx < d_ff;+ idx += 32) {acc += hidden_ptr[idx] * float(w2_ptr[idx]);}// Simdgroup reduction and accumulatefloat result = simd_sum(acc);- if (simd.lane_idx == 0) {+ if (simd.simdgroup_index == 0) {result += float(biases[(ulong)expert_u * (ulong)d_model + (ulong)my_col]);final_acc += prob * result;}}- if (simd.lane_idx == 0) {+ if (simd.simdgroup_index == 0) {y[my_col] = T(final_acc);}}\ No newline at end of filediff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metalindex c7400665f..deab8e40c 100644--- a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal@@ -1,6 +1,7 @@#include <metal_stdlib>#include <metal_simdgroup>-#include "../definitions.metal"+#include "../common/dsl.h"+#include "../common/thread_context.h"#include "moe_commons.h"using namespace metal;@@ -26,7 +27,7 @@ PUBLIC KERNEL(MoeExpertsDecodePassA)(constant float& silu_alpha,device const uint* tile_map,const uint gating_sel SPECIALIZE,- const Simd simd,+ const ThreadContext simd,const uint tile_idx GROUPS(INDIRECT),const uint tid THREADS(128)) {@@ -43,7 +44,7 @@ PUBLIC KERNEL(MoeExpertsDecodePassA)(return;// Each simdgroup outputs one hidden element- const uint h_idx = h_block_idx * 4 + simd.group_idx;+ const uint h_idx = h_block_idx * 4 + simd.threadgroup_index;if (h_idx >= d_ff)return;@@ -65,7 +66,7 @@ PUBLIC KERNEL(MoeExpertsDecodePassA)(const uint vec_iters = d_model / 128;for (uint i = 0; i < vec_iters; ++i) {- uint base_idx = i * 128 + simd.lane_idx * 4;+ uint base_idx = i * 128 + simd.simdgroup_index * 4;device const T* x_vec = reinterpret_cast<device const T*>(x_ptr + base_idx);device const T* w_up_vec =@@ -86,7 +87,7 @@ PUBLIC KERNEL(MoeExpertsDecodePassA)(}// Handle leftover elements- uint leftover_start = vec_iters * 128 + simd.lane_idx;+ uint leftover_start = vec_iters * 128 + simd.simdgroup_index;for (uint idx = leftover_start; idx < d_model; idx += 32) {float xv = float(x_ptr[idx]);acc_up += xv * float(w_up_row[idx]);@@ -102,7 +103,7 @@ PUBLIC KERNEL(MoeExpertsDecodePassA)(}// Lane 0 applies activation and writes result- if (simd.lane_idx == 0) {+ if (simd.simdgroup_index == 0) {float up_val = acc_up + float(up_biases[bias_base + h_idx]);up_val = clamp(up_val, up_clip_min, up_clip_max);@@ -141,7 +142,7 @@ PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(constant uint& d_model,constant uint& d_ff,constant uint& e,- const Simd simd,+ const ThreadContext simd,const uint tgpig_x GROUPS(d_model.div_ceil(SIMDGROUPS_PER_TG)),const uint tgpig_y GROUPS(total_rows),const uint tid THREADS(256)@@ -149,7 +150,7 @@ PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(const uint row_idx = tgpig_y;// Each simdgroup computes one output column- const uint my_col = tgpig_x * SIMDGROUPS_PER_TG + simd.group_idx;+ const uint my_col = tgpig_x * SIMDGROUPS_PER_TG + simd.threadgroup_index;if (my_col >= d_model)return;@@ -170,7 +171,7 @@ PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(const uint k_vec_iters = k_iters / 8;for (uint iter = 0; iter < k_vec_iters; ++iter) {- const uint k_base = iter * (8 * THREADS_PER_SIMD) + simd.lane_idx;+ const uint k_base = iter * (8 * THREADS_PER_SIMD) + simd.simdgroup_index;// hidden: stride-32 per lane, already f32 from Pass Aconst AccumT h0 = hidden[hidden_base + k_base + 0 * THREADS_PER_SIMD];@@ -216,7 +217,7 @@ PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(// Handle remaining full iterationsfor (uint iter = k_vec_iters * 8; iter < k_iters; ++iter) {- const uint k = iter * THREADS_PER_SIMD + simd.lane_idx;+ const uint k = iter * THREADS_PER_SIMD + simd.simdgroup_index;acc =fma(AccumT(hidden[hidden_base + k]),AccumT(w2_all[w2_col_base + k]),@@ -225,8 +226,8 @@ PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(// Handle leftover elements (d_ff % 32)const uint leftover_start = k_iters * THREADS_PER_SIMD;- if (leftover_start + simd.lane_idx < d_ff) {- const uint k = leftover_start + simd.lane_idx;+ if (leftover_start + simd.simdgroup_index < d_ff) {+ const uint k = leftover_start + simd.simdgroup_index;acc =fma(AccumT(hidden[hidden_base + k]),AccumT(w2_all[w2_col_base + k]),@@ -237,7 +238,7 @@ PUBLIC KERNEL(MoeExpertsDecodeDownFused2D)(AccumT result = simd_sum(acc);// Lane 0 writes result- if (simd.lane_idx == 0) {+ if (simd.simdgroup_index == 0) {const ulong bias_idx = (ulong)expert_idx * (ulong)d_model + (ulong)my_col;result += AccumT(down_biases[bias_idx]);diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metalindex 92f9ae0c8..f91b44dbc 100644--- a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_prefill.metal@@ -1,5 +1,6 @@#include <metal_stdlib>-#include "../definitions.metal"+#include "../common/dsl.h"+#include "../common/thread_context.h"#include "../quant_matmul/mma.h"#include "moe_commons.h"@@ -59,7 +60,7 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(const uint n_tile_idx GROUPS(INDIRECT),const uint row_tile_idx GROUPS(INDIRECT),const uint local_tid_x THREADS(128),- const Simd simd+ const ThreadContext simd) {const uint base = row_tile_idx * 3u;const uint expert_idx = tile_map[base + 0u];@@ -105,15 +106,15 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(// simdgroup tile mappingconst uint sg_col_count = Bn / SgBn;- const uint row_sg = simd.group_idx / sg_col_count;- const uint col_sg = simd.group_idx % sg_col_count;+ const uint row_sg = simd.threadgroup_index / sg_col_count;+ const uint col_sg = simd.threadgroup_index % sg_col_count;const uint row_sg_off = row_sg * SgBm;const uint col_sg_off = col_sg * SgBn;// Guard against misconfigured TG sizesconstexpr uint SG_TILE = 8; // simdgroup fragment dimension (8x8)constexpr uint SG_EXPECTED = (Bm / SG_TILE) * (Bn / SG_TILE);- if (simd.group_idx >= SG_EXPECTED)+ if (simd.threadgroup_index >= SG_EXPECTED)return;// partial accumulators (8x8 tiles per simdgroup)@@ -144,7 +145,7 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(constexpr uint total_vecs = Bm * vec_cols; // 16*8 = 128const uint iters = ceil_div(total_vecs, threads_total);- UZU_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (uint t = 0; t < iters; ++t) {const uint i = t * threads_total + lin;if (i < total_vecs) {@@ -158,7 +159,7 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(// Safe to read - at least first element is validconst ulong base =x_base + (ulong)r * (ulong)d_model + (ulong)(k_off + col_base);- UZU_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (uint j = 0; j < vec_size; j++) {dst[j] = (col_base + j < valid_k)? static_cast<T>(float(x_perm[base + j]))@@ -166,7 +167,7 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(}} else {// Row out of bounds or entire vec8 chunk is out of K bounds- UZU_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (uint j = 0; j < vec_size; j++) {dst[j] = static_cast<T>(0.0f);}@@ -183,7 +184,7 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(constexpr uint total_vecs = Bn * vec_cols; // 32*8 = 256const uint iters = ceil_div(total_vecs, threads_total);- UZU_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (uint t = 0; t < iters; ++t) {const uint i = t * threads_total + lin;if (i < total_vecs) {@@ -203,7 +204,7 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(const ulong gate_base = w13_expert_base +(ulong)(d_ff + gcol) * (ulong)d_model +(ulong)(k_off + col_base);- UZU_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (uint j = 0; j < vec_size; j++) {up_dst[j] = (col_base + j < valid_k)? static_cast<T>(float(w13_all[up_base + j]))@@ -216,7 +217,7 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(}} else {// Row out of bounds or entire vec8 chunk is out of K bounds- UZU_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (uint j = 0; j < vec_size; j++) {up_dst[j] = static_cast<T>(0.0f);if (gating_sel > 1u)@@ -230,15 +231,15 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(threadgroup_barrier(mem_flags::mem_threadgroup);// ---- MMA across the BK tile (in simdgroup-sized chunks) ----- UZU_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (uint kk = 0; kk < Bk; kk += SG_TILE) {- UZU_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (uint m_sub = 0; m_sub < SgBm; m_sub += SG_TILE) {const uint r_idx = m_sub / SG_TILE;metal::simdgroup_matrix<T, 8, 8> lhs_frag;simdgroup_load(lhs_frag, Xs, X_LD, ulong2(kk, row_sg_off + m_sub));- UZU_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (uint n_sub = 0; n_sub < SgBn; n_sub += SG_TILE) {const uint c_idx = n_sub / SG_TILE;const uint tile = r_idx * (SgBn / SG_TILE) + c_idx;@@ -294,7 +295,7 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(threadgroup_barrier(mem_flags::mem_threadgroup);// ---- Epilogue: activation/gating → hidden_out (f32) ----- const uint sg_lane_start = simd.group_idx * 32;+ const uint sg_lane_start = simd.threadgroup_index * 32;const uint sg_lane_end = sg_lane_start + 32;if (lin < sg_lane_start || lin >= sg_lane_end)return;@@ -305,10 +306,10 @@ PUBLIC KERNEL(MoeExpertsPrefillPassA)(const uint lane_row = (lane_qid & 4u) + ((sg_lin >> 1) & 3u);const uint lane_col_base = ((lane_qid & 2u) << 1) + ((sg_lin & 1u) << 1);- UZU_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (uint n_sub = 0; n_sub < SgBn; n_sub += SG_TILE) {const uint c_idx = n_sub / SG_TILE;- UZU_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (uint m_sub = 0; m_sub < SgBm; m_sub += SG_TILE) {const uint r_idx = m_sub / SG_TILE;const uint tile = r_idx * (SgBn / SG_TILE) + c_idx;@@ -403,7 +404,7 @@ PUBLIC KERNEL(MoeExpertsPrefillPassB)(threadgroup float Hs[PASSB_BM * (PASSB_BK + PASSB_TG_PAD)],threadgroup T Wk[PASSB_BN * (PASSB_BK + PASSB_TG_PAD)],threadgroup float bias_tile[PASSB_BN],- const Simd simd,+ const ThreadContext simd,const uint n_tile_idx GROUPS(INDIRECT),const uint row_tile_idx GROUPS(INDIRECT),const uint lin THREADS(128)@@ -445,20 +446,20 @@ PUBLIC KERNEL(MoeExpertsPrefillPassB)(const ulong bias_base = (ulong)expert_idx * (ulong)d_model;// 2×2 simdgroup layout for 16×64 output (each sg handles 8×32)- const uint row_sg = simd.group_idx / 2; // 0-1 → rows- const uint col_sg = simd.group_idx % 2; // 0-1 → cols+ const uint row_sg = simd.threadgroup_index / 2; // 0-1 → rows+ const uint col_sg = simd.threadgroup_index % 2; // 0-1 → colsconst uint row_sg_off = row_sg * SgBm;const uint col_sg_off = col_sg * SgBn;// Guard against excessive simdgroupsconstexpr uint SG_EXPECTED = (PASSB_BM / SgBm) * (PASSB_BN / SgBn);- if (simd.group_idx >= SG_EXPECTED)+ if (simd.threadgroup_index >= SG_EXPECTED)return;// 4 accumulators per simdgroup: 1×4 layout of 8×8 tilesconstexpr uint TEMP = (SgBm / SG_TILE) * (SgBn / SG_TILE); // 1×4 = 4metal::simdgroup_float8x8 Out[TEMP];- UZU_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (uint i = 0; i < TEMP; ++i) {Out[i] = metal::make_filled_simdgroup_matrix<float, 8, 8>(0.0f);}@@ -488,12 +489,12 @@ PUBLIC KERNEL(MoeExpertsPrefillPassB)(// Safe to read - at least first element is validdevice const float* my_src =hidden + h_base + (ulong)h_bi * (ulong)d_ff + (ulong)(k_off + h_bj);- UZU_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (uint j = 0; j < H_VEC; j++) {my_dst[j] = (h_bj + j < valid_k) ? my_src[j] : 0.0f;}} else {- UZU_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (uint j = 0; j < H_VEC; j++) {my_dst[j] = 0.0f;}@@ -512,13 +513,13 @@ PUBLIC KERNEL(MoeExpertsPrefillPassB)(device const T* my_src = w2_all + w2_expert_base +(ulong)gcol * (ulong)d_ff +(ulong)(k_off + w_bj);- UZU_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (uint j = 0; j < W_VEC; j++) {my_dst[j] = (w_bj + j < valid_k) ? static_cast<T>(float(my_src[j])): static_cast<T>(0.0f);}} else {- UZU_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (uint j = 0; j < W_VEC; j++) {my_dst[j] = static_cast<T>(0.0f);}@@ -529,7 +530,7 @@ PUBLIC KERNEL(MoeExpertsPrefillPassB)(threadgroup_barrier(mem_flags::mem_threadgroup);// ---- Serpentine MMA: 1×4 tiles per simdgroup ----- UZU_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (uint kk = 0; kk < Bk; kk += SG_TILE) {// Load 1 LHS fragmentmetal::simdgroup_float8x8 lhs;@@ -560,17 +561,17 @@ PUBLIC KERNEL(MoeExpertsPrefillPassB)(threadgroup_barrier(mem_flags::mem_threadgroup);// ---- Writeout with fragment extraction ----- const uint lane_qid = simd.lane_idx >> 2;- const uint lane_row = (lane_qid & 4u) + ((simd.lane_idx >> 1) & 3u);+ const uint lane_qid = simd.simdgroup_index >> 2;+ const uint lane_row = (lane_qid & 4u) + ((simd.simdgroup_index >> 1) & 3u);const uint lane_col_base =- ((lane_qid & 2u) << 1) + ((simd.lane_idx & 1u) << 1);+ ((lane_qid & 2u) << 1) + ((simd.simdgroup_index & 1u) << 1);const uint local_row = row_sg_off + lane_row;if (local_row >= m_rows)return;const ulong out_row = seg_start + row_tg_off + local_row;- UZU_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (uint ni = 0; ni < 4; ni++) { // 4 col tiles per simdgroupconst auto accum = Out[ni].thread_elements();const uint local_col = col_sg_off + ni * 8 + lane_col_base;diff --git a/crates/uzu/src/backends/metal/kernel/moe/finalize.metal b/crates/uzu/src/backends/metal/kernel/moe/finalize.metalindex 435eb592a..f94928258 100644--- a/crates/uzu/src/backends/metal/kernel/moe/finalize.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/finalize.metal@@ -1,5 +1,5 @@#include <metal_stdlib>-#include "../definitions.metal"+#include "../common/dsl.h"#define BM 32#define BN 64diff --git a/crates/uzu/src/backends/metal/kernel/moe/gather.metal b/crates/uzu/src/backends/metal/kernel/moe/gather.metalindex 74f724df0..98365a997 100644--- a/crates/uzu/src/backends/metal/kernel/moe/gather.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/gather.metal@@ -1,5 +1,5 @@#include <metal_stdlib>-#include "../definitions.metal"+#include "../common/dsl.h"// Tiling config for 2D gather#define ROWS_PER_TG 8diff --git a/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal b/crates/uzu/src/backends/metal/kernel/moe/router_topk.metalindex 3c0a29155..ad16c6b40 100644--- a/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/router_topk.metal@@ -1,6 +1,8 @@#include <metal_stdlib>#include <metal_simdgroup>-#include "../definitions.metal"+#include "../common/dsl.h"+#include "../common/thread_context.h"+#include "../common/threadgroup_reduce.h"using namespace metal;constant uint THREADS_PER_TG = 256; // 8 simdgroups@@ -28,7 +30,7 @@ PUBLIC KERNEL(MoeRouterTopK)(threadgroup uint reduce_tmp_u[THREADS_PER_TG],threadgroup uint shared_best_idx[1],threadgroup float shared_best_val[1],- const Simd simd,+ const ThreadContext simd,const uint tgpig_x GROUPS(1),const uint token_idx GROUPS(t),const uint lid THREADS(256)@@ -50,11 +52,12 @@ PUBLIC KERNEL(MoeRouterTopK)(}threadgroup_barrier(mem_flags::mem_threadgroup);- for (uint row = simd.group_idx; row < e; row += simd.groups_per_threadgroup) {+ for (uint row = simd.threadgroup_index; row < e;+ row += simd.simdgroups_per_threadgroup) {const device ScalarT* w_vec = weight + (ulong)row * (ulong)vecs * 4;float4 accum4 = float4(0.0f);- for (uint c = simd.lane_idx; c < vecs; c += 32u) {+ for (uint c = simd.simdgroup_index; c < vecs; c += 32u) {const float4 wv = float4(w_vec[c * 4 + 0],w_vec[c * 4 + 1],diff --git a/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal b/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metalindex b32d13f4f..6cc4b4701 100644--- a/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/scatter_buckets.metal@@ -1,6 +1,7 @@#include <metal_stdlib>#include <metal_atomic>-#include "../definitions.metal"+#include "../common/dsl.h"+#include "../common/thread_context.h"using namespace metal;#define BLOCK_SIZE 256@@ -29,7 +30,7 @@ inline void moe_scatter_buckets_impl(uint3 tgpig,threadgroup _atomic<uint>* sg_counts,threadgroup uint* sg_base,- const thread Simd& simd+ const thread ThreadContext& simd) {if (E == 0 || T == 0 || K == 0)return;@@ -64,7 +65,7 @@ inline void moe_scatter_buckets_impl(if (ue >= e0 && ue < e0 + tile_e) {const uint te = ue - e0;atomic_fetch_add_explicit(- &sg_counts[simd.group_idx * TILE_E + te],+ &sg_counts[simd.threadgroup_index * TILE_E + te],1u,memory_order_relaxed);@@ -102,8 +103,8 @@ inline void moe_scatter_buckets_impl(threadgroup_barrier(mem_flags::mem_threadgroup);// Phase 3: deterministic write with 32-lane sequencing, no device atomics- for (uint step = 0; step < simd.group_size; ++step) {- if (simd.lane_idx == step) {+ for (uint step = 0; step < simd.simdgroup_size; ++step) {+ if (simd.simdgroup_index == step) {const uint t = t_start + lid;if (t < t_end) {const uint base = t * K;@@ -116,7 +117,7 @@ inline void moe_scatter_buckets_impl(const uint tile_id = e0 / TILE_E;const uint base_idx_block =(block_id * num_tiles + tile_id) * TILE_E + te;- const uint idx_sg_te = simd.group_idx * TILE_E + te;+ const uint idx_sg_te = simd.threadgroup_index * TILE_E + te;const uint local = atomic_fetch_add_explicit(&sg_counts[idx_sg_te],1u,@@ -201,7 +202,7 @@ PUBLIC KERNEL(MoeScatterBuckets)(constant uint& num_tiles,threadgroup _atomic<uint> sg_counts[NUM_SG * TILE_E],threadgroup uint sg_base[NUM_SG * TILE_E],- const Simd simd,+ const ThreadContext simd,const uint tgpig_x GROUPS(num_blocks),const uint lid THREADS(256)) {@@ -250,7 +251,7 @@ PUBLIC KERNEL(MoeScatterBucketsMap)(device int* tok2row,threadgroup _atomic<uint> sg_counts[NUM_SG * TILE_E],threadgroup uint sg_base[NUM_SG * TILE_E],- const Simd simd,+ const ThreadContext simd,const uint tgpig_x GROUPS(num_blocks),const uint lid THREADS(256)) {diff --git a/crates/uzu/src/backends/metal/kernel/moe/tiles_map.metal b/crates/uzu/src/backends/metal/kernel/moe/tiles_map.metalindex 05a4cd5a4..a6489393a 100644--- a/crates/uzu/src/backends/metal/kernel/moe/tiles_map.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/tiles_map.metal@@ -1,6 +1,6 @@#include <metal_stdlib>#include <metal_atomic>-#include "../definitions.metal"+#include "../common/dsl.h"#define BM 16diff --git a/crates/uzu/src/backends/metal/kernel/moe/tiles_pass_a.metal b/crates/uzu/src/backends/metal/kernel/moe/tiles_pass_a.metalindex f0de182c1..899c6c168 100644--- a/crates/uzu/src/backends/metal/kernel/moe/tiles_pass_a.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/tiles_pass_a.metal@@ -1,5 +1,5 @@#include <metal_stdlib>-#include "../definitions.metal"+#include "../common/dsl.h"// === Helper kernels for indirect dispatch of Pass A ===diff --git a/crates/uzu/src/backends/metal/kernel/pooling/pooling.metal b/crates/uzu/src/backends/metal/kernel/pooling/pooling.metalindex 185ae7d16..ba4c7a47f 100644--- a/crates/uzu/src/backends/metal/kernel/pooling/pooling.metal+++ b/crates/uzu/src/backends/metal/kernel/pooling/pooling.metal@@ -1,5 +1,5 @@#include <metal_stdlib>-#include "../definitions.metal"+#include "../common/dsl.h"// CLS pooling: Extract first token [batch, seq_len, hidden_dim] → [batch,// hidden_dim]diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/mma.h b/crates/uzu/src/backends/metal/kernel/quant_matmul/mma.hindex 56000c057..162e1ba18 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/mma.h+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/mma.h@@ -1,13 +1,10 @@#pragma once+#include "../common/defines.h"#include <metal_simdgroup>#include <metal_simdgroup_matrix>#include <metal_stdlib>-#define UZU_MTL_CONST static constant constexpr-#define UZU_PRAGMA_UNROLL _Pragma("clang loop unroll(full)")-#define SIMD_SIZE 32-using namespace metal;namespace matmul_utils {@@ -47,11 +44,11 @@ struct BaseMMAFrag {template <typename T>struct BaseMMAFrag<T, 8, 8> {- UZU_MTL_CONST int kFragRows = 8;- UZU_MTL_CONST int kFragCols = 8;- UZU_MTL_CONST int kElemsPerFrag = (kFragRows * kFragCols) / 32;- UZU_MTL_CONST int kElemRows = 1;- UZU_MTL_CONST int kElemCols = 2;+ METAL_CONST int kFragRows = 8;+ METAL_CONST int kFragCols = 8;+ METAL_CONST int kElemsPerFrag = (kFragRows * kFragCols) / 32;+ METAL_CONST int kElemRows = 1;+ METAL_CONST int kElemCols = 2;static_assert(kElemRows * kElemCols == kElemsPerFrag,@@ -77,9 +74,9 @@ struct BaseMMAFrag<T, 8, 8> {StrX str_x,StrY str_y) {- UZU_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (short i = 0; i < kElemRows; i++) {- UZU_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (short j = 0; j < kElemCols; j++) {dst[i * kElemCols + j] = static_cast<T>(src[i * str_x + j * str_y]);}@@ -95,9 +92,9 @@ struct BaseMMAFrag<T, 8, 8> {) {using U = pointer_element_t<DstPtrType>;- UZU_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (short i = 0; i < kElemRows; i++) {- UZU_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (short j = 0; j < kElemCols; j++) {dst[i * str_x + j * str_y] = static_cast<U>(src[i * kElemCols + j]);}@@ -124,9 +121,9 @@ struct BaseMMAFrag<T, 8, 8> {) {using U = pointer_element_t<DstPtrType>;- UZU_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (short i = 0; i < kElemRows; i++) {- UZU_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (short j = 0; j < kElemCols; j++) {if ((off_x + i) < lim_x && (off_y + j) < lim_y) {dst[(off_x + i) * str_x + (off_y + j) * str_y] =@@ -174,18 +171,18 @@ template <struct MMATile {using MMAFrag_t = MMAFrag_;using elem_type = T;- UZU_MTL_CONST int kFragRows = MMAFrag_t::kFragRows;- UZU_MTL_CONST int kFragCols = MMAFrag_t::kFragCols;- UZU_MTL_CONST int kElemsPerFrag = MMAFrag_t::kElemsPerFrag;+ METAL_CONST int kFragRows = MMAFrag_t::kFragRows;+ METAL_CONST int kFragCols = MMAFrag_t::kFragCols;+ METAL_CONST int kElemsPerFrag = MMAFrag_t::kElemsPerFrag;- UZU_MTL_CONST int kTileRows = kTileRows_;- UZU_MTL_CONST int kTileCols = kTileCols_;+ METAL_CONST int kTileRows = kTileRows_;+ METAL_CONST int kTileCols = kTileCols_;- UZU_MTL_CONST int kRows = kTileRows * kFragRows;- UZU_MTL_CONST int kCols = kTileCols * kFragCols;+ METAL_CONST int kRows = kTileRows * kFragRows;+ METAL_CONST int kCols = kTileCols * kFragCols;- UZU_MTL_CONST int kNumFrags = kTileRows * kTileCols;- UZU_MTL_CONST int kElemsPerTile = kNumFrags * kElemsPerFrag;+ METAL_CONST int kNumFrags = kTileRows * kTileCols;+ METAL_CONST int kElemsPerTile = kNumFrags * kElemsPerFrag;typedef typename MMAFrag_t::mat_type mat_type;typedef typename MMAFrag_t::frag_type frag_type;@@ -195,7 +192,7 @@ struct MMATile {METAL_FUNC MMATile() thread {}METAL_FUNC constexpr void clear() {- UZU_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (short i = 0; i < kNumFrags; ++i) {val_frags[i] = frag_type(0);}@@ -222,9 +219,9 @@ struct MMATile {template <typename U, int w_x, int w_y, int str_x, int str_y>METAL_FUNC void load(const threadgroup U* src) {- UZU_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (short i = 0; i < kTileRows; ++i) {- UZU_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (short j = 0; j < kTileCols; ++j) {MMAFrag_t::load(frag_at(i, j),@@ -241,9 +238,9 @@ struct MMATile {template <typename U, int w_x, int w_y>METAL_FUNC void store(device U* dst, const int ld) const {- UZU_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (short i = 0; i < kTileRows; ++i) {- UZU_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (short j = 0; j < kTileCols; ++j) {MMAFrag_t::store(frag_at(i, j),@@ -258,9 +255,9 @@ struct MMATile {// Store results into threadgroup memory (same layout as device variant)template <typename U, int w_x, int w_y>METAL_FUNC void store_tg(threadgroup U* dst, const int ld) const {- UZU_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (short i = 0; i < kTileRows; ++i) {- UZU_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (short j = 0; j < kTileCols; ++j) {MMAFrag_t::store(frag_at(i, j),@@ -278,9 +275,9 @@ struct MMATile {const int ld,const short2 dst_tile_dims) const {- UZU_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (int i = 0; i < kTileRows; ++i) {- UZU_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (int j = 0; j < kTileCols; ++j) {MMAFrag_t::store_safe(frag_at(i, j),@@ -304,12 +301,12 @@ METAL_FUNC void tile_matmad(thread MMATile<U, K, N>& B,thread MMATile<T, M, N>& C) {- UZU_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (short m = 0; m < M; ++m) {- UZU_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (short n = 0; n < N; ++n) {short n_serp = (m % 2) ? (N - 1 - n) : n;- UZU_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (short k = 0; k < K; ++k) {MMATile<T, M, N>::MMAFrag_t::mma(D.frag_at(m, n_serp),@@ -338,30 +335,30 @@ template <typename Epilogue = TransformNone<U, AccumType>>struct BlockMMA {// MMAFrag size- UZU_MTL_CONST short kFragSize = 8;+ METAL_CONST short kFragSize = 8;using MMAFrag_acc_t = BaseMMAFrag<AccumType, kFragSize, kFragSize>;// Warp tile simdgroup matrix strides along M- UZU_MTL_CONST short TM_stride = kFragSize * WM;+ METAL_CONST short TM_stride = kFragSize * WM;// Warp tile simdgroup matrix strides along M- UZU_MTL_CONST short TN_stride = kFragSize * WN;+ METAL_CONST short TN_stride = kFragSize * WN;// Warp tile size along M- UZU_MTL_CONST short TM = BM / (kFragSize * WM);+ METAL_CONST short TM = BM / (kFragSize * WM);// Warp tile size along N- UZU_MTL_CONST short TN = BN / (kFragSize * WN);+ METAL_CONST short TN = BN / (kFragSize * WN);// Threadgroup A strides- UZU_MTL_CONST short A_str_m = transpose_a ? 1 : lda_tgp; // M- UZU_MTL_CONST short A_str_k = transpose_a ? lda_tgp : 1; // K+ METAL_CONST short A_str_m = transpose_a ? 1 : lda_tgp; // M+ METAL_CONST short A_str_k = transpose_a ? lda_tgp : 1; // K// Threadgroup B strides- UZU_MTL_CONST short B_str_k = transpose_b ? 1 : ldb_tgp; // K- UZU_MTL_CONST short B_str_n = transpose_b ? ldb_tgp : 1; // N+ METAL_CONST short B_str_k = transpose_b ? 1 : ldb_tgp; // K+ METAL_CONST short B_str_n = transpose_b ? ldb_tgp : 1; // N// Threadgroup strides along K- UZU_MTL_CONST short tile_stride_a = kFragSize * A_str_k;- UZU_MTL_CONST short tile_stride_b = kFragSize * B_str_k;+ METAL_CONST short tile_stride_a = kFragSize * A_str_k;+ METAL_CONST short tile_stride_b = kFragSize * B_str_k;// Simdgroup matricesMMATile<AccumType, TM, 1, MMAFrag_acc_t> Atile;@@ -403,7 +400,7 @@ struct BlockMMA {Bs += Bs_offset;// Iterate over BK in blocks of kFragSize- UZU_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (short kk = 0; kk < BK; kk += kFragSize) {simdgroup_barrier(mem_flags::mem_none);@@ -426,7 +423,7 @@ struct BlockMMA {/* Store results from simdgroup_matrix results into device memory */METAL_FUNC void store_result(device U* D, const int ldd) {// Apply epilogue- UZU_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (short i = 0; i < decltype(Ctile)::kElemsPerTile; i++) {Ctile.elems()[i] = Epilogue::apply(Ctile.elems()[i]);}@@ -440,7 +437,7 @@ struct BlockMMA {/* Store results into threadgroup memory */METAL_FUNC void store_result_tg(threadgroup U* D, const int ldd) {// Apply epilogue- UZU_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (short i = 0; i < decltype(Ctile)::kElemsPerTile; i++) {Ctile.elems()[i] = Epilogue::apply(Ctile.elems()[i]);}@@ -457,7 +454,7 @@ struct BlockMMA {short2 dst_tile_dims) {// Apply epilogue- UZU_PRAGMA_UNROLL+ METAL_PRAGMA_UNROLLfor (short i = 0; i < decltype(Ctile)::kElemsPerTile; i++) {Ctile.elems()[i] = Epilogue::apply(Ctile.elems()[i]);}@@ -483,7 +480,7 @@ template <int align_M = 1,int align_K = 4>struct BlockLoader {- UZU_MTL_CONST short n_reads = (BM * BK < tgp_size) ? 1 : (BM * BK) / tgp_size;+ METAL_CONST short n_reads = (BM * BK < tgp_size) ? 1 : (BM * BK) / tgp_size;const int src_ld;const short thread_idx;diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm.metalindex f020cc70b..f16fba9bb 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm.metal@@ -1,5 +1,5 @@#include <metal_stdlib>-#include "../definitions.metal"+#include "../common/dsl.h"#include "quant_matmul.metal"template <typename T, int GROUP_SIZE, int BITS>diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metalindex 7f26b135c..c324e936f 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed.metal@@ -1,5 +1,5 @@#include <metal_stdlib>-#include "../definitions.metal"+#include "../common/dsl.h"#include "quant_matmul.metal"template <typename T, int GROUP_SIZE, int BITS>diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metalindex 012359d6b..61d6fc85f 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmm_transposed_64x64.metal@@ -1,5 +1,5 @@#include <metal_stdlib>-#include "../definitions.metal"+#include "../common/dsl.h"#include "quant_matmul.metal"template <typename T, int GROUP_SIZE, int BITS>diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metalindex ea76f8f7c..6bcfc03b9 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv.metal@@ -1,5 +1,5 @@#include <metal_stdlib>-#include "../definitions.metal"+#include "../common/dsl.h"#include "quant_matmul.metal"template <typename T, int GROUP_SIZE, int BITS>diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metalindex 756cb624a..4b3120dee 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qmv_fast.metal@@ -1,5 +1,5 @@#include <metal_stdlib>-#include "../definitions.metal"+#include "../common/dsl.h"#include "quant_matmul.metal"template <typename T, int GROUP_SIZE, int BITS>diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metalindex ed923674d..59657ecff 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal@@ -287,12 +287,12 @@ struct QuantizedBlockLoaderMlx {);static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");- UZU_MTL_CONST short pack_factor = get_pack_factor<bits, 8>();- UZU_MTL_CONST short bytes_per_pack = get_bytes_per_pack<bits>();- UZU_MTL_CONST short BCOLS_PACKED = BCOLS / pack_factor;- UZU_MTL_CONST short n_reads =+ METAL_CONST short pack_factor = get_pack_factor<bits, 8>();+ METAL_CONST short bytes_per_pack = get_bytes_per_pack<bits>();+ METAL_CONST short BCOLS_PACKED = BCOLS / pack_factor;+ METAL_CONST short n_reads =(BCOLS_PACKED * BROWS < tgp_size) ? 1 : (BCOLS_PACKED * BROWS) / tgp_size;- UZU_MTL_CONST short group_steps = group_size / BCOLS;+ METAL_CONST short group_steps = group_size / BCOLS;const int src_ld;const int tile_stride;@@ -422,12 +422,12 @@ struct QuantizedBlockLoaderZp {);static_assert(bits == 4 || bits == 8, "Only int4 and int8 supported");- UZU_MTL_CONST short pack_factor = get_pack_factor<bits, 8>();- UZU_MTL_CONST short bytes_per_pack = get_bytes_per_pack<bits>();- UZU_MTL_CONST short BCOLS_PACKED = BCOLS / pack_factor;- UZU_MTL_CONST short n_reads =+ METAL_CONST short pack_factor = get_pack_factor<bits, 8>();+ METAL_CONST short bytes_per_pack = get_bytes_per_pack<bits>();+ METAL_CONST short BCOLS_PACKED = BCOLS / pack_factor;+ METAL_CONST short n_reads =(BCOLS_PACKED * BROWS < tgp_size) ? 1 : (BCOLS_PACKED * BROWS) / tgp_size;- UZU_MTL_CONST short group_steps = group_size / BCOLS;+ METAL_CONST short group_steps = group_size / BCOLS;const int src_ld;const int groups_per_row;@@ -744,8 +744,8 @@ void qmm_impl(uint simd_gid [[simdgroup_index_in_threadgroup]],uint simd_lid [[thread_index_in_simdgroup]]) {- static_assert(BK >= 32, "BK should be larger than SIMD_SIZE");- static_assert(BK % 32 == 0, "BK should be divisible by SIMD_SIZE");+ static_assert(BK >= 32, "BK should be larger than METAL_SIMD_SIZE");+ static_assert(BK % 32 == 0, "BK should be divisible by METAL_SIMD_SIZE");(void)lid;@@ -953,8 +953,8 @@ void qmm_transposed_impl(uint simd_gid [[simdgroup_index_in_threadgroup]],uint simd_lid [[thread_index_in_simdgroup]]) {- static_assert(BK >= 32, "BK should be larger than SIMD_SIZE");- static_assert(BK % 32 == 0, "BK should be divisible by SIMD_SIZE");+ static_assert(BK >= 32, "BK should be larger than METAL_SIMD_SIZE");+ static_assert(BK % 32 == 0, "BK should be divisible by METAL_SIMD_SIZE");constexpr int WM = 2;constexpr int WN = 2;@@ -1377,7 +1377,7 @@ void qmv_fast_impl(constexpr int pack_factor = get_pack_factor<bits, 32>();constexpr int bytes_per_pack = get_bytes_per_pack<bits, 32>();constexpr int values_per_thread = pack_factor * packs_per_thread;- constexpr int block_size = values_per_thread * SIMD_SIZE;+ constexpr int block_size = values_per_thread * METAL_SIMD_SIZE;constexpr int scale_step_per_thread = group_size / values_per_thread;const device uint8_t* ws = (const device uint8_t*)w;typedef float U;diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/qvm.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/qvm.metalindex e12ac5063..3c389f961 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/qvm.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/qvm.metal@@ -1,5 +1,5 @@#include <metal_stdlib>-#include "../definitions.metal"+#include "../common/dsl.h"#include "quant_matmul.metal"template <typename T, int GROUP_SIZE, int BITS>diff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/qk_norm.metal b/crates/uzu/src/backends/metal/kernel/rms_norm/qk_norm.metalindex 10cb61900..c9d2ae75c 100644--- a/crates/uzu/src/backends/metal/kernel/rms_norm/qk_norm.metal+++ b/crates/uzu/src/backends/metal/kernel/rms_norm/qk_norm.metal@@ -1,9 +1,8 @@#include <metal_stdlib>-#include "../definitions.metal"+#include "../common/defines.h"+#include "../common/dsl.h"using namespace metal;--#define SIMD_SIZE 32#define GRAIN_SIZE 4// QK norm: normalize per-head vectors (small head_dim) efficiently.@@ -31,7 +30,7 @@ PUBLIC KERNEL(QKNorm)(constant bool& full_layer,const uint batch_idx GROUPS(batch_size),const uint head_idx GROUPS(head_count),- const uint lane_id THREADS(SIMD_SIZE),+ const uint lane_id THREADS(METAL_SIMD_SIZE),const bool in_place SPECIALIZE) {if (in_place) {@@ -59,7 +58,7 @@ PUBLIC KERNEL(QKNorm)(// Sum of squares: each lane processes GRAIN_SIZE elements per iteration.for (uint base_i = lane_id * GRAIN_SIZE; base_i < element_count;- base_i += SIMD_SIZE * GRAIN_SIZE) {+ base_i += METAL_SIMD_SIZE * GRAIN_SIZE) {AccumT vals[GRAIN_SIZE];for (uint j = 0; j < GRAIN_SIZE; ++j) {uint i = base_i + j;@@ -80,7 +79,7 @@ PUBLIC KERNEL(QKNorm)(// Normalize + scale.for (uint base_i = lane_id * GRAIN_SIZE; base_i < element_count;- base_i += SIMD_SIZE * GRAIN_SIZE) {+ base_i += METAL_SIMD_SIZE * GRAIN_SIZE) {AccumT vals[GRAIN_SIZE];for (uint j = 0; j < GRAIN_SIZE; ++j) {uint i = base_i + j;diff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metalindex f846ad0d2..eb0d04ac4 100644--- a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal+++ b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal@@ -1,10 +1,12 @@#include <metal_stdlib>-#include "../definitions.metal"+#include "../common/defines.h"+#include "../common/dsl.h"+#include "../common/thread_context.h"+#include "../common/threadgroup_reduce.h"using namespace metal;#define BLOCK_SIZE 1024-#define SIMD_SIZE 32#define GRAIN_SIZE 4template <typename InputT, typename ScaleT, typename OutputT, typename AccumT>@@ -22,8 +24,8 @@ PUBLIC KERNEL(RMSNorm)(constant float& scale_offset,constant bool& full_layer,const bool in_place SPECIALIZE,- threadgroup AccumT shared_sum[SIMD_SIZE],- const Simd simd,+ threadgroup AccumT shared_sum[METAL_SIMD_SIZE],+ const ThreadContext simd,const uint batch_idx GROUPS(batch_size),const uint thread_in_row THREADS(1024)) {diff --git a/crates/uzu/src/backends/metal/kernel/rope/rope.metal b/crates/uzu/src/backends/metal/kernel/rope/rope.metalindex bb4cc6217..5f6274e14 100644--- a/crates/uzu/src/backends/metal/kernel/rope/rope.metal+++ b/crates/uzu/src/backends/metal/kernel/rope/rope.metal@@ -1,5 +1,6 @@#include <metal_stdlib>-#include "../definitions.metal"+#include "../common/dsl.h"+#include "../common/tensor_view.h"template <typename T>inline T applyRopeTransform(diff --git a/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal b/crates/uzu/src/backends/metal/kernel/sampling/argmax.metalindex 89d317024..86939cf03 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal+++ b/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal@@ -1,6 +1,7 @@#include <metal_stdlib>#include <metal_atomic>-#include "../definitions.metal"+#include "../common/dsl.h"+#include "../common/thread_context.h"#include "argmax.h"@@ -26,7 +27,7 @@ static ArgmaxPair threadgroup_cooperative_argmax(ArgmaxPair value,threadgroup ArgmaxPair* shared,const ushort lid,- const thread Simd& simd+ const thread ThreadContext& simd) {// Reduce within simdgroup using manual shuffle operationsArgmaxPair local_result = value;@@ -39,8 +40,8 @@ static ArgmaxPair threadgroup_cooperative_argmax(}// First thread in each simdgroup writes to shared memory- if (simd.lane_idx == 0) {- shared[simd.group_idx] = local_result;+ if (simd.simdgroup_index == 0) {+ shared[simd.threadgroup_index] = local_result;}// Synchronize across the threadgroup@@ -202,7 +203,7 @@ PUBLIC KERNEL(ArgmaxMain)(constant uint& batch_size,constant uint& vocab_size,threadgroup ArgmaxPair shared[BLOCK_SIZE],- const Simd simd,+ const ThreadContext simd,uint batch_idx GROUPS(batch_size),uint vocab_group_idx GROUPS(vocab_size.div_ceil(BLOCK_SIZE * GRAIN_SIZE)),ushort local_id THREADS(BLOCK_SIZE)@@ -246,7 +247,7 @@ PUBLIC KERNEL(ArgmaxFinal)(constant uint& batch_size,constant uint& vocab_size,threadgroup ArgmaxPair shared[BLOCK_SIZE],- const Simd simd,+ const ThreadContext simd,uint batch_idx GROUPS(batch_size),ushort local_id THREADS(BLOCK_SIZE)) {diff --git a/crates/uzu/src/backends/metal/kernel/sampling/bitmask.metal b/crates/uzu/src/backends/metal/kernel/sampling/bitmask.metalindex 010fa690b..3dd8b2011 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/bitmask.metal+++ b/crates/uzu/src/backends/metal/kernel/sampling/bitmask.metal@@ -1,5 +1,5 @@#include <metal_stdlib>-#include "../definitions.metal"+#include "../common/dsl.h"#define BLOCK_SIZE 1024#define GRAIN_SIZE 64diff --git a/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metal b/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metalindex 349e49cbe..6fb5ce481 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metal+++ b/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metal@@ -1,5 +1,5 @@#include <metal_stdlib>-#include "../definitions.metal"+#include "../common/dsl.h"#include "../rng.metal"#define BLOCK_SIZE 1024diff --git a/crates/uzu/src/backends/metal/kernel/sampling/min_p.metal b/crates/uzu/src/backends/metal/kernel/sampling/min_p.metalindex 327195b5a..121a1cca3 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/min_p.metal+++ b/crates/uzu/src/backends/metal/kernel/sampling/min_p.metal@@ -1,5 +1,7 @@#include <metal_stdlib>-#include "../definitions.metal"+#include "../common/dsl.h"+#include "../common/thread_context.h"+#include "../common/threadgroup_reduce.h"#define BLOCK_SIZE 1024#define BLOCK_SIZE_IN_SIMDS (BLOCK_SIZE / 32)@@ -13,7 +15,7 @@ PUBLIC KERNEL(MinP) (constant uint& batch_size,constant uint& vocab_size,constant float& min_p,- const Simd simd,+ const ThreadContext simd,uint batch_idx GROUPS(batch_size),uint thread_idx THREADS(BLOCK_SIZE),const bool in_place SPECIALIZEdiff --git a/crates/uzu/src/backends/metal/kernel/sampling/temperature.metal b/crates/uzu/src/backends/metal/kernel/sampling/temperature.metalindex cba9ff056..73c7bd3e4 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/temperature.metal+++ b/crates/uzu/src/backends/metal/kernel/sampling/temperature.metal@@ -1,5 +1,5 @@#include <metal_stdlib>-#include "../definitions.metal"+#include "../common/dsl.h"#define BLOCK_SIZE 1024#define GRAIN_SIZE 64diff --git a/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal b/crates/uzu/src/backends/metal/kernel/sampling/top_k.metalindex 5113ee31c..d921385b2 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal+++ b/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal@@ -1,5 +1,7 @@#include <metal_stdlib>-#include "../definitions.metal"+#include "../common/dsl.h"+#include "../common/thread_context.h"+#include "../common/threadgroup_reduce.h"#define BLOCK_SIZE 1024#define BLOCK_SIZE_IN_SIMDS (BLOCK_SIZE / 32)@@ -14,7 +16,7 @@ PUBLIC KERNEL(TopK)(constant uint& batch_size,constant uint& vocab_size,constant uint& top_k,- const Simd simd,+ const ThreadContext simd,uint batch_idx GROUPS(batch_size),uint thread_idx THREADS(BLOCK_SIZE),const bool in_place SPECIALIZEdiff --git a/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal b/crates/uzu/src/backends/metal/kernel/sampling/top_p.metalindex f94ddb033..cd4f01774 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal+++ b/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal@@ -1,5 +1,7 @@#include <metal_stdlib>-#include "../definitions.metal"+#include "../common/dsl.h"+#include "../common/thread_context.h"+#include "../common/threadgroup_reduce.h"#define BLOCK_SIZE 1024#define BLOCK_SIZE_IN_SIMDS (BLOCK_SIZE / 32)@@ -15,7 +17,7 @@ PUBLIC KERNEL(TopP) (constant uint& batch_size,constant uint& vocab_size,constant float& top_p,- const Simd simd,+ const ThreadContext simd,uint batch_idx GROUPS(batch_size),uint thread_idx THREADS(BLOCK_SIZE),const bool in_place SPECIALIZEdiff --git a/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal b/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metalindex d4088a527..7b5d757a6 100644--- a/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal+++ b/crates/uzu/src/backends/metal/kernel/short_conv/short_conv.metal@@ -1,5 +1,5 @@#include <metal_stdlib>-#include "../definitions.metal"+#include "../common/dsl.h"template <typename T>VARIANTS(T, float, half, bfloat)diff --git a/crates/uzu/src/backends/metal/kernel/sigmoid/sigmoid.metal b/crates/uzu/src/backends/metal/kernel/sigmoid/sigmoid.metalindex 69d5c400a..4ddc4c8b7 100644--- a/crates/uzu/src/backends/metal/kernel/sigmoid/sigmoid.metal+++ b/crates/uzu/src/backends/metal/kernel/sigmoid/sigmoid.metal@@ -1,5 +1,5 @@#include <metal_stdlib>-#include "../definitions.metal"+#include "../common/dsl.h"template <typename T>VARIANTS(T, float, half, bfloat)diff --git a/crates/uzu/src/backends/metal/kernel/ssm/conv1d.metal b/crates/uzu/src/backends/metal/kernel/ssm/conv1d.metalindex 1ff690c80..0c48c0d01 100644--- a/crates/uzu/src/backends/metal/kernel/ssm/conv1d.metal+++ b/crates/uzu/src/backends/metal/kernel/ssm/conv1d.metal@@ -1,5 +1,5 @@#include <metal_stdlib>-#include "../definitions.metal"+#include "../common/dsl.h"#include "ssm_common.h"template <typename T>diff --git a/crates/uzu/src/backends/metal/kernel/ssm/split_inproj.metal b/crates/uzu/src/backends/metal/kernel/ssm/split_inproj.metalindex b6c6709ad..fb1b85ebd 100644--- a/crates/uzu/src/backends/metal/kernel/ssm/split_inproj.metal+++ b/crates/uzu/src/backends/metal/kernel/ssm/split_inproj.metal@@ -1,5 +1,5 @@#include <metal_stdlib>-#include "../definitions.metal"+#include "../common/dsl.h"template <typename T>VARIANTS(T, float, half, bfloat)diff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal b/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metalindex db7910523..f84606b78 100644--- a/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal+++ b/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal@@ -1,6 +1,7 @@#include <metal_stdlib>#include <metal_simdgroup>-#include "../definitions.metal"+#include "../common/dsl.h"+#include "../common/thread_context.h"#include "ssm_common.h"using namespace metal;@@ -27,7 +28,7 @@ PUBLIC KERNEL(SSDPrefill64)(constant const uint* state_strides,constant const uint& num_heads,constant const uint& head_dim,- const Simd simd,+ const ThreadContext simd,const uint pair_idx GROUPS(num_heads * head_dim),const uint lane_idx THREADS(32)) {@@ -55,7 +56,7 @@ PUBLIC KERNEL(SSDPrefill64)(const float d_scalar = float(d[h_idx]);const uint idx0 = lane_idx;- const uint idx1 = lane_idx + simd.group_size;+ const uint idx1 = lane_idx + simd.simdgroup_size;float state0 = 0.0f;float state1 = 0.0f;@@ -125,7 +126,7 @@ PUBLIC KERNEL(SSDPrefill)(constant const uint* state_strides,constant const uint& num_heads,constant const uint& head_dim,- const Simd simd,+ const ThreadContext simd,const uint pair_idx GROUPS(num_heads * head_dim),const uint lane_idx THREADS(32)) {@@ -153,9 +154,9 @@ PUBLIC KERNEL(SSDPrefill)(const uint cb_group_base = group_idx * cb_group_stride;const float d_scalar = float(d[h_idx]);- const int max_chunks = SSM_PREFILL_MAX_STATE / simd.group_size;+ const int max_chunks = SSM_PREFILL_MAX_STATE / simd.simdgroup_size;const int chunk_count =- (state_dim + int(simd.group_size) - 1) / int(simd.group_size);+ (state_dim + int(simd.simdgroup_size) - 1) / int(simd.simdgroup_size);if (state_dim <= 0 || chunk_count > max_chunks) {return;}@@ -164,7 +165,7 @@ PUBLIC KERNEL(SSDPrefill)(#pragma unrollfor (int chunk = 0; chunk < chunk_count; ++chunk) {- const int idx = chunk * int(simd.group_size) + int(lane_idx);+ const int idx = chunk * int(simd.simdgroup_size) + int(lane_idx);const uint state_idx = state_base + idx * state_inner_stride;lane_states[chunk] = float(state[state_idx]);}@@ -182,7 +183,7 @@ PUBLIC KERNEL(SSDPrefill)(float contrib_sum = 0.0f;#pragma unrollfor (int chunk = 0; chunk < chunk_count; ++chunk) {- const int idx = chunk * int(simd.group_size) + int(lane_idx);+ const int idx = chunk * int(simd.simdgroup_size) + int(lane_idx);const uint cb_idx =cb_group_base + uint(idx) * cb_state_stride + token * cb_token_stride;const float new_state =@@ -199,7 +200,7 @@ PUBLIC KERNEL(SSDPrefill)(#pragma unrollfor (int chunk = 0; chunk < chunk_count; ++chunk) {- const int idx = chunk * int(simd.group_size) + int(lane_idx);+ const int idx = chunk * int(simd.simdgroup_size) + int(lane_idx);const uint state_idx = state_base + uint(idx) * state_inner_stride;state[state_idx] = static_cast<T>(lane_states[chunk]);}diff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal b/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metalindex 33857f7fb..8ddf31f79 100644--- a/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal+++ b/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal@@ -1,5 +1,5 @@#include <metal_stdlib>-#include "../definitions.metal"+#include "../common/dsl.h"#include "ssm_common.h"using namespace metal;diff --git a/crates/uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal b/crates/uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metalindex 48e6dcb6f..184350e6f 100644--- a/crates/uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal+++ b/crates/uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal@@ -1,5 +1,5 @@#include <metal_stdlib>-#include "../definitions.metal"+#include "../common/dsl.h"template <typename T>VARIANTS(T, float, half, bfloat)diff --git a/crates/uzu/src/backends/metal/kernel/tensor_add_swap/tensor_add_swap.metal b/crates/uzu/src/backends/metal/kernel/tensor_add_swap/tensor_add_swap.metalindex 3524e2e77..82a033113 100644--- a/crates/uzu/src/backends/metal/kernel/tensor_add_swap/tensor_add_swap.metal+++ b/crates/uzu/src/backends/metal/kernel/tensor_add_swap/tensor_add_swap.metal@@ -1,5 +1,5 @@#include <metal_stdlib>-#include "../definitions.metal"+#include "../common/dsl.h"template <typename T>VARIANTS(T, float, half, bfloat)diff --git a/crates/uzu/src/backends/metal/kernel/tensor_copy/tensor_copy.metal b/crates/uzu/src/backends/metal/kernel/tensor_copy/tensor_copy.metalindex d619ed30b..052a48efe 100644--- a/crates/uzu/src/backends/metal/kernel/tensor_copy/tensor_copy.metal+++ b/crates/uzu/src/backends/metal/kernel/tensor_copy/tensor_copy.metal@@ -1,5 +1,5 @@#include <metal_stdlib>-#include "../definitions.metal"+#include "../common/dsl.h"template <typename T>VARIANTS(T, float, half, bfloat)diff --git a/crates/uzu/src/backends/metal/kernel/token_copy/token_copy.metal b/crates/uzu/src/backends/metal/kernel/token_copy/token_copy.metalindex f4ebbf08f..e5cc942f6 100644--- a/crates/uzu/src/backends/metal/kernel/token_copy/token_copy.metal+++ b/crates/uzu/src/backends/metal/kernel/token_copy/token_copy.metal@@ -1,5 +1,5 @@#include <metal_stdlib>-#include "../definitions.metal"+#include "../common/dsl.h"/// Copies a single sampled token (u32) to the token_ids buffer (u64)./// Used in async pipeline to pass sampled token to next forward pass.diff --git a/crates/uzu/tests/kernel/matmul/gemv_test.rs b/crates/uzu/tests/kernel/matmul/gemv_test.rsindex 9660f02b9..2381dfe8f 100644--- a/crates/uzu/tests/kernel/matmul/gemv_test.rs+++ b/crates/uzu/tests/kernel/matmul/gemv_test.rs@@ -78,9 +78,9 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {batch: m,input_dim: k,output_dim: n,- lda: k,- ldb: k,- ldd: n,+ leading_dimension_a: k,+ leading_dimension_b: k,+ leading_dimension_d: n,transpose_b: true,};diff --git a/crates/uzu/tests/kernel/matmul/steel_gemm_test.rs b/crates/uzu/tests/kernel/matmul/steel_gemm_test.rsindex 8a539aebe..70c4d305f 100644--- a/crates/uzu/tests/kernel/matmul/steel_gemm_test.rs+++ b/crates/uzu/tests/kernel/matmul/steel_gemm_test.rs@@ -12,7 +12,7 @@ use uzu::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,Context,- gpu_types::GEMMParams,+ gpu_types::GemmParams,kernel::matmul::{GridSize, MatmulArguments,gemm::{GemmDispatchDescriptor, GemmKernel, GemmSpecialization},@@ -81,28 +81,28 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {..base});- let tiles_n = (n + config.block_cols - 1) / config.block_cols;- let tiles_m = (m + config.block_rows - 1) / config.block_rows;+ let threadgroups_per_row = (n + config.block_cols - 1) / config.block_cols;+ let threadgroups_per_column = (m + config.block_rows - 1) / config.block_rows;- let params = GEMMParams {+ let params = GemmParams {M: m,N: n,K: k,- lda: k,- ldb: k,- ldd: n,- tiles_n,- tiles_m,+ leading_dimension_a: k,+ leading_dimension_b: k,+ leading_dimension_d: n,+ threadgroups_per_row,+ threadgroups_per_column,swizzle_log: 0,- gemm_k_iterations_aligned: k / config.block_depth,+ aligned_inner_iterations: k / config.block_depth,};let descriptor = GemmDispatchDescriptor {specialization: config,params,threadgroups: GridSize {- x: tiles_n as usize,- y: tiles_m as usize,+ x: threadgroups_per_row as usize,+ y: threadgroups_per_column as usize,z: 1,},};@@ -126,9 +126,9 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {batch: m,input_dim: k,output_dim: n,- lda: k,- ldb: k,- ldd: n,+ leading_dimension_a: k,+ leading_dimension_b: k,+ leading_dimension_d: n,transpose_b: true,};kernel.encode(&context, &mut arguments, &descriptor, &mut command_buffer).expect("Failed to encode");diff --git a/crates/uzu/tests/matmul_correctness_test.rs b/crates/uzu/tests/matmul_correctness_test.rsindex 3d687081c..802873345 100644--- a/crates/uzu/tests/matmul_correctness_test.rs+++ b/crates/uzu/tests/matmul_correctness_test.rs@@ -38,7 +38,7 @@ fn run_metal_matmul(.new_buffer(m * n * core::mem::size_of::<bf16>(), MTLResourceOptions::STORAGE_MODE_SHARED).expect("Failed to create buffer");- let ldb = if transpose_b {+ let leading_dim_b = if transpose_b {k} else {n@@ -56,9 +56,9 @@ fn run_metal_matmul(batch: m as i32,input_dim: k as i32,output_dim: n as i32,- lda: k as i32,- ldb: ldb as i32,- ldd: n as i32,+ leading_dimension_a: k as i32,+ leading_dimension_b: leading_dim_b as i32,+ leading_dimension_d: n as i32,transpose_b,};let descriptor =diff --git a/crates/uzu/tests/matmul_perf_test/bench.rs b/crates/uzu/tests/matmul_perf_test/bench.rsindex 783e3f324..460f3ed83 100644--- a/crates/uzu/tests/matmul_perf_test/bench.rs+++ b/crates/uzu/tests/matmul_perf_test/bench.rs@@ -46,9 +46,9 @@ pub fn make_arguments<'a>(batch: shape.batch as i32,input_dim: shape.input_dim as i32,output_dim: shape.output_dim as i32,- lda: shape.input_dim as i32,- ldb: shape.input_dim as i32,- ldd: shape.output_dim as i32,+ leading_dimension_a: shape.input_dim as i32,+ leading_dimension_b: shape.input_dim as i32,+ leading_dimension_d: shape.output_dim as i32,transpose_b: true,}}