A high-performance inference engine for AI models
introduce gemm
eugenea11f4d32eb07
diff --git a/Cargo.lock b/Cargo.lockindex a295ab640..c3df3983a 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -323,6 +323,26 @@ version = "0.1.4"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "383d29d513d8764dcdc42ea295d979eb99c3c9f00607b3692cf68a431f7dca72"+[[package]]+name = "bindgen"+version = "0.72.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895"+dependencies = [+ "bitflags 2.9.4",+ "cexpr",+ "clang-sys",+ "itertools 0.13.0",+ "log",+ "prettyplease",+ "proc-macro2",+ "quote",+ "regex",+ "rustc-hash",+ "shlex",+ "syn 2.0.102",+]+[[package]]name = "bitflags"version = "1.3.2"@@ -4071,6 +4091,7 @@ dependencies = [name = "uzu"version = "0.1.7"dependencies = [+ "bindgen","bitflags 2.9.4","block","bytemuck",diff --git a/crates/uzu/Cargo.toml b/crates/uzu/Cargo.tomlindex d10698ffc..211820eb1 100644--- a/crates/uzu/Cargo.toml+++ b/crates/uzu/Cargo.toml@@ -30,6 +30,9 @@ sysinfo = "0.37.2"itertools = "0.14.0"schemars = "1.0.5"+[build-dependencies]+bindgen = "0.72.1"+[dev-dependencies]is_close = "0.1.3"serde_json5 = "0.2.1"diff --git a/crates/uzu/build.rs b/crates/uzu/build.rsindex f3c06e90f..5f22b39ff 100644--- a/crates/uzu/build.rs+++ b/crates/uzu/build.rs@@ -8,6 +8,9 @@ use std::{};fn main() {+ // Generate Rust bindings from shared Metal/C headers+ generate_metal_bindings();+if cfg!(feature = "metal-shaders") {println!("cargo:rerun-if-env-changed=MY_API_LEVEL");compile_metal_shaders();@@ -16,6 +19,39 @@ fn main() {}}+/// Generate Rust bindings from shared C headers using bindgen.+/// These headers are also used by Metal shaders, ensuring type consistency.+fn generate_metal_bindings() {+ let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());+ let matmul_dir = manifest_dir.join("src/backends/metal/kernel/matmul");+ let types_header = matmul_dir.join("shared_types.h");++ // Only regenerate if header changed+ println!("cargo:rerun-if-changed={}", types_header.display());++ let bindings = bindgen::Builder::default()+ .header(types_header.to_string_lossy())+ // Generate for C mode (no Metal-specific stuff)+ .clang_arg("-x")+ .clang_arg("c")+ // Derive common traits+ .derive_default(true)+ .derive_copy(true)+ // Only generate types we care about+ .allowlist_type("GEMMParams")+ .allowlist_type("GEMMAddMMParams")+ // Use core types+ .use_core()+ .generate()+ .expect("Failed to generate bindings from shared_types.h");++ // Write bindings to the source directory so it can be checked in+ let bindings_path = matmul_dir.join("shared_types.rs");+ bindings+ .write_to_file(&bindings_path)+ .expect("Failed to write shared_types.rs");+}+fn compile_metal_shaders() {// Determine target platform (only proceed on Apple platforms)let target = env::var("TARGET").unwrap_or_default();diff --git a/crates/uzu/src/backends/metal/kernel/matmul/arguments.rs b/crates/uzu/src/backends/metal/kernel/matmul/arguments.rsnew file mode 100644index 000000000..65b8ba5e3--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/arguments.rs@@ -0,0 +1,19 @@+use metal::Buffer as MTLBuffer;++#[derive(Debug, Clone, Copy)]+pub struct MatmulArguments<'a> {+ pub a: &'a MTLBuffer,+ pub b: &'a MTLBuffer,+ pub d: &'a MTLBuffer,+ /// M dimension - batch/number of tokens (rows of A, rows of D)+ pub batch: i32,+ /// K dimension - input_dim/reduction dimension (cols of A, rows of B)+ pub input_dim: i32,+ /// N dimension - output_dim (cols of B, cols of D)+ pub output_dim: i32,+ pub lda: i32,+ pub ldb: i32,+ pub ldd: i32,+ /// Number of batched matrix multiplications (z-dimension)+ pub batch_count: i32,+}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/defines.h b/crates/uzu/src/backends/metal/kernel/matmul/defines.hnew file mode 100644index 000000000..07ca0a072--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/defines.h@@ -0,0 +1,80 @@+#pragma once++#include <metal_simdgroup>+#include <metal_simdgroup_matrix>+#include <metal_stdlib>++#define UZU_MTL_CONST static constant constexpr+#define UZU_PRAGMA_UNROLL _Pragma("clang loop unroll(full)")+#define SIMD_SIZE 32++using namespace metal;++namespace uzu {+namespace matmul {++// Pointer element type extraction+template <typename T>+struct pointer_element_t_impl {+ using type = T;+};+template <typename T>+struct pointer_element_t_impl<device T*> {+ using type = T;+};+template <typename T>+struct pointer_element_t_impl<const device T*> {+ using type = T;+};+template <typename T>+struct pointer_element_t_impl<threadgroup T*> {+ using type = T;+};+template <typename T>+using pointer_element_t = typename pointer_element_t_impl<T>::type;++// Accumulator type helper+template <typename T>+struct AccumHelper {+ typedef float accum_type;+};++// Transform operations for epilogue+// Note: apply(x) is static for default store_result usage+// apply(x, c) is non-static for epilogue operations with C matrix+template <typename OutT, typename InT>+struct TransformNone {+ TransformNone(float = 1.0f, float = 0.0f) {}++ static METAL_FUNC OutT apply(InT x) { return static_cast<OutT>(x); }+ METAL_FUNC OutT apply(InT x, OutT) const { return static_cast<OutT>(x); }+};++template <typename OutT, typename InT>+struct TransformAdd {+ TransformAdd(float = 1.0f, float = 1.0f) {}++ static METAL_FUNC OutT apply(InT x) { return static_cast<OutT>(x); }+ METAL_FUNC OutT apply(InT x, OutT c) const {+ return static_cast<OutT>(x) + c;+ }+};++template <typename OutT, typename InT>+struct TransformAxpby {+ const float alpha;+ const float beta;++ TransformAxpby(float alpha_, float beta_) : alpha(alpha_), beta(beta_) {}++ static METAL_FUNC OutT apply(InT x) { return static_cast<OutT>(x); }++ METAL_FUNC OutT apply(InT x, OutT c) const {+ return static_cast<OutT>(+ x * static_cast<InT>(alpha) + (static_cast<OutT>(beta) * c)+ );+ }+};++} // namespace matmul+} // namespace uzudiff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm.h b/crates/uzu/src/backends/metal/kernel/matmul/gemm.hnew file mode 100644index 000000000..4c825a159--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm.h@@ -0,0 +1,300 @@+#pragma once++#include "loader.h"+#include "mma.h"+#include "shared_types.h"++using namespace metal;++namespace uzu {+namespace matmul {++///////////////////////////////////////////////////////////////////////////////+// Loop Alignment Helper+///////////////////////////////////////////////////////////////////////////////++template <bool M_aligned, bool N_aligned, bool K_aligned>+struct LoopAlignment {};++///////////////////////////////////////////////////////////////////////////////+// GEMM Kernel+///////////////////////////////////////////////////////////////////////////////++template <+ typename T,+ typename U,+ int BM,+ int BN,+ int BK,+ int WM,+ int WN,+ bool transpose_a,+ bool transpose_b,+ bool MN_aligned,+ bool K_aligned,+ typename AccumType = float,+ typename Epilogue = TransformNone<U, AccumType>>+struct GEMMKernel {+ UZU_MTL_CONST short tgp_padding_a = 16 / sizeof(T);+ UZU_MTL_CONST short tgp_padding_b = 16 / sizeof(T);+ UZU_MTL_CONST short tgp_mem_size_a =+ transpose_a ? BK * (BM + tgp_padding_a) : BM * (BK + tgp_padding_a);+ UZU_MTL_CONST short tgp_mem_size_b =+ transpose_b ? BN * (BK + tgp_padding_b) : BK * (BN + tgp_padding_b);+ UZU_MTL_CONST short tgp_mem_size = tgp_mem_size_a + tgp_mem_size_b;++ UZU_MTL_CONST short tgp_size = WM * WN * 32;++ using loader_a_t = BlockLoader<+ T,+ transpose_a ? BK : BM,+ transpose_a ? BM : BK,+ transpose_a ? BM + tgp_padding_a : BK + tgp_padding_a,+ !transpose_a,+ tgp_size>;+ using loader_b_t = BlockLoader<+ T,+ transpose_b ? BN : BK,+ transpose_b ? BK : BN,+ transpose_b ? BK + tgp_padding_b : BN + tgp_padding_b,+ transpose_b,+ tgp_size>;+ using mma_t = BlockMMA<+ T,+ U,+ BM,+ BN,+ BK,+ WM,+ WN,+ transpose_a,+ transpose_b,+ transpose_a ? BM + tgp_padding_a : BK + tgp_padding_a,+ transpose_b ? BK + tgp_padding_b : BN + tgp_padding_b,+ AccumType,+ Epilogue>;++ /* Main kernel function */+ template <bool M_aligned, bool N_aligned, bool K_aligned_>+ static METAL_FUNC void gemm_loop(+ threadgroup T* As,+ threadgroup T* Bs,+ const int gemm_k_iterations,+ thread loader_a_t& loader_a,+ thread loader_b_t& loader_b,+ thread mma_t& mma_op,+ thread const short& tgp_bm,+ thread const short& tgp_bn,+ thread const short& lbk,+ LoopAlignment<M_aligned, N_aligned, K_aligned_> l = {}+ ) {+ // Appease the compiler+ (void)l;++ short2 tile_dims_A = transpose_a ? short2(tgp_bm, BK) : short2(BK, tgp_bm);+ short2 tile_dims_B = transpose_b ? short2(BK, tgp_bn) : short2(tgp_bn, BK);++ for (int k = 0; k < gemm_k_iterations; k++) {+ threadgroup_barrier(mem_flags::mem_threadgroup);+ // Load elements into threadgroup+ if (M_aligned) {+ loader_a.load_unsafe();+ } else {+ loader_a.load_safe(tile_dims_A);+ }++ if (N_aligned) {+ loader_b.load_unsafe();+ } else {+ loader_b.load_safe(tile_dims_B);+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ // Multiply and accumulate threadgroup elements+ mma_op.mma(As, Bs);++ // Prepare for next iteration+ loader_a.next();+ loader_b.next();+ }++ if (!K_aligned_) {+ threadgroup_barrier(mem_flags::mem_threadgroup);++ short2 tile_dims_A_last =+ transpose_a ? short2(tgp_bm, lbk) : short2(lbk, tgp_bm);+ short2 tile_dims_B_last =+ transpose_b ? short2(lbk, tgp_bn) : short2(tgp_bn, lbk);++ loader_a.load_safe(tile_dims_A_last);+ loader_b.load_safe(tile_dims_B_last);++ threadgroup_barrier(mem_flags::mem_threadgroup);++ mma_op.mma(As, Bs);+ }+ }++ /* Main kernel function */+ static METAL_FUNC void run(+ const device T* A,+ const device T* B,+ device U* D,+ const constant GEMMParams* params,+ threadgroup T* As,+ threadgroup T* Bs,+ uint simd_lane_id [[thread_index_in_simdgroup]],+ uint simd_group_id [[simdgroup_index_in_threadgroup]],+ uint3 tid [[threadgroup_position_in_grid]],+ uint3 lid [[thread_position_in_threadgroup]]+ ) {+ // Pacifying compiler+ (void)lid;++ const int tid_y = ((tid.y) << params->swizzle_log) ++ ((tid.x) & ((1 << params->swizzle_log) - 1));+ const int tid_x = (tid.x) >> params->swizzle_log;++ if (params->tiles_n <= tid_x || params->tiles_m <= tid_y) {+ return;+ }++ threadgroup_barrier(mem_flags::mem_none);++ // Find block in A, B, C+ const int c_row = tid_y * BM;+ const int c_col = tid_x * BN;+ const size_t c_row_long = size_t(c_row);+ const size_t c_col_long = size_t(c_col);++ A += transpose_a ? c_row_long : c_row_long * params->lda;+ B += transpose_b ? c_col_long * params->ldb : c_col_long;+ D += c_row_long * params->ldd + c_col_long;++ // Prepare threadgroup loading operations+ thread loader_a_t loader_a(A, params->lda, As, simd_group_id, simd_lane_id);+ thread loader_b_t loader_b(B, params->ldb, Bs, simd_group_id, simd_lane_id);++ // Prepare threadgroup mma operation+ thread mma_t mma_op(simd_group_id, simd_lane_id);++ int gemm_k_iterations = params->gemm_k_iterations_aligned;++ ///////////////////////////////////////////////////////////////////////////+ // MNK aligned loop+ if (MN_aligned) {+ for (int k = 0; k < gemm_k_iterations; k++) {+ threadgroup_barrier(mem_flags::mem_threadgroup);+ // Load elements into threadgroup+ loader_a.load_unsafe();+ loader_b.load_unsafe();++ threadgroup_barrier(mem_flags::mem_threadgroup);++ // Multiply and accumulate threadgroup elements+ mma_op.mma(As, Bs);++ // Prepare for next iteration+ loader_a.next();+ loader_b.next();+ }++ threadgroup_barrier(mem_flags::mem_none);++ // Loop tail+ if (!K_aligned) {+ int lbk = params->input_dim - params->gemm_k_iterations_aligned * BK;+ short2 tile_dims_A = transpose_a ? short2(BM, lbk) : short2(lbk, BM);+ short2 tile_dims_B = transpose_b ? short2(lbk, BN) : short2(BN, lbk);++ loader_a.load_safe(tile_dims_A);+ loader_b.load_safe(tile_dims_B);++ threadgroup_barrier(mem_flags::mem_threadgroup);++ mma_op.mma(As, Bs);+ }++ // Store results to device memory+ mma_op.store_result(D, params->ldd);+ return;+ }+ ///////////////////////////////////////////////////////////////////////////+ // MN unaligned loop+ else { // Loop over K - unaligned case+ short tgp_bm = min(BM, params->batch - c_row);+ short tgp_bn = min(BN, params->output_dim - c_col);+ short leftover_bk =+ params->input_dim - params->gemm_k_iterations_aligned * BK;++ if (tgp_bm == BM && tgp_bn == BN) {+ gemm_loop<true, true, K_aligned>(+ As,+ Bs,+ gemm_k_iterations,+ loader_a,+ loader_b,+ mma_op,+ tgp_bm,+ tgp_bn,+ leftover_bk+ );++ mma_op.store_result(D, params->ldd);+ return;++ } else if (tgp_bn == BN) {+ gemm_loop<false, true, K_aligned>(+ As,+ Bs,+ gemm_k_iterations,+ loader_a,+ loader_b,+ mma_op,+ tgp_bm,+ tgp_bn,+ leftover_bk+ );++ mma_op.store_result_safe(D, params->ldd, short2(tgp_bn, tgp_bm));+ return;++ } else if (tgp_bm == BM) {+ gemm_loop<true, false, K_aligned>(+ As,+ Bs,+ gemm_k_iterations,+ loader_a,+ loader_b,+ mma_op,+ tgp_bm,+ tgp_bn,+ leftover_bk+ );++ mma_op.store_result_safe(D, params->ldd, short2(tgp_bn, tgp_bm));+ return;++ } else {+ gemm_loop<false, false, K_aligned>(+ As,+ Bs,+ gemm_k_iterations,+ loader_a,+ loader_b,+ mma_op,+ tgp_bm,+ tgp_bn,+ leftover_bk+ );++ mma_op.store_result_safe(D, params->ldd, short2(tgp_bn, tgp_bm));+ return;+ }+ }+ }+};++} // namespace matmul+} // namespace uzudiff --git a/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/kernel.rsnew file mode 100644index 000000000..2846419a8--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs@@ -0,0 +1,195 @@+use std::collections::HashMap;++use metal::{+ ComputeCommandEncoderRef, ComputePipelineState as MTLComputePipelineState,+ MTLSize,+};++use super::{+ arguments::MatmulArguments, pipeline::PipelineKey,+ shared_types::GEMMParams, transpose::transpose_configuration,+};+use crate::{+ DataType,+ backends::metal::{MTLContext, MTLError},+};++pub struct MatmulKernel {+ dt: DataType,+ transpose_a: bool,+ transpose_b: bool,+ bm: i32,+ bn: i32,+ bk: i32,+ wm: u64,+ wn: u64,+ pipelines: HashMap<PipelineKey, MTLComputePipelineState>,+}++impl MatmulKernel {+ fn dtype_suffix(dt: DataType) -> &'static str {+ match dt {+ DataType::F16 => "f16",+ DataType::BF16 => "bf16",+ DataType::F32 => "f32",+ _ => unreachable!(),+ }+ }++ fn kernel_name(&self) -> String {+ let t = Self::dtype_suffix(self.dt);+ let cfg = transpose_configuration(self.transpose_a, self.transpose_b);+ let tcfg = cfg.as_str();+ // Shapes we currently instantiate in matmul.metal: bm64/bn64/bk16/wm2/wn2+ format!("gemm_{}_{}_bm64_bn64_bk16_wm2_wn2", tcfg, t)+ }++ pub fn new(+ _mtl: &MTLContext,+ dt: DataType,+ transpose_a: bool,+ transpose_b: bool,+ ) -> Result<Self, MTLError> {+ if !matches!(dt, DataType::F16 | DataType::BF16 | DataType::F32) {+ return Err(MTLError::Generic(format!(+ "Unsupported dtype for MatmulKernel: {dt:?}"+ )));+ }++ Ok(Self {+ dt,+ transpose_a,+ transpose_b,+ bm: 64,+ bn: 64,+ bk: 16,+ wm: 2,+ wn: 2,+ pipelines: HashMap::new(),+ })+ }++ fn get_or_compile_pipeline(+ &mut self,+ mtl: &MTLContext,+ name: &str,+ align_m: bool,+ align_n: bool,+ align_k: bool,+ ) -> Result<&MTLComputePipelineState, MTLError> {+ let key = PipelineKey {+ name: name.to_string(),+ align_m,+ align_n,+ align_k,+ };+ if !self.pipelines.contains_key(&key) {+ let fcv = metal::FunctionConstantValues::new();+ // Base specializations+ let has_batch = false;+ let use_out_source = false;+ let do_axpby = false;+ fcv.set_constant_value_at_index(+ &has_batch as *const bool as *const _,+ metal::MTLDataType::Bool,+ 10,+ );+ fcv.set_constant_value_at_index(+ &use_out_source as *const bool as *const _,+ metal::MTLDataType::Bool,+ 100,+ );+ fcv.set_constant_value_at_index(+ &do_axpby as *const bool as *const _,+ metal::MTLDataType::Bool,+ 110,+ );+ fcv.set_constant_value_at_index(+ &align_m as *const bool as *const _,+ metal::MTLDataType::Bool,+ 200,+ );+ fcv.set_constant_value_at_index(+ &align_n as *const bool as *const _,+ metal::MTLDataType::Bool,+ 201,+ );+ fcv.set_constant_value_at_index(+ &align_k as *const bool as *const _,+ metal::MTLDataType::Bool,+ 202,+ );++ let cache_key = format!(+ "{}_am{}_an{}_ak{}",+ name, align_m as u8, align_n as u8, align_k as u8+ );+ let (ps, _) = mtl.compute_pipeline_state_with_reflection_cached(+ &cache_key,+ name,+ Some(&fcv),+ )?;+ self.pipelines.insert(key.clone(), ps);+ }+ Ok(self.pipelines.get(&key).unwrap())+ }++ pub fn encode(+ &mut self,+ mtl: &MTLContext,+ enc: &ComputeCommandEncoderRef,+ args: MatmulArguments,+ ) -> Result<(), MTLError> {+ let kname = self.kernel_name();++ // M = batch, N = output_dim, K = input_dim+ let m = args.batch;+ let n = args.output_dim;+ let k = args.input_dim;++ let am = (m % self.bm) == 0;+ let an = (n % self.bn) == 0;+ let ak = (k % self.bk) == 0;+ let ps = self.get_or_compile_pipeline(mtl, &kname, am, an, ak)?;++ enc.set_compute_pipeline_state(ps);++ // Set buffers (C is elided since use_out_source=false)+ enc.set_buffer(0, Some(args.a), 0);+ enc.set_buffer(1, Some(args.b), 0);+ enc.set_buffer(3, Some(args.d), 0);++ // Params+ let tiles_n = (n + self.bn - 1) / self.bn;+ let tiles_m = (m + self.bm - 1) / self.bm;+ let params = GEMMParams {+ batch: m,+ output_dim: n,+ input_dim: k,+ lda: args.lda,+ ldb: args.ldb,+ ldd: args.ldd,+ tiles_n,+ tiles_m,+ batch_stride_a: (args.lda as i64) * (k as i64),+ batch_stride_b: (args.ldb as i64) * (n as i64),+ batch_stride_d: (args.ldd as i64) * (n as i64),+ swizzle_log: 0,+ gemm_k_iterations_aligned: k / self.bk,+ };+ enc.set_bytes(+ 4,+ std::mem::size_of::<GEMMParams>() as u64,+ ¶ms as *const GEMMParams as *const _,+ );++ // Threadgroup sizing+ let threads_per_tg = MTLSize::new(32, self.wn, self.wm);+ let tg_x = ((n as i64 + self.bn as i64 - 1) / self.bn as i64) as u64;+ let tg_y = ((m as i64 + self.bm as i64 - 1) / self.bm as i64) as u64;+ let tg_z = args.batch_count as u64;+ let tgs = MTLSize::new(tg_x, tg_y, tg_z);+ enc.dispatch_thread_groups(tgs, threads_per_tg);+ Ok(())+ }+}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/loader.h b/crates/uzu/src/backends/metal/kernel/matmul/loader.hnew file mode 100644index 000000000..7abc01db1--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/loader.h@@ -0,0 +1,200 @@+#pragma once++#include "defines.h"++using namespace metal;++namespace uzu {+namespace matmul {++///////////////////////////////////////////////////////////////////////////////+// Block Loader - loads tiles from device memory to threadgroup memory+///////////////////////////////////////////////////////////////////////////////++template <+ typename T,+ short BROWS,+ short BCOLS,+ short dst_ld,+ short reduction_dim,+ short tgp_size,+ short alignment = 1,+ short n_reads = (BCOLS * BROWS) / (tgp_size),+ short TCOLS = BCOLS / n_reads,+ short TROWS = tgp_size / TCOLS>+struct BlockLoader {+ UZU_MTL_CONST short n_rows = (BROWS + TROWS - 1) / TROWS;+ UZU_MTL_CONST short vec_size = n_reads;++ // Leading dimension for src+ const int src_ld;+ const int tile_stride;++ // Thread location indices+ const short thread_idx;+ const short bi;+ const short bj;++ // threadgroup and device memory+ threadgroup T* dst;+ const device T* src;++ struct alignas(alignment * sizeof(T)) ReadVector {+ uint8_t v[sizeof(T) * vec_size];+ };++ /* Constructor */+ METAL_FUNC BlockLoader(+ const device T* src_,+ const int src_ld_,+ threadgroup T* dst_,+ ushort simd_group_id [[simdgroup_index_in_threadgroup]],+ ushort simd_lane_id [[thread_index_in_simdgroup]]+ )+ : src_ld(src_ld_), tile_stride(reduction_dim ? BCOLS : BROWS * src_ld),+ thread_idx(simd_group_id * 32 + simd_lane_id), bi(thread_idx / TCOLS),+ bj(vec_size * (thread_idx % TCOLS)), dst(dst_ + bi * dst_ld + bj),+ src(src_ + bi * src_ld + bj) {}++ /* Apply operation to threadgroup without bound checking */+ template <typename UnaryOp>+ METAL_FUNC void apply_inplace_op(thread const UnaryOp& op) const {+ UZU_PRAGMA_UNROLL+ for (short i = 0; i < BROWS; i += TROWS) {+ UZU_PRAGMA_UNROLL+ for (short j = 0; j < vec_size; j++) {+ dst[i * dst_ld + j] = op.apply(dst[i * dst_ld + j]);+ }+ }+ }++ /* Load from device memory into threadgroup memory - without bound checking */+ METAL_FUNC void load_unsafe() const {+ UZU_PRAGMA_UNROLL+ for (short i = 0; i < BROWS; i += TROWS) {+ *((threadgroup ReadVector*)(&dst[i * dst_ld])) =+ *((const device ReadVector*)(&src[i * src_ld]));+ }+ }++ /* Load from device memory into threadgroup memory - with bound checking */+ METAL_FUNC void load_safe(short2 src_tile_dim) const {+ src_tile_dim = src_tile_dim - short2(bj, bi);++ // Skip loading if thread has no valid reads+ if (src_tile_dim.x <= 0 || src_tile_dim.y <= 0) {+ UZU_PRAGMA_UNROLL+ for (short i = 0; i < BROWS; i += TROWS) {+ UZU_PRAGMA_UNROLL+ for (short j = 0; j < vec_size; j++) {+ dst[i * dst_ld + j] = T(0);+ }+ }+ return;+ }++ // Use fast thread memory for bound checks+ bool tmp_idx[vec_size];+ T tmp_val[vec_size];++ UZU_PRAGMA_UNROLL+ for (short i = 0; i < BROWS; i += TROWS) {+ // Make sure tmp_idx only contains valid indices+ UZU_PRAGMA_UNROLL+ for (short j = 0; j < vec_size; j++) {+ tmp_idx[j] = (i < src_tile_dim.y) && (j < src_tile_dim.x);+ }++ // Read valid indices into tmp_val+ UZU_PRAGMA_UNROLL+ for (short j = 0; j < vec_size; j++) {+ tmp_val[j] = src[(tmp_idx[j] ? i * src_ld + j : 0)];+ }++ // Zero out unneeded values+ UZU_PRAGMA_UNROLL+ for (short j = 0; j < vec_size; j++) {+ tmp_val[j] = tmp_idx[j] ? tmp_val[j] : T(0);+ }++ // Copy values to threadgroup memory+ UZU_PRAGMA_UNROLL+ for (short j = 0; j < vec_size; j++) {+ dst[i * dst_ld + j] = tmp_val[j];+ }+ }+ }++ /* Iteration helper */+ METAL_FUNC void next() { src += tile_stride; }+};++///////////////////////////////////////////////////////////////////////////////+// Simple Block Loader - simpler version for smaller tiles+///////////////////////////////////////////////////////////////////////////////++template <+ typename T,+ int BM,+ int BK,+ int lda_tgp,+ int reduction_dim,+ int tgp_size,+ int align_M = 1,+ int align_K = 4>+struct SimpleBlockLoader {+ UZU_MTL_CONST short n_reads = (BM * BK < tgp_size) ? 1 : (BM * BK) / tgp_size;++ const int src_ld;+ const short thread_idx;+ const short bi;+ const short bj;++ threadgroup T* dst;+ const device T* src;++ SimpleBlockLoader(+ const device T* src_,+ const int src_ld_,+ threadgroup T* dst_,+ ushort simd_group_id [[simdgroup_index_in_threadgroup]],+ ushort simd_lane_id [[thread_index_in_simdgroup]]+ )+ : src_ld(src_ld_), thread_idx(simd_group_id * 32 + simd_lane_id),+ bi(n_reads * thread_idx / BK), bj((n_reads * thread_idx) % BK),+ dst(dst_ + bi * lda_tgp + bj), src(src_ + bi * src_ld + bj) {}++ void load_unsafe() const {+ if (BM * BK < tgp_size && bi >= BM) {+ return;+ }++ for (int i = 0; i < n_reads; i++) {+ dst[i] = src[i];+ }+ }++ void load_safe(short2 src_tile_dim) const {+ if (BM * BK < tgp_size && bi >= BM) {+ return;+ }++ if (bi >= src_tile_dim.y || bj >= src_tile_dim.x) {+ for (int i = 0; i < n_reads; i++) {+ dst[i] = T(0);+ }+ return;+ }++ for (int i = 0; i < n_reads; i++) {+ dst[i] = src[i];+ }+ }++ void next() {+ src += BK; // Advance by tile size+ }+};++} // namespace matmul+} // namespace uzudiff --git a/crates/uzu/src/backends/metal/kernel/matmul/matmul.metal b/crates/uzu/src/backends/metal/kernel/matmul/matmul.metalnew file mode 100644index 000000000..bc5fb5664--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/matmul.metal@@ -0,0 +1,570 @@+// SPDX-License-Identifier: MIT+// General matrix multiplication kernels for uzu+// Adapted from MLX steel GEMM implementation++#include <metal_stdlib>+#include "gemm.h"++using namespace metal;+using namespace uzu::matmul;++///////////////////////////////////////////////////////////////////////////////+// Function constants for compile-time specialization+///////////////////////////////////////////////////////////////////////////////++constant bool has_batch [[function_constant(10)]];++constant bool use_out_source [[function_constant(100)]];+constant bool do_axpby [[function_constant(110)]];++constant bool align_M [[function_constant(200)]];+constant bool align_N [[function_constant(201)]];+constant bool align_K [[function_constant(202)]];++///////////////////////////////////////////////////////////////////////////////+// GEMM kernel template+///////////////////////////////////////////////////////////////////////////////++template <+ typename T,+ int BM,+ int BN,+ int BK,+ int WM,+ int WN,+ bool transpose_a,+ bool transpose_b,+ typename AccumType = float>+[[kernel, max_total_threads_per_threadgroup(WM * WN * 32)]] void gemm(+ const device T* input_a [[buffer(0)]],+ const device T* input_b [[buffer(1)]],+ const device T* input_c [[buffer(2), function_constant(use_out_source)]],+ device T* output [[buffer(3)]],+ const constant GEMMParams* params [[buffer(4)]],+ const constant GEMMAddMMParams* addmm_params+ [[buffer(5), function_constant(use_out_source)]],+ const constant int* batch_shape [[buffer(6), function_constant(has_batch)]],+ const constant int64_t* batch_strides+ [[buffer(7), function_constant(has_batch)]],+ uint simd_lane_id [[thread_index_in_simdgroup]],+ uint simd_group_id [[simdgroup_index_in_threadgroup]],+ uint3 threadgroup_position [[threadgroup_position_in_grid]],+ uint3 thread_position [[thread_position_in_threadgroup]]+) {+ // Pacifying compiler+ (void)thread_position;++ using gemm_kernel = GEMMKernel<+ T,+ T,+ BM,+ BN,+ BK,+ WM,+ WN,+ transpose_a,+ transpose_b,+ true,+ true,+ AccumType>;++ using loader_a_t = typename gemm_kernel::loader_a_t;+ using loader_b_t = typename gemm_kernel::loader_b_t;+ using mma_t = typename gemm_kernel::mma_t;++ // Find block with swizzle+ const int threadgroup_y =+ ((threadgroup_position.y) << params->swizzle_log) ++ ((threadgroup_position.x) & ((1 << params->swizzle_log) - 1));+ const int threadgroup_x = (threadgroup_position.x) >> params->swizzle_log;++ // Exit early if out of bounds+ if (params->tiles_n <= threadgroup_x || params->tiles_m <= threadgroup_y) {+ return;+ }++ // Adjust for batch+ if (has_batch) {+ // Simple batch striding (no broadcast)+ input_a += params->batch_stride_a * threadgroup_position.z;+ input_b += params->batch_stride_b * threadgroup_position.z;++ if (use_out_source) {+ input_c += addmm_params->batch_stride_c * threadgroup_position.z;+ }+ } else {+ input_a += params->batch_stride_a * threadgroup_position.z;+ input_b += params->batch_stride_b * threadgroup_position.z;++ if (use_out_source) {+ input_c += addmm_params->batch_stride_c * threadgroup_position.z;+ }+ }++ output += params->batch_stride_d * threadgroup_position.z;++ // Prepare threadgroup memory+ threadgroup T threadgroup_a[gemm_kernel::tgp_mem_size_a];+ threadgroup T threadgroup_b[gemm_kernel::tgp_mem_size_b];++ threadgroup_barrier(mem_flags::mem_none);++ // Find block in input_a, input_b, input_c+ const int output_row = threadgroup_y * BM;+ const int output_col = threadgroup_x * BN;+ const size_t output_row_long = size_t(output_row);+ const size_t output_col_long = size_t(output_col);++ input_a += transpose_a ? output_row_long : output_row_long * params->lda;+ input_b += transpose_b ? output_col_long * params->ldb : output_col_long;+ output += output_row_long * params->ldd + output_col_long;++ if (use_out_source) {+ input_c += output_row_long * addmm_params->ldc ++ output_col_long * addmm_params->fdc;+ }++ // Prepare threadgroup mma operation+ thread mma_t matrix_multiply_accumulate(simd_group_id, simd_lane_id);++ // Prepare threadgroup loading operations+ thread loader_a_t block_loader_a(+ input_a,+ params->lda,+ threadgroup_a,+ simd_group_id,+ simd_lane_id+ );+ thread loader_b_t block_loader_b(+ input_b,+ params->ldb,+ threadgroup_b,+ simd_group_id,+ simd_lane_id+ );++ // Prepare threadgroup bounds+ const short threadgroup_block_batch =+ align_M ? BM : short(min(BM, params->batch - output_row));+ const short threadgroup_block_output =+ align_N ? BN : short(min(BN, params->output_dim - output_col));++ // Prepare iterations+ int gemm_k_iterations = params->gemm_k_iterations_aligned;++ // Do unaligned K iterations first+ if (!align_K) {+ const int input_aligned_end = params->gemm_k_iterations_aligned * BK;+ const int input_remainder = params->input_dim - input_aligned_end;+ const size_t input_offset_a = transpose_a+ ? params->lda * size_t(input_aligned_end)+ : size_t(input_aligned_end);+ const size_t input_offset_b = transpose_b+ ? size_t(input_aligned_end)+ : params->ldb * size_t(input_aligned_end);++ // Move loader source ahead to end+ block_loader_a.src += input_offset_a;+ block_loader_b.src += input_offset_b;++ // Load tile+ const short2 tile_dimensions_a =+ transpose_a ? short2(threadgroup_block_batch, input_remainder)+ : short2(input_remainder, threadgroup_block_batch);+ const short2 tile_dimensions_b =+ transpose_b ? short2(input_remainder, threadgroup_block_output)+ : short2(threadgroup_block_output, input_remainder);++ block_loader_a.load_safe(tile_dimensions_a);+ block_loader_b.load_safe(tile_dimensions_b);++ threadgroup_barrier(mem_flags::mem_threadgroup);++ // Do matmul+ matrix_multiply_accumulate.mma(threadgroup_a, threadgroup_b);++ // Reset source back to start+ block_loader_a.src -= input_offset_a;+ block_loader_b.src -= input_offset_b;+ }++ const TransformAdd<AccumType, AccumType> epilogue_operation_add(+ addmm_params->alpha,+ addmm_params->beta+ );+ const TransformAxpby<AccumType, AccumType> epilogue_operation_axpby(+ addmm_params->alpha,+ addmm_params->beta+ );++ ///////////////////////////////////////////////////////////////////////////+ // MNK aligned loop+ if (align_M && align_N) {+ // Do gemm+ for (int iteration = 0; iteration < gemm_k_iterations; iteration++) {+ threadgroup_barrier(mem_flags::mem_threadgroup);+ // Load elements into threadgroup+ block_loader_a.load_unsafe();+ block_loader_b.load_unsafe();++ threadgroup_barrier(mem_flags::mem_threadgroup);++ // Multiply and accumulate threadgroup elements+ matrix_multiply_accumulate.mma(threadgroup_a, threadgroup_b);++ // Prepare for next iteration+ block_loader_a.next();+ block_loader_b.next();+ }++ threadgroup_barrier(mem_flags::mem_none);++ // Do epilogue+ if (use_out_source) {+ if (do_axpby) {+ matrix_multiply_accumulate.apply_epilogue(+ input_c,+ addmm_params->ldc,+ addmm_params->fdc,+ epilogue_operation_axpby+ );+ } else {+ matrix_multiply_accumulate.apply_epilogue(+ input_c,+ addmm_params->ldc,+ addmm_params->fdc,+ epilogue_operation_add+ );+ }+ }++ // Store results to device memory+ return matrix_multiply_accumulate.store_result(output, params->ldd);+ }+ ///////////////////////////////////////////////////////////////////////////+ // MN unaligned loop+ else { // Loop over K - unaligned case+ const int leftover_block_input = 0;++ if ((align_M || threadgroup_block_batch == BM) &&+ (align_N || threadgroup_block_output == BN)) {+ // Do gemm+ gemm_kernel::gemm_loop(+ threadgroup_a,+ threadgroup_b,+ gemm_k_iterations,+ block_loader_a,+ block_loader_b,+ matrix_multiply_accumulate,+ threadgroup_block_batch,+ threadgroup_block_output,+ leftover_block_input,+ LoopAlignment<true, true, true>{}+ );++ // Do epilogue+ if (use_out_source) {+ if (do_axpby) {+ matrix_multiply_accumulate.apply_epilogue(+ input_c,+ addmm_params->ldc,+ addmm_params->fdc,+ epilogue_operation_axpby+ );+ } else {+ matrix_multiply_accumulate.apply_epilogue(+ input_c,+ addmm_params->ldc,+ addmm_params->fdc,+ epilogue_operation_add+ );+ }+ }++ // Store results to device memory+ return matrix_multiply_accumulate.store_result(output, params->ldd);++ } else if (align_N || threadgroup_block_output == BN) {+ gemm_kernel::gemm_loop(+ threadgroup_a,+ threadgroup_b,+ gemm_k_iterations,+ block_loader_a,+ block_loader_b,+ matrix_multiply_accumulate,+ threadgroup_block_batch,+ threadgroup_block_output,+ leftover_block_input,+ LoopAlignment<false, true, true>{}+ );++ // Do epilogue+ if (use_out_source) {+ if (do_axpby) {+ matrix_multiply_accumulate.apply_epilogue_safe(+ input_c,+ addmm_params->ldc,+ addmm_params->fdc,+ short2(threadgroup_block_output, threadgroup_block_batch),+ epilogue_operation_axpby+ );+ } else {+ matrix_multiply_accumulate.apply_epilogue_safe(+ input_c,+ addmm_params->ldc,+ addmm_params->fdc,+ short2(threadgroup_block_output, threadgroup_block_batch),+ epilogue_operation_add+ );+ }+ }++ // Store results to device memory+ return matrix_multiply_accumulate.store_result_safe(+ output,+ params->ldd,+ short2(threadgroup_block_output, threadgroup_block_batch)+ );++ } else if (align_M || threadgroup_block_batch == BM) {+ gemm_kernel::gemm_loop(+ threadgroup_a,+ threadgroup_b,+ gemm_k_iterations,+ block_loader_a,+ block_loader_b,+ matrix_multiply_accumulate,+ threadgroup_block_batch,+ threadgroup_block_output,+ leftover_block_input,+ LoopAlignment<true, false, true>{}+ );++ // Do epilogue+ if (use_out_source) {+ if (do_axpby) {+ matrix_multiply_accumulate.apply_epilogue_safe(+ input_c,+ addmm_params->ldc,+ addmm_params->fdc,+ short2(threadgroup_block_output, threadgroup_block_batch),+ epilogue_operation_axpby+ );+ } else {+ matrix_multiply_accumulate.apply_epilogue_safe(+ input_c,+ addmm_params->ldc,+ addmm_params->fdc,+ short2(threadgroup_block_output, threadgroup_block_batch),+ epilogue_operation_add+ );+ }+ }++ // Store results to device memory+ return matrix_multiply_accumulate.store_result_safe(+ output,+ params->ldd,+ short2(threadgroup_block_output, threadgroup_block_batch)+ );++ } else {+ gemm_kernel::gemm_loop(+ threadgroup_a,+ threadgroup_b,+ gemm_k_iterations,+ block_loader_a,+ block_loader_b,+ matrix_multiply_accumulate,+ threadgroup_block_batch,+ threadgroup_block_output,+ leftover_block_input,+ LoopAlignment<false, false, true>{}+ );++ // Do epilogue+ if (use_out_source) {+ if (do_axpby) {+ matrix_multiply_accumulate.apply_epilogue_safe(+ input_c,+ addmm_params->ldc,+ addmm_params->fdc,+ short2(threadgroup_block_output, threadgroup_block_batch),+ epilogue_operation_axpby+ );+ } else {+ matrix_multiply_accumulate.apply_epilogue_safe(+ input_c,+ addmm_params->ldc,+ addmm_params->fdc,+ short2(threadgroup_block_output, threadgroup_block_batch),+ epilogue_operation_add+ );+ }+ }++ // Store results to device memory+ return matrix_multiply_accumulate.store_result_safe(+ output,+ params->ldd,+ short2(threadgroup_block_output, threadgroup_block_batch)+ );+ }+ }+}++///////////////////////////////////////////////////////////////////////////////+// Kernel instantiation macros+///////////////////////////////////////////////////////////////////////////////++#define instantiate_gemm( \+ transpose_name, \+ transpose_a, \+ transpose_b, \+ type_name, \+ element_type, \+ block_batch, \+ block_output, \+ block_input, \+ warp_batch, \+ warp_output \+) \+ template [[host_name( \+ "gemm_" #transpose_name "_" #type_name "_bm" #block_batch \+ "_bn" #block_output "_bk" #block_input "_wm" #warp_batch \+ "_wn" #warp_output \+ )]] [[kernel]] void \+ gemm< \+ element_type, \+ block_batch, \+ block_output, \+ block_input, \+ warp_batch, \+ warp_output, \+ transpose_a, \+ transpose_b, \+ float>( \+ const device element_type* input_a [[buffer(0)]], \+ const device element_type* input_b [[buffer(1)]], \+ const device element_type* input_c \+ [[buffer(2), function_constant(use_out_source)]], \+ device element_type* output [[buffer(3)]], \+ const constant GEMMParams* params [[buffer(4)]], \+ const constant GEMMAddMMParams* addmm_params \+ [[buffer(5), function_constant(use_out_source)]], \+ const constant int* batch_shape \+ [[buffer(6), function_constant(has_batch)]], \+ const constant int64_t* batch_strides \+ [[buffer(7), function_constant(has_batch)]], \+ uint simd_lane_id [[thread_index_in_simdgroup]], \+ uint simd_group_id [[simdgroup_index_in_threadgroup]], \+ uint3 threadgroup_position [[threadgroup_position_in_grid]], \+ uint3 thread_position [[thread_position_in_threadgroup]] \+ );++// clang-format off+#define instantiate_gemm_transpose_helper( \+ type_name, element_type, block_batch, block_output, block_input, warp_batch, warp_output \+) \+ instantiate_gemm(nn, false, false, type_name, element_type, block_batch, block_output, \+ block_input, warp_batch, warp_output) \+ instantiate_gemm(nt, false, true, type_name, element_type, block_batch, block_output, \+ block_input, warp_batch, warp_output) \+ instantiate_gemm(tn, true, false, type_name, element_type, block_batch, block_output, \+ block_input, warp_batch, warp_output) \+ instantiate_gemm(tt, true, true, type_name, element_type, block_batch, block_output, \+ block_input, warp_batch, warp_output)+// clang-format on++// clang-format off+#define instantiate_gemm_shapes_helper(type_name, element_type) \+ instantiate_gemm_transpose_helper(type_name, element_type, 64, 64, 16, 2, 2) \+ instantiate_gemm_transpose_helper(type_name, element_type, 64, 64, 16, 1, 2) \+ instantiate_gemm_transpose_helper(type_name, element_type, 64, 32, 32, 2, 2) \+ instantiate_gemm_transpose_helper(type_name, element_type, 32, 64, 16, 1, 2) \+ instantiate_gemm_transpose_helper(type_name, element_type, 32, 32, 16, 2, 2) \+ instantiate_gemm_transpose_helper(type_name, element_type, 64, 32, 8, 4, 1)+// clang-format on++// Instantiate for float16, bfloat16, and float32+instantiate_gemm_shapes_helper(f16, half);+instantiate_gemm_shapes_helper(bf16, bfloat);+instantiate_gemm_shapes_helper(f32, float);++///////////////////////////////////////////////////////////////////////////////+// Simple GEMM kernel (for smaller matrices or vector operations)+///////////////////////////////////////////////////////////////////////////////++template <typename T>+[[kernel]] void gemv(+ const device T* input_a [[buffer(0)]],+ const device T* input_b [[buffer(1)]],+ device T* output [[buffer(2)]],+ constant int& batch [[buffer(3)]],+ constant int& output_dim [[buffer(4)]],+ constant int& input_dim [[buffer(5)]],+ constant int& leading_dimension_a [[buffer(6)]],+ constant int& leading_dimension_b [[buffer(7)]],+ constant int& leading_dimension_output [[buffer(8)]],+ uint2 thread_position [[thread_position_in_grid]],+ uint2 grid_size [[threads_per_grid]]+) {+ const int row_index = thread_position.y;+ const int column_index = thread_position.x;++ if (row_index >= batch || column_index >= output_dim) {+ return;+ }++ float accumulator = 0.0f;+ for (int input_index = 0; input_index < input_dim; input_index++) {+ accumulator +=+ float(input_a[row_index * leading_dimension_a + input_index]) *+ float(input_b[input_index * leading_dimension_b + column_index]);+ }++ output[row_index * leading_dimension_output + column_index] = T(accumulator);+}++// Simple GEMV instantiations+template [[host_name("gemv_f16")]] [[kernel]] void gemv<half>(+ const device half* input_a [[buffer(0)]],+ const device half* input_b [[buffer(1)]],+ device half* output [[buffer(2)]],+ constant int& batch [[buffer(3)]],+ constant int& output_dim [[buffer(4)]],+ constant int& input_dim [[buffer(5)]],+ constant int& leading_dimension_a [[buffer(6)]],+ constant int& leading_dimension_b [[buffer(7)]],+ constant int& leading_dimension_output [[buffer(8)]],+ uint2 thread_position [[thread_position_in_grid]],+ uint2 grid_size [[threads_per_grid]]+);++template [[host_name("gemv_bf16")]] [[kernel]] void gemv<bfloat>(+ const device bfloat* input_a [[buffer(0)]],+ const device bfloat* input_b [[buffer(1)]],+ device bfloat* output [[buffer(2)]],+ constant int& batch [[buffer(3)]],+ constant int& output_dim [[buffer(4)]],+ constant int& input_dim [[buffer(5)]],+ constant int& leading_dimension_a [[buffer(6)]],+ constant int& leading_dimension_b [[buffer(7)]],+ constant int& leading_dimension_output [[buffer(8)]],+ uint2 thread_position [[thread_position_in_grid]],+ uint2 grid_size [[threads_per_grid]]+);++template [[host_name("gemv_f32")]] [[kernel]] void gemv<float>(+ const device float* input_a [[buffer(0)]],+ const device float* input_b [[buffer(1)]],+ device float* output [[buffer(2)]],+ constant int& batch [[buffer(3)]],+ constant int& output_dim [[buffer(4)]],+ constant int& input_dim [[buffer(5)]],+ constant int& leading_dimension_a [[buffer(6)]],+ constant int& leading_dimension_b [[buffer(7)]],+ constant int& leading_dimension_output [[buffer(8)]],+ uint2 thread_position [[thread_position_in_grid]],+ uint2 grid_size [[threads_per_grid]]+);diff --git a/crates/uzu/src/backends/metal/kernel/matmul/mma.h b/crates/uzu/src/backends/metal/kernel/matmul/mma.hnew file mode 100644index 000000000..741ef4394--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/mma.h@@ -0,0 +1,524 @@+#pragma once++#include "defines.h"++using namespace metal;++namespace uzu {+namespace matmul {++///////////////////////////////////////////////////////////////////////////////+// MMA Fragment - 8x8 simdgroup matrix fragment+///////////////////////////////////////////////////////////////////////////////++template <typename T, int kFragRows_, int kFragCols_>+struct BaseMMAFrag {+ static_assert(+ kFragRows_ == 8,+ "Only 8 x 8 fragment matrices are currently supported"+ );+ static_assert(+ kFragCols_ == 8,+ "Only 8 x 8 fragment matrices are currently supported"+ );+};++template <typename T>+struct BaseMMAFrag<T, 8, 8> {+ UZU_MTL_CONST int kFragRows = 8;+ UZU_MTL_CONST int kFragCols = 8;+ UZU_MTL_CONST int kElemsPerFrag = (kFragRows * kFragCols) / 32;+ UZU_MTL_CONST int kElemRows = 1;+ UZU_MTL_CONST int kElemCols = 2;++ static_assert(+ kElemRows * kElemCols == kElemsPerFrag,+ "MMAFrag shape is not consistent with MMAFrag size"+ );++ typedef metal::simdgroup_matrix<T, kFragRows, kFragCols> mat_type;+ typedef metal::vec<T, kElemsPerFrag> frag_type;++ METAL_FUNC static constexpr short2 get_coord(+ ushort simd_lane_id [[thread_index_in_simdgroup]]+ ) {+ const short qid = simd_lane_id / 4;+ const short fm = (qid & 4) + ((simd_lane_id / 2) % 4);+ const short fn = (qid & 2) * 2 + (simd_lane_id % 2) * 2;+ return short2{fn, fm};+ }++ template <typename SrcPtrType, typename StrX, typename StrY>+ METAL_FUNC static constexpr void load(+ thread frag_type& dst,+ SrcPtrType src,+ StrX str_x,+ StrY str_y+ ) {+ UZU_PRAGMA_UNROLL+ for (short i = 0; i < kElemRows; i++) {+ UZU_PRAGMA_UNROLL+ for (short j = 0; j < kElemCols; j++) {+ dst[i * kElemCols + j] = static_cast<T>(src[i * str_x + j * str_y]);+ }+ }+ }++ template <+ typename SrcPtrType,+ typename StrX,+ typename StrY,+ typename LimX,+ typename LimY,+ typename OffX,+ typename OffY>+ METAL_FUNC static constexpr void load_safe(+ thread frag_type& dst,+ SrcPtrType src,+ StrX str_x,+ StrY str_y,+ LimX lim_x,+ LimY lim_y,+ OffX off_x = 0,+ OffY off_y = 0+ ) {+ UZU_PRAGMA_UNROLL+ for (short i = 0; i < kElemRows; i++) {+ UZU_PRAGMA_UNROLL+ for (short j = 0; j < kElemCols; j++) {+ if ((off_x + i) < lim_x && (off_y + j) < lim_y) {+ dst[i * kElemCols + j] =+ static_cast<T>(src[(off_x + i) * str_x + (off_y + j) * str_y]);+ } else {+ dst[i * kElemCols + j] = T(0);+ }+ }+ }+ }++ template <typename DstPtrType, typename StrX, typename StrY>+ METAL_FUNC static constexpr void store(+ const thread frag_type& src,+ DstPtrType dst,+ StrX str_x,+ StrY str_y+ ) {+ using U = pointer_element_t<DstPtrType>;++ UZU_PRAGMA_UNROLL+ for (short i = 0; i < kElemRows; i++) {+ UZU_PRAGMA_UNROLL+ for (short j = 0; j < kElemCols; j++) {+ dst[i * str_x + j * str_y] = static_cast<U>(src[i * kElemCols + j]);+ }+ }+ }++ template <+ typename DstPtrType,+ typename StrX,+ typename StrY,+ typename LimX,+ typename LimY,+ typename OffX,+ typename OffY>+ METAL_FUNC static constexpr void store_safe(+ const thread frag_type& src,+ DstPtrType dst,+ StrX str_x,+ StrY str_y,+ LimX lim_x,+ LimY lim_y,+ OffX off_x = 0,+ OffY off_y = 0+ ) {+ using U = pointer_element_t<DstPtrType>;++ UZU_PRAGMA_UNROLL+ for (short i = 0; i < kElemRows; i++) {+ UZU_PRAGMA_UNROLL+ for (short j = 0; j < kElemCols; j++) {+ if ((off_x + i) < lim_x && (off_y + j) < lim_y) {+ dst[(off_x + i) * str_x + (off_y + j) * str_y] =+ static_cast<U>(src[i * kElemCols + j]);+ }+ }+ }+ }++ METAL_FUNC static constexpr void mma(+ thread frag_type& D,+ thread frag_type& A,+ thread frag_type& B,+ thread frag_type& C+ ) {+ mat_type D_mat;+ mat_type A_mat;+ mat_type B_mat;+ mat_type C_mat;++ reinterpret_cast<thread frag_type&>(A_mat.thread_elements()) = A;+ reinterpret_cast<thread frag_type&>(B_mat.thread_elements()) = B;+ reinterpret_cast<thread frag_type&>(C_mat.thread_elements()) = C;++ mma(D_mat, A_mat, B_mat, C_mat);++ D = reinterpret_cast<thread frag_type&>(D_mat.thread_elements());+ }++ METAL_FUNC static constexpr void mma(+ thread mat_type& D,+ thread mat_type& A,+ thread mat_type& B,+ thread mat_type& C+ ) {+ simdgroup_multiply_accumulate(D, A, B, C);+ }+};++///////////////////////////////////////////////////////////////////////////////+// MMA Tile - collection of MMA fragments+///////////////////////////////////////////////////////////////////////////////++template <+ typename T,+ int kTileRows_,+ int kTileCols_,+ class MMAFrag_ = BaseMMAFrag<T, 8, 8>>+struct MMATile {+ using MMAFrag_t = MMAFrag_;+ using elem_type = T;+ UZU_MTL_CONST int kFragRows = MMAFrag_t::kFragRows;+ UZU_MTL_CONST int kFragCols = MMAFrag_t::kFragCols;+ UZU_MTL_CONST int kElemsPerFrag = MMAFrag_t::kElemsPerFrag;++ UZU_MTL_CONST int kTileRows = kTileRows_;+ UZU_MTL_CONST int kTileCols = kTileCols_;++ UZU_MTL_CONST int kRows = kTileRows * kFragRows;+ UZU_MTL_CONST int kCols = kTileCols * kFragCols;++ UZU_MTL_CONST int kNumFrags = kTileRows * kTileCols;+ UZU_MTL_CONST int kElemsPerTile = kNumFrags * kElemsPerFrag;++ typedef typename MMAFrag_t::mat_type mat_type;+ typedef typename MMAFrag_t::frag_type frag_type;++ frag_type val_frags[kNumFrags] = {frag_type(0)};++ METAL_FUNC MMATile() thread {}++ METAL_FUNC constexpr void clear() {+ UZU_PRAGMA_UNROLL+ for (short i = 0; i < kNumFrags; ++i) {+ val_frags[i] = frag_type(0);+ }+ }++ METAL_FUNC constexpr thread frag_type& frag_at(const short i, const short j) {+ return val_frags[i * kTileCols + j];+ }++ METAL_FUNC constexpr const thread frag_type& frag_at(+ const short i,+ const short j+ ) const {+ return val_frags[i * kTileCols + j];+ }++ METAL_FUNC thread elem_type* elems() {+ return reinterpret_cast<thread elem_type*>(val_frags);+ }++ METAL_FUNC const thread elem_type* elems() const {+ return reinterpret_cast<const thread elem_type*>(val_frags);+ }++ template <typename U, int w_x, int w_y, int str_x, int str_y>+ METAL_FUNC void load(const threadgroup U* src) {+ UZU_PRAGMA_UNROLL+ for (short i = 0; i < kTileRows; ++i) {+ UZU_PRAGMA_UNROLL+ for (short j = 0; j < kTileCols; ++j) {+ MMAFrag_t::load(+ frag_at(i, j),+ &(+ src[(i * kFragRows) * w_x * str_x ++ (j * kFragCols) * w_y * str_y]+ ),+ str_x,+ str_y+ );+ }+ }+ }++ template <typename U, int w_x, int w_y>+ METAL_FUNC void store(device U* dst, const int ld) const {+ UZU_PRAGMA_UNROLL+ for (short i = 0; i < kTileRows; ++i) {+ UZU_PRAGMA_UNROLL+ for (short j = 0; j < kTileCols; ++j) {+ MMAFrag_t::store(+ frag_at(i, j),+ &(dst[(i * kFragRows) * w_x * ld + (j * kFragCols) * w_y]),+ ld,+ 1+ );+ }+ }+ }++ template <typename U, int w_x, int w_y>+ METAL_FUNC void store_safe(+ device U* dst,+ const int ld,+ const short2 dst_tile_dims+ ) const {+ UZU_PRAGMA_UNROLL+ for (int i = 0; i < kTileRows; ++i) {+ UZU_PRAGMA_UNROLL+ for (int j = 0; j < kTileCols; ++j) {+ MMAFrag_t::store_safe(+ frag_at(i, j),+ dst,+ ld,+ 1,+ dst_tile_dims.y,+ dst_tile_dims.x,+ (i * kFragRows) * w_x,+ (j * kFragCols) * w_y+ );+ }+ }+ }+};++///////////////////////////////////////////////////////////////////////////////+// Tile matrix multiply-accumulate+///////////////////////////////////////////////////////////////////////////////++template <typename T, typename U, int M, int N, int K>+METAL_FUNC void tile_matmad(+ thread MMATile<T, M, N>& D,+ thread MMATile<U, M, K>& A,+ thread MMATile<U, K, N>& B,+ thread MMATile<T, M, N>& C+) {+ UZU_PRAGMA_UNROLL+ for (short m = 0; m < M; ++m) {+ UZU_PRAGMA_UNROLL+ for (short n = 0; n < N; ++n) {+ short n_serp = (m % 2) ? (N - 1 - n) : n;+ UZU_PRAGMA_UNROLL+ for (short k = 0; k < K; ++k) {+ MMATile<T, M, N>::MMAFrag_t::mma(+ D.frag_at(m, n_serp),+ A.frag_at(m, k),+ B.frag_at(k, n_serp),+ C.frag_at(m, n_serp)+ );+ }+ }+ }+}++///////////////////////////////////////////////////////////////////////////////+// Block MMA - manages the GEMM computation for a threadgroup+///////////////////////////////////////////////////////////////////////////////++template <+ typename T,+ typename U,+ int BM,+ int BN,+ int BK,+ int WM,+ int WN,+ bool transpose_a,+ bool transpose_b,+ short lda_tgp,+ short ldb_tgp,+ typename AccumType = float,+ typename Epilogue = TransformNone<U, AccumType>>+struct BlockMMA {+ // MMAFrag size+ UZU_MTL_CONST short kFragSize = 8;+ using MMAFrag_acc_t = BaseMMAFrag<AccumType, kFragSize, kFragSize>;++ // Warp tile simdgroup matrix strides along M+ UZU_MTL_CONST short TM_stride = kFragSize * WM;+ // Warp tile simdgroup matrix strides along N+ UZU_MTL_CONST short TN_stride = kFragSize * WN;++ // Warp tile size along M+ UZU_MTL_CONST short TM = BM / (kFragSize * WM);+ // Warp tile size along N+ UZU_MTL_CONST short TN = BN / (kFragSize * WN);++ // Threadgroup A strides+ UZU_MTL_CONST short A_str_m = transpose_a ? 1 : lda_tgp; // M+ UZU_MTL_CONST short A_str_k = transpose_a ? lda_tgp : 1; // K++ // Threadgroup B strides+ UZU_MTL_CONST short B_str_k = transpose_b ? 1 : ldb_tgp; // K+ UZU_MTL_CONST short B_str_n = transpose_b ? ldb_tgp : 1; // N++ // Threadgroup strides along K+ UZU_MTL_CONST short tile_stride_a = kFragSize * A_str_k;+ UZU_MTL_CONST short tile_stride_b = kFragSize * B_str_k;++ // Simdgroup matrices+ MMATile<AccumType, TM, 1, MMAFrag_acc_t> Atile;+ MMATile<AccumType, 1, TN, MMAFrag_acc_t> Btile;+ MMATile<AccumType, TM, TN, MMAFrag_acc_t> Ctile;++ // Offsets within threadgroup+ short sm;+ short sn;++ short As_offset;+ short Bs_offset;++ /* Constructor */+ METAL_FUNC BlockMMA(+ ushort simd_group_id [[simdgroup_index_in_threadgroup]],+ ushort simd_lane_id [[thread_index_in_simdgroup]]+ ) {+ // Determine thread position in simdgroup matrix+ short tm = kFragSize * (simd_group_id / WN);+ short tn = kFragSize * (simd_group_id % WN);++ short2 simd_coord = MMAFrag_acc_t::get_coord(simd_lane_id);+ sm = simd_coord.y;+ sn = simd_coord.x;++ // Determine thread and simdgroup offset+ As_offset = (tm + sm) * A_str_m + (sn)*A_str_k; // M, K+ Bs_offset = (sm)*B_str_k + (tn + sn) * B_str_n; // K, N++ sm += tm;+ sn += tn;+ }++ /* (BM, BK) X (BK, BN) multiply accumulate function */+ METAL_FUNC void mma(const threadgroup T* As, const threadgroup T* Bs) {+ // Adjust for simdgroup and thread location+ As += As_offset;+ Bs += Bs_offset;++ // Iterate over BK in blocks of kFragSize+ UZU_PRAGMA_UNROLL+ for (short kk = 0; kk < BK; kk += kFragSize) {+ simdgroup_barrier(mem_flags::mem_none);++ Atile.template load<T, WM, 1, A_str_m, A_str_k>(As);++ simdgroup_barrier(mem_flags::mem_none);++ Btile.template load<T, 1, WN, B_str_k, B_str_n>(Bs);++ simdgroup_barrier(mem_flags::mem_none);++ tile_matmad(Ctile, Atile, Btile, Ctile);++ // Progress to next simdgroup tile+ As += tile_stride_a;+ Bs += tile_stride_b;+ }+ }++ /* Store results from simdgroup_matrix results into device memory */+ METAL_FUNC void store_result(device U* D, const int ldd) {+ // Apply epilogue+ UZU_PRAGMA_UNROLL+ for (short i = 0; i < decltype(Ctile)::kElemsPerTile; i++) {+ Ctile.elems()[i] = Epilogue::apply(Ctile.elems()[i]);+ }++ // Adjust for simdgroup and thread location+ D += sm * ldd + sn;++ Ctile.template store<U, WM, WN>(D, ldd);+ }++ METAL_FUNC void store_result_safe(+ device U* D,+ const int ldd,+ short2 dst_tile_dims+ ) {+ // Apply epilogue+ UZU_PRAGMA_UNROLL+ for (short i = 0; i < decltype(Ctile)::kElemsPerTile; i++) {+ Ctile.elems()[i] = Epilogue::apply(Ctile.elems()[i]);+ }++ // Adjust for simdgroup and thread location+ D += sm * ldd + sn;+ dst_tile_dims -= short2(sn, sm);++ if (dst_tile_dims.x <= 0 || dst_tile_dims.y <= 0)+ return;++ Ctile.template store_safe<U, WM, WN>(D, ldd, dst_tile_dims);+ }++ /* Apply epilogue with C matrix */+ template <typename EpilogueOp>+ METAL_FUNC void apply_epilogue(+ const device U* C,+ const int ldc,+ const int fdc,+ thread const EpilogueOp& epilogue_op+ ) {+ const device U* C_ptr = C + sm * ldc + sn * fdc;++ UZU_PRAGMA_UNROLL+ for (short i = 0; i < TM; i++) {+ UZU_PRAGMA_UNROLL+ for (short j = 0; j < TN; j++) {+ thread auto& frag = Ctile.frag_at(i, j);+ UZU_PRAGMA_UNROLL+ for (short k = 0; k < decltype(Ctile)::kElemsPerFrag; k++) {+ short row_offset = (i * kFragSize * WM);+ short col_offset = (j * kFragSize * WN);+ // Get the C element (accounting for fragment layout)+ U c_val = C_ptr[row_offset * ldc + col_offset * fdc + k];+ frag[k] = epilogue_op.apply(frag[k], static_cast<AccumType>(c_val));+ }+ }+ }+ }++ template <typename EpilogueOp>+ METAL_FUNC void apply_epilogue_safe(+ const device U* C,+ const int ldc,+ const int fdc,+ short2 tile_dims,+ thread const EpilogueOp& epilogue_op+ ) {+ const device U* C_ptr = C + sm * ldc + sn * fdc;+ tile_dims -= short2(sn, sm);++ UZU_PRAGMA_UNROLL+ for (short i = 0; i < TM; i++) {+ UZU_PRAGMA_UNROLL+ for (short j = 0; j < TN; j++) {+ thread auto& frag = Ctile.frag_at(i, j);+ short row_offset = (i * kFragSize * WM);+ short col_offset = (j * kFragSize * WN);+ UZU_PRAGMA_UNROLL+ for (short k = 0; k < decltype(Ctile)::kElemsPerFrag; k++) {+ if (row_offset < tile_dims.y && col_offset + k < tile_dims.x) {+ U c_val = C_ptr[row_offset * ldc + col_offset * fdc + k];+ frag[k] = epilogue_op.apply(frag[k], static_cast<AccumType>(c_val));+ }+ }+ }+ }+ }+};++} // namespace matmul+} // namespace uzudiff --git a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/mod.rsnew file mode 100644index 000000000..191479882--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/mod.rs@@ -0,0 +1,9 @@+mod arguments;+mod kernel;+mod pipeline;+#[allow(dead_code, clippy::unnecessary_operation, clippy::identity_op)]+mod shared_types;+mod transpose;++pub use arguments::MatmulArguments;+pub use kernel::MatmulKernel;diff --git a/crates/uzu/src/backends/metal/kernel/matmul/pipeline.rs b/crates/uzu/src/backends/metal/kernel/matmul/pipeline.rsnew file mode 100644index 000000000..801ce989d--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/pipeline.rs@@ -0,0 +1,7 @@+#[derive(Debug, Clone, PartialEq, Eq, Hash)]+pub(super) struct PipelineKey {+ pub name: String,+ pub align_m: bool,+ pub align_n: bool,+ pub align_k: bool,+}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/shared_types.h b/crates/uzu/src/backends/metal/kernel/matmul/shared_types.hnew file mode 100644index 000000000..4a9b28047--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/shared_types.h@@ -0,0 +1,66 @@+// Shared types for GEMM kernel - used by both Metal shaders and Rust (via+// bindgen). This header must be C-compatible for bindgen to parse it.++#pragma once++// Metal and C have different type systems+// __METAL_VERSION__ is defined when compiling with the Metal shader compiler+#ifdef __METAL_VERSION__+// Metal shading language - int64_t comes from metal_stdlib+#include <metal_stdlib>+using namespace metal;++namespace uzu {+namespace matmul {++#else+// C/C++ for bindgen - use stdint types+#include <stdint.h>++#endif++///////////////////////////////////////////////////////////////////////////////+// GEMM Parameters+///////////////////////////////////////////////////////////////////////////////++/// Parameters for the main GEMM kernel.+/// Layout: 8 x int32, 3 x int64, 2 x int32 = 64 bytes total+struct GEMMParams {+ /// M dimension - batch/number of tokens (rows of A, rows of D)+ int batch;+ /// N dimension - output_dim (cols of B, cols of D)+ int output_dim;+ /// K dimension - input_dim/reduction dimension (cols of A, rows of B)+ int input_dim;++ int lda;+ int ldb;+ int ldd;++ int tiles_n;+ int tiles_m;++ int64_t batch_stride_a;+ int64_t batch_stride_b;+ int64_t batch_stride_d;++ int swizzle_log;+ int gemm_k_iterations_aligned;+};++/// Parameters for addmm (alpha * A @ B + beta * C) operations+struct GEMMAddMMParams {+ int ldc;+ int fdc;++ int64_t batch_stride_c;++ float alpha;+ float beta;+};++// Close Metal namespace+#ifdef __METAL_VERSION__+} // namespace matmul+} // namespace uzu+#endifdiff --git a/crates/uzu/src/backends/metal/kernel/matmul/shared_types.rs b/crates/uzu/src/backends/metal/kernel/matmul/shared_types.rsnew file mode 100644index 000000000..1e4f91113--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/shared_types.rs@@ -0,0 +1,83 @@+/* automatically generated by rust-bindgen 0.72.1 */++#[doc = " Parameters for the main GEMM kernel.\n Layout: 8 x int32, 3 x int64, 2 x int32 = 64 bytes total"]+#[repr(C)]+#[derive(Debug, Default, Copy, Clone)]+pub struct GEMMParams {+ #[doc = " M dimension - batch/number of tokens (rows of A, rows of D)"]+ pub batch: ::core::ffi::c_int,+ #[doc = " N dimension - output_dim (cols of B, cols of D)"]+ pub output_dim: ::core::ffi::c_int,+ #[doc = " K dimension - input_dim/reduction dimension (cols of A, rows of B)"]+ pub input_dim: ::core::ffi::c_int,+ pub lda: ::core::ffi::c_int,+ pub ldb: ::core::ffi::c_int,+ pub ldd: ::core::ffi::c_int,+ pub tiles_n: ::core::ffi::c_int,+ pub tiles_m: ::core::ffi::c_int,+ pub batch_stride_a: i64,+ pub batch_stride_b: i64,+ pub batch_stride_d: i64,+ pub swizzle_log: ::core::ffi::c_int,+ pub gemm_k_iterations_aligned: ::core::ffi::c_int,+}+#[allow(clippy::unnecessary_operation, clippy::identity_op)]+const _: () = {+ ["Size of GEMMParams"][::core::mem::size_of::<GEMMParams>() - 64usize];+ ["Alignment of GEMMParams"][::core::mem::align_of::<GEMMParams>() - 8usize];+ ["Offset of field: GEMMParams::batch"]+ [::core::mem::offset_of!(GEMMParams, batch) - 0usize];+ ["Offset of field: GEMMParams::output_dim"]+ [::core::mem::offset_of!(GEMMParams, output_dim) - 4usize];+ ["Offset of field: GEMMParams::input_dim"]+ [::core::mem::offset_of!(GEMMParams, input_dim) - 8usize];+ ["Offset of field: GEMMParams::lda"]+ [::core::mem::offset_of!(GEMMParams, lda) - 12usize];+ ["Offset of field: GEMMParams::ldb"]+ [::core::mem::offset_of!(GEMMParams, ldb) - 16usize];+ ["Offset of field: GEMMParams::ldd"]+ [::core::mem::offset_of!(GEMMParams, ldd) - 20usize];+ ["Offset of field: GEMMParams::tiles_n"]+ [::core::mem::offset_of!(GEMMParams, tiles_n) - 24usize];+ ["Offset of field: GEMMParams::tiles_m"]+ [::core::mem::offset_of!(GEMMParams, tiles_m) - 28usize];+ ["Offset of field: GEMMParams::batch_stride_a"]+ [::core::mem::offset_of!(GEMMParams, batch_stride_a) - 32usize];+ ["Offset of field: GEMMParams::batch_stride_b"]+ [::core::mem::offset_of!(GEMMParams, batch_stride_b) - 40usize];+ ["Offset of field: GEMMParams::batch_stride_d"]+ [::core::mem::offset_of!(GEMMParams, batch_stride_d) - 48usize];+ ["Offset of field: GEMMParams::swizzle_log"]+ [::core::mem::offset_of!(GEMMParams, swizzle_log) - 56usize];+ ["Offset of field: GEMMParams::gemm_k_iterations_aligned"][::core::mem::offset_of!(+ GEMMParams,+ gemm_k_iterations_aligned+ ) - 60usize];+};+#[doc = " Parameters for addmm (alpha * A @ B + beta * C) operations"]+#[repr(C)]+#[derive(Debug, Default, Copy, Clone)]+pub struct GEMMAddMMParams {+ pub ldc: ::core::ffi::c_int,+ pub fdc: ::core::ffi::c_int,+ pub batch_stride_c: i64,+ pub alpha: f32,+ pub beta: f32,+}+#[allow(clippy::unnecessary_operation, clippy::identity_op)]+const _: () = {+ ["Size of GEMMAddMMParams"]+ [::core::mem::size_of::<GEMMAddMMParams>() - 24usize];+ ["Alignment of GEMMAddMMParams"]+ [::core::mem::align_of::<GEMMAddMMParams>() - 8usize];+ ["Offset of field: GEMMAddMMParams::ldc"]+ [::core::mem::offset_of!(GEMMAddMMParams, ldc) - 0usize];+ ["Offset of field: GEMMAddMMParams::fdc"]+ [::core::mem::offset_of!(GEMMAddMMParams, fdc) - 4usize];+ ["Offset of field: GEMMAddMMParams::batch_stride_c"]+ [::core::mem::offset_of!(GEMMAddMMParams, batch_stride_c) - 8usize];+ ["Offset of field: GEMMAddMMParams::alpha"]+ [::core::mem::offset_of!(GEMMAddMMParams, alpha) - 16usize];+ ["Offset of field: GEMMAddMMParams::beta"]+ [::core::mem::offset_of!(GEMMAddMMParams, beta) - 20usize];+};diff --git a/crates/uzu/src/backends/metal/kernel/matmul/transpose.rs b/crates/uzu/src/backends/metal/kernel/matmul/transpose.rsnew file mode 100644index 000000000..d75b4d2c8--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/transpose.rs@@ -0,0 +1,30 @@+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+pub(super) enum TransposeConfiguration {+ NN,+ NT,+ TN,+ TT,+}++impl TransposeConfiguration {+ pub fn as_str(&self) -> &'static str {+ match self {+ TransposeConfiguration::NN => "nn",+ TransposeConfiguration::NT => "nt",+ TransposeConfiguration::TN => "tn",+ TransposeConfiguration::TT => "tt",+ }+ }+}++pub(super) fn transpose_configuration(+ a_t: bool,+ b_t: bool,+) -> TransposeConfiguration {+ match (a_t, b_t) {+ (false, false) => TransposeConfiguration::NN,+ (false, true) => TransposeConfiguration::NT,+ (true, false) => TransposeConfiguration::TN,+ (true, true) => TransposeConfiguration::TT,+ }+}diff --git a/crates/uzu/tests/gemm_test.rs b/crates/uzu/tests/gemm_test.rsnew file mode 100644index 000000000..baa84b764--- /dev/null+++ b/crates/uzu/tests/gemm_test.rs@@ -0,0 +1,244 @@+use half::{bf16, f16};+use metal::{Device, MTLResourceOptions};+use uzu::{+ DataType,+ backends::metal::{+ MTLContext,+ kernel::{MatmulArguments, MatmulKernel},+ },+};++fn create_test_context() -> Option<MTLContext> {+ let device = Device::system_default()?;+ let command_queue = device.new_command_queue();+ MTLContext::new(device, command_queue).ok()+}++fn matmul_ref(+ a: &[f32],+ b: &[f32],+ m: usize,+ k: usize,+ n: usize,+) -> Vec<f32> {+ let mut out = vec![0f32; m * n];+ for i in 0..m {+ for j in 0..n {+ let mut acc = 0f32;+ for p in 0..k {+ acc += a[i * k + p] * b[p * n + j];+ }+ out[i * n + j] = acc;+ }+ }+ out+}++#[test]+fn gemm_f16_64x64() {+ let Some(ctx) = create_test_context() else {+ return;+ }; // Skip if no Metal++ let m = 64;+ let k = 64;+ let n = 64;+ let a_f32: Vec<f32> =+ (0..(m * k)).map(|i| ((i % 13) as f32) * 0.01).collect();+ let b_f32: Vec<f32> =+ (0..(k * n)).map(|i| ((i % 17) as f32) * 0.02 - 0.1).collect();+ let exp = matmul_ref(&a_f32, &b_f32, m, k, n);++ let a: Vec<f16> = a_f32.iter().map(|&v| f16::from_f32(v)).collect();+ let b: Vec<f16> = b_f32.iter().map(|&v| f16::from_f32(v)).collect();++ let a_buf = ctx.device.new_buffer_with_data(+ a.as_ptr() as *const _,+ (a.len() * core::mem::size_of::<f16>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );+ let b_buf = ctx.device.new_buffer_with_data(+ b.as_ptr() as *const _,+ (b.len() * core::mem::size_of::<f16>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );+ let d_buf = ctx.device.new_buffer(+ (m * n * core::mem::size_of::<f16>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );++ let mut kernel = MatmulKernel::new(&ctx, DataType::F16, false, false)+ .expect("kernel new");++ let cb = ctx.command_queue.new_command_buffer().to_owned();+ let enc = cb.new_compute_command_encoder();+ kernel+ .encode(+ &ctx,+ &enc,+ MatmulArguments {+ a: &a_buf,+ b: &b_buf,+ d: &d_buf,+ batch: m as i32,+ input_dim: k as i32,+ output_dim: n as i32,+ lda: k as i32,+ ldb: n as i32,+ ldd: n as i32,+ batch_count: 1,+ },+ )+ .expect("encode");+ enc.end_encoding();+ cb.commit();+ cb.wait_until_completed();++ let got: Vec<f32> = unsafe {+ let ptr = d_buf.contents() as *const f16;+ let slice = core::slice::from_raw_parts(ptr, m * n);+ slice.iter().map(|&h| h.to_f32()).collect()+ };++ for (i, (&e, &g)) in exp.iter().zip(got.iter()).enumerate() {+ let diff = (e - g).abs();+ assert!(diff <= 0.1, "idx {i}: exp {e} got {g} diff {diff}");+ }+}++#[test]+fn gemm_bf16_32x48() {+ let Some(ctx) = create_test_context() else {+ return;+ }; // Skip if no Metal+ let m = 32;+ let k = 48;+ let n = 32;+ let a_f32: Vec<f32> =+ (0..(m * k)).map(|i| ((i % 7) as f32) * 0.03).collect();+ let b_f32: Vec<f32> =+ (0..(k * n)).map(|i| ((i % 5) as f32) * 0.04 - 0.2).collect();+ let exp = matmul_ref(&a_f32, &b_f32, m, k, n);++ let a: Vec<bf16> = a_f32.iter().map(|&v| bf16::from_f32(v)).collect();+ let b: Vec<bf16> = b_f32.iter().map(|&v| bf16::from_f32(v)).collect();++ let a_buf = ctx.device.new_buffer_with_data(+ a.as_ptr() as *const _,+ (a.len() * core::mem::size_of::<bf16>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );+ let b_buf = ctx.device.new_buffer_with_data(+ b.as_ptr() as *const _,+ (b.len() * core::mem::size_of::<bf16>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );+ let d_buf = ctx.device.new_buffer(+ (m * n * core::mem::size_of::<bf16>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );++ let mut kernel = MatmulKernel::new(&ctx, DataType::BF16, false, false)+ .expect("kernel new");++ let cb = ctx.command_queue.new_command_buffer().to_owned();+ let enc = cb.new_compute_command_encoder();+ kernel+ .encode(+ &ctx,+ &enc,+ MatmulArguments {+ a: &a_buf,+ b: &b_buf,+ d: &d_buf,+ batch: m as i32,+ input_dim: k as i32,+ output_dim: n as i32,+ lda: k as i32,+ ldb: n as i32,+ ldd: n as i32,+ batch_count: 1,+ },+ )+ .expect("encode");+ enc.end_encoding();+ cb.commit();+ cb.wait_until_completed();++ let got: Vec<f32> = unsafe {+ let ptr = d_buf.contents() as *const bf16;+ let slice = core::slice::from_raw_parts(ptr, m * n);+ slice.iter().map(|&h| h.to_f32()).collect()+ };++ for (i, (&e, &g)) in exp.iter().zip(got.iter()).enumerate() {+ let diff = (e - g).abs();+ assert!(diff <= 0.2, "idx {i}: exp {e} got {g} diff {diff}");+ }+}++#[test]+fn gemm_f32_16x20() {+ let Some(ctx) = create_test_context() else {+ return;+ }; // Skip if no Metal+ let m = 16;+ let k = 20;+ let n = 16;+ let a: Vec<f32> = (0..(m * k)).map(|i| ((i % 11) as f32) * 0.05).collect();+ let b: Vec<f32> =+ (0..(k * n)).map(|i| ((i % 9) as f32) * 0.02 - 0.3).collect();+ let exp = matmul_ref(&a, &b, m, k, n);++ let a_buf = ctx.device.new_buffer_with_data(+ a.as_ptr() as *const _,+ (a.len() * core::mem::size_of::<f32>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );+ let b_buf = ctx.device.new_buffer_with_data(+ b.as_ptr() as *const _,+ (b.len() * core::mem::size_of::<f32>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );+ let d_buf = ctx.device.new_buffer(+ (m * n * core::mem::size_of::<f32>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );++ let mut kernel = MatmulKernel::new(&ctx, DataType::F32, false, false)+ .expect("kernel new");+ let cb = ctx.command_queue.new_command_buffer().to_owned();+ let enc = cb.new_compute_command_encoder();+ kernel+ .encode(+ &ctx,+ &enc,+ MatmulArguments {+ a: &a_buf,+ b: &b_buf,+ d: &d_buf,+ batch: m as i32,+ input_dim: k as i32,+ output_dim: n as i32,+ lda: k as i32,+ ldb: n as i32,+ ldd: n as i32,+ batch_count: 1,+ },+ )+ .expect("encode");+ enc.end_encoding();+ cb.commit();+ cb.wait_until_completed();++ let got: Vec<f32> = unsafe {+ let ptr = d_buf.contents() as *const f32;+ let slice = core::slice::from_raw_parts(ptr, m * n);+ slice.to_vec()+ };++ for (i, (&e, &g)) in exp.iter().zip(got.iter()).enumerate() {+ let diff = (e - g).abs();+ assert!(diff <= 1e-3, "idx {i}: exp {e} got {g} diff {diff}");+ }+}