Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

introduce gemm

eugenea11f4d32eb07
diff --git a/Cargo.lock b/Cargo.lock
index a295ab640..c3df3983a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -323,6 +323,26 @@ version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "383d29d513d8764dcdc42ea295d979eb99c3c9f00607b3692cf68a431f7dca72"
+[[package]]
+name = "bindgen"
+version = "0.72.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895"
+dependencies = [
+ "bitflags 2.9.4",
+ "cexpr",
+ "clang-sys",
+ "itertools 0.13.0",
+ "log",
+ "prettyplease",
+ "proc-macro2",
+ "quote",
+ "regex",
+ "rustc-hash",
+ "shlex",
+ "syn 2.0.102",
+]
+
[[package]]
name = "bitflags"
version = "1.3.2"
@@ -4071,6 +4091,7 @@ dependencies = [
name = "uzu"
version = "0.1.7"
dependencies = [
+ "bindgen",
"bitflags 2.9.4",
"block",
"bytemuck",
diff --git a/crates/uzu/Cargo.toml b/crates/uzu/Cargo.toml
index d10698ffc..211820eb1 100644
--- a/crates/uzu/Cargo.toml
+++ b/crates/uzu/Cargo.toml
@@ -30,6 +30,9 @@ sysinfo = "0.37.2"
itertools = "0.14.0"
schemars = "1.0.5"
+[build-dependencies]
+bindgen = "0.72.1"
+
[dev-dependencies]
is_close = "0.1.3"
serde_json5 = "0.2.1"
diff --git a/crates/uzu/build.rs b/crates/uzu/build.rs
index f3c06e90f..5f22b39ff 100644
--- a/crates/uzu/build.rs
+++ b/crates/uzu/build.rs
@@ -8,6 +8,9 @@ use std::{
};
fn main() {
+ // Generate Rust bindings from shared Metal/C headers
+ generate_metal_bindings();
+
if cfg!(feature = "metal-shaders") {
println!("cargo:rerun-if-env-changed=MY_API_LEVEL");
compile_metal_shaders();
@@ -16,6 +19,39 @@ fn main() {
}
}
+/// Generate Rust bindings from shared C headers using bindgen.
+/// These headers are also used by Metal shaders, ensuring type consistency.
+fn generate_metal_bindings() {
+ let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());
+ let matmul_dir = manifest_dir.join("src/backends/metal/kernel/matmul");
+ let types_header = matmul_dir.join("shared_types.h");
+
+ // Only regenerate if header changed
+ println!("cargo:rerun-if-changed={}", types_header.display());
+
+ let bindings = bindgen::Builder::default()
+ .header(types_header.to_string_lossy())
+ // Generate for C mode (no Metal-specific stuff)
+ .clang_arg("-x")
+ .clang_arg("c")
+ // Derive common traits
+ .derive_default(true)
+ .derive_copy(true)
+ // Only generate types we care about
+ .allowlist_type("GEMMParams")
+ .allowlist_type("GEMMAddMMParams")
+ // Use core types
+ .use_core()
+ .generate()
+ .expect("Failed to generate bindings from shared_types.h");
+
+ // Write bindings to the source directory so it can be checked in
+ let bindings_path = matmul_dir.join("shared_types.rs");
+ bindings
+ .write_to_file(&bindings_path)
+ .expect("Failed to write shared_types.rs");
+}
+
fn compile_metal_shaders() {
// Determine target platform (only proceed on Apple platforms)
let target = env::var("TARGET").unwrap_or_default();
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/arguments.rs b/crates/uzu/src/backends/metal/kernel/matmul/arguments.rs
new file mode 100644
index 000000000..65b8ba5e3
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/arguments.rs
@@ -0,0 +1,19 @@
+use metal::Buffer as MTLBuffer;
+
+#[derive(Debug, Clone, Copy)]
+pub struct MatmulArguments<'a> {
+ pub a: &'a MTLBuffer,
+ pub b: &'a MTLBuffer,
+ pub d: &'a MTLBuffer,
+ /// M dimension - batch/number of tokens (rows of A, rows of D)
+ pub batch: i32,
+ /// K dimension - input_dim/reduction dimension (cols of A, rows of B)
+ pub input_dim: i32,
+ /// N dimension - output_dim (cols of B, cols of D)
+ pub output_dim: i32,
+ pub lda: i32,
+ pub ldb: i32,
+ pub ldd: i32,
+ /// Number of batched matrix multiplications (z-dimension)
+ pub batch_count: i32,
+}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/defines.h b/crates/uzu/src/backends/metal/kernel/matmul/defines.h
new file mode 100644
index 000000000..07ca0a072
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/defines.h
@@ -0,0 +1,80 @@
+#pragma once
+
+#include <metal_simdgroup>
+#include <metal_simdgroup_matrix>
+#include <metal_stdlib>
+
+#define UZU_MTL_CONST static constant constexpr
+#define UZU_PRAGMA_UNROLL _Pragma("clang loop unroll(full)")
+#define SIMD_SIZE 32
+
+using namespace metal;
+
+namespace uzu {
+namespace matmul {
+
+// Pointer element type extraction
+template <typename T>
+struct pointer_element_t_impl {
+ using type = T;
+};
+template <typename T>
+struct pointer_element_t_impl<device T*> {
+ using type = T;
+};
+template <typename T>
+struct pointer_element_t_impl<const device T*> {
+ using type = T;
+};
+template <typename T>
+struct pointer_element_t_impl<threadgroup T*> {
+ using type = T;
+};
+template <typename T>
+using pointer_element_t = typename pointer_element_t_impl<T>::type;
+
+// Accumulator type helper
+template <typename T>
+struct AccumHelper {
+ typedef float accum_type;
+};
+
+// Transform operations for epilogue
+// Note: apply(x) is static for default store_result usage
+// apply(x, c) is non-static for epilogue operations with C matrix
+template <typename OutT, typename InT>
+struct TransformNone {
+ TransformNone(float = 1.0f, float = 0.0f) {}
+
+ static METAL_FUNC OutT apply(InT x) { return static_cast<OutT>(x); }
+ METAL_FUNC OutT apply(InT x, OutT) const { return static_cast<OutT>(x); }
+};
+
+template <typename OutT, typename InT>
+struct TransformAdd {
+ TransformAdd(float = 1.0f, float = 1.0f) {}
+
+ static METAL_FUNC OutT apply(InT x) { return static_cast<OutT>(x); }
+ METAL_FUNC OutT apply(InT x, OutT c) const {
+ return static_cast<OutT>(x) + c;
+ }
+};
+
+template <typename OutT, typename InT>
+struct TransformAxpby {
+ const float alpha;
+ const float beta;
+
+ TransformAxpby(float alpha_, float beta_) : alpha(alpha_), beta(beta_) {}
+
+ static METAL_FUNC OutT apply(InT x) { return static_cast<OutT>(x); }
+
+ METAL_FUNC OutT apply(InT x, OutT c) const {
+ return static_cast<OutT>(
+ x * static_cast<InT>(alpha) + (static_cast<OutT>(beta) * c)
+ );
+ }
+};
+
+} // namespace matmul
+} // namespace uzu
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm.h b/crates/uzu/src/backends/metal/kernel/matmul/gemm.h
new file mode 100644
index 000000000..4c825a159
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm.h
@@ -0,0 +1,300 @@
+#pragma once
+
+#include "loader.h"
+#include "mma.h"
+#include "shared_types.h"
+
+using namespace metal;
+
+namespace uzu {
+namespace matmul {
+
+///////////////////////////////////////////////////////////////////////////////
+// Loop Alignment Helper
+///////////////////////////////////////////////////////////////////////////////
+
+template <bool M_aligned, bool N_aligned, bool K_aligned>
+struct LoopAlignment {};
+
+///////////////////////////////////////////////////////////////////////////////
+// GEMM Kernel
+///////////////////////////////////////////////////////////////////////////////
+
+template <
+ typename T,
+ typename U,
+ int BM,
+ int BN,
+ int BK,
+ int WM,
+ int WN,
+ bool transpose_a,
+ bool transpose_b,
+ bool MN_aligned,
+ bool K_aligned,
+ typename AccumType = float,
+ typename Epilogue = TransformNone<U, AccumType>>
+struct GEMMKernel {
+ UZU_MTL_CONST short tgp_padding_a = 16 / sizeof(T);
+ UZU_MTL_CONST short tgp_padding_b = 16 / sizeof(T);
+ UZU_MTL_CONST short tgp_mem_size_a =
+ transpose_a ? BK * (BM + tgp_padding_a) : BM * (BK + tgp_padding_a);
+ UZU_MTL_CONST short tgp_mem_size_b =
+ transpose_b ? BN * (BK + tgp_padding_b) : BK * (BN + tgp_padding_b);
+ UZU_MTL_CONST short tgp_mem_size = tgp_mem_size_a + tgp_mem_size_b;
+
+ UZU_MTL_CONST short tgp_size = WM * WN * 32;
+
+ using loader_a_t = BlockLoader<
+ T,
+ transpose_a ? BK : BM,
+ transpose_a ? BM : BK,
+ transpose_a ? BM + tgp_padding_a : BK + tgp_padding_a,
+ !transpose_a,
+ tgp_size>;
+ using loader_b_t = BlockLoader<
+ T,
+ transpose_b ? BN : BK,
+ transpose_b ? BK : BN,
+ transpose_b ? BK + tgp_padding_b : BN + tgp_padding_b,
+ transpose_b,
+ tgp_size>;
+ using mma_t = BlockMMA<
+ T,
+ U,
+ BM,
+ BN,
+ BK,
+ WM,
+ WN,
+ transpose_a,
+ transpose_b,
+ transpose_a ? BM + tgp_padding_a : BK + tgp_padding_a,
+ transpose_b ? BK + tgp_padding_b : BN + tgp_padding_b,
+ AccumType,
+ Epilogue>;
+
+ /* Main kernel function */
+ template <bool M_aligned, bool N_aligned, bool K_aligned_>
+ static METAL_FUNC void gemm_loop(
+ threadgroup T* As,
+ threadgroup T* Bs,
+ const int gemm_k_iterations,
+ thread loader_a_t& loader_a,
+ thread loader_b_t& loader_b,
+ thread mma_t& mma_op,
+ thread const short& tgp_bm,
+ thread const short& tgp_bn,
+ thread const short& lbk,
+ LoopAlignment<M_aligned, N_aligned, K_aligned_> l = {}
+ ) {
+ // Appease the compiler
+ (void)l;
+
+ short2 tile_dims_A = transpose_a ? short2(tgp_bm, BK) : short2(BK, tgp_bm);
+ short2 tile_dims_B = transpose_b ? short2(BK, tgp_bn) : short2(tgp_bn, BK);
+
+ for (int k = 0; k < gemm_k_iterations; k++) {
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ // Load elements into threadgroup
+ if (M_aligned) {
+ loader_a.load_unsafe();
+ } else {
+ loader_a.load_safe(tile_dims_A);
+ }
+
+ if (N_aligned) {
+ loader_b.load_unsafe();
+ } else {
+ loader_b.load_safe(tile_dims_B);
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // Multiply and accumulate threadgroup elements
+ mma_op.mma(As, Bs);
+
+ // Prepare for next iteration
+ loader_a.next();
+ loader_b.next();
+ }
+
+ if (!K_aligned_) {
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ short2 tile_dims_A_last =
+ transpose_a ? short2(tgp_bm, lbk) : short2(lbk, tgp_bm);
+ short2 tile_dims_B_last =
+ transpose_b ? short2(lbk, tgp_bn) : short2(tgp_bn, lbk);
+
+ loader_a.load_safe(tile_dims_A_last);
+ loader_b.load_safe(tile_dims_B_last);
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ mma_op.mma(As, Bs);
+ }
+ }
+
+ /* Main kernel function */
+ static METAL_FUNC void run(
+ const device T* A,
+ const device T* B,
+ device U* D,
+ const constant GEMMParams* params,
+ threadgroup T* As,
+ threadgroup T* Bs,
+ uint simd_lane_id [[thread_index_in_simdgroup]],
+ uint simd_group_id [[simdgroup_index_in_threadgroup]],
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint3 lid [[thread_position_in_threadgroup]]
+ ) {
+ // Pacifying compiler
+ (void)lid;
+
+ const int tid_y = ((tid.y) << params->swizzle_log) +
+ ((tid.x) & ((1 << params->swizzle_log) - 1));
+ const int tid_x = (tid.x) >> params->swizzle_log;
+
+ if (params->tiles_n <= tid_x || params->tiles_m <= tid_y) {
+ return;
+ }
+
+ threadgroup_barrier(mem_flags::mem_none);
+
+ // Find block in A, B, C
+ const int c_row = tid_y * BM;
+ const int c_col = tid_x * BN;
+ const size_t c_row_long = size_t(c_row);
+ const size_t c_col_long = size_t(c_col);
+
+ A += transpose_a ? c_row_long : c_row_long * params->lda;
+ B += transpose_b ? c_col_long * params->ldb : c_col_long;
+ D += c_row_long * params->ldd + c_col_long;
+
+ // Prepare threadgroup loading operations
+ thread loader_a_t loader_a(A, params->lda, As, simd_group_id, simd_lane_id);
+ thread loader_b_t loader_b(B, params->ldb, Bs, simd_group_id, simd_lane_id);
+
+ // Prepare threadgroup mma operation
+ thread mma_t mma_op(simd_group_id, simd_lane_id);
+
+ int gemm_k_iterations = params->gemm_k_iterations_aligned;
+
+ ///////////////////////////////////////////////////////////////////////////
+ // MNK aligned loop
+ if (MN_aligned) {
+ for (int k = 0; k < gemm_k_iterations; k++) {
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ // Load elements into threadgroup
+ loader_a.load_unsafe();
+ loader_b.load_unsafe();
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // Multiply and accumulate threadgroup elements
+ mma_op.mma(As, Bs);
+
+ // Prepare for next iteration
+ loader_a.next();
+ loader_b.next();
+ }
+
+ threadgroup_barrier(mem_flags::mem_none);
+
+ // Loop tail
+ if (!K_aligned) {
+ int lbk = params->input_dim - params->gemm_k_iterations_aligned * BK;
+ short2 tile_dims_A = transpose_a ? short2(BM, lbk) : short2(lbk, BM);
+ short2 tile_dims_B = transpose_b ? short2(lbk, BN) : short2(BN, lbk);
+
+ loader_a.load_safe(tile_dims_A);
+ loader_b.load_safe(tile_dims_B);
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ mma_op.mma(As, Bs);
+ }
+
+ // Store results to device memory
+ mma_op.store_result(D, params->ldd);
+ return;
+ }
+ ///////////////////////////////////////////////////////////////////////////
+ // MN unaligned loop
+ else { // Loop over K - unaligned case
+ short tgp_bm = min(BM, params->batch - c_row);
+ short tgp_bn = min(BN, params->output_dim - c_col);
+ short leftover_bk =
+ params->input_dim - params->gemm_k_iterations_aligned * BK;
+
+ if (tgp_bm == BM && tgp_bn == BN) {
+ gemm_loop<true, true, K_aligned>(
+ As,
+ Bs,
+ gemm_k_iterations,
+ loader_a,
+ loader_b,
+ mma_op,
+ tgp_bm,
+ tgp_bn,
+ leftover_bk
+ );
+
+ mma_op.store_result(D, params->ldd);
+ return;
+
+ } else if (tgp_bn == BN) {
+ gemm_loop<false, true, K_aligned>(
+ As,
+ Bs,
+ gemm_k_iterations,
+ loader_a,
+ loader_b,
+ mma_op,
+ tgp_bm,
+ tgp_bn,
+ leftover_bk
+ );
+
+ mma_op.store_result_safe(D, params->ldd, short2(tgp_bn, tgp_bm));
+ return;
+
+ } else if (tgp_bm == BM) {
+ gemm_loop<true, false, K_aligned>(
+ As,
+ Bs,
+ gemm_k_iterations,
+ loader_a,
+ loader_b,
+ mma_op,
+ tgp_bm,
+ tgp_bn,
+ leftover_bk
+ );
+
+ mma_op.store_result_safe(D, params->ldd, short2(tgp_bn, tgp_bm));
+ return;
+
+ } else {
+ gemm_loop<false, false, K_aligned>(
+ As,
+ Bs,
+ gemm_k_iterations,
+ loader_a,
+ loader_b,
+ mma_op,
+ tgp_bm,
+ tgp_bn,
+ leftover_bk
+ );
+
+ mma_op.store_result_safe(D, params->ldd, short2(tgp_bn, tgp_bm));
+ return;
+ }
+ }
+ }
+};
+
+} // namespace matmul
+} // namespace uzu
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs
new file mode 100644
index 000000000..2846419a8
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs
@@ -0,0 +1,195 @@
+use std::collections::HashMap;
+
+use metal::{
+ ComputeCommandEncoderRef, ComputePipelineState as MTLComputePipelineState,
+ MTLSize,
+};
+
+use super::{
+ arguments::MatmulArguments, pipeline::PipelineKey,
+ shared_types::GEMMParams, transpose::transpose_configuration,
+};
+use crate::{
+ DataType,
+ backends::metal::{MTLContext, MTLError},
+};
+
+pub struct MatmulKernel {
+ dt: DataType,
+ transpose_a: bool,
+ transpose_b: bool,
+ bm: i32,
+ bn: i32,
+ bk: i32,
+ wm: u64,
+ wn: u64,
+ pipelines: HashMap<PipelineKey, MTLComputePipelineState>,
+}
+
+impl MatmulKernel {
+ fn dtype_suffix(dt: DataType) -> &'static str {
+ match dt {
+ DataType::F16 => "f16",
+ DataType::BF16 => "bf16",
+ DataType::F32 => "f32",
+ _ => unreachable!(),
+ }
+ }
+
+ fn kernel_name(&self) -> String {
+ let t = Self::dtype_suffix(self.dt);
+ let cfg = transpose_configuration(self.transpose_a, self.transpose_b);
+ let tcfg = cfg.as_str();
+ // Shapes we currently instantiate in matmul.metal: bm64/bn64/bk16/wm2/wn2
+ format!("gemm_{}_{}_bm64_bn64_bk16_wm2_wn2", tcfg, t)
+ }
+
+ pub fn new(
+ _mtl: &MTLContext,
+ dt: DataType,
+ transpose_a: bool,
+ transpose_b: bool,
+ ) -> Result<Self, MTLError> {
+ if !matches!(dt, DataType::F16 | DataType::BF16 | DataType::F32) {
+ return Err(MTLError::Generic(format!(
+ "Unsupported dtype for MatmulKernel: {dt:?}"
+ )));
+ }
+
+ Ok(Self {
+ dt,
+ transpose_a,
+ transpose_b,
+ bm: 64,
+ bn: 64,
+ bk: 16,
+ wm: 2,
+ wn: 2,
+ pipelines: HashMap::new(),
+ })
+ }
+
+ fn get_or_compile_pipeline(
+ &mut self,
+ mtl: &MTLContext,
+ name: &str,
+ align_m: bool,
+ align_n: bool,
+ align_k: bool,
+ ) -> Result<&MTLComputePipelineState, MTLError> {
+ let key = PipelineKey {
+ name: name.to_string(),
+ align_m,
+ align_n,
+ align_k,
+ };
+ if !self.pipelines.contains_key(&key) {
+ let fcv = metal::FunctionConstantValues::new();
+ // Base specializations
+ let has_batch = false;
+ let use_out_source = false;
+ let do_axpby = false;
+ fcv.set_constant_value_at_index(
+ &has_batch as *const bool as *const _,
+ metal::MTLDataType::Bool,
+ 10,
+ );
+ fcv.set_constant_value_at_index(
+ &use_out_source as *const bool as *const _,
+ metal::MTLDataType::Bool,
+ 100,
+ );
+ fcv.set_constant_value_at_index(
+ &do_axpby as *const bool as *const _,
+ metal::MTLDataType::Bool,
+ 110,
+ );
+ fcv.set_constant_value_at_index(
+ &align_m as *const bool as *const _,
+ metal::MTLDataType::Bool,
+ 200,
+ );
+ fcv.set_constant_value_at_index(
+ &align_n as *const bool as *const _,
+ metal::MTLDataType::Bool,
+ 201,
+ );
+ fcv.set_constant_value_at_index(
+ &align_k as *const bool as *const _,
+ metal::MTLDataType::Bool,
+ 202,
+ );
+
+ let cache_key = format!(
+ "{}_am{}_an{}_ak{}",
+ name, align_m as u8, align_n as u8, align_k as u8
+ );
+ let (ps, _) = mtl.compute_pipeline_state_with_reflection_cached(
+ &cache_key,
+ name,
+ Some(&fcv),
+ )?;
+ self.pipelines.insert(key.clone(), ps);
+ }
+ Ok(self.pipelines.get(&key).unwrap())
+ }
+
+ pub fn encode(
+ &mut self,
+ mtl: &MTLContext,
+ enc: &ComputeCommandEncoderRef,
+ args: MatmulArguments,
+ ) -> Result<(), MTLError> {
+ let kname = self.kernel_name();
+
+ // M = batch, N = output_dim, K = input_dim
+ let m = args.batch;
+ let n = args.output_dim;
+ let k = args.input_dim;
+
+ let am = (m % self.bm) == 0;
+ let an = (n % self.bn) == 0;
+ let ak = (k % self.bk) == 0;
+ let ps = self.get_or_compile_pipeline(mtl, &kname, am, an, ak)?;
+
+ enc.set_compute_pipeline_state(ps);
+
+ // Set buffers (C is elided since use_out_source=false)
+ enc.set_buffer(0, Some(args.a), 0);
+ enc.set_buffer(1, Some(args.b), 0);
+ enc.set_buffer(3, Some(args.d), 0);
+
+ // Params
+ let tiles_n = (n + self.bn - 1) / self.bn;
+ let tiles_m = (m + self.bm - 1) / self.bm;
+ let params = GEMMParams {
+ batch: m,
+ output_dim: n,
+ input_dim: k,
+ lda: args.lda,
+ ldb: args.ldb,
+ ldd: args.ldd,
+ tiles_n,
+ tiles_m,
+ batch_stride_a: (args.lda as i64) * (k as i64),
+ batch_stride_b: (args.ldb as i64) * (n as i64),
+ batch_stride_d: (args.ldd as i64) * (n as i64),
+ swizzle_log: 0,
+ gemm_k_iterations_aligned: k / self.bk,
+ };
+ enc.set_bytes(
+ 4,
+ std::mem::size_of::<GEMMParams>() as u64,
+ &params as *const GEMMParams as *const _,
+ );
+
+ // Threadgroup sizing
+ let threads_per_tg = MTLSize::new(32, self.wn, self.wm);
+ let tg_x = ((n as i64 + self.bn as i64 - 1) / self.bn as i64) as u64;
+ let tg_y = ((m as i64 + self.bm as i64 - 1) / self.bm as i64) as u64;
+ let tg_z = args.batch_count as u64;
+ let tgs = MTLSize::new(tg_x, tg_y, tg_z);
+ enc.dispatch_thread_groups(tgs, threads_per_tg);
+ Ok(())
+ }
+}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/loader.h b/crates/uzu/src/backends/metal/kernel/matmul/loader.h
new file mode 100644
index 000000000..7abc01db1
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/loader.h
@@ -0,0 +1,200 @@
+#pragma once
+
+#include "defines.h"
+
+using namespace metal;
+
+namespace uzu {
+namespace matmul {
+
+///////////////////////////////////////////////////////////////////////////////
+// Block Loader - loads tiles from device memory to threadgroup memory
+///////////////////////////////////////////////////////////////////////////////
+
+template <
+ typename T,
+ short BROWS,
+ short BCOLS,
+ short dst_ld,
+ short reduction_dim,
+ short tgp_size,
+ short alignment = 1,
+ short n_reads = (BCOLS * BROWS) / (tgp_size),
+ short TCOLS = BCOLS / n_reads,
+ short TROWS = tgp_size / TCOLS>
+struct BlockLoader {
+ UZU_MTL_CONST short n_rows = (BROWS + TROWS - 1) / TROWS;
+ UZU_MTL_CONST short vec_size = n_reads;
+
+ // Leading dimension for src
+ const int src_ld;
+ const int tile_stride;
+
+ // Thread location indices
+ const short thread_idx;
+ const short bi;
+ const short bj;
+
+ // threadgroup and device memory
+ threadgroup T* dst;
+ const device T* src;
+
+ struct alignas(alignment * sizeof(T)) ReadVector {
+ uint8_t v[sizeof(T) * vec_size];
+ };
+
+ /* Constructor */
+ METAL_FUNC BlockLoader(
+ const device T* src_,
+ const int src_ld_,
+ threadgroup T* dst_,
+ ushort simd_group_id [[simdgroup_index_in_threadgroup]],
+ ushort simd_lane_id [[thread_index_in_simdgroup]]
+ )
+ : src_ld(src_ld_), tile_stride(reduction_dim ? BCOLS : BROWS * src_ld),
+ thread_idx(simd_group_id * 32 + simd_lane_id), bi(thread_idx / TCOLS),
+ bj(vec_size * (thread_idx % TCOLS)), dst(dst_ + bi * dst_ld + bj),
+ src(src_ + bi * src_ld + bj) {}
+
+ /* Apply operation to threadgroup without bound checking */
+ template <typename UnaryOp>
+ METAL_FUNC void apply_inplace_op(thread const UnaryOp& op) const {
+ UZU_PRAGMA_UNROLL
+ for (short i = 0; i < BROWS; i += TROWS) {
+ UZU_PRAGMA_UNROLL
+ for (short j = 0; j < vec_size; j++) {
+ dst[i * dst_ld + j] = op.apply(dst[i * dst_ld + j]);
+ }
+ }
+ }
+
+ /* Load from device memory into threadgroup memory - without bound checking */
+ METAL_FUNC void load_unsafe() const {
+ UZU_PRAGMA_UNROLL
+ for (short i = 0; i < BROWS; i += TROWS) {
+ *((threadgroup ReadVector*)(&dst[i * dst_ld])) =
+ *((const device ReadVector*)(&src[i * src_ld]));
+ }
+ }
+
+ /* Load from device memory into threadgroup memory - with bound checking */
+ METAL_FUNC void load_safe(short2 src_tile_dim) const {
+ src_tile_dim = src_tile_dim - short2(bj, bi);
+
+ // Skip loading if thread has no valid reads
+ if (src_tile_dim.x <= 0 || src_tile_dim.y <= 0) {
+ UZU_PRAGMA_UNROLL
+ for (short i = 0; i < BROWS; i += TROWS) {
+ UZU_PRAGMA_UNROLL
+ for (short j = 0; j < vec_size; j++) {
+ dst[i * dst_ld + j] = T(0);
+ }
+ }
+ return;
+ }
+
+ // Use fast thread memory for bound checks
+ bool tmp_idx[vec_size];
+ T tmp_val[vec_size];
+
+ UZU_PRAGMA_UNROLL
+ for (short i = 0; i < BROWS; i += TROWS) {
+ // Make sure tmp_idx only contains valid indices
+ UZU_PRAGMA_UNROLL
+ for (short j = 0; j < vec_size; j++) {
+ tmp_idx[j] = (i < src_tile_dim.y) && (j < src_tile_dim.x);
+ }
+
+ // Read valid indices into tmp_val
+ UZU_PRAGMA_UNROLL
+ for (short j = 0; j < vec_size; j++) {
+ tmp_val[j] = src[(tmp_idx[j] ? i * src_ld + j : 0)];
+ }
+
+ // Zero out unneeded values
+ UZU_PRAGMA_UNROLL
+ for (short j = 0; j < vec_size; j++) {
+ tmp_val[j] = tmp_idx[j] ? tmp_val[j] : T(0);
+ }
+
+ // Copy values to threadgroup memory
+ UZU_PRAGMA_UNROLL
+ for (short j = 0; j < vec_size; j++) {
+ dst[i * dst_ld + j] = tmp_val[j];
+ }
+ }
+ }
+
+ /* Iteration helper */
+ METAL_FUNC void next() { src += tile_stride; }
+};
+
+///////////////////////////////////////////////////////////////////////////////
+// Simple Block Loader - simpler version for smaller tiles
+///////////////////////////////////////////////////////////////////////////////
+
+template <
+ typename T,
+ int BM,
+ int BK,
+ int lda_tgp,
+ int reduction_dim,
+ int tgp_size,
+ int align_M = 1,
+ int align_K = 4>
+struct SimpleBlockLoader {
+ UZU_MTL_CONST short n_reads = (BM * BK < tgp_size) ? 1 : (BM * BK) / tgp_size;
+
+ const int src_ld;
+ const short thread_idx;
+ const short bi;
+ const short bj;
+
+ threadgroup T* dst;
+ const device T* src;
+
+ SimpleBlockLoader(
+ const device T* src_,
+ const int src_ld_,
+ threadgroup T* dst_,
+ ushort simd_group_id [[simdgroup_index_in_threadgroup]],
+ ushort simd_lane_id [[thread_index_in_simdgroup]]
+ )
+ : src_ld(src_ld_), thread_idx(simd_group_id * 32 + simd_lane_id),
+ bi(n_reads * thread_idx / BK), bj((n_reads * thread_idx) % BK),
+ dst(dst_ + bi * lda_tgp + bj), src(src_ + bi * src_ld + bj) {}
+
+ void load_unsafe() const {
+ if (BM * BK < tgp_size && bi >= BM) {
+ return;
+ }
+
+ for (int i = 0; i < n_reads; i++) {
+ dst[i] = src[i];
+ }
+ }
+
+ void load_safe(short2 src_tile_dim) const {
+ if (BM * BK < tgp_size && bi >= BM) {
+ return;
+ }
+
+ if (bi >= src_tile_dim.y || bj >= src_tile_dim.x) {
+ for (int i = 0; i < n_reads; i++) {
+ dst[i] = T(0);
+ }
+ return;
+ }
+
+ for (int i = 0; i < n_reads; i++) {
+ dst[i] = src[i];
+ }
+ }
+
+ void next() {
+ src += BK; // Advance by tile size
+ }
+};
+
+} // namespace matmul
+} // namespace uzu
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/matmul.metal b/crates/uzu/src/backends/metal/kernel/matmul/matmul.metal
new file mode 100644
index 000000000..bc5fb5664
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/matmul.metal
@@ -0,0 +1,570 @@
+// SPDX-License-Identifier: MIT
+// General matrix multiplication kernels for uzu
+// Adapted from MLX steel GEMM implementation
+
+#include <metal_stdlib>
+#include "gemm.h"
+
+using namespace metal;
+using namespace uzu::matmul;
+
+///////////////////////////////////////////////////////////////////////////////
+// Function constants for compile-time specialization
+///////////////////////////////////////////////////////////////////////////////
+
+constant bool has_batch [[function_constant(10)]];
+
+constant bool use_out_source [[function_constant(100)]];
+constant bool do_axpby [[function_constant(110)]];
+
+constant bool align_M [[function_constant(200)]];
+constant bool align_N [[function_constant(201)]];
+constant bool align_K [[function_constant(202)]];
+
+///////////////////////////////////////////////////////////////////////////////
+// GEMM kernel template
+///////////////////////////////////////////////////////////////////////////////
+
+template <
+ typename T,
+ int BM,
+ int BN,
+ int BK,
+ int WM,
+ int WN,
+ bool transpose_a,
+ bool transpose_b,
+ typename AccumType = float>
+[[kernel, max_total_threads_per_threadgroup(WM * WN * 32)]] void gemm(
+ const device T* input_a [[buffer(0)]],
+ const device T* input_b [[buffer(1)]],
+ const device T* input_c [[buffer(2), function_constant(use_out_source)]],
+ device T* output [[buffer(3)]],
+ const constant GEMMParams* params [[buffer(4)]],
+ const constant GEMMAddMMParams* addmm_params
+ [[buffer(5), function_constant(use_out_source)]],
+ const constant int* batch_shape [[buffer(6), function_constant(has_batch)]],
+ const constant int64_t* batch_strides
+ [[buffer(7), function_constant(has_batch)]],
+ uint simd_lane_id [[thread_index_in_simdgroup]],
+ uint simd_group_id [[simdgroup_index_in_threadgroup]],
+ uint3 threadgroup_position [[threadgroup_position_in_grid]],
+ uint3 thread_position [[thread_position_in_threadgroup]]
+) {
+ // Pacifying compiler
+ (void)thread_position;
+
+ using gemm_kernel = GEMMKernel<
+ T,
+ T,
+ BM,
+ BN,
+ BK,
+ WM,
+ WN,
+ transpose_a,
+ transpose_b,
+ true,
+ true,
+ AccumType>;
+
+ using loader_a_t = typename gemm_kernel::loader_a_t;
+ using loader_b_t = typename gemm_kernel::loader_b_t;
+ using mma_t = typename gemm_kernel::mma_t;
+
+ // Find block with swizzle
+ const int threadgroup_y =
+ ((threadgroup_position.y) << params->swizzle_log) +
+ ((threadgroup_position.x) & ((1 << params->swizzle_log) - 1));
+ const int threadgroup_x = (threadgroup_position.x) >> params->swizzle_log;
+
+ // Exit early if out of bounds
+ if (params->tiles_n <= threadgroup_x || params->tiles_m <= threadgroup_y) {
+ return;
+ }
+
+ // Adjust for batch
+ if (has_batch) {
+ // Simple batch striding (no broadcast)
+ input_a += params->batch_stride_a * threadgroup_position.z;
+ input_b += params->batch_stride_b * threadgroup_position.z;
+
+ if (use_out_source) {
+ input_c += addmm_params->batch_stride_c * threadgroup_position.z;
+ }
+ } else {
+ input_a += params->batch_stride_a * threadgroup_position.z;
+ input_b += params->batch_stride_b * threadgroup_position.z;
+
+ if (use_out_source) {
+ input_c += addmm_params->batch_stride_c * threadgroup_position.z;
+ }
+ }
+
+ output += params->batch_stride_d * threadgroup_position.z;
+
+ // Prepare threadgroup memory
+ threadgroup T threadgroup_a[gemm_kernel::tgp_mem_size_a];
+ threadgroup T threadgroup_b[gemm_kernel::tgp_mem_size_b];
+
+ threadgroup_barrier(mem_flags::mem_none);
+
+ // Find block in input_a, input_b, input_c
+ const int output_row = threadgroup_y * BM;
+ const int output_col = threadgroup_x * BN;
+ const size_t output_row_long = size_t(output_row);
+ const size_t output_col_long = size_t(output_col);
+
+ input_a += transpose_a ? output_row_long : output_row_long * params->lda;
+ input_b += transpose_b ? output_col_long * params->ldb : output_col_long;
+ output += output_row_long * params->ldd + output_col_long;
+
+ if (use_out_source) {
+ input_c += output_row_long * addmm_params->ldc +
+ output_col_long * addmm_params->fdc;
+ }
+
+ // Prepare threadgroup mma operation
+ thread mma_t matrix_multiply_accumulate(simd_group_id, simd_lane_id);
+
+ // Prepare threadgroup loading operations
+ thread loader_a_t block_loader_a(
+ input_a,
+ params->lda,
+ threadgroup_a,
+ simd_group_id,
+ simd_lane_id
+ );
+ thread loader_b_t block_loader_b(
+ input_b,
+ params->ldb,
+ threadgroup_b,
+ simd_group_id,
+ simd_lane_id
+ );
+
+ // Prepare threadgroup bounds
+ const short threadgroup_block_batch =
+ align_M ? BM : short(min(BM, params->batch - output_row));
+ const short threadgroup_block_output =
+ align_N ? BN : short(min(BN, params->output_dim - output_col));
+
+ // Prepare iterations
+ int gemm_k_iterations = params->gemm_k_iterations_aligned;
+
+ // Do unaligned K iterations first
+ if (!align_K) {
+ const int input_aligned_end = params->gemm_k_iterations_aligned * BK;
+ const int input_remainder = params->input_dim - input_aligned_end;
+ const size_t input_offset_a = transpose_a
+ ? params->lda * size_t(input_aligned_end)
+ : size_t(input_aligned_end);
+ const size_t input_offset_b = transpose_b
+ ? size_t(input_aligned_end)
+ : params->ldb * size_t(input_aligned_end);
+
+ // Move loader source ahead to end
+ block_loader_a.src += input_offset_a;
+ block_loader_b.src += input_offset_b;
+
+ // Load tile
+ const short2 tile_dimensions_a =
+ transpose_a ? short2(threadgroup_block_batch, input_remainder)
+ : short2(input_remainder, threadgroup_block_batch);
+ const short2 tile_dimensions_b =
+ transpose_b ? short2(input_remainder, threadgroup_block_output)
+ : short2(threadgroup_block_output, input_remainder);
+
+ block_loader_a.load_safe(tile_dimensions_a);
+ block_loader_b.load_safe(tile_dimensions_b);
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // Do matmul
+ matrix_multiply_accumulate.mma(threadgroup_a, threadgroup_b);
+
+ // Reset source back to start
+ block_loader_a.src -= input_offset_a;
+ block_loader_b.src -= input_offset_b;
+ }
+
+ const TransformAdd<AccumType, AccumType> epilogue_operation_add(
+ addmm_params->alpha,
+ addmm_params->beta
+ );
+ const TransformAxpby<AccumType, AccumType> epilogue_operation_axpby(
+ addmm_params->alpha,
+ addmm_params->beta
+ );
+
+ ///////////////////////////////////////////////////////////////////////////
+ // MNK aligned loop
+ if (align_M && align_N) {
+ // Do gemm
+ for (int iteration = 0; iteration < gemm_k_iterations; iteration++) {
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ // Load elements into threadgroup
+ block_loader_a.load_unsafe();
+ block_loader_b.load_unsafe();
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // Multiply and accumulate threadgroup elements
+ matrix_multiply_accumulate.mma(threadgroup_a, threadgroup_b);
+
+ // Prepare for next iteration
+ block_loader_a.next();
+ block_loader_b.next();
+ }
+
+ threadgroup_barrier(mem_flags::mem_none);
+
+ // Do epilogue
+ if (use_out_source) {
+ if (do_axpby) {
+ matrix_multiply_accumulate.apply_epilogue(
+ input_c,
+ addmm_params->ldc,
+ addmm_params->fdc,
+ epilogue_operation_axpby
+ );
+ } else {
+ matrix_multiply_accumulate.apply_epilogue(
+ input_c,
+ addmm_params->ldc,
+ addmm_params->fdc,
+ epilogue_operation_add
+ );
+ }
+ }
+
+ // Store results to device memory
+ return matrix_multiply_accumulate.store_result(output, params->ldd);
+ }
+ ///////////////////////////////////////////////////////////////////////////
+ // MN unaligned loop
+ else { // Loop over K - unaligned case
+ const int leftover_block_input = 0;
+
+ if ((align_M || threadgroup_block_batch == BM) &&
+ (align_N || threadgroup_block_output == BN)) {
+ // Do gemm
+ gemm_kernel::gemm_loop(
+ threadgroup_a,
+ threadgroup_b,
+ gemm_k_iterations,
+ block_loader_a,
+ block_loader_b,
+ matrix_multiply_accumulate,
+ threadgroup_block_batch,
+ threadgroup_block_output,
+ leftover_block_input,
+ LoopAlignment<true, true, true>{}
+ );
+
+ // Do epilogue
+ if (use_out_source) {
+ if (do_axpby) {
+ matrix_multiply_accumulate.apply_epilogue(
+ input_c,
+ addmm_params->ldc,
+ addmm_params->fdc,
+ epilogue_operation_axpby
+ );
+ } else {
+ matrix_multiply_accumulate.apply_epilogue(
+ input_c,
+ addmm_params->ldc,
+ addmm_params->fdc,
+ epilogue_operation_add
+ );
+ }
+ }
+
+ // Store results to device memory
+ return matrix_multiply_accumulate.store_result(output, params->ldd);
+
+ } else if (align_N || threadgroup_block_output == BN) {
+ gemm_kernel::gemm_loop(
+ threadgroup_a,
+ threadgroup_b,
+ gemm_k_iterations,
+ block_loader_a,
+ block_loader_b,
+ matrix_multiply_accumulate,
+ threadgroup_block_batch,
+ threadgroup_block_output,
+ leftover_block_input,
+ LoopAlignment<false, true, true>{}
+ );
+
+ // Do epilogue
+ if (use_out_source) {
+ if (do_axpby) {
+ matrix_multiply_accumulate.apply_epilogue_safe(
+ input_c,
+ addmm_params->ldc,
+ addmm_params->fdc,
+ short2(threadgroup_block_output, threadgroup_block_batch),
+ epilogue_operation_axpby
+ );
+ } else {
+ matrix_multiply_accumulate.apply_epilogue_safe(
+ input_c,
+ addmm_params->ldc,
+ addmm_params->fdc,
+ short2(threadgroup_block_output, threadgroup_block_batch),
+ epilogue_operation_add
+ );
+ }
+ }
+
+ // Store results to device memory
+ return matrix_multiply_accumulate.store_result_safe(
+ output,
+ params->ldd,
+ short2(threadgroup_block_output, threadgroup_block_batch)
+ );
+
+ } else if (align_M || threadgroup_block_batch == BM) {
+ gemm_kernel::gemm_loop(
+ threadgroup_a,
+ threadgroup_b,
+ gemm_k_iterations,
+ block_loader_a,
+ block_loader_b,
+ matrix_multiply_accumulate,
+ threadgroup_block_batch,
+ threadgroup_block_output,
+ leftover_block_input,
+ LoopAlignment<true, false, true>{}
+ );
+
+ // Do epilogue
+ if (use_out_source) {
+ if (do_axpby) {
+ matrix_multiply_accumulate.apply_epilogue_safe(
+ input_c,
+ addmm_params->ldc,
+ addmm_params->fdc,
+ short2(threadgroup_block_output, threadgroup_block_batch),
+ epilogue_operation_axpby
+ );
+ } else {
+ matrix_multiply_accumulate.apply_epilogue_safe(
+ input_c,
+ addmm_params->ldc,
+ addmm_params->fdc,
+ short2(threadgroup_block_output, threadgroup_block_batch),
+ epilogue_operation_add
+ );
+ }
+ }
+
+ // Store results to device memory
+ return matrix_multiply_accumulate.store_result_safe(
+ output,
+ params->ldd,
+ short2(threadgroup_block_output, threadgroup_block_batch)
+ );
+
+ } else {
+ gemm_kernel::gemm_loop(
+ threadgroup_a,
+ threadgroup_b,
+ gemm_k_iterations,
+ block_loader_a,
+ block_loader_b,
+ matrix_multiply_accumulate,
+ threadgroup_block_batch,
+ threadgroup_block_output,
+ leftover_block_input,
+ LoopAlignment<false, false, true>{}
+ );
+
+ // Do epilogue
+ if (use_out_source) {
+ if (do_axpby) {
+ matrix_multiply_accumulate.apply_epilogue_safe(
+ input_c,
+ addmm_params->ldc,
+ addmm_params->fdc,
+ short2(threadgroup_block_output, threadgroup_block_batch),
+ epilogue_operation_axpby
+ );
+ } else {
+ matrix_multiply_accumulate.apply_epilogue_safe(
+ input_c,
+ addmm_params->ldc,
+ addmm_params->fdc,
+ short2(threadgroup_block_output, threadgroup_block_batch),
+ epilogue_operation_add
+ );
+ }
+ }
+
+ // Store results to device memory
+ return matrix_multiply_accumulate.store_result_safe(
+ output,
+ params->ldd,
+ short2(threadgroup_block_output, threadgroup_block_batch)
+ );
+ }
+ }
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// Kernel instantiation macros
+///////////////////////////////////////////////////////////////////////////////
+
+#define instantiate_gemm( \
+ transpose_name, \
+ transpose_a, \
+ transpose_b, \
+ type_name, \
+ element_type, \
+ block_batch, \
+ block_output, \
+ block_input, \
+ warp_batch, \
+ warp_output \
+) \
+ template [[host_name( \
+ "gemm_" #transpose_name "_" #type_name "_bm" #block_batch \
+ "_bn" #block_output "_bk" #block_input "_wm" #warp_batch \
+ "_wn" #warp_output \
+ )]] [[kernel]] void \
+ gemm< \
+ element_type, \
+ block_batch, \
+ block_output, \
+ block_input, \
+ warp_batch, \
+ warp_output, \
+ transpose_a, \
+ transpose_b, \
+ float>( \
+ const device element_type* input_a [[buffer(0)]], \
+ const device element_type* input_b [[buffer(1)]], \
+ const device element_type* input_c \
+ [[buffer(2), function_constant(use_out_source)]], \
+ device element_type* output [[buffer(3)]], \
+ const constant GEMMParams* params [[buffer(4)]], \
+ const constant GEMMAddMMParams* addmm_params \
+ [[buffer(5), function_constant(use_out_source)]], \
+ const constant int* batch_shape \
+ [[buffer(6), function_constant(has_batch)]], \
+ const constant int64_t* batch_strides \
+ [[buffer(7), function_constant(has_batch)]], \
+ uint simd_lane_id [[thread_index_in_simdgroup]], \
+ uint simd_group_id [[simdgroup_index_in_threadgroup]], \
+ uint3 threadgroup_position [[threadgroup_position_in_grid]], \
+ uint3 thread_position [[thread_position_in_threadgroup]] \
+ );
+
+// clang-format off
+#define instantiate_gemm_transpose_helper( \
+ type_name, element_type, block_batch, block_output, block_input, warp_batch, warp_output \
+) \
+ instantiate_gemm(nn, false, false, type_name, element_type, block_batch, block_output, \
+ block_input, warp_batch, warp_output) \
+ instantiate_gemm(nt, false, true, type_name, element_type, block_batch, block_output, \
+ block_input, warp_batch, warp_output) \
+ instantiate_gemm(tn, true, false, type_name, element_type, block_batch, block_output, \
+ block_input, warp_batch, warp_output) \
+ instantiate_gemm(tt, true, true, type_name, element_type, block_batch, block_output, \
+ block_input, warp_batch, warp_output)
+// clang-format on
+
+// clang-format off
+#define instantiate_gemm_shapes_helper(type_name, element_type) \
+ instantiate_gemm_transpose_helper(type_name, element_type, 64, 64, 16, 2, 2) \
+ instantiate_gemm_transpose_helper(type_name, element_type, 64, 64, 16, 1, 2) \
+ instantiate_gemm_transpose_helper(type_name, element_type, 64, 32, 32, 2, 2) \
+ instantiate_gemm_transpose_helper(type_name, element_type, 32, 64, 16, 1, 2) \
+ instantiate_gemm_transpose_helper(type_name, element_type, 32, 32, 16, 2, 2) \
+ instantiate_gemm_transpose_helper(type_name, element_type, 64, 32, 8, 4, 1)
+// clang-format on
+
+// Instantiate for float16, bfloat16, and float32
+instantiate_gemm_shapes_helper(f16, half);
+instantiate_gemm_shapes_helper(bf16, bfloat);
+instantiate_gemm_shapes_helper(f32, float);
+
+///////////////////////////////////////////////////////////////////////////////
+// Simple GEMM kernel (for smaller matrices or vector operations)
+///////////////////////////////////////////////////////////////////////////////
+
+template <typename T>
+[[kernel]] void gemv(
+ const device T* input_a [[buffer(0)]],
+ const device T* input_b [[buffer(1)]],
+ device T* output [[buffer(2)]],
+ constant int& batch [[buffer(3)]],
+ constant int& output_dim [[buffer(4)]],
+ constant int& input_dim [[buffer(5)]],
+ constant int& leading_dimension_a [[buffer(6)]],
+ constant int& leading_dimension_b [[buffer(7)]],
+ constant int& leading_dimension_output [[buffer(8)]],
+ uint2 thread_position [[thread_position_in_grid]],
+ uint2 grid_size [[threads_per_grid]]
+) {
+ const int row_index = thread_position.y;
+ const int column_index = thread_position.x;
+
+ if (row_index >= batch || column_index >= output_dim) {
+ return;
+ }
+
+ float accumulator = 0.0f;
+ for (int input_index = 0; input_index < input_dim; input_index++) {
+ accumulator +=
+ float(input_a[row_index * leading_dimension_a + input_index]) *
+ float(input_b[input_index * leading_dimension_b + column_index]);
+ }
+
+ output[row_index * leading_dimension_output + column_index] = T(accumulator);
+}
+
+// Simple GEMV instantiations
+template [[host_name("gemv_f16")]] [[kernel]] void gemv<half>(
+ const device half* input_a [[buffer(0)]],
+ const device half* input_b [[buffer(1)]],
+ device half* output [[buffer(2)]],
+ constant int& batch [[buffer(3)]],
+ constant int& output_dim [[buffer(4)]],
+ constant int& input_dim [[buffer(5)]],
+ constant int& leading_dimension_a [[buffer(6)]],
+ constant int& leading_dimension_b [[buffer(7)]],
+ constant int& leading_dimension_output [[buffer(8)]],
+ uint2 thread_position [[thread_position_in_grid]],
+ uint2 grid_size [[threads_per_grid]]
+);
+
+template [[host_name("gemv_bf16")]] [[kernel]] void gemv<bfloat>(
+ const device bfloat* input_a [[buffer(0)]],
+ const device bfloat* input_b [[buffer(1)]],
+ device bfloat* output [[buffer(2)]],
+ constant int& batch [[buffer(3)]],
+ constant int& output_dim [[buffer(4)]],
+ constant int& input_dim [[buffer(5)]],
+ constant int& leading_dimension_a [[buffer(6)]],
+ constant int& leading_dimension_b [[buffer(7)]],
+ constant int& leading_dimension_output [[buffer(8)]],
+ uint2 thread_position [[thread_position_in_grid]],
+ uint2 grid_size [[threads_per_grid]]
+);
+
+template [[host_name("gemv_f32")]] [[kernel]] void gemv<float>(
+ const device float* input_a [[buffer(0)]],
+ const device float* input_b [[buffer(1)]],
+ device float* output [[buffer(2)]],
+ constant int& batch [[buffer(3)]],
+ constant int& output_dim [[buffer(4)]],
+ constant int& input_dim [[buffer(5)]],
+ constant int& leading_dimension_a [[buffer(6)]],
+ constant int& leading_dimension_b [[buffer(7)]],
+ constant int& leading_dimension_output [[buffer(8)]],
+ uint2 thread_position [[thread_position_in_grid]],
+ uint2 grid_size [[threads_per_grid]]
+);
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/mma.h b/crates/uzu/src/backends/metal/kernel/matmul/mma.h
new file mode 100644
index 000000000..741ef4394
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/mma.h
@@ -0,0 +1,524 @@
+#pragma once
+
+#include "defines.h"
+
+using namespace metal;
+
+namespace uzu {
+namespace matmul {
+
+///////////////////////////////////////////////////////////////////////////////
+// MMA Fragment - 8x8 simdgroup matrix fragment
+///////////////////////////////////////////////////////////////////////////////
+
+template <typename T, int kFragRows_, int kFragCols_>
+struct BaseMMAFrag {
+ static_assert(
+ kFragRows_ == 8,
+ "Only 8 x 8 fragment matrices are currently supported"
+ );
+ static_assert(
+ kFragCols_ == 8,
+ "Only 8 x 8 fragment matrices are currently supported"
+ );
+};
+
+template <typename T>
+struct BaseMMAFrag<T, 8, 8> {
+ UZU_MTL_CONST int kFragRows = 8;
+ UZU_MTL_CONST int kFragCols = 8;
+ UZU_MTL_CONST int kElemsPerFrag = (kFragRows * kFragCols) / 32;
+ UZU_MTL_CONST int kElemRows = 1;
+ UZU_MTL_CONST int kElemCols = 2;
+
+ static_assert(
+ kElemRows * kElemCols == kElemsPerFrag,
+ "MMAFrag shape is not consistent with MMAFrag size"
+ );
+
+ typedef metal::simdgroup_matrix<T, kFragRows, kFragCols> mat_type;
+ typedef metal::vec<T, kElemsPerFrag> frag_type;
+
+ METAL_FUNC static constexpr short2 get_coord(
+ ushort simd_lane_id [[thread_index_in_simdgroup]]
+ ) {
+ const short qid = simd_lane_id / 4;
+ const short fm = (qid & 4) + ((simd_lane_id / 2) % 4);
+ const short fn = (qid & 2) * 2 + (simd_lane_id % 2) * 2;
+ return short2{fn, fm};
+ }
+
+ template <typename SrcPtrType, typename StrX, typename StrY>
+ METAL_FUNC static constexpr void load(
+ thread frag_type& dst,
+ SrcPtrType src,
+ StrX str_x,
+ StrY str_y
+ ) {
+ UZU_PRAGMA_UNROLL
+ for (short i = 0; i < kElemRows; i++) {
+ UZU_PRAGMA_UNROLL
+ for (short j = 0; j < kElemCols; j++) {
+ dst[i * kElemCols + j] = static_cast<T>(src[i * str_x + j * str_y]);
+ }
+ }
+ }
+
+ template <
+ typename SrcPtrType,
+ typename StrX,
+ typename StrY,
+ typename LimX,
+ typename LimY,
+ typename OffX,
+ typename OffY>
+ METAL_FUNC static constexpr void load_safe(
+ thread frag_type& dst,
+ SrcPtrType src,
+ StrX str_x,
+ StrY str_y,
+ LimX lim_x,
+ LimY lim_y,
+ OffX off_x = 0,
+ OffY off_y = 0
+ ) {
+ UZU_PRAGMA_UNROLL
+ for (short i = 0; i < kElemRows; i++) {
+ UZU_PRAGMA_UNROLL
+ for (short j = 0; j < kElemCols; j++) {
+ if ((off_x + i) < lim_x && (off_y + j) < lim_y) {
+ dst[i * kElemCols + j] =
+ static_cast<T>(src[(off_x + i) * str_x + (off_y + j) * str_y]);
+ } else {
+ dst[i * kElemCols + j] = T(0);
+ }
+ }
+ }
+ }
+
+ template <typename DstPtrType, typename StrX, typename StrY>
+ METAL_FUNC static constexpr void store(
+ const thread frag_type& src,
+ DstPtrType dst,
+ StrX str_x,
+ StrY str_y
+ ) {
+ using U = pointer_element_t<DstPtrType>;
+
+ UZU_PRAGMA_UNROLL
+ for (short i = 0; i < kElemRows; i++) {
+ UZU_PRAGMA_UNROLL
+ for (short j = 0; j < kElemCols; j++) {
+ dst[i * str_x + j * str_y] = static_cast<U>(src[i * kElemCols + j]);
+ }
+ }
+ }
+
+ template <
+ typename DstPtrType,
+ typename StrX,
+ typename StrY,
+ typename LimX,
+ typename LimY,
+ typename OffX,
+ typename OffY>
+ METAL_FUNC static constexpr void store_safe(
+ const thread frag_type& src,
+ DstPtrType dst,
+ StrX str_x,
+ StrY str_y,
+ LimX lim_x,
+ LimY lim_y,
+ OffX off_x = 0,
+ OffY off_y = 0
+ ) {
+ using U = pointer_element_t<DstPtrType>;
+
+ UZU_PRAGMA_UNROLL
+ for (short i = 0; i < kElemRows; i++) {
+ UZU_PRAGMA_UNROLL
+ for (short j = 0; j < kElemCols; j++) {
+ if ((off_x + i) < lim_x && (off_y + j) < lim_y) {
+ dst[(off_x + i) * str_x + (off_y + j) * str_y] =
+ static_cast<U>(src[i * kElemCols + j]);
+ }
+ }
+ }
+ }
+
+ METAL_FUNC static constexpr void mma(
+ thread frag_type& D,
+ thread frag_type& A,
+ thread frag_type& B,
+ thread frag_type& C
+ ) {
+ mat_type D_mat;
+ mat_type A_mat;
+ mat_type B_mat;
+ mat_type C_mat;
+
+ reinterpret_cast<thread frag_type&>(A_mat.thread_elements()) = A;
+ reinterpret_cast<thread frag_type&>(B_mat.thread_elements()) = B;
+ reinterpret_cast<thread frag_type&>(C_mat.thread_elements()) = C;
+
+ mma(D_mat, A_mat, B_mat, C_mat);
+
+ D = reinterpret_cast<thread frag_type&>(D_mat.thread_elements());
+ }
+
+ METAL_FUNC static constexpr void mma(
+ thread mat_type& D,
+ thread mat_type& A,
+ thread mat_type& B,
+ thread mat_type& C
+ ) {
+ simdgroup_multiply_accumulate(D, A, B, C);
+ }
+};
+
+///////////////////////////////////////////////////////////////////////////////
+// MMA Tile - collection of MMA fragments
+///////////////////////////////////////////////////////////////////////////////
+
+template <
+ typename T,
+ int kTileRows_,
+ int kTileCols_,
+ class MMAFrag_ = BaseMMAFrag<T, 8, 8>>
+struct MMATile {
+ using MMAFrag_t = MMAFrag_;
+ using elem_type = T;
+ UZU_MTL_CONST int kFragRows = MMAFrag_t::kFragRows;
+ UZU_MTL_CONST int kFragCols = MMAFrag_t::kFragCols;
+ UZU_MTL_CONST int kElemsPerFrag = MMAFrag_t::kElemsPerFrag;
+
+ UZU_MTL_CONST int kTileRows = kTileRows_;
+ UZU_MTL_CONST int kTileCols = kTileCols_;
+
+ UZU_MTL_CONST int kRows = kTileRows * kFragRows;
+ UZU_MTL_CONST int kCols = kTileCols * kFragCols;
+
+ UZU_MTL_CONST int kNumFrags = kTileRows * kTileCols;
+ UZU_MTL_CONST int kElemsPerTile = kNumFrags * kElemsPerFrag;
+
+ typedef typename MMAFrag_t::mat_type mat_type;
+ typedef typename MMAFrag_t::frag_type frag_type;
+
+ frag_type val_frags[kNumFrags] = {frag_type(0)};
+
+ METAL_FUNC MMATile() thread {}
+
+ METAL_FUNC constexpr void clear() {
+ UZU_PRAGMA_UNROLL
+ for (short i = 0; i < kNumFrags; ++i) {
+ val_frags[i] = frag_type(0);
+ }
+ }
+
+ METAL_FUNC constexpr thread frag_type& frag_at(const short i, const short j) {
+ return val_frags[i * kTileCols + j];
+ }
+
+ METAL_FUNC constexpr const thread frag_type& frag_at(
+ const short i,
+ const short j
+ ) const {
+ return val_frags[i * kTileCols + j];
+ }
+
+ METAL_FUNC thread elem_type* elems() {
+ return reinterpret_cast<thread elem_type*>(val_frags);
+ }
+
+ METAL_FUNC const thread elem_type* elems() const {
+ return reinterpret_cast<const thread elem_type*>(val_frags);
+ }
+
+ template <typename U, int w_x, int w_y, int str_x, int str_y>
+ METAL_FUNC void load(const threadgroup U* src) {
+ UZU_PRAGMA_UNROLL
+ for (short i = 0; i < kTileRows; ++i) {
+ UZU_PRAGMA_UNROLL
+ for (short j = 0; j < kTileCols; ++j) {
+ MMAFrag_t::load(
+ frag_at(i, j),
+ &(
+ src[(i * kFragRows) * w_x * str_x +
+ (j * kFragCols) * w_y * str_y]
+ ),
+ str_x,
+ str_y
+ );
+ }
+ }
+ }
+
+ template <typename U, int w_x, int w_y>
+ METAL_FUNC void store(device U* dst, const int ld) const {
+ UZU_PRAGMA_UNROLL
+ for (short i = 0; i < kTileRows; ++i) {
+ UZU_PRAGMA_UNROLL
+ for (short j = 0; j < kTileCols; ++j) {
+ MMAFrag_t::store(
+ frag_at(i, j),
+ &(dst[(i * kFragRows) * w_x * ld + (j * kFragCols) * w_y]),
+ ld,
+ 1
+ );
+ }
+ }
+ }
+
+ template <typename U, int w_x, int w_y>
+ METAL_FUNC void store_safe(
+ device U* dst,
+ const int ld,
+ const short2 dst_tile_dims
+ ) const {
+ UZU_PRAGMA_UNROLL
+ for (int i = 0; i < kTileRows; ++i) {
+ UZU_PRAGMA_UNROLL
+ for (int j = 0; j < kTileCols; ++j) {
+ MMAFrag_t::store_safe(
+ frag_at(i, j),
+ dst,
+ ld,
+ 1,
+ dst_tile_dims.y,
+ dst_tile_dims.x,
+ (i * kFragRows) * w_x,
+ (j * kFragCols) * w_y
+ );
+ }
+ }
+ }
+};
+
+///////////////////////////////////////////////////////////////////////////////
+// Tile matrix multiply-accumulate
+///////////////////////////////////////////////////////////////////////////////
+
+template <typename T, typename U, int M, int N, int K>
+METAL_FUNC void tile_matmad(
+ thread MMATile<T, M, N>& D,
+ thread MMATile<U, M, K>& A,
+ thread MMATile<U, K, N>& B,
+ thread MMATile<T, M, N>& C
+) {
+ UZU_PRAGMA_UNROLL
+ for (short m = 0; m < M; ++m) {
+ UZU_PRAGMA_UNROLL
+ for (short n = 0; n < N; ++n) {
+ short n_serp = (m % 2) ? (N - 1 - n) : n;
+ UZU_PRAGMA_UNROLL
+ for (short k = 0; k < K; ++k) {
+ MMATile<T, M, N>::MMAFrag_t::mma(
+ D.frag_at(m, n_serp),
+ A.frag_at(m, k),
+ B.frag_at(k, n_serp),
+ C.frag_at(m, n_serp)
+ );
+ }
+ }
+ }
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// Block MMA - manages the GEMM computation for a threadgroup
+///////////////////////////////////////////////////////////////////////////////
+
+template <
+ typename T,
+ typename U,
+ int BM,
+ int BN,
+ int BK,
+ int WM,
+ int WN,
+ bool transpose_a,
+ bool transpose_b,
+ short lda_tgp,
+ short ldb_tgp,
+ typename AccumType = float,
+ typename Epilogue = TransformNone<U, AccumType>>
+struct BlockMMA {
+ // MMAFrag size
+ UZU_MTL_CONST short kFragSize = 8;
+ using MMAFrag_acc_t = BaseMMAFrag<AccumType, kFragSize, kFragSize>;
+
+ // Warp tile simdgroup matrix strides along M
+ UZU_MTL_CONST short TM_stride = kFragSize * WM;
+ // Warp tile simdgroup matrix strides along N
+ UZU_MTL_CONST short TN_stride = kFragSize * WN;
+
+ // Warp tile size along M
+ UZU_MTL_CONST short TM = BM / (kFragSize * WM);
+ // Warp tile size along N
+ UZU_MTL_CONST short TN = BN / (kFragSize * WN);
+
+ // Threadgroup A strides
+ UZU_MTL_CONST short A_str_m = transpose_a ? 1 : lda_tgp; // M
+ UZU_MTL_CONST short A_str_k = transpose_a ? lda_tgp : 1; // K
+
+ // Threadgroup B strides
+ UZU_MTL_CONST short B_str_k = transpose_b ? 1 : ldb_tgp; // K
+ UZU_MTL_CONST short B_str_n = transpose_b ? ldb_tgp : 1; // N
+
+ // Threadgroup strides along K
+ UZU_MTL_CONST short tile_stride_a = kFragSize * A_str_k;
+ UZU_MTL_CONST short tile_stride_b = kFragSize * B_str_k;
+
+ // Simdgroup matrices
+ MMATile<AccumType, TM, 1, MMAFrag_acc_t> Atile;
+ MMATile<AccumType, 1, TN, MMAFrag_acc_t> Btile;
+ MMATile<AccumType, TM, TN, MMAFrag_acc_t> Ctile;
+
+ // Offsets within threadgroup
+ short sm;
+ short sn;
+
+ short As_offset;
+ short Bs_offset;
+
+ /* Constructor */
+ METAL_FUNC BlockMMA(
+ ushort simd_group_id [[simdgroup_index_in_threadgroup]],
+ ushort simd_lane_id [[thread_index_in_simdgroup]]
+ ) {
+ // Determine thread position in simdgroup matrix
+ short tm = kFragSize * (simd_group_id / WN);
+ short tn = kFragSize * (simd_group_id % WN);
+
+ short2 simd_coord = MMAFrag_acc_t::get_coord(simd_lane_id);
+ sm = simd_coord.y;
+ sn = simd_coord.x;
+
+ // Determine thread and simdgroup offset
+ As_offset = (tm + sm) * A_str_m + (sn)*A_str_k; // M, K
+ Bs_offset = (sm)*B_str_k + (tn + sn) * B_str_n; // K, N
+
+ sm += tm;
+ sn += tn;
+ }
+
+ /* (BM, BK) X (BK, BN) multiply accumulate function */
+ METAL_FUNC void mma(const threadgroup T* As, const threadgroup T* Bs) {
+ // Adjust for simdgroup and thread location
+ As += As_offset;
+ Bs += Bs_offset;
+
+ // Iterate over BK in blocks of kFragSize
+ UZU_PRAGMA_UNROLL
+ for (short kk = 0; kk < BK; kk += kFragSize) {
+ simdgroup_barrier(mem_flags::mem_none);
+
+ Atile.template load<T, WM, 1, A_str_m, A_str_k>(As);
+
+ simdgroup_barrier(mem_flags::mem_none);
+
+ Btile.template load<T, 1, WN, B_str_k, B_str_n>(Bs);
+
+ simdgroup_barrier(mem_flags::mem_none);
+
+ tile_matmad(Ctile, Atile, Btile, Ctile);
+
+ // Progress to next simdgroup tile
+ As += tile_stride_a;
+ Bs += tile_stride_b;
+ }
+ }
+
+ /* Store results from simdgroup_matrix results into device memory */
+ METAL_FUNC void store_result(device U* D, const int ldd) {
+ // Apply epilogue
+ UZU_PRAGMA_UNROLL
+ for (short i = 0; i < decltype(Ctile)::kElemsPerTile; i++) {
+ Ctile.elems()[i] = Epilogue::apply(Ctile.elems()[i]);
+ }
+
+ // Adjust for simdgroup and thread location
+ D += sm * ldd + sn;
+
+ Ctile.template store<U, WM, WN>(D, ldd);
+ }
+
+ METAL_FUNC void store_result_safe(
+ device U* D,
+ const int ldd,
+ short2 dst_tile_dims
+ ) {
+ // Apply epilogue
+ UZU_PRAGMA_UNROLL
+ for (short i = 0; i < decltype(Ctile)::kElemsPerTile; i++) {
+ Ctile.elems()[i] = Epilogue::apply(Ctile.elems()[i]);
+ }
+
+ // Adjust for simdgroup and thread location
+ D += sm * ldd + sn;
+ dst_tile_dims -= short2(sn, sm);
+
+ if (dst_tile_dims.x <= 0 || dst_tile_dims.y <= 0)
+ return;
+
+ Ctile.template store_safe<U, WM, WN>(D, ldd, dst_tile_dims);
+ }
+
+ /* Apply epilogue with C matrix */
+ template <typename EpilogueOp>
+ METAL_FUNC void apply_epilogue(
+ const device U* C,
+ const int ldc,
+ const int fdc,
+ thread const EpilogueOp& epilogue_op
+ ) {
+ const device U* C_ptr = C + sm * ldc + sn * fdc;
+
+ UZU_PRAGMA_UNROLL
+ for (short i = 0; i < TM; i++) {
+ UZU_PRAGMA_UNROLL
+ for (short j = 0; j < TN; j++) {
+ thread auto& frag = Ctile.frag_at(i, j);
+ UZU_PRAGMA_UNROLL
+ for (short k = 0; k < decltype(Ctile)::kElemsPerFrag; k++) {
+ short row_offset = (i * kFragSize * WM);
+ short col_offset = (j * kFragSize * WN);
+ // Get the C element (accounting for fragment layout)
+ U c_val = C_ptr[row_offset * ldc + col_offset * fdc + k];
+ frag[k] = epilogue_op.apply(frag[k], static_cast<AccumType>(c_val));
+ }
+ }
+ }
+ }
+
+ template <typename EpilogueOp>
+ METAL_FUNC void apply_epilogue_safe(
+ const device U* C,
+ const int ldc,
+ const int fdc,
+ short2 tile_dims,
+ thread const EpilogueOp& epilogue_op
+ ) {
+ const device U* C_ptr = C + sm * ldc + sn * fdc;
+ tile_dims -= short2(sn, sm);
+
+ UZU_PRAGMA_UNROLL
+ for (short i = 0; i < TM; i++) {
+ UZU_PRAGMA_UNROLL
+ for (short j = 0; j < TN; j++) {
+ thread auto& frag = Ctile.frag_at(i, j);
+ short row_offset = (i * kFragSize * WM);
+ short col_offset = (j * kFragSize * WN);
+ UZU_PRAGMA_UNROLL
+ for (short k = 0; k < decltype(Ctile)::kElemsPerFrag; k++) {
+ if (row_offset < tile_dims.y && col_offset + k < tile_dims.x) {
+ U c_val = C_ptr[row_offset * ldc + col_offset * fdc + k];
+ frag[k] = epilogue_op.apply(frag[k], static_cast<AccumType>(c_val));
+ }
+ }
+ }
+ }
+ }
+};
+
+} // namespace matmul
+} // namespace uzu
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/mod.rs
new file mode 100644
index 000000000..191479882
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/mod.rs
@@ -0,0 +1,9 @@
+mod arguments;
+mod kernel;
+mod pipeline;
+#[allow(dead_code, clippy::unnecessary_operation, clippy::identity_op)]
+mod shared_types;
+mod transpose;
+
+pub use arguments::MatmulArguments;
+pub use kernel::MatmulKernel;
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/pipeline.rs b/crates/uzu/src/backends/metal/kernel/matmul/pipeline.rs
new file mode 100644
index 000000000..801ce989d
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/pipeline.rs
@@ -0,0 +1,7 @@
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub(super) struct PipelineKey {
+ pub name: String,
+ pub align_m: bool,
+ pub align_n: bool,
+ pub align_k: bool,
+}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/shared_types.h b/crates/uzu/src/backends/metal/kernel/matmul/shared_types.h
new file mode 100644
index 000000000..4a9b28047
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/shared_types.h
@@ -0,0 +1,66 @@
+// Shared types for GEMM kernel - used by both Metal shaders and Rust (via
+// bindgen). This header must be C-compatible for bindgen to parse it.
+
+#pragma once
+
+// Metal and C have different type systems
+// __METAL_VERSION__ is defined when compiling with the Metal shader compiler
+#ifdef __METAL_VERSION__
+// Metal shading language - int64_t comes from metal_stdlib
+#include <metal_stdlib>
+using namespace metal;
+
+namespace uzu {
+namespace matmul {
+
+#else
+// C/C++ for bindgen - use stdint types
+#include <stdint.h>
+
+#endif
+
+///////////////////////////////////////////////////////////////////////////////
+// GEMM Parameters
+///////////////////////////////////////////////////////////////////////////////
+
+/// Parameters for the main GEMM kernel.
+/// Layout: 8 x int32, 3 x int64, 2 x int32 = 64 bytes total
+struct GEMMParams {
+ /// M dimension - batch/number of tokens (rows of A, rows of D)
+ int batch;
+ /// N dimension - output_dim (cols of B, cols of D)
+ int output_dim;
+ /// K dimension - input_dim/reduction dimension (cols of A, rows of B)
+ int input_dim;
+
+ int lda;
+ int ldb;
+ int ldd;
+
+ int tiles_n;
+ int tiles_m;
+
+ int64_t batch_stride_a;
+ int64_t batch_stride_b;
+ int64_t batch_stride_d;
+
+ int swizzle_log;
+ int gemm_k_iterations_aligned;
+};
+
+/// Parameters for addmm (alpha * A @ B + beta * C) operations
+struct GEMMAddMMParams {
+ int ldc;
+ int fdc;
+
+ int64_t batch_stride_c;
+
+ float alpha;
+ float beta;
+};
+
+// Close Metal namespace
+#ifdef __METAL_VERSION__
+} // namespace matmul
+} // namespace uzu
+#endif
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/shared_types.rs b/crates/uzu/src/backends/metal/kernel/matmul/shared_types.rs
new file mode 100644
index 000000000..1e4f91113
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/shared_types.rs
@@ -0,0 +1,83 @@
+/* automatically generated by rust-bindgen 0.72.1 */
+
+#[doc = " Parameters for the main GEMM kernel.\n Layout: 8 x int32, 3 x int64, 2 x int32 = 64 bytes total"]
+#[repr(C)]
+#[derive(Debug, Default, Copy, Clone)]
+pub struct GEMMParams {
+ #[doc = " M dimension - batch/number of tokens (rows of A, rows of D)"]
+ pub batch: ::core::ffi::c_int,
+ #[doc = " N dimension - output_dim (cols of B, cols of D)"]
+ pub output_dim: ::core::ffi::c_int,
+ #[doc = " K dimension - input_dim/reduction dimension (cols of A, rows of B)"]
+ pub input_dim: ::core::ffi::c_int,
+ pub lda: ::core::ffi::c_int,
+ pub ldb: ::core::ffi::c_int,
+ pub ldd: ::core::ffi::c_int,
+ pub tiles_n: ::core::ffi::c_int,
+ pub tiles_m: ::core::ffi::c_int,
+ pub batch_stride_a: i64,
+ pub batch_stride_b: i64,
+ pub batch_stride_d: i64,
+ pub swizzle_log: ::core::ffi::c_int,
+ pub gemm_k_iterations_aligned: ::core::ffi::c_int,
+}
+#[allow(clippy::unnecessary_operation, clippy::identity_op)]
+const _: () = {
+ ["Size of GEMMParams"][::core::mem::size_of::<GEMMParams>() - 64usize];
+ ["Alignment of GEMMParams"][::core::mem::align_of::<GEMMParams>() - 8usize];
+ ["Offset of field: GEMMParams::batch"]
+ [::core::mem::offset_of!(GEMMParams, batch) - 0usize];
+ ["Offset of field: GEMMParams::output_dim"]
+ [::core::mem::offset_of!(GEMMParams, output_dim) - 4usize];
+ ["Offset of field: GEMMParams::input_dim"]
+ [::core::mem::offset_of!(GEMMParams, input_dim) - 8usize];
+ ["Offset of field: GEMMParams::lda"]
+ [::core::mem::offset_of!(GEMMParams, lda) - 12usize];
+ ["Offset of field: GEMMParams::ldb"]
+ [::core::mem::offset_of!(GEMMParams, ldb) - 16usize];
+ ["Offset of field: GEMMParams::ldd"]
+ [::core::mem::offset_of!(GEMMParams, ldd) - 20usize];
+ ["Offset of field: GEMMParams::tiles_n"]
+ [::core::mem::offset_of!(GEMMParams, tiles_n) - 24usize];
+ ["Offset of field: GEMMParams::tiles_m"]
+ [::core::mem::offset_of!(GEMMParams, tiles_m) - 28usize];
+ ["Offset of field: GEMMParams::batch_stride_a"]
+ [::core::mem::offset_of!(GEMMParams, batch_stride_a) - 32usize];
+ ["Offset of field: GEMMParams::batch_stride_b"]
+ [::core::mem::offset_of!(GEMMParams, batch_stride_b) - 40usize];
+ ["Offset of field: GEMMParams::batch_stride_d"]
+ [::core::mem::offset_of!(GEMMParams, batch_stride_d) - 48usize];
+ ["Offset of field: GEMMParams::swizzle_log"]
+ [::core::mem::offset_of!(GEMMParams, swizzle_log) - 56usize];
+ ["Offset of field: GEMMParams::gemm_k_iterations_aligned"][::core::mem::offset_of!(
+ GEMMParams,
+ gemm_k_iterations_aligned
+ ) - 60usize];
+};
+#[doc = " Parameters for addmm (alpha * A @ B + beta * C) operations"]
+#[repr(C)]
+#[derive(Debug, Default, Copy, Clone)]
+pub struct GEMMAddMMParams {
+ pub ldc: ::core::ffi::c_int,
+ pub fdc: ::core::ffi::c_int,
+ pub batch_stride_c: i64,
+ pub alpha: f32,
+ pub beta: f32,
+}
+#[allow(clippy::unnecessary_operation, clippy::identity_op)]
+const _: () = {
+ ["Size of GEMMAddMMParams"]
+ [::core::mem::size_of::<GEMMAddMMParams>() - 24usize];
+ ["Alignment of GEMMAddMMParams"]
+ [::core::mem::align_of::<GEMMAddMMParams>() - 8usize];
+ ["Offset of field: GEMMAddMMParams::ldc"]
+ [::core::mem::offset_of!(GEMMAddMMParams, ldc) - 0usize];
+ ["Offset of field: GEMMAddMMParams::fdc"]
+ [::core::mem::offset_of!(GEMMAddMMParams, fdc) - 4usize];
+ ["Offset of field: GEMMAddMMParams::batch_stride_c"]
+ [::core::mem::offset_of!(GEMMAddMMParams, batch_stride_c) - 8usize];
+ ["Offset of field: GEMMAddMMParams::alpha"]
+ [::core::mem::offset_of!(GEMMAddMMParams, alpha) - 16usize];
+ ["Offset of field: GEMMAddMMParams::beta"]
+ [::core::mem::offset_of!(GEMMAddMMParams, beta) - 20usize];
+};
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/transpose.rs b/crates/uzu/src/backends/metal/kernel/matmul/transpose.rs
new file mode 100644
index 000000000..d75b4d2c8
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/transpose.rs
@@ -0,0 +1,30 @@
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub(super) enum TransposeConfiguration {
+ NN,
+ NT,
+ TN,
+ TT,
+}
+
+impl TransposeConfiguration {
+ pub fn as_str(&self) -> &'static str {
+ match self {
+ TransposeConfiguration::NN => "nn",
+ TransposeConfiguration::NT => "nt",
+ TransposeConfiguration::TN => "tn",
+ TransposeConfiguration::TT => "tt",
+ }
+ }
+}
+
+pub(super) fn transpose_configuration(
+ a_t: bool,
+ b_t: bool,
+) -> TransposeConfiguration {
+ match (a_t, b_t) {
+ (false, false) => TransposeConfiguration::NN,
+ (false, true) => TransposeConfiguration::NT,
+ (true, false) => TransposeConfiguration::TN,
+ (true, true) => TransposeConfiguration::TT,
+ }
+}
diff --git a/crates/uzu/tests/gemm_test.rs b/crates/uzu/tests/gemm_test.rs
new file mode 100644
index 000000000..baa84b764
--- /dev/null
+++ b/crates/uzu/tests/gemm_test.rs
@@ -0,0 +1,244 @@
+use half::{bf16, f16};
+use metal::{Device, MTLResourceOptions};
+use uzu::{
+ DataType,
+ backends::metal::{
+ MTLContext,
+ kernel::{MatmulArguments, MatmulKernel},
+ },
+};
+
+fn create_test_context() -> Option<MTLContext> {
+ let device = Device::system_default()?;
+ let command_queue = device.new_command_queue();
+ MTLContext::new(device, command_queue).ok()
+}
+
+fn matmul_ref(
+ a: &[f32],
+ b: &[f32],
+ m: usize,
+ k: usize,
+ n: usize,
+) -> Vec<f32> {
+ let mut out = vec![0f32; m * n];
+ for i in 0..m {
+ for j in 0..n {
+ let mut acc = 0f32;
+ for p in 0..k {
+ acc += a[i * k + p] * b[p * n + j];
+ }
+ out[i * n + j] = acc;
+ }
+ }
+ out
+}
+
+#[test]
+fn gemm_f16_64x64() {
+ let Some(ctx) = create_test_context() else {
+ return;
+ }; // Skip if no Metal
+
+ let m = 64;
+ let k = 64;
+ let n = 64;
+ let a_f32: Vec<f32> =
+ (0..(m * k)).map(|i| ((i % 13) as f32) * 0.01).collect();
+ let b_f32: Vec<f32> =
+ (0..(k * n)).map(|i| ((i % 17) as f32) * 0.02 - 0.1).collect();
+ let exp = matmul_ref(&a_f32, &b_f32, m, k, n);
+
+ let a: Vec<f16> = a_f32.iter().map(|&v| f16::from_f32(v)).collect();
+ let b: Vec<f16> = b_f32.iter().map(|&v| f16::from_f32(v)).collect();
+
+ let a_buf = ctx.device.new_buffer_with_data(
+ a.as_ptr() as *const _,
+ (a.len() * core::mem::size_of::<f16>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+ let b_buf = ctx.device.new_buffer_with_data(
+ b.as_ptr() as *const _,
+ (b.len() * core::mem::size_of::<f16>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+ let d_buf = ctx.device.new_buffer(
+ (m * n * core::mem::size_of::<f16>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+
+ let mut kernel = MatmulKernel::new(&ctx, DataType::F16, false, false)
+ .expect("kernel new");
+
+ let cb = ctx.command_queue.new_command_buffer().to_owned();
+ let enc = cb.new_compute_command_encoder();
+ kernel
+ .encode(
+ &ctx,
+ &enc,
+ MatmulArguments {
+ a: &a_buf,
+ b: &b_buf,
+ d: &d_buf,
+ batch: m as i32,
+ input_dim: k as i32,
+ output_dim: n as i32,
+ lda: k as i32,
+ ldb: n as i32,
+ ldd: n as i32,
+ batch_count: 1,
+ },
+ )
+ .expect("encode");
+ enc.end_encoding();
+ cb.commit();
+ cb.wait_until_completed();
+
+ let got: Vec<f32> = unsafe {
+ let ptr = d_buf.contents() as *const f16;
+ let slice = core::slice::from_raw_parts(ptr, m * n);
+ slice.iter().map(|&h| h.to_f32()).collect()
+ };
+
+ for (i, (&e, &g)) in exp.iter().zip(got.iter()).enumerate() {
+ let diff = (e - g).abs();
+ assert!(diff <= 0.1, "idx {i}: exp {e} got {g} diff {diff}");
+ }
+}
+
+#[test]
+fn gemm_bf16_32x48() {
+ let Some(ctx) = create_test_context() else {
+ return;
+ }; // Skip if no Metal
+ let m = 32;
+ let k = 48;
+ let n = 32;
+ let a_f32: Vec<f32> =
+ (0..(m * k)).map(|i| ((i % 7) as f32) * 0.03).collect();
+ let b_f32: Vec<f32> =
+ (0..(k * n)).map(|i| ((i % 5) as f32) * 0.04 - 0.2).collect();
+ let exp = matmul_ref(&a_f32, &b_f32, m, k, n);
+
+ let a: Vec<bf16> = a_f32.iter().map(|&v| bf16::from_f32(v)).collect();
+ let b: Vec<bf16> = b_f32.iter().map(|&v| bf16::from_f32(v)).collect();
+
+ let a_buf = ctx.device.new_buffer_with_data(
+ a.as_ptr() as *const _,
+ (a.len() * core::mem::size_of::<bf16>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+ let b_buf = ctx.device.new_buffer_with_data(
+ b.as_ptr() as *const _,
+ (b.len() * core::mem::size_of::<bf16>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+ let d_buf = ctx.device.new_buffer(
+ (m * n * core::mem::size_of::<bf16>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+
+ let mut kernel = MatmulKernel::new(&ctx, DataType::BF16, false, false)
+ .expect("kernel new");
+
+ let cb = ctx.command_queue.new_command_buffer().to_owned();
+ let enc = cb.new_compute_command_encoder();
+ kernel
+ .encode(
+ &ctx,
+ &enc,
+ MatmulArguments {
+ a: &a_buf,
+ b: &b_buf,
+ d: &d_buf,
+ batch: m as i32,
+ input_dim: k as i32,
+ output_dim: n as i32,
+ lda: k as i32,
+ ldb: n as i32,
+ ldd: n as i32,
+ batch_count: 1,
+ },
+ )
+ .expect("encode");
+ enc.end_encoding();
+ cb.commit();
+ cb.wait_until_completed();
+
+ let got: Vec<f32> = unsafe {
+ let ptr = d_buf.contents() as *const bf16;
+ let slice = core::slice::from_raw_parts(ptr, m * n);
+ slice.iter().map(|&h| h.to_f32()).collect()
+ };
+
+ for (i, (&e, &g)) in exp.iter().zip(got.iter()).enumerate() {
+ let diff = (e - g).abs();
+ assert!(diff <= 0.2, "idx {i}: exp {e} got {g} diff {diff}");
+ }
+}
+
+#[test]
+fn gemm_f32_16x20() {
+ let Some(ctx) = create_test_context() else {
+ return;
+ }; // Skip if no Metal
+ let m = 16;
+ let k = 20;
+ let n = 16;
+ let a: Vec<f32> = (0..(m * k)).map(|i| ((i % 11) as f32) * 0.05).collect();
+ let b: Vec<f32> =
+ (0..(k * n)).map(|i| ((i % 9) as f32) * 0.02 - 0.3).collect();
+ let exp = matmul_ref(&a, &b, m, k, n);
+
+ let a_buf = ctx.device.new_buffer_with_data(
+ a.as_ptr() as *const _,
+ (a.len() * core::mem::size_of::<f32>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+ let b_buf = ctx.device.new_buffer_with_data(
+ b.as_ptr() as *const _,
+ (b.len() * core::mem::size_of::<f32>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+ let d_buf = ctx.device.new_buffer(
+ (m * n * core::mem::size_of::<f32>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+
+ let mut kernel = MatmulKernel::new(&ctx, DataType::F32, false, false)
+ .expect("kernel new");
+ let cb = ctx.command_queue.new_command_buffer().to_owned();
+ let enc = cb.new_compute_command_encoder();
+ kernel
+ .encode(
+ &ctx,
+ &enc,
+ MatmulArguments {
+ a: &a_buf,
+ b: &b_buf,
+ d: &d_buf,
+ batch: m as i32,
+ input_dim: k as i32,
+ output_dim: n as i32,
+ lda: k as i32,
+ ldb: n as i32,
+ ldd: n as i32,
+ batch_count: 1,
+ },
+ )
+ .expect("encode");
+ enc.end_encoding();
+ cb.commit();
+ cb.wait_until_completed();
+
+ let got: Vec<f32> = unsafe {
+ let ptr = d_buf.contents() as *const f32;
+ let slice = core::slice::from_raw_parts(ptr, m * n);
+ slice.to_vec()
+ };
+
+ for (i, (&e, &g)) in exp.iter().zip(got.iter()).enumerate() {
+ let diff = (e - g).abs();
+ assert!(diff <= 1e-3, "idx {i}: exp {e} got {g} diff {diff}");
+ }
+}