Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

wip

eugene010e14efa68b
diff --git a/Cargo.lock b/Cargo.lock
index b5369e596..c3df3983a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2191,6 +2191,19 @@ dependencies = [
"cxx",
]
+[[package]]
+name = "mpsgraph"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b3d4f4ee48894ff2d83808e8a0333f00eb7576375a5a77e8132f10ebb2a1f86"
+dependencies = [
+ "bitflags 2.9.4",
+ "block2",
+ "metal",
+ "objc2",
+ "objc2-foundation",
+]
+
[[package]]
name = "multer"
version = "3.1.0"
@@ -4090,6 +4103,7 @@ dependencies = [
"metal",
"minijinja",
"minijinja-contrib",
+ "mpsgraph",
"ndarray",
"num-traits",
"objc2",
diff --git a/crates/uzu/Cargo.toml b/crates/uzu/Cargo.toml
index 211820eb1..c7e47cf0f 100644
--- a/crates/uzu/Cargo.toml
+++ b/crates/uzu/Cargo.toml
@@ -37,6 +37,7 @@ bindgen = "0.72.1"
is_close = "0.1.3"
serde_json5 = "0.2.1"
schemars = { version = "1.0.5", features = ["derive"] }
+mpsgraph = "0.2.0"
[features]
default = ["metal-shaders"]
diff --git a/crates/uzu/examples/matmul_benchmark.rs b/crates/uzu/examples/matmul_benchmark.rs
new file mode 100644
index 000000000..671c5de27
--- /dev/null
+++ b/crates/uzu/examples/matmul_benchmark.rs
@@ -0,0 +1,400 @@
+//! Matmul benchmark comparing Metal GEMM kernel vs MPSGraph
+//!
+//! Run with: cargo run --release -p uzu --example matmul_benchmark
+
+use std::{collections::HashMap, io::Write, time::Instant};
+
+use half::bf16;
+use metal::{Device, MTLResourceOptions};
+use mpsgraph::{
+ CommandBuffer, Device as MPSDevice, ExecutableExecutionDescriptor, Graph,
+ ShapedType, TensorData,
+};
+use objc2::rc::autoreleasepool;
+use uzu::{
+ DataType,
+ backends::metal::{
+ MTLContext,
+ kernel::{MatmulArguments, MatmulKernel},
+ },
+};
+
+struct BenchmarkResult {
+ name: &'static str,
+ m: usize,
+ k: usize,
+ n: usize,
+ metal_ms: f64,
+ mpsgraph_ms: f64,
+ metal_gflops: f64,
+ mpsgraph_gflops: f64,
+}
+
+fn compute_gflops(
+ m: usize,
+ k: usize,
+ n: usize,
+ time_ms: f64,
+) -> f64 {
+ let flops = 2.0 * (m as f64) * (k as f64) * (n as f64);
+ flops / (time_ms * 1e6)
+}
+
+fn benchmark_metal_gemm(
+ ctx: &MTLContext,
+ m: usize,
+ k: usize,
+ n: usize,
+ warmup_iters: usize,
+ bench_iters: usize,
+) -> f64 {
+ let a: Vec<bf16> = (0..(m * k))
+ .map(|i| bf16::from_f32(((i % 13) as f32) * 0.01))
+ .collect();
+ let b: Vec<bf16> = (0..(k * n))
+ .map(|i| bf16::from_f32(((i % 17) as f32) * 0.02 - 0.1))
+ .collect();
+
+ let a_buf = ctx.device.new_buffer_with_data(
+ a.as_ptr() as *const _,
+ (a.len() * core::mem::size_of::<bf16>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+ let b_buf = ctx.device.new_buffer_with_data(
+ b.as_ptr() as *const _,
+ (b.len() * core::mem::size_of::<bf16>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+ let d_buf = ctx.device.new_buffer(
+ (m * n * core::mem::size_of::<bf16>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+
+ let mut kernel = MatmulKernel::new(ctx, DataType::BF16, false, false)
+ .expect("kernel new");
+
+ // Warmup
+ for _ in 0..warmup_iters {
+ let cb = ctx.command_queue.new_command_buffer().to_owned();
+ let enc = cb.new_compute_command_encoder();
+ kernel
+ .encode(
+ ctx,
+ &enc,
+ MatmulArguments {
+ a: &a_buf,
+ b: &b_buf,
+ d: &d_buf,
+ batch: m as i32,
+ input_dim: k as i32,
+ output_dim: n as i32,
+ lda: k as i32,
+ ldb: n as i32,
+ ldd: n as i32,
+ batch_count: 1,
+ },
+ )
+ .expect("encode");
+ enc.end_encoding();
+ cb.commit();
+ cb.wait_until_completed();
+ }
+
+ // Benchmark
+ let start = Instant::now();
+ for _ in 0..bench_iters {
+ let cb = ctx.command_queue.new_command_buffer().to_owned();
+ let enc = cb.new_compute_command_encoder();
+ kernel
+ .encode(
+ ctx,
+ &enc,
+ MatmulArguments {
+ a: &a_buf,
+ b: &b_buf,
+ d: &d_buf,
+ batch: m as i32,
+ input_dim: k as i32,
+ output_dim: n as i32,
+ lda: k as i32,
+ ldb: n as i32,
+ ldd: n as i32,
+ batch_count: 1,
+ },
+ )
+ .expect("encode");
+ enc.end_encoding();
+ cb.commit();
+ cb.wait_until_completed();
+ }
+ let elapsed = start.elapsed();
+ (elapsed.as_secs_f64() * 1000.0) / bench_iters as f64
+}
+
+fn benchmark_mpsgraph_matmul(
+ device: &metal::Device,
+ m: usize,
+ k: usize,
+ n: usize,
+ warmup_iters: usize,
+ bench_iters: usize,
+) -> f64 {
+ autoreleasepool(|_| {
+ print!("graph ");
+ let _ = std::io::stdout().flush();
+ let graph = Graph::new();
+
+ // Create input placeholders
+ let a_shape = [m as isize, k as isize];
+ let b_shape = [k as isize, n as isize];
+
+ let a_placeholder = graph.placeholder(
+ Some(&a_shape),
+ mpsgraph::DataType::BFloat16,
+ Some("a"),
+ );
+ let b_placeholder = graph.placeholder(
+ Some(&b_shape),
+ mpsgraph::DataType::BFloat16,
+ Some("b"),
+ );
+
+ // Matrix multiplication
+ let result =
+ graph.matrix_multiplication(&a_placeholder, &b_placeholder, None);
+
+ // Create shaped types for compilation
+ let a_shaped_type = ShapedType::new_with_shape_data_type(
+ Some(&a_shape),
+ mpsgraph::DataType::BFloat16,
+ );
+ let b_shaped_type = ShapedType::new_with_shape_data_type(
+ Some(&b_shape),
+ mpsgraph::DataType::BFloat16,
+ );
+
+ let feeds = HashMap::from([
+ (&*a_placeholder, &*a_shaped_type),
+ (&*b_placeholder, &*b_shaped_type),
+ ]);
+
+ // Compile
+ print!("compile ");
+ let _ = std::io::stdout().flush();
+ let mps_device = MPSDevice::with_device(device);
+ let executable =
+ graph.compile(&mps_device, &feeds, &[&result], None, None);
+ print!("ok ");
+
+ // Create execution descriptor
+ let execution_descriptor = ExecutableExecutionDescriptor::new();
+
+ // Create input data buffers
+ let a: Vec<bf16> = (0..(m * k))
+ .map(|i| bf16::from_f32(((i % 13) as f32) * 0.01))
+ .collect();
+ let b: Vec<bf16> = (0..(k * n))
+ .map(|i| bf16::from_f32(((i % 17) as f32) * 0.02 - 0.1))
+ .collect();
+
+ let a_buf = device.new_buffer_with_data(
+ a.as_ptr() as *const _,
+ (a.len() * core::mem::size_of::<bf16>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+ let b_buf = device.new_buffer_with_data(
+ b.as_ptr() as *const _,
+ (b.len() * core::mem::size_of::<bf16>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+
+ // Create output buffer
+ let output_buf = device.new_buffer(
+ (m * n * core::mem::size_of::<bf16>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+
+ let command_queue = device.new_command_queue();
+
+ // Create tensor data wrappers
+ let a_tensor_data = TensorData::new_with_mtl_buffer(
+ &a_buf,
+ &[m, k],
+ mpsgraph::DataType::BFloat16,
+ None,
+ );
+ let b_tensor_data = TensorData::new_with_mtl_buffer(
+ &b_buf,
+ &[k, n],
+ mpsgraph::DataType::BFloat16,
+ None,
+ );
+ let output_tensor_data = TensorData::new_with_mtl_buffer(
+ &output_buf,
+ &[m, n],
+ mpsgraph::DataType::BFloat16,
+ None,
+ );
+
+ let inputs: &[&TensorData] = &[&a_tensor_data, &b_tensor_data];
+ let outputs: &[&TensorData] = &[&output_tensor_data];
+
+ // Warmup
+ execution_descriptor.set_enable_commit_and_continue(true);
+ print!("warmup ");
+ let _ = std::io::stdout().flush();
+ for _ in 0..warmup_iters {
+ let command_buffer =
+ CommandBuffer::from_command_queue(&command_queue);
+ let root_cb = command_buffer.root_command_buffer().to_owned();
+ let _ = executable.encode_to_command_buffer(
+ &command_buffer,
+ inputs,
+ Some(outputs),
+ Some(&execution_descriptor),
+ );
+ command_buffer.commit_and_continue();
+ root_cb.wait_until_completed();
+ }
+ print!("ok ");
+ let _ = std::io::stdout().flush();
+
+ // Benchmark
+ print!("bench ");
+ let _ = std::io::stdout().flush();
+ let start = Instant::now();
+ for _ in 0..bench_iters {
+ let command_buffer =
+ CommandBuffer::from_command_queue(&command_queue);
+ let root_cb = command_buffer.root_command_buffer().to_owned();
+ let _ = executable.encode_to_command_buffer(
+ &command_buffer,
+ inputs,
+ Some(outputs),
+ Some(&execution_descriptor),
+ );
+ command_buffer.commit_and_continue();
+ root_cb.wait_until_completed();
+ }
+ let elapsed = start.elapsed();
+ print!("done ");
+ let _ = std::io::stdout().flush();
+ (elapsed.as_secs_f64() * 1000.0) / bench_iters as f64
+ })
+}
+
+fn run_benchmark(
+ ctx: &MTLContext,
+ name: &'static str,
+ m: usize,
+ k: usize,
+ n: usize,
+ warmup: usize,
+ iters: usize,
+) -> BenchmarkResult {
+ print!("[Metal] ");
+ let _ = std::io::stdout().flush();
+ let metal_ms = benchmark_metal_gemm(ctx, m, k, n, warmup, iters);
+ print!("{:.2}ms ", metal_ms);
+ let _ = std::io::stdout().flush();
+
+ print!("[MPS] ");
+ let _ = std::io::stdout().flush();
+ let mpsgraph_ms =
+ benchmark_mpsgraph_matmul(&ctx.device, m, k, n, warmup, iters);
+ print!("{:.2}ms ", mpsgraph_ms);
+ let _ = std::io::stdout().flush();
+
+ BenchmarkResult {
+ name,
+ m,
+ k,
+ n,
+ metal_ms,
+ mpsgraph_ms,
+ metal_gflops: compute_gflops(m, k, n, metal_ms),
+ mpsgraph_gflops: compute_gflops(m, k, n, mpsgraph_ms),
+ }
+}
+
+fn main() {
+ let device = Device::system_default().expect("No Metal device found");
+ let command_queue = device.new_command_queue();
+ let ctx = MTLContext::new(device, command_queue)
+ .expect("Failed to create MTLContext");
+
+ println!("Metal GEMM vs MPSGraph Matmul Benchmark");
+ println!("Device: {}", ctx.device.name());
+ println!("========================================\n");
+
+ let warmup = 10;
+ let iters = 50;
+
+ // Different tensor shapes to test - typical LLM shapes
+ let shapes: &[(&str, usize, usize, usize)] = &[
+ ("GEMV small (1 token)", 1, 2048, 2048),
+ ("GEMV medium (1 token)", 1, 4096, 4096),
+ ("GEMV large (1 token)", 1, 8192, 8192),
+ ("Small batch (8 tokens)", 8, 2048, 2048),
+ ("Medium batch (32 tokens)", 32, 2048, 2048),
+ ("Large batch (128 tokens)", 128, 2048, 2048),
+ ("Prefill small (256 tokens)", 256, 2048, 2048),
+ ("Prefill medium (512 tokens)", 512, 2048, 2048),
+ ("Prefill large (1024 tokens)", 1024, 2048, 2048),
+ ("Square small", 256, 256, 256),
+ ("Square medium", 512, 512, 512),
+ ("Square large", 1024, 1024, 1024),
+ ("MLP up (1 token)", 1, 2048, 8192),
+ ("MLP down (1 token)", 1, 8192, 2048),
+ ("MLP up (128 tokens)", 128, 2048, 8192),
+ ("MLP down (128 tokens)", 128, 8192, 2048),
+ ];
+
+ println!(
+ "{:<30} {:>6} {:>6} {:>6} {:>10} {:>10} {:>10} {:>10} {:>8}",
+ "Shape",
+ "M",
+ "K",
+ "N",
+ "Metal(ms)",
+ "MPS(ms)",
+ "Metal(GF)",
+ "MPS(GF)",
+ "Speedup"
+ );
+ println!("{}", "-".repeat(110));
+
+ for (name, m, k, n) in shapes {
+ // Print progress indicator
+ print!("Running {}... ", name);
+ let _ = std::io::stdout().flush();
+
+ let result = run_benchmark(&ctx, name, *m, *k, *n, warmup, iters);
+ let speedup = result.metal_ms / result.mpsgraph_ms;
+ let speedup_str = if speedup > 1.0 {
+ format!("{:.2}x ✓", speedup)
+ } else {
+ format!("{:.2}x", speedup)
+ };
+
+ // Clear line and print result
+ print!("\r");
+ println!(
+ "{:<30} {:>6} {:>6} {:>6} {:>10.4} {:>10.4} {:>10.1} {:>10.1} {:>8}",
+ result.name,
+ result.m,
+ result.k,
+ result.n,
+ result.metal_ms,
+ result.mpsgraph_ms,
+ result.metal_gflops,
+ result.mpsgraph_gflops,
+ speedup_str
+ );
+ let _ = std::io::stdout().flush();
+ }
+
+ println!("\n=== Summary ===");
+ println!("Speedup > 1.0 means MPSGraph is faster");
+ println!("Speedup < 1.0 means Metal GEMM is faster");
+}
diff --git a/crates/uzu/src/backends/metal/context.rs b/crates/uzu/src/backends/metal/context.rs
index 5ef3e1712..659c8d03e 100644
--- a/crates/uzu/src/backends/metal/context.rs
+++ b/crates/uzu/src/backends/metal/context.rs
@@ -1,7 +1,7 @@
#![allow(dead_code)]
include!(concat!(env!("OUT_DIR"), "/metal_lib.rs"));
-use std::{cell::RefCell, collections::HashMap, rc::Rc};
+use std::{cell::RefCell, collections::HashMap, rc::Rc, env};
use metal::{
CommandQueue as MTLCommandQueue,
@@ -14,9 +14,147 @@ use super::{
};
use crate::{DataType, DeviceContext, array::array_size_in_bytes};
+/// Apple GPU architecture generation.
+/// Based on Apple GPU family naming convention (e.g., "applegpu_g13p").
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum DeviceGeneration {
+ Gen13, // M1 family
+ Gen14, // M2 family
+ Gen15, // M3 family
+ Gen16, // M3 Pro/Max with enhanced features
+ Gen17, // M4 family (NAX capable)
+ Unknown(u8),
+}
+
+impl DeviceGeneration {
+ fn from_generation_number(r#gen: u8) -> Self {
+ match r#gen {
+ 13 => Self::Gen13,
+ 14 => Self::Gen14,
+ 15 => Self::Gen15,
+ 16 => Self::Gen16,
+ 17 => Self::Gen17,
+ n => Self::Unknown(n),
+ }
+ }
+
+ pub fn generation_number(&self) -> u8 {
+ match self {
+ Self::Gen13 => 13,
+ Self::Gen14 => 14,
+ Self::Gen15 => 15,
+ Self::Gen16 => 16,
+ Self::Gen17 => 17,
+ Self::Unknown(n) => *n,
+ }
+ }
+}
+
+/// Device performance class based on the last character of architecture name.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum DeviceClass {
+ Phone, // 'p' - iPhone/iPad integrated
+ Integrated, // 'g' - Mac integrated GPU
+ Desktop, // 'd' - Mac Pro/Max discrete-class GPU
+ Unknown(char),
+}
+
+impl DeviceClass {
+ fn from_arch_suffix(c: char) -> Self {
+ match c {
+ 'p' => Self::Phone,
+ 'g' => Self::Integrated,
+ 'd' => Self::Desktop,
+ other => Self::Unknown(other),
+ }
+ }
+
+ pub fn is_high_performance(&self) -> bool {
+ matches!(self, Self::Desktop)
+ }
+}
+
+/// Complete device architecture information.
+#[derive(Debug, Clone)]
+pub struct DeviceArchitecture {
+ pub generation: DeviceGeneration,
+ pub device_class: DeviceClass,
+ pub arch_string: String,
+}
+
+impl DeviceArchitecture {
+ pub fn from_device(device: &MTLDevice) -> Self {
+ let arch_string = device.name().to_string();
+
+ // Extract generation from architecture name
+ // Format is typically like "Apple M3 Pro" or the GPU name "applegpu_g15d"
+ let (generation, device_class) =
+ Self::parse_architecture_info(&arch_string);
+
+ Self {
+ generation,
+ device_class,
+ arch_string,
+ }
+ }
+
+ fn parse_architecture_info(
+ arch: &str,
+ ) -> (DeviceGeneration, DeviceClass) {
+ // Try to extract from GPU architecture string if available
+ // MLX uses device_->architecture()->name() which gives "applegpu_gXXY"
+ // where XX is generation and Y is class (p/g/d)
+ // The metal crate's device.name() gives the marketing name like "Apple M3 Pro"
+
+ // For now, we'll use a heuristic based on the device name
+ let generation = if arch.contains("M4") {
+ DeviceGeneration::Gen17
+ } else if arch.contains("M3") {
+ if arch.contains("Max") || arch.contains("Ultra") {
+ DeviceGeneration::Gen16
+ } else {
+ DeviceGeneration::Gen15
+ }
+ } else if arch.contains("M2") {
+ DeviceGeneration::Gen14
+ } else if arch.contains("M1") {
+ DeviceGeneration::Gen13
+ } else {
+ DeviceGeneration::Unknown(0)
+ };
+
+ let device_class = if arch.contains("Max")
+ || arch.contains("Ultra")
+ || arch.contains("Pro")
+ {
+ DeviceClass::Desktop
+ } else if arch.contains("iPhone") || arch.contains("iPad") {
+ DeviceClass::Phone
+ } else {
+ DeviceClass::Integrated
+ };
+
+ (generation, device_class)
+ }
+
+ /// Returns true if NAX (New Accelerator eXtensions) is available.
+ /// NAX requires M4 or later (generation >= 17) and macOS 26.2+.
+ /// Since we can't check macOS version at compile time in Rust,
+ /// we check generation only - the kernels will fail gracefully if unavailable.
+ pub fn is_nax_available(&self) -> bool {
+ self.generation.generation_number() >= 17
+ }
+
+ /// Returns true if this is a high-performance device (Pro/Max class).
+ pub fn is_high_performance(&self) -> bool {
+ self.device_class.is_high_performance()
+ }
+}
+
pub struct MTLContext {
pub device: MTLDevice,
pub command_queue: MTLCommandQueue,
+ pub architecture: DeviceArchitecture,
library: MTLLibrary,
pipeline_cache: RefCell<HashMap<String, MTLComputePipelineState>>,
}
@@ -36,14 +174,44 @@ impl MTLContext {
},
};
+ let architecture = DeviceArchitecture::from_device(&device);
+
Ok(Self {
device,
command_queue,
+ architecture,
library,
pipeline_cache: RefCell::new(HashMap::new()),
})
}
+ /// Returns true if NAX kernels are available on this device.
+ pub fn is_nax_available(&self) -> bool {
+ self.architecture.is_nax_available()
+ }
+
+ /// Returns true if this is a high-performance device (Pro/Max class).
+ pub fn is_high_performance(&self) -> bool {
+ self.architecture.is_high_performance()
+ }
+
+ /// Returns the device generation for tile size selection.
+ pub fn device_generation(&self) -> DeviceGeneration {
+ self.architecture.generation
+ }
+
+ /// Returns the device class for performance tuning.
+ pub fn device_class(&self) -> DeviceClass {
+ self.architecture.device_class
+ }
+
+ /// TF32 toggle similar to MLX env::enable_tf32().
+ pub fn tf32_enabled(&self) -> bool {
+ env::var("UZU_TF32")
+ .map(|v| v == "1" || v.eq_ignore_ascii_case("true"))
+ .unwrap_or(false)
+ }
+
pub fn compute_pipeline_state(
&self,
function_name: &str,
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv.h b/crates/uzu/src/backends/metal/kernel/matmul/gemv.h
new file mode 100644
index 000000000..2b0da5b02
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv.h
@@ -0,0 +1,19 @@
+// Declarations for GEMV kernels (Metal).
+// Implementations are in gemv.metal; names must match Rust dispatch.
+
+#pragma once
+
+#ifdef __METAL_VERSION__
+// No host-side declarations needed; compiled by Metal.
+#else
+// Host-side string literals for kernel names.
+#define GEMV_F16_ROWS2 "gemv_f16_rows2"
+#define GEMV_F16_ROWS4 "gemv_f16_rows4"
+#define GEMV_F16_ROWS8 "gemv_f16_rows8"
+
+#define GEMV_BF16_ROWS2 "gemv_bf16_rows2"
+#define GEMV_BF16_ROWS4 "gemv_bf16_rows4"
+#define GEMV_BF16_ROWS8 "gemv_bf16_rows8"
+#endif
+
+
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemv.metal
new file mode 100644
index 000000000..c3ff84e4f
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv.metal
@@ -0,0 +1,280 @@
+#include <metal_stdlib>
+using namespace metal;
+
+// Tiled GEMV with vector blocking in threadgroup memory.
+// Signature matches existing Rust dispatch (no bias/axpby).
+template <typename T, typename AccT, ushort ROWS_PER_TG, ushort BK>
+METAL_FUNC void gemv_impl_tiled(
+ const device T* __restrict matrix [[buffer(0)]],
+ const device T* __restrict vector [[buffer(1)]],
+ device T* __restrict output [[buffer(2)]],
+ constant uint& k [[buffer(3)]],
+ constant uint& n [[buffer(4)]],
+ constant uint& ldb [[buffer(5)]],
+ constant uint& lda [[buffer(6)]],
+ constant uint& ldd [[buffer(7)]],
+ uint3 tgid [[threadgroup_position_in_grid]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]],
+ uint tid [[thread_index_in_threadgroup]],
+ threadgroup AccT* vec_tile) {
+ const ushort TG_THREADS = ROWS_PER_TG * 32;
+ if (simd_gid >= ROWS_PER_TG) {
+ return;
+ }
+ const uint row = tgid.x * ROWS_PER_TG + simd_gid;
+ if (row >= n) {
+ return;
+ }
+ const uint batch = tgid.z;
+
+ const device T* row_ptr = matrix + row * ldb;
+ const device T* vec_ptr = vector + batch * lda;
+ device T* out_ptr = output + batch * ldd;
+
+ AccT acc = static_cast<AccT>(0);
+
+ // Cooperative load of vector tile into threadgroup memory.
+ for (uint k_block = 0; k_block < k; k_block += BK) {
+ const uint remaining = k - k_block;
+ const uint tile_elems = remaining < BK ? remaining : BK;
+
+ // Each thread loads at most one element of the vector tile.
+ if (tid < tile_elems && tid < TG_THREADS) {
+ vec_tile[tid] = static_cast<AccT>(vec_ptr[k_block + tid]);
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // Accumulate over this tile.
+ for (uint idx = simd_lid; idx < tile_elems; idx += 32) {
+ acc += static_cast<AccT>(row_ptr[k_block + idx]) * vec_tile[idx];
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+
+ // Reduce within simdgroup.
+ for (ushort offset = 16; offset > 0; offset >>= 1) {
+ acc += simd_shuffle_down(acc, offset);
+ }
+
+ if (simd_lid == 0) {
+ out_ptr[row] = static_cast<T>(acc);
+ }
+}
+
+#define DECL_GEMV(NAME, TYPE, ACC, ROWS, BK) \
+ [[kernel, max_total_threads_per_threadgroup(ROWS * 32)]] void NAME( \
+ const device TYPE* matrix [[buffer(0)]], \
+ const device TYPE* vector [[buffer(1)]], \
+ device TYPE* output [[buffer(2)]], \
+ constant uint& k [[buffer(3)]], \
+ constant uint& n [[buffer(4)]], \
+ constant uint& ldb [[buffer(5)]], \
+ constant uint& lda [[buffer(6)]], \
+ constant uint& ldd [[buffer(7)]], \
+ uint3 tgid [[threadgroup_position_in_grid]], \
+ uint simd_gid [[simdgroup_index_in_threadgroup]], \
+ uint simd_lid [[thread_index_in_simdgroup]], \
+ uint tid [[thread_index_in_threadgroup]]) { \
+ threadgroup ACC vec_tile[BK]; \
+ gemv_impl_tiled<TYPE, ACC, ROWS, BK>( \
+ matrix, vector, output, k, n, ldb, lda, ldd, \
+ tgid, simd_gid, simd_lid, tid, vec_tile); \
+ }
+
+// half variants
+DECL_GEMV(gemv_f16_rows2, half, float, 2, 128)
+DECL_GEMV(gemv_f16_rows4, half, float, 4, 128)
+DECL_GEMV(gemv_f16_rows8, half, float, 8, 128)
+
+// bfloat16 variants
+DECL_GEMV(gemv_bf16_rows2, bfloat, float, 2, 128)
+DECL_GEMV(gemv_bf16_rows4, bfloat, float, 4, 128)
+DECL_GEMV(gemv_bf16_rows8, bfloat, float, 8, 128)
+#include <metal_stdlib>
+using namespace metal;
+
+#define MLX_MTL_PRAGMA_UNROLL _Pragma("unroll")
+
+// Simplified MLX-style GEMV without bias/axpby; compact signature.
+template <
+ typename T,
+ const int BM,
+ const int BN,
+ const int SM,
+ const int SN,
+ const int TM,
+ const int TN>
+struct GEMVKernel {
+ enum : int {
+ threadsM = BM * SM,
+ threadsN = BN * SN,
+ blockM = threadsM * TM,
+ blockN = threadsN * TN,
+ };
+
+ static_assert(SM * SN == 32, "simdgroup must be 32 threads");
+ static_assert(SN == 4 || SN == 8 || SN == 16 || SN == 32, "SN must divide 32");
+
+ template <typename U = T>
+ static METAL_FUNC void load_safe(
+ const device T* src,
+ thread U dst[TN],
+ const int src_offset,
+ const int src_size) {
+ if (src_offset + TN <= src_size) {
+ MLX_MTL_PRAGMA_UNROLL
+ for (int tn = 0; tn < TN; tn++) {
+ dst[tn] = static_cast<U>(src[src_offset + tn]);
+ }
+ } else {
+ MLX_MTL_PRAGMA_UNROLL
+ for (int tn = 0; tn < TN; tn++) {
+ dst[tn] = src_offset + tn < src_size
+ ? static_cast<U>(src[src_offset + tn])
+ : U(0);
+ }
+ }
+ }
+
+ static METAL_FUNC void run(
+ const device T* mat [[buffer(0)]],
+ const device T* in_vec [[buffer(1)]],
+ device T* out_vec [[buffer(2)]],
+ const constant int& in_vec_size [[buffer(3)]],
+ const constant int& out_vec_size [[buffer(4)]],
+ const constant int& matrix_ld [[buffer(5)]],
+ threadgroup float* tgp_memory [[threadgroup(0)]],
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint3 lid [[thread_position_in_threadgroup]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]) {
+ (void)lid;
+
+ thread float result[TM] = {0};
+ thread T inter[TN];
+ thread float v_coeff[TN];
+
+ const int thrM = SN != 32 ? simd_lid / SN : 0;
+ const int thrN = SN != 32 ? simd_lid % SN : int(simd_lid);
+
+ const int sgN = BN != 1 ? (simd_gid % BN) : 0;
+
+ const int simdM = BN != 1 ? SM * (simd_gid / BN) : int(SM * simd_gid);
+ const int simdN = BN != 1 ? SN * (simd_gid % BN) : 0;
+
+ int bm = (simdM + thrM) * TM;
+ int bn = (simdN + thrN) * TN;
+
+ int out_row = tid.x * blockM + bm;
+ if (out_row >= out_vec_size) {
+ return;
+ }
+ out_row = out_row + TM <= out_vec_size ? out_row : out_vec_size - TM;
+
+ mat += out_row * matrix_ld;
+
+ const int loop_stride = blockN;
+ const int in_size = in_vec_size;
+ const int n_iter = in_size / loop_stride;
+ const int leftover = in_size - loop_stride * n_iter;
+
+ for (int i = 0; i < n_iter; ++i) {
+ load_safe<float>(in_vec, v_coeff, bn, TN);
+
+ int mat_offset = 0;
+ MLX_MTL_PRAGMA_UNROLL
+ for (int tm = 0; tm < TM; tm++) {
+ load_safe(mat, inter, mat_offset + bn, TN);
+ MLX_MTL_PRAGMA_UNROLL
+ for (int tn = 0; tn < TN; tn++) {
+ result[tm] += inter[tn] * v_coeff[tn];
+ }
+ mat_offset += matrix_ld;
+ }
+
+ bn += blockN;
+ }
+
+ if (leftover > 0) {
+ load_safe<float>(in_vec, v_coeff, bn, in_size);
+
+ MLX_MTL_PRAGMA_UNROLL
+ for (int tm = 0; tm < TM; tm++) {
+ load_safe(&mat[tm * matrix_ld], inter, bn, in_size);
+ MLX_MTL_PRAGMA_UNROLL
+ for (int tn = 0; tn < TN; tn++) {
+ result[tm] += inter[tn] * v_coeff[tn];
+ }
+ }
+ }
+
+ MLX_MTL_PRAGMA_UNROLL
+ for (int tm = 0; tm < TM; tm++) {
+ MLX_MTL_PRAGMA_UNROLL
+ for (ushort sn = (SN / 2); sn >= 1; sn >>= 1) {
+ result[tm] += simd_shuffle_down(result[tm], sn);
+ }
+ }
+
+ if (BN > 1) {
+ threadgroup float* tgp_results = tgp_memory + sgN * (blockM + TM) + bm;
+ if (thrN == 0) {
+ MLX_MTL_PRAGMA_UNROLL
+ for (int tm = 0; tm < TM; tm++) {
+ tgp_results[tm] = result[tm];
+ }
+
+ threadgroup_barrier(mem_flags::mem_none);
+
+ if (sgN == 0) {
+ MLX_MTL_PRAGMA_UNROLL
+ for (int sgn = 1; sgn < BN; sgn++) {
+ MLX_MTL_PRAGMA_UNROLL
+ for (int tm = 0; tm < TM; tm++) {
+ result[tm] += tgp_results[sgn * (blockM + TM) + tm];
+ }
+ }
+ }
+ }
+ }
+
+ if (simdN == 0 && thrN == 0) {
+ MLX_MTL_PRAGMA_UNROLL
+ for (int tm = 0; tm < TM; tm++) {
+ out_vec[out_row + tm] = static_cast<T>(result[tm]);
+ }
+ }
+ }
+};
+
+#define DECL_GEMV(name, itype, bm, bn, sm, sn, tm, tn) \
+ [[kernel, max_total_threads_per_threadgroup(bm * sm * bn * sn)]] void \
+ name( \
+ const device itype* mat [[buffer(0)]], \
+ const device itype* in_vec [[buffer(1)]], \
+ device itype* out_vec [[buffer(2)]], \
+ const constant int& in_vec_size [[buffer(3)]], \
+ const constant int& out_vec_size [[buffer(4)]], \
+ const constant int& matrix_ld [[buffer(5)]], \
+ uint3 tid [[threadgroup_position_in_grid]], \
+ uint3 lid [[thread_position_in_threadgroup]], \
+ uint simd_gid [[simdgroup_index_in_threadgroup]], \
+ uint simd_lid [[thread_index_in_simdgroup]], \
+ threadgroup float* tgp_memory [[threadgroup(0)]]) { \
+ GEMVKernel<itype, bm, bn, sm, sn, tm, tn>::run( \
+ mat, in_vec, out_vec, in_vec_size, out_vec_size, \
+ matrix_ld, tgp_memory, \
+ tid, lid, simd_gid, simd_lid); \
+ }
+
+#define DECL_GEMV_BLOCKS(name, itype) \
+ DECL_GEMV(name##_bm2_bn1_sm4_sn8_tm1_tn4, itype, 2, 1, 4, 8, 1, 4) \
+ DECL_GEMV(name##_bm2_bn1_sm4_sn8_tm4_tn4, itype, 2, 1, 4, 8, 4, 4) \
+ DECL_GEMV(name##_bm2_bn1_sm2_sn16_tm1_tn4, itype, 2, 1, 2, 16, 1, 4) \
+ DECL_GEMV(name##_bm2_bn1_sm2_sn16_tm4_tn4, itype, 2, 1, 2, 16, 4, 4) \
+ DECL_GEMV(name##_bm4_bn1_sm2_sn16_tm4_tn4, itype, 4, 1, 2, 16, 4, 4)
+
+DECL_GEMV_BLOCKS(gemv_f32, float)
+DECL_GEMV_BLOCKS(gemv_f16, half)
+DECL_GEMV_BLOCKS(gemv_bf16, bfloat)
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv.rs
new file mode 100644
index 000000000..abcea511c
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv.rs
@@ -0,0 +1,124 @@
+use std::collections::HashMap;
+
+use metal::{
+ ComputeCommandEncoderRef, ComputePipelineState as MTLComputePipelineState,
+ MTLSize,
+};
+
+use super::arguments::MatmulArguments;
+use crate::{
+ DataType,
+ backends::metal::{MTLContext, MTLError},
+};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+struct PipelineKey {
+ rows_per_group: u32,
+}
+
+pub struct GemvKernel {
+ dt: DataType,
+ pipelines: HashMap<PipelineKey, MTLComputePipelineState>,
+}
+
+impl GemvKernel {
+ pub fn new(dt: DataType) -> Self {
+ Self {
+ dt,
+ pipelines: HashMap::new(),
+ }
+ }
+
+ fn kernel_name(
+ &self,
+ rows: u32,
+ ) -> Result<String, MTLError> {
+ let base = match self.dt {
+ DataType::F16 => "gemv_f16",
+ DataType::BF16 => "gemv_bf16",
+ _ => {
+ return Err(MTLError::Generic(format!(
+ "Unsupported dtype for GEMV: {:?}",
+ self.dt
+ )));
+ },
+ };
+ Ok(format!("{base}_rows{rows}"))
+ }
+
+ fn get_or_compile_pipeline(
+ &mut self,
+ mtl: &MTLContext,
+ rows: u32,
+ ) -> Result<&MTLComputePipelineState, MTLError> {
+ let key = PipelineKey {
+ rows_per_group: rows,
+ };
+ if !self.pipelines.contains_key(&key) {
+ let name = self.kernel_name(rows)?;
+ let ps = mtl.compute_pipeline_state(&name, None)?;
+ self.pipelines.insert(key, ps);
+ }
+ Ok(self.pipelines.get(&key).unwrap())
+ }
+
+ pub fn encode(
+ &mut self,
+ mtl: &MTLContext,
+ enc: &ComputeCommandEncoderRef,
+ args: MatmulArguments,
+ rows_per_group: u32,
+ ) -> Result<(), MTLError> {
+ let rows_per_group = rows_per_group.max(1);
+ let ps = self.get_or_compile_pipeline(mtl, rows_per_group)?;
+ enc.set_compute_pipeline_state(ps);
+
+ // Bind buffers: matrix (B), vector (A), output (D)
+ enc.set_buffer(0, Some(args.b), 0);
+ enc.set_buffer(1, Some(args.a), 0);
+ enc.set_buffer(2, Some(args.d), 0);
+
+ // Scalars
+ let k = args.input_dim as u32;
+ let n = args.output_dim as u32;
+ let ldb = args.ldb as u32;
+ let lda = args.lda as u32;
+ let ldd = args.ldd as u32;
+
+ enc.set_bytes(
+ 3,
+ std::mem::size_of::<u32>() as u64,
+ &k as *const u32 as *const std::ffi::c_void,
+ );
+ enc.set_bytes(
+ 4,
+ std::mem::size_of::<u32>() as u64,
+ &n as *const u32 as *const std::ffi::c_void,
+ );
+ enc.set_bytes(
+ 5,
+ std::mem::size_of::<u32>() as u64,
+ &ldb as *const u32 as *const std::ffi::c_void,
+ );
+ enc.set_bytes(
+ 6,
+ std::mem::size_of::<u32>() as u64,
+ &lda as *const u32 as *const std::ffi::c_void,
+ );
+ enc.set_bytes(
+ 7,
+ std::mem::size_of::<u32>() as u64,
+ &ldd as *const u32 as *const std::ffi::c_void,
+ );
+
+ // Threadgroup sizing
+ let threads_per_tg = MTLSize::new((rows_per_group * 32) as u64, 1, 1);
+ let tg_x = ((n + rows_per_group - 1) / rows_per_group) as u64;
+ let tg_y = 1;
+ let tg_z = args.batch_count as u64;
+ let tgs = MTLSize::new(tg_x, tg_y, tg_z);
+
+ enc.dispatch_thread_groups(tgs, threads_per_tg);
+ Ok(())
+ }
+}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv_masked.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemv_masked.metal
new file mode 100644
index 000000000..c6bbe437b
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv_masked.metal
@@ -0,0 +1,150 @@
+// Masked and gather GEMV variants (simplified). Masks/indices are honored only
+// for output write; data loads remain dense. This keeps parity with MLX kernel
+// signatures while avoiding redundant specialization.
+
+#include <metal_stdlib>
+using namespace metal;
+
+template <typename T, typename AccT, ushort ROWS>
+METAL_FUNC void gemv_masked_impl(
+ const device T* __restrict matrix [[buffer(0)]],
+ const device T* __restrict vector [[buffer(1)]],
+ const device uchar* __restrict out_mask [[buffer(2)]],
+ const device uchar* __restrict op_mask [[buffer(3)]],
+ device T* __restrict output [[buffer(4)]],
+ constant uint& k [[buffer(5)]],
+ constant uint& n [[buffer(6)]],
+ constant uint& ldb [[buffer(7)]],
+ constant uint& lda [[buffer(8)]],
+ constant uint& ldd [[buffer(9)]],
+ uint3 tgid [[threadgroup_position_in_grid]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]) {
+ if (simd_gid >= ROWS) {
+ return;
+ }
+ const uint row = tgid.x * ROWS + simd_gid;
+ if (row >= n) {
+ return;
+ }
+ const uint batch = tgid.z;
+ const device T* row_ptr = matrix + row * ldb;
+ const device T* vec_ptr = vector + batch * lda;
+ device T* out_ptr = output + batch * ldd;
+
+ AccT acc = static_cast<AccT>(0);
+ for (uint idx = simd_lid; idx < k; idx += 32) {
+ // If op_mask provided, zero out masked elements
+ const bool op_on = (op_mask == nullptr) || (op_mask[row] != 0);
+ if (op_on) {
+ acc += static_cast<AccT>(row_ptr[idx]) *
+ static_cast<AccT>(vec_ptr[idx]);
+ }
+ }
+
+ for (ushort offset = 16; offset > 0; offset >>= 1) {
+ acc += simd_shuffle_down(acc, offset);
+ }
+
+ if (simd_lid == 0) {
+ if (out_mask == nullptr || out_mask[row] != 0) {
+ out_ptr[row] = static_cast<T>(acc);
+ }
+ }
+}
+
+template <typename T, typename AccT, ushort ROWS>
+METAL_FUNC void gemv_gather_impl(
+ const device T* __restrict matrix [[buffer(0)]],
+ const device T* __restrict vector [[buffer(1)]],
+ const device uint* __restrict vec_indices [[buffer(2)]],
+ const device uint* __restrict mat_indices [[buffer(3)]],
+ device T* __restrict output [[buffer(4)]],
+ constant uint& k [[buffer(5)]],
+ constant uint& n [[buffer(6)]],
+ constant uint& ldb [[buffer(7)]],
+ constant uint& lda [[buffer(8)]],
+ constant uint& ldd [[buffer(9)]],
+ uint3 tgid [[threadgroup_position_in_grid]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]) {
+ if (simd_gid >= ROWS) {
+ return;
+ }
+ const uint row = tgid.x * ROWS + simd_gid;
+ if (row >= n) {
+ return;
+ }
+ const uint batch = tgid.z;
+ const device T* base_row = matrix + row * ldb;
+ device T* out_ptr = output + batch * ldd;
+
+ AccT acc = static_cast<AccT>(0);
+ for (uint idx = simd_lid; idx < k; idx += 32) {
+ const uint col = mat_indices ? mat_indices[idx] : idx;
+ const uint vec_idx = vec_indices ? vec_indices[idx] : idx;
+ acc += static_cast<AccT>(base_row[col]) *
+ static_cast<AccT>(vector[batch * lda + vec_idx]);
+ }
+
+ for (ushort offset = 16; offset > 0; offset >>= 1) {
+ acc += simd_shuffle_down(acc, offset);
+ }
+
+ if (simd_lid == 0) {
+ out_ptr[row] = static_cast<T>(acc);
+ }
+}
+
+#define DECL_GEMV_MASKED(NAME, TYPE, ACC, ROWS) \
+ [[kernel, max_total_threads_per_threadgroup(ROWS * 32)]] void NAME( \
+ const device TYPE* matrix [[buffer(0)]], \
+ const device TYPE* vector [[buffer(1)]], \
+ const device uchar* out_mask [[buffer(2)]], \
+ const device uchar* op_mask [[buffer(3)]], \
+ device TYPE* output [[buffer(4)]], \
+ constant uint& k [[buffer(5)]], \
+ constant uint& n [[buffer(6)]], \
+ constant uint& ldb [[buffer(7)]], \
+ constant uint& lda [[buffer(8)]], \
+ constant uint& ldd [[buffer(9)]], \
+ uint3 tgid [[threadgroup_position_in_grid]], \
+ uint simd_gid [[simdgroup_index_in_threadgroup]], \
+ uint simd_lid [[thread_index_in_simdgroup]]) { \
+ gemv_masked_impl<TYPE, ACC, ROWS>( \
+ matrix, vector, out_mask, op_mask, output, \
+ k, n, ldb, lda, ldd, tgid, simd_gid, simd_lid); \
+ }
+
+#define DECL_GEMV_GATHER(NAME, TYPE, ACC, ROWS) \
+ [[kernel, max_total_threads_per_threadgroup(ROWS * 32)]] void NAME( \
+ const device TYPE* matrix [[buffer(0)]], \
+ const device TYPE* vector [[buffer(1)]], \
+ const device uint* vec_indices [[buffer(2)]], \
+ const device uint* mat_indices [[buffer(3)]], \
+ device TYPE* output [[buffer(4)]], \
+ constant uint& k [[buffer(5)]], \
+ constant uint& n [[buffer(6)]], \
+ constant uint& ldb [[buffer(7)]], \
+ constant uint& lda [[buffer(8)]], \
+ constant uint& ldd [[buffer(9)]], \
+ uint3 tgid [[threadgroup_position_in_grid]], \
+ uint simd_gid [[simdgroup_index_in_threadgroup]], \
+ uint simd_lid [[thread_index_in_simdgroup]]) { \
+ gemv_gather_impl<TYPE, ACC, ROWS>( \
+ matrix, vector, vec_indices, mat_indices, output, \
+ k, n, ldb, lda, ldd, tgid, simd_gid, simd_lid); \
+ }
+
+// Masked instantiations
+DECL_GEMV_MASKED(gemv_masked_f16_rows4, half, float, 4)
+DECL_GEMV_MASKED(gemv_masked_f16_rows8, half, float, 8)
+DECL_GEMV_MASKED(gemv_masked_bf16_rows4, bfloat, float, 4)
+DECL_GEMV_MASKED(gemv_masked_bf16_rows8, bfloat, float, 8)
+
+// Gather instantiations
+DECL_GEMV_GATHER(gemv_gather_f16_rows4, half, float, 4)
+DECL_GEMV_GATHER(gemv_gather_f16_rows8, half, float, 8)
+DECL_GEMV_GATHER(gemv_gather_bf16_rows4, bfloat, float, 4)
+DECL_GEMV_GATHER(gemv_gather_bf16_rows8, bfloat, float, 8)
+// Masked/gather GEMV kernels are stubbed out (not used). Keeping file empty to avoid shader compilation errors.
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs
index 2846419a8..2e2f99419 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs
@@ -6,8 +6,9 @@ use metal::{
};
use super::{
- arguments::MatmulArguments, pipeline::PipelineKey,
- shared_types::GEMMParams, transpose::transpose_configuration,
+ arguments::MatmulArguments, gemv::GemvKernel, pipeline::PipelineKey,
+ shared_types::GEMMParams, splitk::SplitKGemm,
+ transpose::transpose_configuration,
};
use crate::{
DataType,
@@ -18,11 +19,8 @@ pub struct MatmulKernel {
dt: DataType,
transpose_a: bool,
transpose_b: bool,
- bm: i32,
- bn: i32,
- bk: i32,
- wm: u64,
- wn: u64,
+ gemv: Option<GemvKernel>,
+ splitk: Option<SplitKGemm>,
pipelines: HashMap<PipelineKey, MTLComputePipelineState>,
}
@@ -36,12 +34,21 @@ impl MatmulKernel {
}
}
- fn kernel_name(&self) -> String {
+ fn kernel_name(
+ &self,
+ bm: i32,
+ bn: i32,
+ bk: i32,
+ wm: u64,
+ wn: u64,
+ ) -> String {
let t = Self::dtype_suffix(self.dt);
let cfg = transpose_configuration(self.transpose_a, self.transpose_b);
let tcfg = cfg.as_str();
- // Shapes we currently instantiate in matmul.metal: bm64/bn64/bk16/wm2/wn2
- format!("gemm_{}_{}_bm64_bn64_bk16_wm2_wn2", tcfg, t)
+ format!(
+ "gemm_{}_{}_bm{}_bn{}_bk{}_wm{}_wn{}",
+ tcfg, t, bm, bn, bk, wm, wn
+ )
}
pub fn new(
@@ -60,15 +67,201 @@ impl MatmulKernel {
dt,
transpose_a,
transpose_b,
- bm: 64,
- bn: 64,
- bk: 16,
- wm: 2,
- wn: 2,
+ gemv: None,
+ splitk: None,
pipelines: HashMap::new(),
})
}
+ fn select_gemv_rows(
+ &self,
+ output_dim: i32,
+ ) -> u32 {
+ if output_dim >= 2048 {
+ 8
+ } else if output_dim >= 512 {
+ 4
+ } else {
+ 2
+ }
+ }
+
+ fn maybe_use_gemv(
+ &mut self,
+ mtl: &MTLContext,
+ enc: &ComputeCommandEncoderRef,
+ args: MatmulArguments,
+ ) -> Result<bool, MTLError> {
+ // Apply GEMV when batch (M) is 1 or output_dim (N) is 1, and when A is not transposed.
+ let m = args.batch;
+ let n = args.output_dim;
+ if self.transpose_a {
+ return Ok(false);
+ }
+ if m != 1 && n != 1 {
+ return Ok(false);
+ }
+ let rows = self.select_gemv_rows(n);
+ let gemv = self.gemv.get_or_insert_with(|| GemvKernel::new(self.dt));
+ gemv.encode(mtl, enc, args, rows)?;
+ Ok(true)
+ }
+
+ fn maybe_use_splitk(
+ &mut self,
+ mtl: &MTLContext,
+ enc: &ComputeCommandEncoderRef,
+ args: MatmulArguments,
+ ) -> Result<bool, MTLError> {
+ let m = args.batch;
+ let n = args.output_dim;
+ let k = args.input_dim;
+ let batch_count = args.batch_count;
+
+ let splitk =
+ self.splitk.get_or_insert_with(|| SplitKGemm::new(self.dt));
+ if splitk.should_use_splitk(m, n, k, batch_count) {
+ splitk.encode(mtl, enc, args)?;
+ return Ok(true);
+ }
+ Ok(false)
+ }
+
+ fn select_tiles(
+ &self,
+ mtl: &MTLContext,
+ args: &MatmulArguments,
+ ) -> (i32, i32, i32, u64, u64) {
+ use crate::backends::metal::DeviceClass;
+ let mut bm = 64;
+ let mut bn = 64;
+ let mut bk = 16;
+ let mut wm = 2;
+ let mut wn = 2;
+
+ let m = args.batch;
+ let n = args.output_dim;
+ let k = args.input_dim;
+ let batch_size_out = args.batch_count;
+ let large_mat =
+ (batch_size_out as i64) * (m as i64) * (n as i64) >= (1_i64 << 20);
+
+ // Prefer NAX tiles when available (mirrors MLX fused_nax path).
+ if mtl.is_nax_available()
+ && (!matches!(self.dt, DataType::F32) || mtl.tf32_enabled())
+ {
+ if large_mat {
+ return (128, 128, 32, 2, 2);
+ }
+ return (64, 64, 32, 2, 2);
+ }
+ let devc = match mtl.architecture.device_class {
+ DeviceClass::Desktop => 'd',
+ DeviceClass::Integrated => 'g',
+ DeviceClass::Phone => 'p',
+ DeviceClass::Unknown(_) => 'g',
+ };
+
+ let is_float = matches!(self.dt, DataType::F32);
+ let prefer_halfish = !is_float || mtl.tf32_enabled();
+
+ if devc == 'g' || devc == 'p' {
+ if prefer_halfish {
+ if !self.transpose_a && self.transpose_b {
+ bm = 64;
+ bn = 32;
+ bk = 32;
+ wm = 2;
+ wn = 2;
+ } else {
+ bm = 64;
+ bn = 64;
+ bk = 16;
+ wm = 1;
+ wn = 2;
+ }
+ } else if !self.transpose_a && self.transpose_b {
+ bm = 32;
+ bn = 64;
+ bk = 16;
+ wm = 1;
+ wn = 2;
+ } else {
+ bm = 64;
+ bn = 32;
+ bk = 32;
+ wm = 2;
+ wn = 2;
+ }
+ } else if devc == 'd' {
+ if large_mat {
+ if prefer_halfish {
+ if 2 * std::cmp::max(m, n) > k {
+ bm = 64;
+ bn = 64;
+ bk = 16;
+ wm = 1;
+ wn = 2;
+ } else if !self.transpose_a && self.transpose_b {
+ bm = 64;
+ bn = 32;
+ bk = 32;
+ wm = 2;
+ wn = 2;
+ } else {
+ bm = 32;
+ bn = 64;
+ bk = 16;
+ wm = 1;
+ wn = 2;
+ }
+ } else if !self.transpose_a && self.transpose_b {
+ bm = 32;
+ bn = 64;
+ bk = 16;
+ wm = 1;
+ wn = 2;
+ } else {
+ bm = 64;
+ bn = 32;
+ bk = 32;
+ wm = 2;
+ wn = 2;
+ }
+ } else {
+ if prefer_halfish {
+ if !self.transpose_a && self.transpose_b {
+ bm = 64;
+ bn = 32;
+ bk = 32;
+ wm = 2;
+ wn = 2;
+ } else {
+ bm = 64;
+ bn = 64;
+ bk = 16;
+ wm = 1;
+ wn = 2;
+ }
+ } else if !self.transpose_a && self.transpose_b {
+ bm = 32;
+ bn = 64;
+ bk = 16;
+ wm = 1;
+ wn = 2;
+ } else {
+ bm = 64;
+ bn = 32;
+ bk = 32;
+ wm = 2;
+ wn = 2;
+ }
+ }
+ }
+
+ (bm, bn, bk, wm, wn)
+ }
+
fn get_or_compile_pipeline(
&mut self,
mtl: &MTLContext,
@@ -138,18 +331,31 @@ impl MatmulKernel {
&mut self,
mtl: &MTLContext,
enc: &ComputeCommandEncoderRef,
- args: MatmulArguments,
+ mut args: MatmulArguments,
) -> Result<(), MTLError> {
- let kname = self.kernel_name();
+ self.apply_batch_collapse(&mut args);
+
+ // Try GEMV fast path for decode shapes.
+ if self.maybe_use_gemv(mtl, enc, args)? {
+ return Ok(());
+ }
+
+ // Try Split-K for small M*N and large K.
+ if self.maybe_use_splitk(mtl, enc, args)? {
+ return Ok(());
+ }
+
+ let (bm, bn, bk, wm, wn) = self.select_tiles(mtl, &args);
+ let kname = self.kernel_name(bm, bn, bk, wm, wn);
// M = batch, N = output_dim, K = input_dim
let m = args.batch;
let n = args.output_dim;
let k = args.input_dim;
- let am = (m % self.bm) == 0;
- let an = (n % self.bn) == 0;
- let ak = (k % self.bk) == 0;
+ let am = (m % bm) == 0;
+ let an = (n % bn) == 0;
+ let ak = (k % bk) == 0;
let ps = self.get_or_compile_pipeline(mtl, &kname, am, an, ak)?;
enc.set_compute_pipeline_state(ps);
@@ -160,8 +366,18 @@ impl MatmulKernel {
enc.set_buffer(3, Some(args.d), 0);
// Params
- let tiles_n = (n + self.bn - 1) / self.bn;
- let tiles_m = (m + self.bm - 1) / self.bm;
+ let tiles_n = (n + bn - 1) / bn;
+ let tiles_m = (m + bm - 1) / bm;
+ // Swizzle like MLX: small tm -> no swizzle, otherwise simple 2-way.
+ let swizzle_log = if tiles_m <= 3 {
+ 0
+ } else {
+ 1
+ };
+
+ let tile = 1 << swizzle_log;
+ let tm_swizzled = (tiles_m + tile - 1) / tile;
+ let tn_swizzled = tiles_n * tile;
let params = GEMMParams {
batch: m,
output_dim: n,
@@ -169,13 +385,13 @@ impl MatmulKernel {
lda: args.lda,
ldb: args.ldb,
ldd: args.ldd,
- tiles_n,
- tiles_m,
+ tiles_n: tn_swizzled,
+ tiles_m: tm_swizzled,
batch_stride_a: (args.lda as i64) * (k as i64),
batch_stride_b: (args.ldb as i64) * (n as i64),
batch_stride_d: (args.ldd as i64) * (n as i64),
- swizzle_log: 0,
- gemm_k_iterations_aligned: k / self.bk,
+ swizzle_log,
+ gemm_k_iterations_aligned: k / bk,
};
enc.set_bytes(
4,
@@ -184,12 +400,29 @@ impl MatmulKernel {
);
// Threadgroup sizing
- let threads_per_tg = MTLSize::new(32, self.wn, self.wm);
- let tg_x = ((n as i64 + self.bn as i64 - 1) / self.bn as i64) as u64;
- let tg_y = ((m as i64 + self.bm as i64 - 1) / self.bm as i64) as u64;
+ let threads_per_tg = MTLSize::new(32, wn, wm);
+ let tg_x = tn_swizzled as u64;
+ let tg_y = tm_swizzled as u64;
let tg_z = args.batch_count as u64;
let tgs = MTLSize::new(tg_x, tg_y, tg_z);
enc.dispatch_thread_groups(tgs, threads_per_tg);
Ok(())
}
+
+ fn apply_batch_collapse(
+ &self,
+ args: &mut MatmulArguments,
+ ) {
+ if self.transpose_a {
+ return;
+ }
+ if args.batch_count <= 1 {
+ return;
+ }
+ // Collapse batch_count into M when A is contiguous and B is broadcast.
+ if args.lda == args.input_dim && self.transpose_b {
+ args.batch *= args.batch_count;
+ args.batch_count = 1;
+ }
+ }
}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/matmul.metal b/crates/uzu/src/backends/metal/kernel/matmul/matmul.metal
index bc5fb5664..de3e3dd5e 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/matmul.metal
+++ b/crates/uzu/src/backends/metal/kernel/matmul/matmul.metal
@@ -483,7 +483,10 @@ template <
instantiate_gemm_transpose_helper(type_name, element_type, 64, 32, 32, 2, 2) \
instantiate_gemm_transpose_helper(type_name, element_type, 32, 64, 16, 1, 2) \
instantiate_gemm_transpose_helper(type_name, element_type, 32, 32, 16, 2, 2) \
- instantiate_gemm_transpose_helper(type_name, element_type, 64, 32, 8, 4, 1)
+ instantiate_gemm_transpose_helper(type_name, element_type, 64, 32, 8, 4, 1) \
+ /* NAX-oriented larger tiles (safe BK=32) */ \
+ instantiate_gemm_transpose_helper(type_name, element_type, 64, 64, 32, 2, 2) \
+ instantiate_gemm_transpose_helper(type_name, element_type, 128, 128, 32, 2, 2)
// clang-format on
// Instantiate for float16, bfloat16, and float32
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/mod.rs
index 191479882..95d9c8adf 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/mod.rs
@@ -1,9 +1,13 @@
mod arguments;
+mod gemv;
mod kernel;
mod pipeline;
#[allow(dead_code, clippy::unnecessary_operation, clippy::identity_op)]
mod shared_types;
+mod splitk;
mod transpose;
pub use arguments::MatmulArguments;
+pub use gemv::GemvKernel;
pub use kernel::MatmulKernel;
+pub use splitk::SplitKGemm;
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/splitk.metal b/crates/uzu/src/backends/metal/kernel/matmul/splitk.metal
new file mode 100644
index 000000000..bdfec8f94
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/splitk.metal
@@ -0,0 +1,125 @@
+// Split-K GEMM partial accumulation kernels (simplified).
+// Partial products are accumulated into a float buffer; a second kernel
+// converts to the target type.
+
+#include <metal_stdlib>
+using namespace metal;
+
+template <typename T, typename AccT>
+METAL_FUNC void splitk_partial_impl(
+ const device T* __restrict a [[buffer(0)]],
+ const device T* __restrict b [[buffer(1)]],
+ device atomic_float* __restrict accum [[buffer(2)]],
+ constant uint& m [[buffer(3)]],
+ constant uint& n [[buffer(4)]],
+ constant uint& k [[buffer(5)]],
+ constant uint& lda [[buffer(6)]],
+ constant uint& ldb [[buffer(7)]],
+ constant uint& ldd [[buffer(8)]],
+ constant uint& k_start [[buffer(9)]],
+ constant uint& k_end [[buffer(10)]],
+ uint3 tgid [[threadgroup_position_in_grid]],
+ uint3 tid [[thread_position_in_threadgroup]]) {
+ const uint col = tgid.x * 8 + tid.x;
+ const uint row = tgid.y * 8 + tid.y;
+ const uint batch = tgid.z;
+
+ if (row >= m || col >= n) {
+ return;
+ }
+
+ const device T* a_row = a + batch * (lda * m) + row * lda;
+ const device T* b_col = b + col * ldb;
+
+ AccT acc = static_cast<AccT>(0);
+ for (uint kk = k_start; kk < k_end; kk++) {
+ acc += static_cast<AccT>(a_row[kk]) *
+ static_cast<AccT>(b_col[kk]);
+ }
+
+ const uint out_index = batch * (ldd * m) + row * ldd + col;
+ atomic_fetch_add_explicit(&accum[out_index], acc, memory_order_relaxed);
+}
+
+template <typename T>
+METAL_FUNC void splitk_convert_impl(
+ const device atomic_float* __restrict accum [[buffer(0)]],
+ device T* __restrict out [[buffer(1)]],
+ constant uint& m [[buffer(2)]],
+ constant uint& n [[buffer(3)]],
+ constant uint& ldd [[buffer(4)]],
+ uint3 tgid [[threadgroup_position_in_grid]],
+ uint3 tid [[thread_position_in_threadgroup]]) {
+ const uint col = tgid.x * 8 + tid.x;
+ const uint row = tgid.y * 8 + tid.y;
+ const uint batch = tgid.z;
+
+ if (row >= m || col >= n) {
+ return;
+ }
+
+ const uint idx = batch * (ldd * m) + row * ldd + col;
+ const float v = atomic_load_explicit(&accum[idx], memory_order_relaxed);
+ out[idx] = static_cast<T>(v);
+}
+
+// Kernel entry points
+[[kernel, max_total_threads_per_threadgroup(256)]] void splitk_partial_f16(
+ const device half* a [[buffer(0)]],
+ const device half* b [[buffer(1)]],
+ device atomic_float* accum [[buffer(2)]],
+ constant uint& m [[buffer(3)]],
+ constant uint& n [[buffer(4)]],
+ constant uint& k [[buffer(5)]],
+ constant uint& lda [[buffer(6)]],
+ constant uint& ldb [[buffer(7)]],
+ constant uint& ldd [[buffer(8)]],
+ constant uint& k_start [[buffer(9)]],
+ constant uint& k_end [[buffer(10)]],
+ uint3 tgid [[threadgroup_position_in_grid]],
+ uint3 tid [[thread_position_in_threadgroup]]) {
+ splitk_partial_impl<half, float>(
+ a, b, accum, m, n, k, lda, ldb, ldd, k_start, k_end, tgid, tid);
+}
+
+[[kernel, max_total_threads_per_threadgroup(256)]] void splitk_partial_bf16(
+ const device bfloat* a [[buffer(0)]],
+ const device bfloat* b [[buffer(1)]],
+ device atomic_float* accum [[buffer(2)]],
+ constant uint& m [[buffer(3)]],
+ constant uint& n [[buffer(4)]],
+ constant uint& k [[buffer(5)]],
+ constant uint& lda [[buffer(6)]],
+ constant uint& ldb [[buffer(7)]],
+ constant uint& ldd [[buffer(8)]],
+ constant uint& k_start [[buffer(9)]],
+ constant uint& k_end [[buffer(10)]],
+ uint3 tgid [[threadgroup_position_in_grid]],
+ uint3 tid [[thread_position_in_threadgroup]]) {
+ splitk_partial_impl<bfloat, float>(
+ a, b, accum, m, n, k, lda, ldb, ldd, k_start, k_end, tgid, tid);
+}
+
+[[kernel, max_total_threads_per_threadgroup(256)]] void splitk_convert_f16(
+ const device atomic_float* accum [[buffer(0)]],
+ device half* out [[buffer(1)]],
+ constant uint& m [[buffer(2)]],
+ constant uint& n [[buffer(3)]],
+ constant uint& ldd [[buffer(4)]],
+ uint3 tgid [[threadgroup_position_in_grid]],
+ uint3 tid [[thread_position_in_threadgroup]]) {
+ splitk_convert_impl<half>(accum, out, m, n, ldd, tgid, tid);
+}
+
+[[kernel, max_total_threads_per_threadgroup(256)]] void splitk_convert_bf16(
+ const device atomic_float* accum [[buffer(0)]],
+ device bfloat* out [[buffer(1)]],
+ constant uint& m [[buffer(2)]],
+ constant uint& n [[buffer(3)]],
+ constant uint& ldd [[buffer(4)]],
+ uint3 tgid [[threadgroup_position_in_grid]],
+ uint3 tid [[thread_position_in_threadgroup]]) {
+ splitk_convert_impl<bfloat>(accum, out, m, n, ldd, tgid, tid);
+}
+
+
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/splitk.rs b/crates/uzu/src/backends/metal/kernel/matmul/splitk.rs
new file mode 100644
index 000000000..3601567b5
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/splitk.rs
@@ -0,0 +1,186 @@
+use std::collections::HashMap;
+
+use metal::{
+ ComputeCommandEncoderRef, ComputePipelineState as MTLComputePipelineState,
+ MTLSize,
+};
+
+use super::arguments::MatmulArguments;
+use crate::{
+ DataType,
+ backends::metal::{MTLContext, MTLError},
+};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+enum SplitKKernelKind {
+ Partial,
+ Convert,
+}
+
+pub struct SplitKGemm {
+ dt: DataType,
+ pipelines: HashMap<SplitKKernelKind, MTLComputePipelineState>,
+}
+
+impl SplitKGemm {
+ pub fn new(dt: DataType) -> Self {
+ Self {
+ dt,
+ pipelines: HashMap::new(),
+ }
+ }
+
+ fn kernel_name(
+ &self,
+ kind: SplitKKernelKind,
+ ) -> Result<&'static str, MTLError> {
+ match (self.dt, kind) {
+ (DataType::F16, SplitKKernelKind::Partial) => {
+ Ok("splitk_partial_f16")
+ },
+ (DataType::F16, SplitKKernelKind::Convert) => {
+ Ok("splitk_convert_f16")
+ },
+ (DataType::BF16, SplitKKernelKind::Partial) => {
+ Ok("splitk_partial_bf16")
+ },
+ (DataType::BF16, SplitKKernelKind::Convert) => {
+ Ok("splitk_convert_bf16")
+ },
+ _ => Err(MTLError::Generic(format!(
+ "Unsupported dtype for Split-K: {:?}",
+ self.dt
+ ))),
+ }
+ }
+
+ fn pipeline(
+ &mut self,
+ mtl: &MTLContext,
+ kind: SplitKKernelKind,
+ ) -> Result<&MTLComputePipelineState, MTLError> {
+ if !self.pipelines.contains_key(&kind) {
+ let name = self.kernel_name(kind)?;
+ let ps = mtl.compute_pipeline_state(name, None)?;
+ self.pipelines.insert(kind, ps);
+ }
+ Ok(self.pipelines.get(&kind).unwrap())
+ }
+
+ /// Heuristic from plan: use Split-K when batch_size == 1, small M*N but large K.
+ pub fn should_use_splitk(
+ &self,
+ m: i32,
+ n: i32,
+ k: i32,
+ batch_count: i32,
+ ) -> bool {
+ if batch_count != 1 {
+ return false;
+ }
+ if m <= 0 || n <= 0 || k <= 0 {
+ return false;
+ }
+ let tm = m / 16;
+ let tn = n / 16;
+ let tk = k / 16;
+ (tm * tn) <= 32 && tk >= 8
+ }
+
+ pub fn encode(
+ &mut self,
+ mtl: &MTLContext,
+ enc: &ComputeCommandEncoderRef,
+ args: MatmulArguments,
+ ) -> Result<(), MTLError> {
+ let m = args.batch as u32;
+ let n = args.output_dim as u32;
+ let k = args.input_dim as u32;
+
+ // Choose split count (fixed 4-way)
+ let split_k = 4u32;
+ let chunk = (k + split_k - 1) / split_k;
+
+ // Temporary accumulation buffer (float)
+ let output_len = (args.batch_count as u32) * m * (args.ldd as u32);
+ let accum_bytes = (output_len as usize) * std::mem::size_of::<f32>();
+ let accum_buf = mtl.device.new_buffer(
+ accum_bytes as u64,
+ metal::MTLResourceOptions::StorageModeShared,
+ );
+ unsafe {
+ // Zero initialize
+ std::ptr::write_bytes(
+ accum_buf.contents() as *mut u8,
+ 0,
+ accum_bytes,
+ );
+ }
+
+ let partial_ps = self.pipeline(mtl, SplitKKernelKind::Partial)?;
+
+ let threads = MTLSize::new(8, 8, 1);
+ let tg_x = (n as u64 + 7) / 8;
+ let tg_y = (m as u64 + 7) / 8;
+ let tg_z = args.batch_count as u64;
+ let grid = MTLSize::new(tg_x, tg_y, tg_z);
+
+ for split in 0..split_k {
+ let k_start = split * chunk;
+ let k_end = ((split + 1) * chunk).min(k);
+
+ enc.set_compute_pipeline_state(partial_ps);
+ enc.set_buffer(0, Some(args.a), 0);
+ enc.set_buffer(1, Some(args.b), 0);
+ enc.set_buffer(2, Some(&accum_buf), 0);
+
+ enc.set_bytes(3, 4, &m as *const u32 as *const std::ffi::c_void);
+ enc.set_bytes(4, 4, &n as *const u32 as *const std::ffi::c_void);
+ enc.set_bytes(5, 4, &k as *const u32 as *const std::ffi::c_void);
+ enc.set_bytes(
+ 6,
+ 4,
+ &(args.lda as u32) as *const u32 as *const std::ffi::c_void,
+ );
+ enc.set_bytes(
+ 7,
+ 4,
+ &(args.ldb as u32) as *const u32 as *const std::ffi::c_void,
+ );
+ enc.set_bytes(
+ 8,
+ 4,
+ &(args.ldd as u32) as *const u32 as *const std::ffi::c_void,
+ );
+ enc.set_bytes(
+ 9,
+ 4,
+ &k_start as *const u32 as *const std::ffi::c_void,
+ );
+ enc.set_bytes(
+ 10,
+ 4,
+ &k_end as *const u32 as *const std::ffi::c_void,
+ );
+
+ enc.dispatch_thread_groups(grid, threads);
+ }
+
+ // Convert to target type
+ let convert_ps = self.pipeline(mtl, SplitKKernelKind::Convert)?;
+ enc.set_compute_pipeline_state(convert_ps);
+ enc.set_buffer(0, Some(&accum_buf), 0);
+ enc.set_buffer(1, Some(args.d), 0);
+
+ enc.set_bytes(2, 4, &m as *const u32 as *const std::ffi::c_void);
+ enc.set_bytes(3, 4, &n as *const u32 as *const std::ffi::c_void);
+ enc.set_bytes(
+ 4,
+ 4,
+ &(args.ldd as u32) as *const u32 as *const std::ffi::c_void,
+ );
+
+ enc.dispatch_thread_groups(grid, threads);
+ Ok(())
+ }
+}
diff --git a/crates/uzu/src/backends/metal/mod.rs b/crates/uzu/src/backends/metal/mod.rs
index c55780fed..04d9d248e 100644
--- a/crates/uzu/src/backends/metal/mod.rs
+++ b/crates/uzu/src/backends/metal/mod.rs
@@ -17,7 +17,9 @@ pub mod utils;
pub use array::MetalArray;
pub use buffer_allocator::BufferAllocator;
pub use classifier_context::ClassifierContext;
-pub use context::MTLContext;
+pub use context::{
+ DeviceArchitecture, DeviceClass, DeviceGeneration, MTLContext,
+};
pub use encodable_block::Decoder;
pub use error::MTLError;
pub use forward_pass::{CacheLayers, ForwardPassState, ModelShape};
diff --git a/crates/uzu/tests/gemv_kernel_test.rs b/crates/uzu/tests/gemv_kernel_test.rs
new file mode 100644
index 000000000..044d3f09b
--- /dev/null
+++ b/crates/uzu/tests/gemv_kernel_test.rs
@@ -0,0 +1,73 @@
+use half::f16;
+use metal::MTLResourceOptions;
+use uzu::{
+ DataType,
+ backends::metal::{MTLContext, MatmulArguments, MatmulKernel},
+};
+
+fn make_context() -> MTLContext {
+ let device = metal::Device::system_default().expect("metal device");
+ let queue = device.new_command_queue();
+ MTLContext::new(device, queue).expect("ctx")
+}
+
+#[test]
+#[ignore = "Requires Apple Silicon + Metal runtime"]
+fn gemv_single_token_runs() {
+ let ctx = make_context();
+
+ // One token (M=1), K=4, N=4
+ let a_vals: [f16; 4] = [1.0, 2.0, 3.0, 4.0].map(f16::from_f32);
+ let b_vals: [f16; 16] = (0..16)
+ .map(|i| f16::from_f32((i as f32) * 0.1))
+ .collect::<Vec<_>>()
+ .try_into()
+ .unwrap();
+
+ let a_buf = ctx.device.new_buffer_with_data(
+ a_vals.as_ptr() as *const _,
+ (a_vals.len() * std::mem::size_of::<f16>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+ let b_buf = ctx.device.new_buffer_with_data(
+ b_vals.as_ptr() as *const _,
+ (b_vals.len() * std::mem::size_of::<f16>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+ let d_buf = ctx.device.new_buffer(
+ (4 * std::mem::size_of::<f16>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+
+ let args = MatmulArguments {
+ a: &a_buf,
+ b: &b_buf,
+ d: &d_buf,
+ batch: 1,
+ input_dim: 4,
+ output_dim: 4,
+ lda: 4,
+ ldb: 4,
+ ldd: 4,
+ batch_count: 1,
+ };
+
+ let mut kernel =
+ MatmulKernel::new(&ctx, DataType::F16, false, true).unwrap();
+ let command_buffer = ctx.command_queue.new_command_buffer().to_owned();
+ let encoder = command_buffer.new_compute_command_encoder();
+ kernel.encode(&ctx, &encoder, args).unwrap();
+ encoder.end_encoding();
+ command_buffer.commit();
+ command_buffer.wait_until_completed();
+
+ // Read back output
+ let out_ptr = d_buf.contents() as *const f16;
+ let out = unsafe { std::slice::from_raw_parts(out_ptr, 4) };
+ let cpu: f32 = a_vals
+ .iter()
+ .zip(b_vals.chunks_exact(4))
+ .map(|(a, row)| a.to_f32() * row[0].to_f32())
+ .sum();
+ assert!((out[0].to_f32() - cpu).abs() < 1e-2);
+}
diff --git a/crates/uzu/tests/nax_gemm_test.rs b/crates/uzu/tests/nax_gemm_test.rs
new file mode 100644
index 000000000..a605cc40f
--- /dev/null
+++ b/crates/uzu/tests/nax_gemm_test.rs
@@ -0,0 +1,69 @@
+use half::bf16;
+use metal::MTLResourceOptions;
+use uzu::{
+ DataType,
+ backends::metal::{MTLContext, MatmulArguments, MatmulKernel},
+};
+
+fn make_context() -> MTLContext {
+ let device = metal::Device::system_default().expect("metal device");
+ let queue = device.new_command_queue();
+ MTLContext::new(device, queue).expect("ctx")
+}
+
+#[test]
+#[ignore = "Requires Apple Silicon + Metal runtime"]
+fn nax_path_runs_when_available() {
+ let ctx = make_context();
+
+ let m = 64usize;
+ let n = 128usize;
+ let k = 256usize;
+
+ let a_vals: Vec<bf16> =
+ (0..(m * k)).map(|i| bf16::from_f32((i % 11) as f32 * 0.01)).collect();
+ let b_vals: Vec<bf16> = (0..(k * n))
+ .map(|i| bf16::from_f32((i % 13) as f32 * 0.02 - 0.05))
+ .collect();
+
+ let a_buf = ctx.device.new_buffer_with_data(
+ a_vals.as_ptr() as *const _,
+ (a_vals.len() * std::mem::size_of::<bf16>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+ let b_buf = ctx.device.new_buffer_with_data(
+ b_vals.as_ptr() as *const _,
+ (b_vals.len() * std::mem::size_of::<bf16>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+ let d_buf = ctx.device.new_buffer(
+ (m * n * std::mem::size_of::<bf16>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+
+ let args = MatmulArguments {
+ a: &a_buf,
+ b: &b_buf,
+ d: &d_buf,
+ batch: m as i32,
+ input_dim: k as i32,
+ output_dim: n as i32,
+ lda: k as i32,
+ ldb: n as i32,
+ ldd: n as i32,
+ batch_count: 1,
+ };
+
+ let mut kernel =
+ MatmulKernel::new(&ctx, DataType::BF16, false, true).unwrap();
+ let command_buffer = ctx.command_queue.new_command_buffer().to_owned();
+ let encoder = command_buffer.new_compute_command_encoder();
+ kernel.encode(&ctx, &encoder, args).unwrap();
+ encoder.end_encoding();
+ command_buffer.commit();
+ command_buffer.wait_until_completed();
+
+ let out_ptr = d_buf.contents() as *const bf16;
+ let out = unsafe { std::slice::from_raw_parts(out_ptr, m * n) };
+ assert!(!out[0].to_f32().is_nan());
+}
diff --git a/crates/uzu/tests/splitk_gemm_test.rs b/crates/uzu/tests/splitk_gemm_test.rs
new file mode 100644
index 000000000..1969aac3f
--- /dev/null
+++ b/crates/uzu/tests/splitk_gemm_test.rs
@@ -0,0 +1,71 @@
+use half::f16;
+use metal::MTLResourceOptions;
+use uzu::{
+ DataType,
+ backends::metal::{MTLContext, MatmulArguments, MatmulKernel},
+};
+
+fn make_context() -> MTLContext {
+ let device = metal::Device::system_default().expect("metal device");
+ let queue = device.new_command_queue();
+ MTLContext::new(device, queue).expect("ctx")
+}
+
+#[test]
+#[ignore = "Requires Apple Silicon + Metal runtime"]
+fn splitk_triggers_for_large_k() {
+ let ctx = make_context();
+
+ // M=32, N=32, K large to trigger split-k heuristic.
+ let m = 32usize;
+ let n = 32usize;
+ let k = 512usize;
+
+ let a_vals: Vec<f16> =
+ (0..(m * k)).map(|i| f16::from_f32((i % 17) as f32 * 0.01)).collect();
+ let b_vals: Vec<f16> = (0..(k * n))
+ .map(|i| f16::from_f32((i % 23) as f32 * 0.02 - 0.1))
+ .collect();
+
+ let a_buf = ctx.device.new_buffer_with_data(
+ a_vals.as_ptr() as *const _,
+ (a_vals.len() * std::mem::size_of::<f16>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+ let b_buf = ctx.device.new_buffer_with_data(
+ b_vals.as_ptr() as *const _,
+ (b_vals.len() * std::mem::size_of::<f16>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+ let d_buf = ctx.device.new_buffer(
+ (m * n * std::mem::size_of::<f16>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+
+ let args = MatmulArguments {
+ a: &a_buf,
+ b: &b_buf,
+ d: &d_buf,
+ batch: m as i32,
+ input_dim: k as i32,
+ output_dim: n as i32,
+ lda: k as i32,
+ ldb: n as i32,
+ ldd: n as i32,
+ batch_count: 1,
+ };
+
+ let mut kernel =
+ MatmulKernel::new(&ctx, DataType::F16, false, true).unwrap();
+ let command_buffer = ctx.command_queue.new_command_buffer().to_owned();
+ let encoder = command_buffer.new_compute_command_encoder();
+ kernel.encode(&ctx, &encoder, args).unwrap();
+ encoder.end_encoding();
+ command_buffer.commit();
+ command_buffer.wait_until_completed();
+
+ // Basic sanity: output buffer was written (check first element non-NaN)
+ let out_ptr = d_buf.contents() as *const f16;
+ let out = unsafe { std::slice::from_raw_parts(out_ptr, m * n) };
+ assert!(!out[0].to_f32().is_nan());
+}