A high-performance inference engine for AI models
wip
eugene010e14efa68b
diff --git a/Cargo.lock b/Cargo.lockindex b5369e596..c3df3983a 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2191,6 +2191,19 @@ dependencies = ["cxx",]+[[package]]+name = "mpsgraph"+version = "0.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0b3d4f4ee48894ff2d83808e8a0333f00eb7576375a5a77e8132f10ebb2a1f86"+dependencies = [+ "bitflags 2.9.4",+ "block2",+ "metal",+ "objc2",+ "objc2-foundation",+]+[[package]]name = "multer"version = "3.1.0"@@ -4090,6 +4103,7 @@ dependencies = ["metal","minijinja","minijinja-contrib",+ "mpsgraph","ndarray","num-traits","objc2",diff --git a/crates/uzu/Cargo.toml b/crates/uzu/Cargo.tomlindex 211820eb1..c7e47cf0f 100644--- a/crates/uzu/Cargo.toml+++ b/crates/uzu/Cargo.toml@@ -37,6 +37,7 @@ bindgen = "0.72.1"is_close = "0.1.3"serde_json5 = "0.2.1"schemars = { version = "1.0.5", features = ["derive"] }+mpsgraph = "0.2.0"[features]default = ["metal-shaders"]diff --git a/crates/uzu/examples/matmul_benchmark.rs b/crates/uzu/examples/matmul_benchmark.rsnew file mode 100644index 000000000..671c5de27--- /dev/null+++ b/crates/uzu/examples/matmul_benchmark.rs@@ -0,0 +1,400 @@+//! Matmul benchmark comparing Metal GEMM kernel vs MPSGraph+//!+//! Run with: cargo run --release -p uzu --example matmul_benchmark++use std::{collections::HashMap, io::Write, time::Instant};++use half::bf16;+use metal::{Device, MTLResourceOptions};+use mpsgraph::{+ CommandBuffer, Device as MPSDevice, ExecutableExecutionDescriptor, Graph,+ ShapedType, TensorData,+};+use objc2::rc::autoreleasepool;+use uzu::{+ DataType,+ backends::metal::{+ MTLContext,+ kernel::{MatmulArguments, MatmulKernel},+ },+};++struct BenchmarkResult {+ name: &'static str,+ m: usize,+ k: usize,+ n: usize,+ metal_ms: f64,+ mpsgraph_ms: f64,+ metal_gflops: f64,+ mpsgraph_gflops: f64,+}++fn compute_gflops(+ m: usize,+ k: usize,+ n: usize,+ time_ms: f64,+) -> f64 {+ let flops = 2.0 * (m as f64) * (k as f64) * (n as f64);+ flops / (time_ms * 1e6)+}++fn benchmark_metal_gemm(+ ctx: &MTLContext,+ m: usize,+ k: usize,+ n: usize,+ warmup_iters: usize,+ bench_iters: usize,+) -> f64 {+ let a: Vec<bf16> = (0..(m * k))+ .map(|i| bf16::from_f32(((i % 13) as f32) * 0.01))+ .collect();+ let b: Vec<bf16> = (0..(k * n))+ .map(|i| bf16::from_f32(((i % 17) as f32) * 0.02 - 0.1))+ .collect();++ let a_buf = ctx.device.new_buffer_with_data(+ a.as_ptr() as *const _,+ (a.len() * core::mem::size_of::<bf16>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );+ let b_buf = ctx.device.new_buffer_with_data(+ b.as_ptr() as *const _,+ (b.len() * core::mem::size_of::<bf16>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );+ let d_buf = ctx.device.new_buffer(+ (m * n * core::mem::size_of::<bf16>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );++ let mut kernel = MatmulKernel::new(ctx, DataType::BF16, false, false)+ .expect("kernel new");++ // Warmup+ for _ in 0..warmup_iters {+ let cb = ctx.command_queue.new_command_buffer().to_owned();+ let enc = cb.new_compute_command_encoder();+ kernel+ .encode(+ ctx,+ &enc,+ MatmulArguments {+ a: &a_buf,+ b: &b_buf,+ d: &d_buf,+ batch: m as i32,+ input_dim: k as i32,+ output_dim: n as i32,+ lda: k as i32,+ ldb: n as i32,+ ldd: n as i32,+ batch_count: 1,+ },+ )+ .expect("encode");+ enc.end_encoding();+ cb.commit();+ cb.wait_until_completed();+ }++ // Benchmark+ let start = Instant::now();+ for _ in 0..bench_iters {+ let cb = ctx.command_queue.new_command_buffer().to_owned();+ let enc = cb.new_compute_command_encoder();+ kernel+ .encode(+ ctx,+ &enc,+ MatmulArguments {+ a: &a_buf,+ b: &b_buf,+ d: &d_buf,+ batch: m as i32,+ input_dim: k as i32,+ output_dim: n as i32,+ lda: k as i32,+ ldb: n as i32,+ ldd: n as i32,+ batch_count: 1,+ },+ )+ .expect("encode");+ enc.end_encoding();+ cb.commit();+ cb.wait_until_completed();+ }+ let elapsed = start.elapsed();+ (elapsed.as_secs_f64() * 1000.0) / bench_iters as f64+}++fn benchmark_mpsgraph_matmul(+ device: &metal::Device,+ m: usize,+ k: usize,+ n: usize,+ warmup_iters: usize,+ bench_iters: usize,+) -> f64 {+ autoreleasepool(|_| {+ print!("graph ");+ let _ = std::io::stdout().flush();+ let graph = Graph::new();++ // Create input placeholders+ let a_shape = [m as isize, k as isize];+ let b_shape = [k as isize, n as isize];++ let a_placeholder = graph.placeholder(+ Some(&a_shape),+ mpsgraph::DataType::BFloat16,+ Some("a"),+ );+ let b_placeholder = graph.placeholder(+ Some(&b_shape),+ mpsgraph::DataType::BFloat16,+ Some("b"),+ );++ // Matrix multiplication+ let result =+ graph.matrix_multiplication(&a_placeholder, &b_placeholder, None);++ // Create shaped types for compilation+ let a_shaped_type = ShapedType::new_with_shape_data_type(+ Some(&a_shape),+ mpsgraph::DataType::BFloat16,+ );+ let b_shaped_type = ShapedType::new_with_shape_data_type(+ Some(&b_shape),+ mpsgraph::DataType::BFloat16,+ );++ let feeds = HashMap::from([+ (&*a_placeholder, &*a_shaped_type),+ (&*b_placeholder, &*b_shaped_type),+ ]);++ // Compile+ print!("compile ");+ let _ = std::io::stdout().flush();+ let mps_device = MPSDevice::with_device(device);+ let executable =+ graph.compile(&mps_device, &feeds, &[&result], None, None);+ print!("ok ");++ // Create execution descriptor+ let execution_descriptor = ExecutableExecutionDescriptor::new();++ // Create input data buffers+ let a: Vec<bf16> = (0..(m * k))+ .map(|i| bf16::from_f32(((i % 13) as f32) * 0.01))+ .collect();+ let b: Vec<bf16> = (0..(k * n))+ .map(|i| bf16::from_f32(((i % 17) as f32) * 0.02 - 0.1))+ .collect();++ let a_buf = device.new_buffer_with_data(+ a.as_ptr() as *const _,+ (a.len() * core::mem::size_of::<bf16>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );+ let b_buf = device.new_buffer_with_data(+ b.as_ptr() as *const _,+ (b.len() * core::mem::size_of::<bf16>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );++ // Create output buffer+ let output_buf = device.new_buffer(+ (m * n * core::mem::size_of::<bf16>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );++ let command_queue = device.new_command_queue();++ // Create tensor data wrappers+ let a_tensor_data = TensorData::new_with_mtl_buffer(+ &a_buf,+ &[m, k],+ mpsgraph::DataType::BFloat16,+ None,+ );+ let b_tensor_data = TensorData::new_with_mtl_buffer(+ &b_buf,+ &[k, n],+ mpsgraph::DataType::BFloat16,+ None,+ );+ let output_tensor_data = TensorData::new_with_mtl_buffer(+ &output_buf,+ &[m, n],+ mpsgraph::DataType::BFloat16,+ None,+ );++ let inputs: &[&TensorData] = &[&a_tensor_data, &b_tensor_data];+ let outputs: &[&TensorData] = &[&output_tensor_data];++ // Warmup+ execution_descriptor.set_enable_commit_and_continue(true);+ print!("warmup ");+ let _ = std::io::stdout().flush();+ for _ in 0..warmup_iters {+ let command_buffer =+ CommandBuffer::from_command_queue(&command_queue);+ let root_cb = command_buffer.root_command_buffer().to_owned();+ let _ = executable.encode_to_command_buffer(+ &command_buffer,+ inputs,+ Some(outputs),+ Some(&execution_descriptor),+ );+ command_buffer.commit_and_continue();+ root_cb.wait_until_completed();+ }+ print!("ok ");+ let _ = std::io::stdout().flush();++ // Benchmark+ print!("bench ");+ let _ = std::io::stdout().flush();+ let start = Instant::now();+ for _ in 0..bench_iters {+ let command_buffer =+ CommandBuffer::from_command_queue(&command_queue);+ let root_cb = command_buffer.root_command_buffer().to_owned();+ let _ = executable.encode_to_command_buffer(+ &command_buffer,+ inputs,+ Some(outputs),+ Some(&execution_descriptor),+ );+ command_buffer.commit_and_continue();+ root_cb.wait_until_completed();+ }+ let elapsed = start.elapsed();+ print!("done ");+ let _ = std::io::stdout().flush();+ (elapsed.as_secs_f64() * 1000.0) / bench_iters as f64+ })+}++fn run_benchmark(+ ctx: &MTLContext,+ name: &'static str,+ m: usize,+ k: usize,+ n: usize,+ warmup: usize,+ iters: usize,+) -> BenchmarkResult {+ print!("[Metal] ");+ let _ = std::io::stdout().flush();+ let metal_ms = benchmark_metal_gemm(ctx, m, k, n, warmup, iters);+ print!("{:.2}ms ", metal_ms);+ let _ = std::io::stdout().flush();++ print!("[MPS] ");+ let _ = std::io::stdout().flush();+ let mpsgraph_ms =+ benchmark_mpsgraph_matmul(&ctx.device, m, k, n, warmup, iters);+ print!("{:.2}ms ", mpsgraph_ms);+ let _ = std::io::stdout().flush();++ BenchmarkResult {+ name,+ m,+ k,+ n,+ metal_ms,+ mpsgraph_ms,+ metal_gflops: compute_gflops(m, k, n, metal_ms),+ mpsgraph_gflops: compute_gflops(m, k, n, mpsgraph_ms),+ }+}++fn main() {+ let device = Device::system_default().expect("No Metal device found");+ let command_queue = device.new_command_queue();+ let ctx = MTLContext::new(device, command_queue)+ .expect("Failed to create MTLContext");++ println!("Metal GEMM vs MPSGraph Matmul Benchmark");+ println!("Device: {}", ctx.device.name());+ println!("========================================\n");++ let warmup = 10;+ let iters = 50;++ // Different tensor shapes to test - typical LLM shapes+ let shapes: &[(&str, usize, usize, usize)] = &[+ ("GEMV small (1 token)", 1, 2048, 2048),+ ("GEMV medium (1 token)", 1, 4096, 4096),+ ("GEMV large (1 token)", 1, 8192, 8192),+ ("Small batch (8 tokens)", 8, 2048, 2048),+ ("Medium batch (32 tokens)", 32, 2048, 2048),+ ("Large batch (128 tokens)", 128, 2048, 2048),+ ("Prefill small (256 tokens)", 256, 2048, 2048),+ ("Prefill medium (512 tokens)", 512, 2048, 2048),+ ("Prefill large (1024 tokens)", 1024, 2048, 2048),+ ("Square small", 256, 256, 256),+ ("Square medium", 512, 512, 512),+ ("Square large", 1024, 1024, 1024),+ ("MLP up (1 token)", 1, 2048, 8192),+ ("MLP down (1 token)", 1, 8192, 2048),+ ("MLP up (128 tokens)", 128, 2048, 8192),+ ("MLP down (128 tokens)", 128, 8192, 2048),+ ];++ println!(+ "{:<30} {:>6} {:>6} {:>6} {:>10} {:>10} {:>10} {:>10} {:>8}",+ "Shape",+ "M",+ "K",+ "N",+ "Metal(ms)",+ "MPS(ms)",+ "Metal(GF)",+ "MPS(GF)",+ "Speedup"+ );+ println!("{}", "-".repeat(110));++ for (name, m, k, n) in shapes {+ // Print progress indicator+ print!("Running {}... ", name);+ let _ = std::io::stdout().flush();++ let result = run_benchmark(&ctx, name, *m, *k, *n, warmup, iters);+ let speedup = result.metal_ms / result.mpsgraph_ms;+ let speedup_str = if speedup > 1.0 {+ format!("{:.2}x ✓", speedup)+ } else {+ format!("{:.2}x", speedup)+ };++ // Clear line and print result+ print!("\r");+ println!(+ "{:<30} {:>6} {:>6} {:>6} {:>10.4} {:>10.4} {:>10.1} {:>10.1} {:>8}",+ result.name,+ result.m,+ result.k,+ result.n,+ result.metal_ms,+ result.mpsgraph_ms,+ result.metal_gflops,+ result.mpsgraph_gflops,+ speedup_str+ );+ let _ = std::io::stdout().flush();+ }++ println!("\n=== Summary ===");+ println!("Speedup > 1.0 means MPSGraph is faster");+ println!("Speedup < 1.0 means Metal GEMM is faster");+}diff --git a/crates/uzu/src/backends/metal/context.rs b/crates/uzu/src/backends/metal/context.rsindex 5ef3e1712..659c8d03e 100644--- a/crates/uzu/src/backends/metal/context.rs+++ b/crates/uzu/src/backends/metal/context.rs@@ -1,7 +1,7 @@#![allow(dead_code)]include!(concat!(env!("OUT_DIR"), "/metal_lib.rs"));-use std::{cell::RefCell, collections::HashMap, rc::Rc};+use std::{cell::RefCell, collections::HashMap, rc::Rc, env};use metal::{CommandQueue as MTLCommandQueue,@@ -14,9 +14,147 @@ use super::{};use crate::{DataType, DeviceContext, array::array_size_in_bytes};+/// Apple GPU architecture generation.+/// Based on Apple GPU family naming convention (e.g., "applegpu_g13p").+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum DeviceGeneration {+ Gen13, // M1 family+ Gen14, // M2 family+ Gen15, // M3 family+ Gen16, // M3 Pro/Max with enhanced features+ Gen17, // M4 family (NAX capable)+ Unknown(u8),+}++impl DeviceGeneration {+ fn from_generation_number(r#gen: u8) -> Self {+ match r#gen {+ 13 => Self::Gen13,+ 14 => Self::Gen14,+ 15 => Self::Gen15,+ 16 => Self::Gen16,+ 17 => Self::Gen17,+ n => Self::Unknown(n),+ }+ }++ pub fn generation_number(&self) -> u8 {+ match self {+ Self::Gen13 => 13,+ Self::Gen14 => 14,+ Self::Gen15 => 15,+ Self::Gen16 => 16,+ Self::Gen17 => 17,+ Self::Unknown(n) => *n,+ }+ }+}++/// Device performance class based on the last character of architecture name.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum DeviceClass {+ Phone, // 'p' - iPhone/iPad integrated+ Integrated, // 'g' - Mac integrated GPU+ Desktop, // 'd' - Mac Pro/Max discrete-class GPU+ Unknown(char),+}++impl DeviceClass {+ fn from_arch_suffix(c: char) -> Self {+ match c {+ 'p' => Self::Phone,+ 'g' => Self::Integrated,+ 'd' => Self::Desktop,+ other => Self::Unknown(other),+ }+ }++ pub fn is_high_performance(&self) -> bool {+ matches!(self, Self::Desktop)+ }+}++/// Complete device architecture information.+#[derive(Debug, Clone)]+pub struct DeviceArchitecture {+ pub generation: DeviceGeneration,+ pub device_class: DeviceClass,+ pub arch_string: String,+}++impl DeviceArchitecture {+ pub fn from_device(device: &MTLDevice) -> Self {+ let arch_string = device.name().to_string();++ // Extract generation from architecture name+ // Format is typically like "Apple M3 Pro" or the GPU name "applegpu_g15d"+ let (generation, device_class) =+ Self::parse_architecture_info(&arch_string);++ Self {+ generation,+ device_class,+ arch_string,+ }+ }++ fn parse_architecture_info(+ arch: &str,+ ) -> (DeviceGeneration, DeviceClass) {+ // Try to extract from GPU architecture string if available+ // MLX uses device_->architecture()->name() which gives "applegpu_gXXY"+ // where XX is generation and Y is class (p/g/d)+ // The metal crate's device.name() gives the marketing name like "Apple M3 Pro"++ // For now, we'll use a heuristic based on the device name+ let generation = if arch.contains("M4") {+ DeviceGeneration::Gen17+ } else if arch.contains("M3") {+ if arch.contains("Max") || arch.contains("Ultra") {+ DeviceGeneration::Gen16+ } else {+ DeviceGeneration::Gen15+ }+ } else if arch.contains("M2") {+ DeviceGeneration::Gen14+ } else if arch.contains("M1") {+ DeviceGeneration::Gen13+ } else {+ DeviceGeneration::Unknown(0)+ };++ let device_class = if arch.contains("Max")+ || arch.contains("Ultra")+ || arch.contains("Pro")+ {+ DeviceClass::Desktop+ } else if arch.contains("iPhone") || arch.contains("iPad") {+ DeviceClass::Phone+ } else {+ DeviceClass::Integrated+ };++ (generation, device_class)+ }++ /// Returns true if NAX (New Accelerator eXtensions) is available.+ /// NAX requires M4 or later (generation >= 17) and macOS 26.2+.+ /// Since we can't check macOS version at compile time in Rust,+ /// we check generation only - the kernels will fail gracefully if unavailable.+ pub fn is_nax_available(&self) -> bool {+ self.generation.generation_number() >= 17+ }++ /// Returns true if this is a high-performance device (Pro/Max class).+ pub fn is_high_performance(&self) -> bool {+ self.device_class.is_high_performance()+ }+}+pub struct MTLContext {pub device: MTLDevice,pub command_queue: MTLCommandQueue,+ pub architecture: DeviceArchitecture,library: MTLLibrary,pipeline_cache: RefCell<HashMap<String, MTLComputePipelineState>>,}@@ -36,14 +174,44 @@ impl MTLContext {},};+ let architecture = DeviceArchitecture::from_device(&device);+Ok(Self {device,command_queue,+ architecture,library,pipeline_cache: RefCell::new(HashMap::new()),})}+ /// Returns true if NAX kernels are available on this device.+ pub fn is_nax_available(&self) -> bool {+ self.architecture.is_nax_available()+ }++ /// Returns true if this is a high-performance device (Pro/Max class).+ pub fn is_high_performance(&self) -> bool {+ self.architecture.is_high_performance()+ }++ /// Returns the device generation for tile size selection.+ pub fn device_generation(&self) -> DeviceGeneration {+ self.architecture.generation+ }++ /// Returns the device class for performance tuning.+ pub fn device_class(&self) -> DeviceClass {+ self.architecture.device_class+ }++ /// TF32 toggle similar to MLX env::enable_tf32().+ pub fn tf32_enabled(&self) -> bool {+ env::var("UZU_TF32")+ .map(|v| v == "1" || v.eq_ignore_ascii_case("true"))+ .unwrap_or(false)+ }+pub fn compute_pipeline_state(&self,function_name: &str,diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv.h b/crates/uzu/src/backends/metal/kernel/matmul/gemv.hnew file mode 100644index 000000000..2b0da5b02--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv.h@@ -0,0 +1,19 @@+// Declarations for GEMV kernels (Metal).+// Implementations are in gemv.metal; names must match Rust dispatch.++#pragma once++#ifdef __METAL_VERSION__+// No host-side declarations needed; compiled by Metal.+#else+// Host-side string literals for kernel names.+#define GEMV_F16_ROWS2 "gemv_f16_rows2"+#define GEMV_F16_ROWS4 "gemv_f16_rows4"+#define GEMV_F16_ROWS8 "gemv_f16_rows8"++#define GEMV_BF16_ROWS2 "gemv_bf16_rows2"+#define GEMV_BF16_ROWS4 "gemv_bf16_rows4"+#define GEMV_BF16_ROWS8 "gemv_bf16_rows8"+#endif++diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemv.metalnew file mode 100644index 000000000..c3ff84e4f--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv.metal@@ -0,0 +1,280 @@+#include <metal_stdlib>+using namespace metal;++// Tiled GEMV with vector blocking in threadgroup memory.+// Signature matches existing Rust dispatch (no bias/axpby).+template <typename T, typename AccT, ushort ROWS_PER_TG, ushort BK>+METAL_FUNC void gemv_impl_tiled(+ const device T* __restrict matrix [[buffer(0)]],+ const device T* __restrict vector [[buffer(1)]],+ device T* __restrict output [[buffer(2)]],+ constant uint& k [[buffer(3)]],+ constant uint& n [[buffer(4)]],+ constant uint& ldb [[buffer(5)]],+ constant uint& lda [[buffer(6)]],+ constant uint& ldd [[buffer(7)]],+ uint3 tgid [[threadgroup_position_in_grid]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]],+ uint tid [[thread_index_in_threadgroup]],+ threadgroup AccT* vec_tile) {+ const ushort TG_THREADS = ROWS_PER_TG * 32;+ if (simd_gid >= ROWS_PER_TG) {+ return;+ }+ const uint row = tgid.x * ROWS_PER_TG + simd_gid;+ if (row >= n) {+ return;+ }+ const uint batch = tgid.z;++ const device T* row_ptr = matrix + row * ldb;+ const device T* vec_ptr = vector + batch * lda;+ device T* out_ptr = output + batch * ldd;++ AccT acc = static_cast<AccT>(0);++ // Cooperative load of vector tile into threadgroup memory.+ for (uint k_block = 0; k_block < k; k_block += BK) {+ const uint remaining = k - k_block;+ const uint tile_elems = remaining < BK ? remaining : BK;++ // Each thread loads at most one element of the vector tile.+ if (tid < tile_elems && tid < TG_THREADS) {+ vec_tile[tid] = static_cast<AccT>(vec_ptr[k_block + tid]);+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);++ // Accumulate over this tile.+ for (uint idx = simd_lid; idx < tile_elems; idx += 32) {+ acc += static_cast<AccT>(row_ptr[k_block + idx]) * vec_tile[idx];+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }++ // Reduce within simdgroup.+ for (ushort offset = 16; offset > 0; offset >>= 1) {+ acc += simd_shuffle_down(acc, offset);+ }++ if (simd_lid == 0) {+ out_ptr[row] = static_cast<T>(acc);+ }+}++#define DECL_GEMV(NAME, TYPE, ACC, ROWS, BK) \+ [[kernel, max_total_threads_per_threadgroup(ROWS * 32)]] void NAME( \+ const device TYPE* matrix [[buffer(0)]], \+ const device TYPE* vector [[buffer(1)]], \+ device TYPE* output [[buffer(2)]], \+ constant uint& k [[buffer(3)]], \+ constant uint& n [[buffer(4)]], \+ constant uint& ldb [[buffer(5)]], \+ constant uint& lda [[buffer(6)]], \+ constant uint& ldd [[buffer(7)]], \+ uint3 tgid [[threadgroup_position_in_grid]], \+ uint simd_gid [[simdgroup_index_in_threadgroup]], \+ uint simd_lid [[thread_index_in_simdgroup]], \+ uint tid [[thread_index_in_threadgroup]]) { \+ threadgroup ACC vec_tile[BK]; \+ gemv_impl_tiled<TYPE, ACC, ROWS, BK>( \+ matrix, vector, output, k, n, ldb, lda, ldd, \+ tgid, simd_gid, simd_lid, tid, vec_tile); \+ }++// half variants+DECL_GEMV(gemv_f16_rows2, half, float, 2, 128)+DECL_GEMV(gemv_f16_rows4, half, float, 4, 128)+DECL_GEMV(gemv_f16_rows8, half, float, 8, 128)++// bfloat16 variants+DECL_GEMV(gemv_bf16_rows2, bfloat, float, 2, 128)+DECL_GEMV(gemv_bf16_rows4, bfloat, float, 4, 128)+DECL_GEMV(gemv_bf16_rows8, bfloat, float, 8, 128)+#include <metal_stdlib>+using namespace metal;++#define MLX_MTL_PRAGMA_UNROLL _Pragma("unroll")++// Simplified MLX-style GEMV without bias/axpby; compact signature.+template <+ typename T,+ const int BM,+ const int BN,+ const int SM,+ const int SN,+ const int TM,+ const int TN>+struct GEMVKernel {+ enum : int {+ threadsM = BM * SM,+ threadsN = BN * SN,+ blockM = threadsM * TM,+ blockN = threadsN * TN,+ };++ static_assert(SM * SN == 32, "simdgroup must be 32 threads");+ static_assert(SN == 4 || SN == 8 || SN == 16 || SN == 32, "SN must divide 32");++ template <typename U = T>+ static METAL_FUNC void load_safe(+ const device T* src,+ thread U dst[TN],+ const int src_offset,+ const int src_size) {+ if (src_offset + TN <= src_size) {+ MLX_MTL_PRAGMA_UNROLL+ for (int tn = 0; tn < TN; tn++) {+ dst[tn] = static_cast<U>(src[src_offset + tn]);+ }+ } else {+ MLX_MTL_PRAGMA_UNROLL+ for (int tn = 0; tn < TN; tn++) {+ dst[tn] = src_offset + tn < src_size+ ? static_cast<U>(src[src_offset + tn])+ : U(0);+ }+ }+ }++ static METAL_FUNC void run(+ const device T* mat [[buffer(0)]],+ const device T* in_vec [[buffer(1)]],+ device T* out_vec [[buffer(2)]],+ const constant int& in_vec_size [[buffer(3)]],+ const constant int& out_vec_size [[buffer(4)]],+ const constant int& matrix_ld [[buffer(5)]],+ threadgroup float* tgp_memory [[threadgroup(0)]],+ uint3 tid [[threadgroup_position_in_grid]],+ uint3 lid [[thread_position_in_threadgroup]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]) {+ (void)lid;++ thread float result[TM] = {0};+ thread T inter[TN];+ thread float v_coeff[TN];++ const int thrM = SN != 32 ? simd_lid / SN : 0;+ const int thrN = SN != 32 ? simd_lid % SN : int(simd_lid);++ const int sgN = BN != 1 ? (simd_gid % BN) : 0;++ const int simdM = BN != 1 ? SM * (simd_gid / BN) : int(SM * simd_gid);+ const int simdN = BN != 1 ? SN * (simd_gid % BN) : 0;++ int bm = (simdM + thrM) * TM;+ int bn = (simdN + thrN) * TN;++ int out_row = tid.x * blockM + bm;+ if (out_row >= out_vec_size) {+ return;+ }+ out_row = out_row + TM <= out_vec_size ? out_row : out_vec_size - TM;++ mat += out_row * matrix_ld;++ const int loop_stride = blockN;+ const int in_size = in_vec_size;+ const int n_iter = in_size / loop_stride;+ const int leftover = in_size - loop_stride * n_iter;++ for (int i = 0; i < n_iter; ++i) {+ load_safe<float>(in_vec, v_coeff, bn, TN);++ int mat_offset = 0;+ MLX_MTL_PRAGMA_UNROLL+ for (int tm = 0; tm < TM; tm++) {+ load_safe(mat, inter, mat_offset + bn, TN);+ MLX_MTL_PRAGMA_UNROLL+ for (int tn = 0; tn < TN; tn++) {+ result[tm] += inter[tn] * v_coeff[tn];+ }+ mat_offset += matrix_ld;+ }++ bn += blockN;+ }++ if (leftover > 0) {+ load_safe<float>(in_vec, v_coeff, bn, in_size);++ MLX_MTL_PRAGMA_UNROLL+ for (int tm = 0; tm < TM; tm++) {+ load_safe(&mat[tm * matrix_ld], inter, bn, in_size);+ MLX_MTL_PRAGMA_UNROLL+ for (int tn = 0; tn < TN; tn++) {+ result[tm] += inter[tn] * v_coeff[tn];+ }+ }+ }++ MLX_MTL_PRAGMA_UNROLL+ for (int tm = 0; tm < TM; tm++) {+ MLX_MTL_PRAGMA_UNROLL+ for (ushort sn = (SN / 2); sn >= 1; sn >>= 1) {+ result[tm] += simd_shuffle_down(result[tm], sn);+ }+ }++ if (BN > 1) {+ threadgroup float* tgp_results = tgp_memory + sgN * (blockM + TM) + bm;+ if (thrN == 0) {+ MLX_MTL_PRAGMA_UNROLL+ for (int tm = 0; tm < TM; tm++) {+ tgp_results[tm] = result[tm];+ }++ threadgroup_barrier(mem_flags::mem_none);++ if (sgN == 0) {+ MLX_MTL_PRAGMA_UNROLL+ for (int sgn = 1; sgn < BN; sgn++) {+ MLX_MTL_PRAGMA_UNROLL+ for (int tm = 0; tm < TM; tm++) {+ result[tm] += tgp_results[sgn * (blockM + TM) + tm];+ }+ }+ }+ }+ }++ if (simdN == 0 && thrN == 0) {+ MLX_MTL_PRAGMA_UNROLL+ for (int tm = 0; tm < TM; tm++) {+ out_vec[out_row + tm] = static_cast<T>(result[tm]);+ }+ }+ }+};++#define DECL_GEMV(name, itype, bm, bn, sm, sn, tm, tn) \+ [[kernel, max_total_threads_per_threadgroup(bm * sm * bn * sn)]] void \+ name( \+ const device itype* mat [[buffer(0)]], \+ const device itype* in_vec [[buffer(1)]], \+ device itype* out_vec [[buffer(2)]], \+ const constant int& in_vec_size [[buffer(3)]], \+ const constant int& out_vec_size [[buffer(4)]], \+ const constant int& matrix_ld [[buffer(5)]], \+ uint3 tid [[threadgroup_position_in_grid]], \+ uint3 lid [[thread_position_in_threadgroup]], \+ uint simd_gid [[simdgroup_index_in_threadgroup]], \+ uint simd_lid [[thread_index_in_simdgroup]], \+ threadgroup float* tgp_memory [[threadgroup(0)]]) { \+ GEMVKernel<itype, bm, bn, sm, sn, tm, tn>::run( \+ mat, in_vec, out_vec, in_vec_size, out_vec_size, \+ matrix_ld, tgp_memory, \+ tid, lid, simd_gid, simd_lid); \+ }++#define DECL_GEMV_BLOCKS(name, itype) \+ DECL_GEMV(name##_bm2_bn1_sm4_sn8_tm1_tn4, itype, 2, 1, 4, 8, 1, 4) \+ DECL_GEMV(name##_bm2_bn1_sm4_sn8_tm4_tn4, itype, 2, 1, 4, 8, 4, 4) \+ DECL_GEMV(name##_bm2_bn1_sm2_sn16_tm1_tn4, itype, 2, 1, 2, 16, 1, 4) \+ DECL_GEMV(name##_bm2_bn1_sm2_sn16_tm4_tn4, itype, 2, 1, 2, 16, 4, 4) \+ DECL_GEMV(name##_bm4_bn1_sm2_sn16_tm4_tn4, itype, 4, 1, 2, 16, 4, 4)++DECL_GEMV_BLOCKS(gemv_f32, float)+DECL_GEMV_BLOCKS(gemv_f16, half)+DECL_GEMV_BLOCKS(gemv_bf16, bfloat)diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv.rsnew file mode 100644index 000000000..abcea511c--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv.rs@@ -0,0 +1,124 @@+use std::collections::HashMap;++use metal::{+ ComputeCommandEncoderRef, ComputePipelineState as MTLComputePipelineState,+ MTLSize,+};++use super::arguments::MatmulArguments;+use crate::{+ DataType,+ backends::metal::{MTLContext, MTLError},+};++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+struct PipelineKey {+ rows_per_group: u32,+}++pub struct GemvKernel {+ dt: DataType,+ pipelines: HashMap<PipelineKey, MTLComputePipelineState>,+}++impl GemvKernel {+ pub fn new(dt: DataType) -> Self {+ Self {+ dt,+ pipelines: HashMap::new(),+ }+ }++ fn kernel_name(+ &self,+ rows: u32,+ ) -> Result<String, MTLError> {+ let base = match self.dt {+ DataType::F16 => "gemv_f16",+ DataType::BF16 => "gemv_bf16",+ _ => {+ return Err(MTLError::Generic(format!(+ "Unsupported dtype for GEMV: {:?}",+ self.dt+ )));+ },+ };+ Ok(format!("{base}_rows{rows}"))+ }++ fn get_or_compile_pipeline(+ &mut self,+ mtl: &MTLContext,+ rows: u32,+ ) -> Result<&MTLComputePipelineState, MTLError> {+ let key = PipelineKey {+ rows_per_group: rows,+ };+ if !self.pipelines.contains_key(&key) {+ let name = self.kernel_name(rows)?;+ let ps = mtl.compute_pipeline_state(&name, None)?;+ self.pipelines.insert(key, ps);+ }+ Ok(self.pipelines.get(&key).unwrap())+ }++ pub fn encode(+ &mut self,+ mtl: &MTLContext,+ enc: &ComputeCommandEncoderRef,+ args: MatmulArguments,+ rows_per_group: u32,+ ) -> Result<(), MTLError> {+ let rows_per_group = rows_per_group.max(1);+ let ps = self.get_or_compile_pipeline(mtl, rows_per_group)?;+ enc.set_compute_pipeline_state(ps);++ // Bind buffers: matrix (B), vector (A), output (D)+ enc.set_buffer(0, Some(args.b), 0);+ enc.set_buffer(1, Some(args.a), 0);+ enc.set_buffer(2, Some(args.d), 0);++ // Scalars+ let k = args.input_dim as u32;+ let n = args.output_dim as u32;+ let ldb = args.ldb as u32;+ let lda = args.lda as u32;+ let ldd = args.ldd as u32;++ enc.set_bytes(+ 3,+ std::mem::size_of::<u32>() as u64,+ &k as *const u32 as *const std::ffi::c_void,+ );+ enc.set_bytes(+ 4,+ std::mem::size_of::<u32>() as u64,+ &n as *const u32 as *const std::ffi::c_void,+ );+ enc.set_bytes(+ 5,+ std::mem::size_of::<u32>() as u64,+ &ldb as *const u32 as *const std::ffi::c_void,+ );+ enc.set_bytes(+ 6,+ std::mem::size_of::<u32>() as u64,+ &lda as *const u32 as *const std::ffi::c_void,+ );+ enc.set_bytes(+ 7,+ std::mem::size_of::<u32>() as u64,+ &ldd as *const u32 as *const std::ffi::c_void,+ );++ // Threadgroup sizing+ let threads_per_tg = MTLSize::new((rows_per_group * 32) as u64, 1, 1);+ let tg_x = ((n + rows_per_group - 1) / rows_per_group) as u64;+ let tg_y = 1;+ let tg_z = args.batch_count as u64;+ let tgs = MTLSize::new(tg_x, tg_y, tg_z);++ enc.dispatch_thread_groups(tgs, threads_per_tg);+ Ok(())+ }+}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv_masked.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemv_masked.metalnew file mode 100644index 000000000..c6bbe437b--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv_masked.metal@@ -0,0 +1,150 @@+// Masked and gather GEMV variants (simplified). Masks/indices are honored only+// for output write; data loads remain dense. This keeps parity with MLX kernel+// signatures while avoiding redundant specialization.++#include <metal_stdlib>+using namespace metal;++template <typename T, typename AccT, ushort ROWS>+METAL_FUNC void gemv_masked_impl(+ const device T* __restrict matrix [[buffer(0)]],+ const device T* __restrict vector [[buffer(1)]],+ const device uchar* __restrict out_mask [[buffer(2)]],+ const device uchar* __restrict op_mask [[buffer(3)]],+ device T* __restrict output [[buffer(4)]],+ constant uint& k [[buffer(5)]],+ constant uint& n [[buffer(6)]],+ constant uint& ldb [[buffer(7)]],+ constant uint& lda [[buffer(8)]],+ constant uint& ldd [[buffer(9)]],+ uint3 tgid [[threadgroup_position_in_grid]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]) {+ if (simd_gid >= ROWS) {+ return;+ }+ const uint row = tgid.x * ROWS + simd_gid;+ if (row >= n) {+ return;+ }+ const uint batch = tgid.z;+ const device T* row_ptr = matrix + row * ldb;+ const device T* vec_ptr = vector + batch * lda;+ device T* out_ptr = output + batch * ldd;++ AccT acc = static_cast<AccT>(0);+ for (uint idx = simd_lid; idx < k; idx += 32) {+ // If op_mask provided, zero out masked elements+ const bool op_on = (op_mask == nullptr) || (op_mask[row] != 0);+ if (op_on) {+ acc += static_cast<AccT>(row_ptr[idx]) *+ static_cast<AccT>(vec_ptr[idx]);+ }+ }++ for (ushort offset = 16; offset > 0; offset >>= 1) {+ acc += simd_shuffle_down(acc, offset);+ }++ if (simd_lid == 0) {+ if (out_mask == nullptr || out_mask[row] != 0) {+ out_ptr[row] = static_cast<T>(acc);+ }+ }+}++template <typename T, typename AccT, ushort ROWS>+METAL_FUNC void gemv_gather_impl(+ const device T* __restrict matrix [[buffer(0)]],+ const device T* __restrict vector [[buffer(1)]],+ const device uint* __restrict vec_indices [[buffer(2)]],+ const device uint* __restrict mat_indices [[buffer(3)]],+ device T* __restrict output [[buffer(4)]],+ constant uint& k [[buffer(5)]],+ constant uint& n [[buffer(6)]],+ constant uint& ldb [[buffer(7)]],+ constant uint& lda [[buffer(8)]],+ constant uint& ldd [[buffer(9)]],+ uint3 tgid [[threadgroup_position_in_grid]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]) {+ if (simd_gid >= ROWS) {+ return;+ }+ const uint row = tgid.x * ROWS + simd_gid;+ if (row >= n) {+ return;+ }+ const uint batch = tgid.z;+ const device T* base_row = matrix + row * ldb;+ device T* out_ptr = output + batch * ldd;++ AccT acc = static_cast<AccT>(0);+ for (uint idx = simd_lid; idx < k; idx += 32) {+ const uint col = mat_indices ? mat_indices[idx] : idx;+ const uint vec_idx = vec_indices ? vec_indices[idx] : idx;+ acc += static_cast<AccT>(base_row[col]) *+ static_cast<AccT>(vector[batch * lda + vec_idx]);+ }++ for (ushort offset = 16; offset > 0; offset >>= 1) {+ acc += simd_shuffle_down(acc, offset);+ }++ if (simd_lid == 0) {+ out_ptr[row] = static_cast<T>(acc);+ }+}++#define DECL_GEMV_MASKED(NAME, TYPE, ACC, ROWS) \+ [[kernel, max_total_threads_per_threadgroup(ROWS * 32)]] void NAME( \+ const device TYPE* matrix [[buffer(0)]], \+ const device TYPE* vector [[buffer(1)]], \+ const device uchar* out_mask [[buffer(2)]], \+ const device uchar* op_mask [[buffer(3)]], \+ device TYPE* output [[buffer(4)]], \+ constant uint& k [[buffer(5)]], \+ constant uint& n [[buffer(6)]], \+ constant uint& ldb [[buffer(7)]], \+ constant uint& lda [[buffer(8)]], \+ constant uint& ldd [[buffer(9)]], \+ uint3 tgid [[threadgroup_position_in_grid]], \+ uint simd_gid [[simdgroup_index_in_threadgroup]], \+ uint simd_lid [[thread_index_in_simdgroup]]) { \+ gemv_masked_impl<TYPE, ACC, ROWS>( \+ matrix, vector, out_mask, op_mask, output, \+ k, n, ldb, lda, ldd, tgid, simd_gid, simd_lid); \+ }++#define DECL_GEMV_GATHER(NAME, TYPE, ACC, ROWS) \+ [[kernel, max_total_threads_per_threadgroup(ROWS * 32)]] void NAME( \+ const device TYPE* matrix [[buffer(0)]], \+ const device TYPE* vector [[buffer(1)]], \+ const device uint* vec_indices [[buffer(2)]], \+ const device uint* mat_indices [[buffer(3)]], \+ device TYPE* output [[buffer(4)]], \+ constant uint& k [[buffer(5)]], \+ constant uint& n [[buffer(6)]], \+ constant uint& ldb [[buffer(7)]], \+ constant uint& lda [[buffer(8)]], \+ constant uint& ldd [[buffer(9)]], \+ uint3 tgid [[threadgroup_position_in_grid]], \+ uint simd_gid [[simdgroup_index_in_threadgroup]], \+ uint simd_lid [[thread_index_in_simdgroup]]) { \+ gemv_gather_impl<TYPE, ACC, ROWS>( \+ matrix, vector, vec_indices, mat_indices, output, \+ k, n, ldb, lda, ldd, tgid, simd_gid, simd_lid); \+ }++// Masked instantiations+DECL_GEMV_MASKED(gemv_masked_f16_rows4, half, float, 4)+DECL_GEMV_MASKED(gemv_masked_f16_rows8, half, float, 8)+DECL_GEMV_MASKED(gemv_masked_bf16_rows4, bfloat, float, 4)+DECL_GEMV_MASKED(gemv_masked_bf16_rows8, bfloat, float, 8)++// Gather instantiations+DECL_GEMV_GATHER(gemv_gather_f16_rows4, half, float, 4)+DECL_GEMV_GATHER(gemv_gather_f16_rows8, half, float, 8)+DECL_GEMV_GATHER(gemv_gather_bf16_rows4, bfloat, float, 4)+DECL_GEMV_GATHER(gemv_gather_bf16_rows8, bfloat, float, 8)+// Masked/gather GEMV kernels are stubbed out (not used). Keeping file empty to avoid shader compilation errors.diff --git a/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/kernel.rsindex 2846419a8..2e2f99419 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs@@ -6,8 +6,9 @@ use metal::{};use super::{- arguments::MatmulArguments, pipeline::PipelineKey,- shared_types::GEMMParams, transpose::transpose_configuration,+ arguments::MatmulArguments, gemv::GemvKernel, pipeline::PipelineKey,+ shared_types::GEMMParams, splitk::SplitKGemm,+ transpose::transpose_configuration,};use crate::{DataType,@@ -18,11 +19,8 @@ pub struct MatmulKernel {dt: DataType,transpose_a: bool,transpose_b: bool,- bm: i32,- bn: i32,- bk: i32,- wm: u64,- wn: u64,+ gemv: Option<GemvKernel>,+ splitk: Option<SplitKGemm>,pipelines: HashMap<PipelineKey, MTLComputePipelineState>,}@@ -36,12 +34,21 @@ impl MatmulKernel {}}- fn kernel_name(&self) -> String {+ fn kernel_name(+ &self,+ bm: i32,+ bn: i32,+ bk: i32,+ wm: u64,+ wn: u64,+ ) -> String {let t = Self::dtype_suffix(self.dt);let cfg = transpose_configuration(self.transpose_a, self.transpose_b);let tcfg = cfg.as_str();- // Shapes we currently instantiate in matmul.metal: bm64/bn64/bk16/wm2/wn2- format!("gemm_{}_{}_bm64_bn64_bk16_wm2_wn2", tcfg, t)+ format!(+ "gemm_{}_{}_bm{}_bn{}_bk{}_wm{}_wn{}",+ tcfg, t, bm, bn, bk, wm, wn+ )}pub fn new(@@ -60,15 +67,201 @@ impl MatmulKernel {dt,transpose_a,transpose_b,- bm: 64,- bn: 64,- bk: 16,- wm: 2,- wn: 2,+ gemv: None,+ splitk: None,pipelines: HashMap::new(),})}+ fn select_gemv_rows(+ &self,+ output_dim: i32,+ ) -> u32 {+ if output_dim >= 2048 {+ 8+ } else if output_dim >= 512 {+ 4+ } else {+ 2+ }+ }++ fn maybe_use_gemv(+ &mut self,+ mtl: &MTLContext,+ enc: &ComputeCommandEncoderRef,+ args: MatmulArguments,+ ) -> Result<bool, MTLError> {+ // Apply GEMV when batch (M) is 1 or output_dim (N) is 1, and when A is not transposed.+ let m = args.batch;+ let n = args.output_dim;+ if self.transpose_a {+ return Ok(false);+ }+ if m != 1 && n != 1 {+ return Ok(false);+ }+ let rows = self.select_gemv_rows(n);+ let gemv = self.gemv.get_or_insert_with(|| GemvKernel::new(self.dt));+ gemv.encode(mtl, enc, args, rows)?;+ Ok(true)+ }++ fn maybe_use_splitk(+ &mut self,+ mtl: &MTLContext,+ enc: &ComputeCommandEncoderRef,+ args: MatmulArguments,+ ) -> Result<bool, MTLError> {+ let m = args.batch;+ let n = args.output_dim;+ let k = args.input_dim;+ let batch_count = args.batch_count;++ let splitk =+ self.splitk.get_or_insert_with(|| SplitKGemm::new(self.dt));+ if splitk.should_use_splitk(m, n, k, batch_count) {+ splitk.encode(mtl, enc, args)?;+ return Ok(true);+ }+ Ok(false)+ }++ fn select_tiles(+ &self,+ mtl: &MTLContext,+ args: &MatmulArguments,+ ) -> (i32, i32, i32, u64, u64) {+ use crate::backends::metal::DeviceClass;+ let mut bm = 64;+ let mut bn = 64;+ let mut bk = 16;+ let mut wm = 2;+ let mut wn = 2;++ let m = args.batch;+ let n = args.output_dim;+ let k = args.input_dim;+ let batch_size_out = args.batch_count;+ let large_mat =+ (batch_size_out as i64) * (m as i64) * (n as i64) >= (1_i64 << 20);++ // Prefer NAX tiles when available (mirrors MLX fused_nax path).+ if mtl.is_nax_available()+ && (!matches!(self.dt, DataType::F32) || mtl.tf32_enabled())+ {+ if large_mat {+ return (128, 128, 32, 2, 2);+ }+ return (64, 64, 32, 2, 2);+ }+ let devc = match mtl.architecture.device_class {+ DeviceClass::Desktop => 'd',+ DeviceClass::Integrated => 'g',+ DeviceClass::Phone => 'p',+ DeviceClass::Unknown(_) => 'g',+ };++ let is_float = matches!(self.dt, DataType::F32);+ let prefer_halfish = !is_float || mtl.tf32_enabled();++ if devc == 'g' || devc == 'p' {+ if prefer_halfish {+ if !self.transpose_a && self.transpose_b {+ bm = 64;+ bn = 32;+ bk = 32;+ wm = 2;+ wn = 2;+ } else {+ bm = 64;+ bn = 64;+ bk = 16;+ wm = 1;+ wn = 2;+ }+ } else if !self.transpose_a && self.transpose_b {+ bm = 32;+ bn = 64;+ bk = 16;+ wm = 1;+ wn = 2;+ } else {+ bm = 64;+ bn = 32;+ bk = 32;+ wm = 2;+ wn = 2;+ }+ } else if devc == 'd' {+ if large_mat {+ if prefer_halfish {+ if 2 * std::cmp::max(m, n) > k {+ bm = 64;+ bn = 64;+ bk = 16;+ wm = 1;+ wn = 2;+ } else if !self.transpose_a && self.transpose_b {+ bm = 64;+ bn = 32;+ bk = 32;+ wm = 2;+ wn = 2;+ } else {+ bm = 32;+ bn = 64;+ bk = 16;+ wm = 1;+ wn = 2;+ }+ } else if !self.transpose_a && self.transpose_b {+ bm = 32;+ bn = 64;+ bk = 16;+ wm = 1;+ wn = 2;+ } else {+ bm = 64;+ bn = 32;+ bk = 32;+ wm = 2;+ wn = 2;+ }+ } else {+ if prefer_halfish {+ if !self.transpose_a && self.transpose_b {+ bm = 64;+ bn = 32;+ bk = 32;+ wm = 2;+ wn = 2;+ } else {+ bm = 64;+ bn = 64;+ bk = 16;+ wm = 1;+ wn = 2;+ }+ } else if !self.transpose_a && self.transpose_b {+ bm = 32;+ bn = 64;+ bk = 16;+ wm = 1;+ wn = 2;+ } else {+ bm = 64;+ bn = 32;+ bk = 32;+ wm = 2;+ wn = 2;+ }+ }+ }++ (bm, bn, bk, wm, wn)+ }+fn get_or_compile_pipeline(&mut self,mtl: &MTLContext,@@ -138,18 +331,31 @@ impl MatmulKernel {&mut self,mtl: &MTLContext,enc: &ComputeCommandEncoderRef,- args: MatmulArguments,+ mut args: MatmulArguments,) -> Result<(), MTLError> {- let kname = self.kernel_name();+ self.apply_batch_collapse(&mut args);++ // Try GEMV fast path for decode shapes.+ if self.maybe_use_gemv(mtl, enc, args)? {+ return Ok(());+ }++ // Try Split-K for small M*N and large K.+ if self.maybe_use_splitk(mtl, enc, args)? {+ return Ok(());+ }++ let (bm, bn, bk, wm, wn) = self.select_tiles(mtl, &args);+ let kname = self.kernel_name(bm, bn, bk, wm, wn);// M = batch, N = output_dim, K = input_dimlet m = args.batch;let n = args.output_dim;let k = args.input_dim;- let am = (m % self.bm) == 0;- let an = (n % self.bn) == 0;- let ak = (k % self.bk) == 0;+ let am = (m % bm) == 0;+ let an = (n % bn) == 0;+ let ak = (k % bk) == 0;let ps = self.get_or_compile_pipeline(mtl, &kname, am, an, ak)?;enc.set_compute_pipeline_state(ps);@@ -160,8 +366,18 @@ impl MatmulKernel {enc.set_buffer(3, Some(args.d), 0);// Params- let tiles_n = (n + self.bn - 1) / self.bn;- let tiles_m = (m + self.bm - 1) / self.bm;+ let tiles_n = (n + bn - 1) / bn;+ let tiles_m = (m + bm - 1) / bm;+ // Swizzle like MLX: small tm -> no swizzle, otherwise simple 2-way.+ let swizzle_log = if tiles_m <= 3 {+ 0+ } else {+ 1+ };++ let tile = 1 << swizzle_log;+ let tm_swizzled = (tiles_m + tile - 1) / tile;+ let tn_swizzled = tiles_n * tile;let params = GEMMParams {batch: m,output_dim: n,@@ -169,13 +385,13 @@ impl MatmulKernel {lda: args.lda,ldb: args.ldb,ldd: args.ldd,- tiles_n,- tiles_m,+ tiles_n: tn_swizzled,+ tiles_m: tm_swizzled,batch_stride_a: (args.lda as i64) * (k as i64),batch_stride_b: (args.ldb as i64) * (n as i64),batch_stride_d: (args.ldd as i64) * (n as i64),- swizzle_log: 0,- gemm_k_iterations_aligned: k / self.bk,+ swizzle_log,+ gemm_k_iterations_aligned: k / bk,};enc.set_bytes(4,@@ -184,12 +400,29 @@ impl MatmulKernel {);// Threadgroup sizing- let threads_per_tg = MTLSize::new(32, self.wn, self.wm);- let tg_x = ((n as i64 + self.bn as i64 - 1) / self.bn as i64) as u64;- let tg_y = ((m as i64 + self.bm as i64 - 1) / self.bm as i64) as u64;+ let threads_per_tg = MTLSize::new(32, wn, wm);+ let tg_x = tn_swizzled as u64;+ let tg_y = tm_swizzled as u64;let tg_z = args.batch_count as u64;let tgs = MTLSize::new(tg_x, tg_y, tg_z);enc.dispatch_thread_groups(tgs, threads_per_tg);Ok(())}++ fn apply_batch_collapse(+ &self,+ args: &mut MatmulArguments,+ ) {+ if self.transpose_a {+ return;+ }+ if args.batch_count <= 1 {+ return;+ }+ // Collapse batch_count into M when A is contiguous and B is broadcast.+ if args.lda == args.input_dim && self.transpose_b {+ args.batch *= args.batch_count;+ args.batch_count = 1;+ }+ }}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/matmul.metal b/crates/uzu/src/backends/metal/kernel/matmul/matmul.metalindex bc5fb5664..de3e3dd5e 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/matmul.metal+++ b/crates/uzu/src/backends/metal/kernel/matmul/matmul.metal@@ -483,7 +483,10 @@ template <instantiate_gemm_transpose_helper(type_name, element_type, 64, 32, 32, 2, 2) \instantiate_gemm_transpose_helper(type_name, element_type, 32, 64, 16, 1, 2) \instantiate_gemm_transpose_helper(type_name, element_type, 32, 32, 16, 2, 2) \- instantiate_gemm_transpose_helper(type_name, element_type, 64, 32, 8, 4, 1)+ instantiate_gemm_transpose_helper(type_name, element_type, 64, 32, 8, 4, 1) \+ /* NAX-oriented larger tiles (safe BK=32) */ \+ instantiate_gemm_transpose_helper(type_name, element_type, 64, 64, 32, 2, 2) \+ instantiate_gemm_transpose_helper(type_name, element_type, 128, 128, 32, 2, 2)// clang-format on// Instantiate for float16, bfloat16, and float32diff --git a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/mod.rsindex 191479882..95d9c8adf 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/mod.rs@@ -1,9 +1,13 @@mod arguments;+mod gemv;mod kernel;mod pipeline;#[allow(dead_code, clippy::unnecessary_operation, clippy::identity_op)]mod shared_types;+mod splitk;mod transpose;pub use arguments::MatmulArguments;+pub use gemv::GemvKernel;pub use kernel::MatmulKernel;+pub use splitk::SplitKGemm;diff --git a/crates/uzu/src/backends/metal/kernel/matmul/splitk.metal b/crates/uzu/src/backends/metal/kernel/matmul/splitk.metalnew file mode 100644index 000000000..bdfec8f94--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/splitk.metal@@ -0,0 +1,125 @@+// Split-K GEMM partial accumulation kernels (simplified).+// Partial products are accumulated into a float buffer; a second kernel+// converts to the target type.++#include <metal_stdlib>+using namespace metal;++template <typename T, typename AccT>+METAL_FUNC void splitk_partial_impl(+ const device T* __restrict a [[buffer(0)]],+ const device T* __restrict b [[buffer(1)]],+ device atomic_float* __restrict accum [[buffer(2)]],+ constant uint& m [[buffer(3)]],+ constant uint& n [[buffer(4)]],+ constant uint& k [[buffer(5)]],+ constant uint& lda [[buffer(6)]],+ constant uint& ldb [[buffer(7)]],+ constant uint& ldd [[buffer(8)]],+ constant uint& k_start [[buffer(9)]],+ constant uint& k_end [[buffer(10)]],+ uint3 tgid [[threadgroup_position_in_grid]],+ uint3 tid [[thread_position_in_threadgroup]]) {+ const uint col = tgid.x * 8 + tid.x;+ const uint row = tgid.y * 8 + tid.y;+ const uint batch = tgid.z;++ if (row >= m || col >= n) {+ return;+ }++ const device T* a_row = a + batch * (lda * m) + row * lda;+ const device T* b_col = b + col * ldb;++ AccT acc = static_cast<AccT>(0);+ for (uint kk = k_start; kk < k_end; kk++) {+ acc += static_cast<AccT>(a_row[kk]) *+ static_cast<AccT>(b_col[kk]);+ }++ const uint out_index = batch * (ldd * m) + row * ldd + col;+ atomic_fetch_add_explicit(&accum[out_index], acc, memory_order_relaxed);+}++template <typename T>+METAL_FUNC void splitk_convert_impl(+ const device atomic_float* __restrict accum [[buffer(0)]],+ device T* __restrict out [[buffer(1)]],+ constant uint& m [[buffer(2)]],+ constant uint& n [[buffer(3)]],+ constant uint& ldd [[buffer(4)]],+ uint3 tgid [[threadgroup_position_in_grid]],+ uint3 tid [[thread_position_in_threadgroup]]) {+ const uint col = tgid.x * 8 + tid.x;+ const uint row = tgid.y * 8 + tid.y;+ const uint batch = tgid.z;++ if (row >= m || col >= n) {+ return;+ }++ const uint idx = batch * (ldd * m) + row * ldd + col;+ const float v = atomic_load_explicit(&accum[idx], memory_order_relaxed);+ out[idx] = static_cast<T>(v);+}++// Kernel entry points+[[kernel, max_total_threads_per_threadgroup(256)]] void splitk_partial_f16(+ const device half* a [[buffer(0)]],+ const device half* b [[buffer(1)]],+ device atomic_float* accum [[buffer(2)]],+ constant uint& m [[buffer(3)]],+ constant uint& n [[buffer(4)]],+ constant uint& k [[buffer(5)]],+ constant uint& lda [[buffer(6)]],+ constant uint& ldb [[buffer(7)]],+ constant uint& ldd [[buffer(8)]],+ constant uint& k_start [[buffer(9)]],+ constant uint& k_end [[buffer(10)]],+ uint3 tgid [[threadgroup_position_in_grid]],+ uint3 tid [[thread_position_in_threadgroup]]) {+ splitk_partial_impl<half, float>(+ a, b, accum, m, n, k, lda, ldb, ldd, k_start, k_end, tgid, tid);+}++[[kernel, max_total_threads_per_threadgroup(256)]] void splitk_partial_bf16(+ const device bfloat* a [[buffer(0)]],+ const device bfloat* b [[buffer(1)]],+ device atomic_float* accum [[buffer(2)]],+ constant uint& m [[buffer(3)]],+ constant uint& n [[buffer(4)]],+ constant uint& k [[buffer(5)]],+ constant uint& lda [[buffer(6)]],+ constant uint& ldb [[buffer(7)]],+ constant uint& ldd [[buffer(8)]],+ constant uint& k_start [[buffer(9)]],+ constant uint& k_end [[buffer(10)]],+ uint3 tgid [[threadgroup_position_in_grid]],+ uint3 tid [[thread_position_in_threadgroup]]) {+ splitk_partial_impl<bfloat, float>(+ a, b, accum, m, n, k, lda, ldb, ldd, k_start, k_end, tgid, tid);+}++[[kernel, max_total_threads_per_threadgroup(256)]] void splitk_convert_f16(+ const device atomic_float* accum [[buffer(0)]],+ device half* out [[buffer(1)]],+ constant uint& m [[buffer(2)]],+ constant uint& n [[buffer(3)]],+ constant uint& ldd [[buffer(4)]],+ uint3 tgid [[threadgroup_position_in_grid]],+ uint3 tid [[thread_position_in_threadgroup]]) {+ splitk_convert_impl<half>(accum, out, m, n, ldd, tgid, tid);+}++[[kernel, max_total_threads_per_threadgroup(256)]] void splitk_convert_bf16(+ const device atomic_float* accum [[buffer(0)]],+ device bfloat* out [[buffer(1)]],+ constant uint& m [[buffer(2)]],+ constant uint& n [[buffer(3)]],+ constant uint& ldd [[buffer(4)]],+ uint3 tgid [[threadgroup_position_in_grid]],+ uint3 tid [[thread_position_in_threadgroup]]) {+ splitk_convert_impl<bfloat>(accum, out, m, n, ldd, tgid, tid);+}++diff --git a/crates/uzu/src/backends/metal/kernel/matmul/splitk.rs b/crates/uzu/src/backends/metal/kernel/matmul/splitk.rsnew file mode 100644index 000000000..3601567b5--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/splitk.rs@@ -0,0 +1,186 @@+use std::collections::HashMap;++use metal::{+ ComputeCommandEncoderRef, ComputePipelineState as MTLComputePipelineState,+ MTLSize,+};++use super::arguments::MatmulArguments;+use crate::{+ DataType,+ backends::metal::{MTLContext, MTLError},+};++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+enum SplitKKernelKind {+ Partial,+ Convert,+}++pub struct SplitKGemm {+ dt: DataType,+ pipelines: HashMap<SplitKKernelKind, MTLComputePipelineState>,+}++impl SplitKGemm {+ pub fn new(dt: DataType) -> Self {+ Self {+ dt,+ pipelines: HashMap::new(),+ }+ }++ fn kernel_name(+ &self,+ kind: SplitKKernelKind,+ ) -> Result<&'static str, MTLError> {+ match (self.dt, kind) {+ (DataType::F16, SplitKKernelKind::Partial) => {+ Ok("splitk_partial_f16")+ },+ (DataType::F16, SplitKKernelKind::Convert) => {+ Ok("splitk_convert_f16")+ },+ (DataType::BF16, SplitKKernelKind::Partial) => {+ Ok("splitk_partial_bf16")+ },+ (DataType::BF16, SplitKKernelKind::Convert) => {+ Ok("splitk_convert_bf16")+ },+ _ => Err(MTLError::Generic(format!(+ "Unsupported dtype for Split-K: {:?}",+ self.dt+ ))),+ }+ }++ fn pipeline(+ &mut self,+ mtl: &MTLContext,+ kind: SplitKKernelKind,+ ) -> Result<&MTLComputePipelineState, MTLError> {+ if !self.pipelines.contains_key(&kind) {+ let name = self.kernel_name(kind)?;+ let ps = mtl.compute_pipeline_state(name, None)?;+ self.pipelines.insert(kind, ps);+ }+ Ok(self.pipelines.get(&kind).unwrap())+ }++ /// Heuristic from plan: use Split-K when batch_size == 1, small M*N but large K.+ pub fn should_use_splitk(+ &self,+ m: i32,+ n: i32,+ k: i32,+ batch_count: i32,+ ) -> bool {+ if batch_count != 1 {+ return false;+ }+ if m <= 0 || n <= 0 || k <= 0 {+ return false;+ }+ let tm = m / 16;+ let tn = n / 16;+ let tk = k / 16;+ (tm * tn) <= 32 && tk >= 8+ }++ pub fn encode(+ &mut self,+ mtl: &MTLContext,+ enc: &ComputeCommandEncoderRef,+ args: MatmulArguments,+ ) -> Result<(), MTLError> {+ let m = args.batch as u32;+ let n = args.output_dim as u32;+ let k = args.input_dim as u32;++ // Choose split count (fixed 4-way)+ let split_k = 4u32;+ let chunk = (k + split_k - 1) / split_k;++ // Temporary accumulation buffer (float)+ let output_len = (args.batch_count as u32) * m * (args.ldd as u32);+ let accum_bytes = (output_len as usize) * std::mem::size_of::<f32>();+ let accum_buf = mtl.device.new_buffer(+ accum_bytes as u64,+ metal::MTLResourceOptions::StorageModeShared,+ );+ unsafe {+ // Zero initialize+ std::ptr::write_bytes(+ accum_buf.contents() as *mut u8,+ 0,+ accum_bytes,+ );+ }++ let partial_ps = self.pipeline(mtl, SplitKKernelKind::Partial)?;++ let threads = MTLSize::new(8, 8, 1);+ let tg_x = (n as u64 + 7) / 8;+ let tg_y = (m as u64 + 7) / 8;+ let tg_z = args.batch_count as u64;+ let grid = MTLSize::new(tg_x, tg_y, tg_z);++ for split in 0..split_k {+ let k_start = split * chunk;+ let k_end = ((split + 1) * chunk).min(k);++ enc.set_compute_pipeline_state(partial_ps);+ enc.set_buffer(0, Some(args.a), 0);+ enc.set_buffer(1, Some(args.b), 0);+ enc.set_buffer(2, Some(&accum_buf), 0);++ enc.set_bytes(3, 4, &m as *const u32 as *const std::ffi::c_void);+ enc.set_bytes(4, 4, &n as *const u32 as *const std::ffi::c_void);+ enc.set_bytes(5, 4, &k as *const u32 as *const std::ffi::c_void);+ enc.set_bytes(+ 6,+ 4,+ &(args.lda as u32) as *const u32 as *const std::ffi::c_void,+ );+ enc.set_bytes(+ 7,+ 4,+ &(args.ldb as u32) as *const u32 as *const std::ffi::c_void,+ );+ enc.set_bytes(+ 8,+ 4,+ &(args.ldd as u32) as *const u32 as *const std::ffi::c_void,+ );+ enc.set_bytes(+ 9,+ 4,+ &k_start as *const u32 as *const std::ffi::c_void,+ );+ enc.set_bytes(+ 10,+ 4,+ &k_end as *const u32 as *const std::ffi::c_void,+ );++ enc.dispatch_thread_groups(grid, threads);+ }++ // Convert to target type+ let convert_ps = self.pipeline(mtl, SplitKKernelKind::Convert)?;+ enc.set_compute_pipeline_state(convert_ps);+ enc.set_buffer(0, Some(&accum_buf), 0);+ enc.set_buffer(1, Some(args.d), 0);++ enc.set_bytes(2, 4, &m as *const u32 as *const std::ffi::c_void);+ enc.set_bytes(3, 4, &n as *const u32 as *const std::ffi::c_void);+ enc.set_bytes(+ 4,+ 4,+ &(args.ldd as u32) as *const u32 as *const std::ffi::c_void,+ );++ enc.dispatch_thread_groups(grid, threads);+ Ok(())+ }+}diff --git a/crates/uzu/src/backends/metal/mod.rs b/crates/uzu/src/backends/metal/mod.rsindex c55780fed..04d9d248e 100644--- a/crates/uzu/src/backends/metal/mod.rs+++ b/crates/uzu/src/backends/metal/mod.rs@@ -17,7 +17,9 @@ pub mod utils;pub use array::MetalArray;pub use buffer_allocator::BufferAllocator;pub use classifier_context::ClassifierContext;-pub use context::MTLContext;+pub use context::{+ DeviceArchitecture, DeviceClass, DeviceGeneration, MTLContext,+};pub use encodable_block::Decoder;pub use error::MTLError;pub use forward_pass::{CacheLayers, ForwardPassState, ModelShape};diff --git a/crates/uzu/tests/gemv_kernel_test.rs b/crates/uzu/tests/gemv_kernel_test.rsnew file mode 100644index 000000000..044d3f09b--- /dev/null+++ b/crates/uzu/tests/gemv_kernel_test.rs@@ -0,0 +1,73 @@+use half::f16;+use metal::MTLResourceOptions;+use uzu::{+ DataType,+ backends::metal::{MTLContext, MatmulArguments, MatmulKernel},+};++fn make_context() -> MTLContext {+ let device = metal::Device::system_default().expect("metal device");+ let queue = device.new_command_queue();+ MTLContext::new(device, queue).expect("ctx")+}++#[test]+#[ignore = "Requires Apple Silicon + Metal runtime"]+fn gemv_single_token_runs() {+ let ctx = make_context();++ // One token (M=1), K=4, N=4+ let a_vals: [f16; 4] = [1.0, 2.0, 3.0, 4.0].map(f16::from_f32);+ let b_vals: [f16; 16] = (0..16)+ .map(|i| f16::from_f32((i as f32) * 0.1))+ .collect::<Vec<_>>()+ .try_into()+ .unwrap();++ let a_buf = ctx.device.new_buffer_with_data(+ a_vals.as_ptr() as *const _,+ (a_vals.len() * std::mem::size_of::<f16>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );+ let b_buf = ctx.device.new_buffer_with_data(+ b_vals.as_ptr() as *const _,+ (b_vals.len() * std::mem::size_of::<f16>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );+ let d_buf = ctx.device.new_buffer(+ (4 * std::mem::size_of::<f16>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );++ let args = MatmulArguments {+ a: &a_buf,+ b: &b_buf,+ d: &d_buf,+ batch: 1,+ input_dim: 4,+ output_dim: 4,+ lda: 4,+ ldb: 4,+ ldd: 4,+ batch_count: 1,+ };++ let mut kernel =+ MatmulKernel::new(&ctx, DataType::F16, false, true).unwrap();+ let command_buffer = ctx.command_queue.new_command_buffer().to_owned();+ let encoder = command_buffer.new_compute_command_encoder();+ kernel.encode(&ctx, &encoder, args).unwrap();+ encoder.end_encoding();+ command_buffer.commit();+ command_buffer.wait_until_completed();++ // Read back output+ let out_ptr = d_buf.contents() as *const f16;+ let out = unsafe { std::slice::from_raw_parts(out_ptr, 4) };+ let cpu: f32 = a_vals+ .iter()+ .zip(b_vals.chunks_exact(4))+ .map(|(a, row)| a.to_f32() * row[0].to_f32())+ .sum();+ assert!((out[0].to_f32() - cpu).abs() < 1e-2);+}diff --git a/crates/uzu/tests/nax_gemm_test.rs b/crates/uzu/tests/nax_gemm_test.rsnew file mode 100644index 000000000..a605cc40f--- /dev/null+++ b/crates/uzu/tests/nax_gemm_test.rs@@ -0,0 +1,69 @@+use half::bf16;+use metal::MTLResourceOptions;+use uzu::{+ DataType,+ backends::metal::{MTLContext, MatmulArguments, MatmulKernel},+};++fn make_context() -> MTLContext {+ let device = metal::Device::system_default().expect("metal device");+ let queue = device.new_command_queue();+ MTLContext::new(device, queue).expect("ctx")+}++#[test]+#[ignore = "Requires Apple Silicon + Metal runtime"]+fn nax_path_runs_when_available() {+ let ctx = make_context();++ let m = 64usize;+ let n = 128usize;+ let k = 256usize;++ let a_vals: Vec<bf16> =+ (0..(m * k)).map(|i| bf16::from_f32((i % 11) as f32 * 0.01)).collect();+ let b_vals: Vec<bf16> = (0..(k * n))+ .map(|i| bf16::from_f32((i % 13) as f32 * 0.02 - 0.05))+ .collect();++ let a_buf = ctx.device.new_buffer_with_data(+ a_vals.as_ptr() as *const _,+ (a_vals.len() * std::mem::size_of::<bf16>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );+ let b_buf = ctx.device.new_buffer_with_data(+ b_vals.as_ptr() as *const _,+ (b_vals.len() * std::mem::size_of::<bf16>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );+ let d_buf = ctx.device.new_buffer(+ (m * n * std::mem::size_of::<bf16>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );++ let args = MatmulArguments {+ a: &a_buf,+ b: &b_buf,+ d: &d_buf,+ batch: m as i32,+ input_dim: k as i32,+ output_dim: n as i32,+ lda: k as i32,+ ldb: n as i32,+ ldd: n as i32,+ batch_count: 1,+ };++ let mut kernel =+ MatmulKernel::new(&ctx, DataType::BF16, false, true).unwrap();+ let command_buffer = ctx.command_queue.new_command_buffer().to_owned();+ let encoder = command_buffer.new_compute_command_encoder();+ kernel.encode(&ctx, &encoder, args).unwrap();+ encoder.end_encoding();+ command_buffer.commit();+ command_buffer.wait_until_completed();++ let out_ptr = d_buf.contents() as *const bf16;+ let out = unsafe { std::slice::from_raw_parts(out_ptr, m * n) };+ assert!(!out[0].to_f32().is_nan());+}diff --git a/crates/uzu/tests/splitk_gemm_test.rs b/crates/uzu/tests/splitk_gemm_test.rsnew file mode 100644index 000000000..1969aac3f--- /dev/null+++ b/crates/uzu/tests/splitk_gemm_test.rs@@ -0,0 +1,71 @@+use half::f16;+use metal::MTLResourceOptions;+use uzu::{+ DataType,+ backends::metal::{MTLContext, MatmulArguments, MatmulKernel},+};++fn make_context() -> MTLContext {+ let device = metal::Device::system_default().expect("metal device");+ let queue = device.new_command_queue();+ MTLContext::new(device, queue).expect("ctx")+}++#[test]+#[ignore = "Requires Apple Silicon + Metal runtime"]+fn splitk_triggers_for_large_k() {+ let ctx = make_context();++ // M=32, N=32, K large to trigger split-k heuristic.+ let m = 32usize;+ let n = 32usize;+ let k = 512usize;++ let a_vals: Vec<f16> =+ (0..(m * k)).map(|i| f16::from_f32((i % 17) as f32 * 0.01)).collect();+ let b_vals: Vec<f16> = (0..(k * n))+ .map(|i| f16::from_f32((i % 23) as f32 * 0.02 - 0.1))+ .collect();++ let a_buf = ctx.device.new_buffer_with_data(+ a_vals.as_ptr() as *const _,+ (a_vals.len() * std::mem::size_of::<f16>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );+ let b_buf = ctx.device.new_buffer_with_data(+ b_vals.as_ptr() as *const _,+ (b_vals.len() * std::mem::size_of::<f16>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );+ let d_buf = ctx.device.new_buffer(+ (m * n * std::mem::size_of::<f16>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );++ let args = MatmulArguments {+ a: &a_buf,+ b: &b_buf,+ d: &d_buf,+ batch: m as i32,+ input_dim: k as i32,+ output_dim: n as i32,+ lda: k as i32,+ ldb: n as i32,+ ldd: n as i32,+ batch_count: 1,+ };++ let mut kernel =+ MatmulKernel::new(&ctx, DataType::F16, false, true).unwrap();+ let command_buffer = ctx.command_queue.new_command_buffer().to_owned();+ let encoder = command_buffer.new_compute_command_encoder();+ kernel.encode(&ctx, &encoder, args).unwrap();+ encoder.end_encoding();+ command_buffer.commit();+ command_buffer.wait_until_completed();++ // Basic sanity: output buffer was written (check first element non-NaN)+ let out_ptr = d_buf.contents() as *const f16;+ let out = unsafe { std::slice::from_raw_parts(out_ptr, m * n) };+ assert!(!out[0].to_f32().is_nan());+}