A high-performance inference engine for AI models
speed up gemv
eugene42f39032fe27
diff --git a/crates/uzu/examples/matmul_benchmark.rs b/crates/uzu/examples/matmul_benchmark.rsindex 9790c354f..4a9d38f9d 100644--- a/crates/uzu/examples/matmul_benchmark.rs+++ b/crates/uzu/examples/matmul_benchmark.rs@@ -48,10 +48,12 @@ fn benchmark_metal_gemm(warmup_iters: usize,bench_iters: usize,) -> f64 {+ // A is [M, K] row-majorlet a: Vec<bf16> = (0..(m * k)).map(|i| bf16::from_f32(((i % 13) as f32) * 0.01)).collect();- let b: Vec<bf16> = (0..(k * n))+ // B is [N, K] row-major (transpose_b = true, like LLM weight matrices)+ let b: Vec<bf16> = (0..(n * k)).map(|i| bf16::from_f32(((i % 17) as f32) * 0.02 - 0.1)).collect();@@ -70,7 +72,8 @@ fn benchmark_metal_gemm(MTLResourceOptions::StorageModeShared,);- let mut kernel = MatmulKernel::new(ctx, DataType::BF16, false, false)+ // transpose_b = true: B is stored as [N, K], matching LLM weight layout+ let mut kernel = MatmulKernel::new(ctx, DataType::BF16, false, true).expect("kernel new");// Warmup@@ -89,7 +92,7 @@ fn benchmark_metal_gemm(input_dim: k as i32,output_dim: n as i32,lda: k as i32,- ldb: n as i32,+ ldb: k as i32, // B is [N, K], stride = Kldd: n as i32,batch_count: 1,},@@ -117,7 +120,7 @@ fn benchmark_metal_gemm(input_dim: k as i32,output_dim: n as i32,lda: k as i32,- ldb: n as i32,+ ldb: k as i32, // B is [N, K], stride = Kldd: n as i32,batch_count: 1,},diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv.h b/crates/uzu/src/backends/metal/kernel/matmul/gemv.hdeleted file mode 100644index 2b0da5b02..000000000--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv.h+++ /dev/null@@ -1,19 +0,0 @@-// Declarations for GEMV kernels (Metal).-// Implementations are in gemv.metal; names must match Rust dispatch.--#pragma once--#ifdef __METAL_VERSION__-// No host-side declarations needed; compiled by Metal.-#else-// Host-side string literals for kernel names.-#define GEMV_F16_ROWS2 "gemv_f16_rows2"-#define GEMV_F16_ROWS4 "gemv_f16_rows4"-#define GEMV_F16_ROWS8 "gemv_f16_rows8"--#define GEMV_BF16_ROWS2 "gemv_bf16_rows2"-#define GEMV_BF16_ROWS4 "gemv_bf16_rows4"-#define GEMV_BF16_ROWS8 "gemv_bf16_rows8"-#endif--diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemv.metalindex d1ee2ea1f..5ba80b972 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv.metal+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv.metal@@ -1,344 +1,209 @@// SPDX-License-Identifier: MIT-// Optimized GEMV kernels:-// - Multi-row output per simdgroup with input vector reuse-// - Vectorized loads (4 elements per thread)-// - simd_sum for fast K-dimension reduction-// - Register-only (no threadgroup memory)-//-// Two variants:-// - gemv_fast_*: 4 rows per simdgroup, best for large K (≥4096)-// - gemv_wide_*: 16 rows per simdgroup, best for small K with large N (MLP up pattern)#include <metal_stdlib>using namespace metal;-// Elements loaded per thread per K iteration (4 elements vectorized)-#define ELEMS_PER_THREAD 4--// Templated GEMV implementation with configurable rows per simdgroup-// y = A * x where A is [N, K] row-major-template <typename T, typename AccT, int ROWS>-METAL_FUNC void gemv_impl(- const device T* __restrict A,- const device T* __restrict x,- device T* __restrict y,- const int K,- const int N,- const int lda,+template <typename T, typename AccT, int ROWS_PER_SIMDGROUP, int ELEMS_PER_THREAD>+METAL_FUNC void gemv_multirow(+ const device T* __restrict weights,+ const device T* __restrict input,+ device T* __restrict output,+ const int input_dim,+ const int output_dim,+ const int weight_stride,const int batch_idx,- const int batch_stride_x,- const int batch_stride_y,+ const int input_batch_stride,+ const int output_batch_stride,uint row_base,uint simd_lid) {- // Block size in K dimension: 32 threads * 4 elements = 128constexpr int BLOCK_K = 32 * ELEMS_PER_THREAD;- // Position pointers for this batch- x += batch_idx * batch_stride_x;- y += batch_idx * batch_stride_y;+ input += batch_idx * input_batch_stride;+ output += batch_idx * output_batch_stride;- // Bounds check- if (row_base >= static_cast<uint>(N)) {+ if (row_base >= static_cast<uint>(output_dim)) {return;}- const int rows_to_compute = min(ROWS, N - static_cast<int>(row_base));+ const int rows_to_compute = min(ROWS_PER_SIMDGROUP, output_dim - static_cast<int>(row_base));- // Accumulators in registers- AccT result[ROWS];- #pragma unroll(ROWS)- for (int r = 0; r < ROWS; ++r) {- result[r] = AccT(0);+ AccT accumulators[ROWS_PER_SIMDGROUP];+ #pragma unroll(ROWS_PER_SIMDGROUP)+ for (int r = 0; r < ROWS_PER_SIMDGROUP; ++r) {+ accumulators[r] = AccT(0);}- // Input vector cache- AccT x_cache[ELEMS_PER_THREAD];+ AccT input_cache[ELEMS_PER_THREAD];- // Weight row pointers- const device T* row_ptrs[ROWS];- #pragma unroll(ROWS)- for (int r = 0; r < ROWS; ++r) {- row_ptrs[r] = A + (row_base + r) * lda;+ const device T* weight_row_ptrs[ROWS_PER_SIMDGROUP];+ #pragma unroll(ROWS_PER_SIMDGROUP)+ for (int r = 0; r < ROWS_PER_SIMDGROUP; ++r) {+ weight_row_ptrs[r] = weights + (row_base + r) * weight_stride;}- // Main K loopint k = 0;- for (; k + BLOCK_K <= K; k += BLOCK_K) {+ for (; k + BLOCK_K <= input_dim; k += BLOCK_K) {const int thread_k = k + simd_lid * ELEMS_PER_THREAD;- // Load input vector chunk#pragma unroll(ELEMS_PER_THREAD)for (int e = 0; e < ELEMS_PER_THREAD; ++e) {- x_cache[e] = static_cast<AccT>(x[thread_k + e]);+ input_cache[e] = static_cast<AccT>(input[thread_k + e]);}- // Dot product for each row- #pragma unroll(ROWS)- for (int r = 0; r < ROWS; ++r) {+ #pragma unroll(ROWS_PER_SIMDGROUP)+ for (int r = 0; r < ROWS_PER_SIMDGROUP; ++r) {AccT dot = 0;#pragma unroll(ELEMS_PER_THREAD)for (int e = 0; e < ELEMS_PER_THREAD; ++e) {- dot += x_cache[e] * static_cast<AccT>(row_ptrs[r][thread_k + e]);+ dot += input_cache[e] * static_cast<AccT>(weight_row_ptrs[r][thread_k + e]);}- result[r] += dot;+ accumulators[r] += dot;}}- // Handle leftover K elements- if (k < K) {+ if (k < input_dim) {const int thread_k = k + simd_lid * ELEMS_PER_THREAD;- const int remaining = K - k;+ const int remaining = input_dim - k;#pragma unroll(ELEMS_PER_THREAD)for (int e = 0; e < ELEMS_PER_THREAD; ++e) {const int idx = simd_lid * ELEMS_PER_THREAD + e;- x_cache[e] = (idx < remaining) ? static_cast<AccT>(x[thread_k + e]) : AccT(0);+ input_cache[e] = (idx < remaining) ? static_cast<AccT>(input[thread_k + e]) : AccT(0);}- #pragma unroll(ROWS)- for (int r = 0; r < ROWS; ++r) {+ #pragma unroll(ROWS_PER_SIMDGROUP)+ for (int r = 0; r < ROWS_PER_SIMDGROUP; ++r) {AccT dot = 0;#pragma unroll(ELEMS_PER_THREAD)for (int e = 0; e < ELEMS_PER_THREAD; ++e) {const int idx = simd_lid * ELEMS_PER_THREAD + e;if (idx < remaining) {- dot += x_cache[e] * static_cast<AccT>(row_ptrs[r][thread_k + e]);+ dot += input_cache[e] * static_cast<AccT>(weight_row_ptrs[r][thread_k + e]);}}- result[r] += dot;+ accumulators[r] += dot;}}- // Reduce across simdgroup- #pragma unroll(ROWS)- for (int r = 0; r < ROWS; ++r) {- result[r] = simd_sum(result[r]);+ #pragma unroll(ROWS_PER_SIMDGROUP)+ for (int r = 0; r < ROWS_PER_SIMDGROUP; ++r) {+ accumulators[r] = simd_sum(accumulators[r]);}- // Write outputif (simd_lid == 0) {- #pragma unroll(ROWS)+ #pragma unroll(ROWS_PER_SIMDGROUP)for (int r = 0; r < rows_to_compute; ++r) {- y[row_base + r] = static_cast<T>(result[r]);+ output[row_base + r] = static_cast<T>(accumulators[r]);}}}-// Standard GEMV: 4 rows per simdgroup (best for large K)-#define DECL_GEMV_FAST(NAME, TYPE, ACC) \- [[kernel]] void NAME( \- const device TYPE* A [[buffer(0)]], \- const device TYPE* x [[buffer(1)]], \- device TYPE* y [[buffer(2)]], \- constant int& K [[buffer(3)]], \- constant int& N [[buffer(4)]], \- constant int& lda [[buffer(5)]], \- constant int& batch_stride_x [[buffer(6)]], \- constant int& batch_stride_y [[buffer(7)]], \- uint3 tgid [[threadgroup_position_in_grid]], \- uint simd_lid [[thread_index_in_simdgroup]]) { \- constexpr int ROWS = 4; \- const uint row_base = tgid.x * ROWS; \- gemv_impl<TYPE, ACC, ROWS>( \- A, x, y, K, N, lda, tgid.z, batch_stride_x, batch_stride_y, \- row_base, simd_lid); \+#define DECL_GEMV_FAST(NAME, TYPE, ACC) \+ [[kernel]] void NAME( \+ const device TYPE* weights [[buffer(0)]], \+ const device TYPE* input [[buffer(1)]], \+ device TYPE* output [[buffer(2)]], \+ constant int& input_dim [[buffer(3)]], \+ constant int& output_dim [[buffer(4)]], \+ constant int& weight_stride [[buffer(5)]], \+ constant int& input_batch_stride [[buffer(6)]], \+ constant int& output_batch_stride [[buffer(7)]], \+ uint3 tgid [[threadgroup_position_in_grid]], \+ uint simd_lid [[thread_index_in_simdgroup]]) { \+ constexpr int ROWS_PER_SIMDGROUP = 4; \+ constexpr int ELEMS_PER_THREAD = 4; \+ const uint row_base = tgid.x * ROWS_PER_SIMDGROUP; \+ gemv_multirow<TYPE, ACC, ROWS_PER_SIMDGROUP, ELEMS_PER_THREAD>( \+ weights, input, output, input_dim, output_dim, weight_stride, \+ tgid.z, input_batch_stride, output_batch_stride, row_base, simd_lid); \}-// Wide GEMV: 8 rows per simdgroup (balanced for small K, large N)-#define DECL_GEMV_WIDE(NAME, TYPE, ACC) \- [[kernel]] void NAME( \- const device TYPE* A [[buffer(0)]], \- const device TYPE* x [[buffer(1)]], \- device TYPE* y [[buffer(2)]], \- constant int& K [[buffer(3)]], \- constant int& N [[buffer(4)]], \- constant int& lda [[buffer(5)]], \- constant int& batch_stride_x [[buffer(6)]], \- constant int& batch_stride_y [[buffer(7)]], \- uint3 tgid [[threadgroup_position_in_grid]], \- uint simd_lid [[thread_index_in_simdgroup]]) { \- constexpr int ROWS = 8; \- const uint row_base = tgid.x * ROWS; \- gemv_impl<TYPE, ACC, ROWS>( \- A, x, y, K, N, lda, tgid.z, batch_stride_x, batch_stride_y, \- row_base, simd_lid); \- }--// Instantiate standard (4-row) kernelsDECL_GEMV_FAST(gemv_fast_f16, half, float)DECL_GEMV_FAST(gemv_fast_bf16, bfloat, float)DECL_GEMV_FAST(gemv_fast_f32, float, float)-// Instantiate wide (8-row) kernels-DECL_GEMV_WIDE(gemv_wide_f16, half, float)-DECL_GEMV_WIDE(gemv_wide_bf16, bfloat, float)-DECL_GEMV_WIDE(gemv_wide_f32, float, float)--// =============================================================================-// Parallel GEMV: Multiple simdgroups per threadgroup with shared input vector-// Best for small K with large N (reduces threadgroup count, amortizes overhead)-// =============================================================================--template <typename T, typename AccT, int SIMDGROUPS_PER_TG, int BK>-[[kernel, max_total_threads_per_threadgroup(SIMDGROUPS_PER_TG * 32)]]-void gemv_parallel_impl(- const device T* __restrict A [[buffer(0)]],- const device T* __restrict x [[buffer(1)]],- device T* __restrict y [[buffer(2)]],- constant int& K [[buffer(3)]],- constant int& N [[buffer(4)]],- constant int& lda [[buffer(5)]],- constant int& batch_stride_x [[buffer(6)]],- constant int& batch_stride_y [[buffer(7)]],- uint3 tgid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]],- uint tid [[thread_index_in_threadgroup]]) {-- // Shared memory for input vector tile- threadgroup AccT x_shared[BK];-- // Position pointers for this batch- const int batch_idx = tgid.z;- x += batch_idx * batch_stride_x;- y += batch_idx * batch_stride_y;-- // Each simdgroup computes one output row- const uint row = tgid.x * SIMDGROUPS_PER_TG + simd_gid;- if (row >= static_cast<uint>(N)) {+template <typename T, typename AccT, ushort ROWS_PER_THREADGROUP, ushort TILE_K>+METAL_FUNC void gemv_tiled(+ const device T* __restrict weights,+ const device T* __restrict input,+ device T* __restrict output,+ constant uint& input_dim,+ constant uint& output_dim,+ constant uint& weight_stride,+ constant uint& input_stride,+ constant uint& output_stride,+ uint3 tgid,+ uint simd_gid,+ uint simd_lid,+ uint tid,+ threadgroup AccT* input_tile) {++ const ushort THREADS_PER_THREADGROUP = ROWS_PER_THREADGROUP * 32;+ if (simd_gid >= ROWS_PER_THREADGROUP) {return;}- const device T* row_ptr = A + row * lda;- AccT acc = AccT(0);-- // Process K in tiles- for (int k_start = 0; k_start < K; k_start += BK) {- const int remaining = K - k_start;- const int tile_k = remaining < BK ? remaining : BK;-- // Cooperative load of input vector tile into shared memory- for (int idx = tid; idx < tile_k; idx += SIMDGROUPS_PER_TG * 32) {- x_shared[idx] = static_cast<AccT>(x[k_start + idx]);- }- threadgroup_barrier(mem_flags::mem_threadgroup);-- // Each thread in simdgroup processes part of the tile- for (int idx = simd_lid; idx < tile_k; idx += 32) {- acc += static_cast<AccT>(row_ptr[k_start + idx]) * x_shared[idx];- }- threadgroup_barrier(mem_flags::mem_threadgroup);- }-- // Reduce within simdgroup- acc = simd_sum(acc);-- // Write output- if (simd_lid == 0) {- y[row] = static_cast<T>(acc);- }-}--// Parallel kernels: 32 simdgroups per threadgroup, BK=1024 for maximum occupancy-template [[host_name("gemv_parallel_f16")]] kernel void gemv_parallel_impl<half, float, 32, 1024>(- const device half*, const device half*, device half*,- constant int&, constant int&, constant int&, constant int&, constant int&,- uint3, uint, uint, uint);-template [[host_name("gemv_parallel_bf16")]] kernel void gemv_parallel_impl<bfloat, float, 32, 1024>(- const device bfloat*, const device bfloat*, device bfloat*,- constant int&, constant int&, constant int&, constant int&, constant int&,- uint3, uint, uint, uint);-template [[host_name("gemv_parallel_f32")]] kernel void gemv_parallel_impl<float, float, 32, 1024>(- const device float*, const device float*, device float*,- constant int&, constant int&, constant int&, constant int&, constant int&,- uint3, uint, uint, uint);--// =============================================================================-// Legacy kernels for backwards compatibility (used by existing Rust dispatcher)-// =============================================================================--template <typename T, typename AccT, ushort ROWS_PER_TG, ushort BK>-METAL_FUNC void gemv_impl_tiled(- const device T* __restrict matrix [[buffer(0)]],- const device T* __restrict vector [[buffer(1)]],- device T* __restrict output [[buffer(2)]],- constant uint& k [[buffer(3)]],- constant uint& n [[buffer(4)]],- constant uint& ldb [[buffer(5)]],- constant uint& lda [[buffer(6)]],- constant uint& ldd [[buffer(7)]],- uint3 tgid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]],- uint tid [[thread_index_in_threadgroup]],- threadgroup AccT* vec_tile) {- const ushort TG_THREADS = ROWS_PER_TG * 32;- if (simd_gid >= ROWS_PER_TG) {- return;- }- const uint row = tgid.x * ROWS_PER_TG + simd_gid;- if (row >= n) {+ const uint row = tgid.x * ROWS_PER_THREADGROUP + simd_gid;+ if (row >= output_dim) {return;}- const uint batch = tgid.z;- const device T* row_ptr = matrix + row * ldb;- const device T* vec_ptr = vector + batch * lda;- device T* out_ptr = output + batch * ldd;+ const uint batch = tgid.z;+ const device T* weight_row = weights + row * weight_stride;+ const device T* input_ptr = input + batch * input_stride;+ device T* output_ptr = output + batch * output_stride;- AccT acc = static_cast<AccT>(0);+ AccT accumulator = AccT(0);- for (uint k_block = 0; k_block < k; k_block += BK) {- const uint remaining = k - k_block;- const uint tile_elems = remaining < BK ? remaining : BK;+ for (uint k_block = 0; k_block < input_dim; k_block += TILE_K) {+ const uint remaining = input_dim - k_block;+ const uint tile_size = remaining < TILE_K ? remaining : TILE_K;- if (tid < tile_elems && tid < TG_THREADS) {- vec_tile[tid] = static_cast<AccT>(vec_ptr[k_block + tid]);+ if (tid < tile_size && tid < THREADS_PER_THREADGROUP) {+ input_tile[tid] = static_cast<AccT>(input_ptr[k_block + tid]);}threadgroup_barrier(mem_flags::mem_threadgroup);- for (uint idx = simd_lid; idx < tile_elems; idx += 32) {- acc += static_cast<AccT>(row_ptr[k_block + idx]) * vec_tile[idx];+ for (uint idx = simd_lid; idx < tile_size; idx += 32) {+ accumulator += static_cast<AccT>(weight_row[k_block + idx]) * input_tile[idx];}threadgroup_barrier(mem_flags::mem_threadgroup);}for (ushort offset = 16; offset > 0; offset >>= 1) {- acc += simd_shuffle_down(acc, offset);+ accumulator += simd_shuffle_down(accumulator, offset);}if (simd_lid == 0) {- out_ptr[row] = static_cast<T>(acc);+ output_ptr[row] = static_cast<T>(accumulator);}}-#define DECL_GEMV_LEGACY(NAME, TYPE, ACC, ROWS, BK) \- [[kernel, max_total_threads_per_threadgroup(ROWS * 32)]] void NAME( \- const device TYPE* matrix [[buffer(0)]], \- const device TYPE* vector [[buffer(1)]], \- device TYPE* output [[buffer(2)]], \- constant uint& k [[buffer(3)]], \- constant uint& n [[buffer(4)]], \- constant uint& ldb [[buffer(5)]], \- constant uint& lda [[buffer(6)]], \- constant uint& ldd [[buffer(7)]], \- uint3 tgid [[threadgroup_position_in_grid]], \- uint simd_gid [[simdgroup_index_in_threadgroup]], \- uint simd_lid [[thread_index_in_simdgroup]], \- uint tid [[thread_index_in_threadgroup]]) { \- threadgroup ACC vec_tile[BK]; \- gemv_impl_tiled<TYPE, ACC, ROWS, BK>( \- matrix, vector, output, k, n, ldb, lda, ldd, \- tgid, simd_gid, simd_lid, tid, vec_tile); \- }--// Legacy half variants-DECL_GEMV_LEGACY(gemv_f16_rows2, half, float, 2, 128)-DECL_GEMV_LEGACY(gemv_f16_rows4, half, float, 4, 128)-DECL_GEMV_LEGACY(gemv_f16_rows8, half, float, 8, 128)--// Legacy bfloat16 variants-DECL_GEMV_LEGACY(gemv_bf16_rows2, bfloat, float, 2, 128)-DECL_GEMV_LEGACY(gemv_bf16_rows4, bfloat, float, 4, 128)-DECL_GEMV_LEGACY(gemv_bf16_rows8, bfloat, float, 8, 128)+#define DECL_GEMV_TILED(NAME, TYPE, ACC, ROWS, TILE_K) \+ [[kernel, max_total_threads_per_threadgroup(ROWS * 32)]] void NAME( \+ const device TYPE* weights [[buffer(0)]], \+ const device TYPE* input [[buffer(1)]], \+ device TYPE* output [[buffer(2)]], \+ constant uint& input_dim [[buffer(3)]], \+ constant uint& output_dim [[buffer(4)]], \+ constant uint& weight_stride [[buffer(5)]], \+ constant uint& input_stride [[buffer(6)]], \+ constant uint& output_stride [[buffer(7)]], \+ uint3 tgid [[threadgroup_position_in_grid]], \+ uint simd_gid [[simdgroup_index_in_threadgroup]], \+ uint simd_lid [[thread_index_in_simdgroup]], \+ uint tid [[thread_index_in_threadgroup]]) { \+ threadgroup ACC input_tile[TILE_K]; \+ gemv_tiled<TYPE, ACC, ROWS, TILE_K>( \+ weights, input, output, input_dim, output_dim, weight_stride, \+ input_stride, output_stride, tgid, simd_gid, simd_lid, tid, input_tile); \+ }++DECL_GEMV_TILED(gemv_f16_rows2, half, float, 2, 128)+DECL_GEMV_TILED(gemv_f16_rows4, half, float, 4, 128)+DECL_GEMV_TILED(gemv_f16_rows8, half, float, 8, 128)++DECL_GEMV_TILED(gemv_bf16_rows2, bfloat, float, 2, 128)+DECL_GEMV_TILED(gemv_bf16_rows4, bfloat, float, 4, 128)+DECL_GEMV_TILED(gemv_bf16_rows8, bfloat, float, 8, 128)diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv.rsindex f3ed8f0b1..ad43ee52a 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv.rs@@ -12,430 +12,224 @@ use crate::{};#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]-enum PipelineKind {- Legacy {- rows_per_group: u32,+enum PipelineVariant {+ Tiled {+ rows_per_threadgroup: u32,},Fast,- Wide,- Parallel,}pub struct GemvKernel {- dt: DataType,- pipelines: HashMap<PipelineKind, MTLComputePipelineState>,+ data_type: DataType,+ pipelines: HashMap<PipelineVariant, MTLComputePipelineState>,}impl GemvKernel {- pub fn new(dt: DataType) -> Self {+ pub fn new(data_type: DataType) -> Self {Self {- dt,+ data_type,pipelines: HashMap::new(),}}fn fast_kernel_name(&self) -> Result<String, MTLError> {- match self.dt {+ match self.data_type {DataType::F16 => Ok("gemv_fast_f16".to_string()),DataType::BF16 => Ok("gemv_fast_bf16".to_string()),DataType::F32 => Ok("gemv_fast_f32".to_string()),_ => Err(MTLError::Generic(format!(- "Unsupported dtype for fast GEMV: {:?}",- self.dt+ "Unsupported data type for fast GEMV: {:?}",+ self.data_type))),}}- fn wide_kernel_name(&self) -> Result<String, MTLError> {- match self.dt {- DataType::F16 => Ok("gemv_wide_f16".to_string()),- DataType::BF16 => Ok("gemv_wide_bf16".to_string()),- DataType::F32 => Ok("gemv_wide_f32".to_string()),- _ => Err(MTLError::Generic(format!(- "Unsupported dtype for wide GEMV: {:?}",- self.dt- ))),- }- }-- fn parallel_kernel_name(&self) -> Result<String, MTLError> {- match self.dt {- DataType::F16 => Ok("gemv_parallel_f16".to_string()),- DataType::BF16 => Ok("gemv_parallel_bf16".to_string()),- DataType::F32 => Ok("gemv_parallel_f32".to_string()),- _ => Err(MTLError::Generic(format!(- "Unsupported dtype for parallel GEMV: {:?}",- self.dt- ))),- }- }-- fn legacy_kernel_name(+ fn tiled_kernel_name(&self,- rows: u32,+ rows_per_threadgroup: u32,) -> Result<String, MTLError> {- let base = match self.dt {+ let prefix = match self.data_type {DataType::F16 => "gemv_f16",DataType::BF16 => "gemv_bf16",_ => {return Err(MTLError::Generic(format!(- "Unsupported dtype for legacy GEMV: {:?}",- self.dt+ "Unsupported data type for tiled GEMV: {:?}",+ self.data_type)));},};- Ok(format!("{base}_rows{rows}"))+ Ok(format!("{prefix}_rows{rows_per_threadgroup}"))}fn get_fast_pipeline(&mut self,- mtl: &MTLContext,- ) -> Result<&MTLComputePipelineState, MTLError> {- let key = PipelineKind::Fast;- if !self.pipelines.contains_key(&key) {- let name = self.fast_kernel_name()?;- let ps = mtl.compute_pipeline_state(&name, None)?;- self.pipelines.insert(key, ps);- }- Ok(self.pipelines.get(&key).unwrap())- }-- fn get_wide_pipeline(- &mut self,- mtl: &MTLContext,+ context: &MTLContext,) -> Result<&MTLComputePipelineState, MTLError> {- let key = PipelineKind::Wide;- if !self.pipelines.contains_key(&key) {- let name = self.wide_kernel_name()?;- let ps = mtl.compute_pipeline_state(&name, None)?;- self.pipelines.insert(key, ps);+ let variant = PipelineVariant::Fast;+ if !self.pipelines.contains_key(&variant) {+ let kernel_name = self.fast_kernel_name()?;+ let pipeline =+ context.compute_pipeline_state(&kernel_name, None)?;+ self.pipelines.insert(variant, pipeline);}- Ok(self.pipelines.get(&key).unwrap())+ Ok(self.pipelines.get(&variant).unwrap())}- fn get_parallel_pipeline(+ fn get_tiled_pipeline(&mut self,- mtl: &MTLContext,+ context: &MTLContext,+ rows_per_threadgroup: u32,) -> Result<&MTLComputePipelineState, MTLError> {- let key = PipelineKind::Parallel;- if !self.pipelines.contains_key(&key) {- let name = self.parallel_kernel_name()?;- let ps = mtl.compute_pipeline_state(&name, None)?;- self.pipelines.insert(key, ps);- }- Ok(self.pipelines.get(&key).unwrap())- }-- fn get_legacy_pipeline(- &mut self,- mtl: &MTLContext,- rows: u32,- ) -> Result<&MTLComputePipelineState, MTLError> {- let key = PipelineKind::Legacy {- rows_per_group: rows,+ let variant = PipelineVariant::Tiled {+ rows_per_threadgroup,};- if !self.pipelines.contains_key(&key) {- let name = self.legacy_kernel_name(rows)?;- let ps = mtl.compute_pipeline_state(&name, None)?;- self.pipelines.insert(key, ps);+ if !self.pipelines.contains_key(&variant) {+ let kernel_name = self.tiled_kernel_name(rows_per_threadgroup)?;+ let pipeline =+ context.compute_pipeline_state(&kernel_name, None)?;+ self.pipelines.insert(variant, pipeline);}- Ok(self.pipelines.get(&key).unwrap())+ Ok(self.pipelines.get(&variant).unwrap())}- /// Check if wide GEMV is applicable.- /// Currently disabled as it doesn't improve MLP up pattern.- fn should_use_wide(- &self,- _args: &MatmulArguments,- ) -> bool {- false- }-- /// Check if parallel GEMV is applicable.- /// Parallel kernel (8 simdgroups/TG) is best for small K with large N.- fn should_use_parallel(+ fn can_use_fast_kernel(&self,args: &MatmulArguments,) -> bool {- let k = args.input_dim;- let n = args.output_dim;- // Use parallel kernel when:- // - K is small (< 4096) so threadgroup memory can hold a good chunk- // - N is large (≥ 4096) so we benefit from reduced TG count- // - K is aligned for vectorized loads- k < 4096 && n >= 4096 && k % 4 == 0 && k >= 128- }-- /// Check if fast GEMV is applicable.- /// Fast kernel works best when K is large enough for vectorized loads.- fn can_use_fast(- &self,- args: &MatmulArguments,- ) -> bool {- // Fast kernel uses BLOCK_K = 128 (32 threads * 4 elements)- // Works well when K >= 128 and is reasonably alignedargs.input_dim >= 128 && args.input_dim % 4 == 0}- /// Encode using the fast optimized GEMV kernel.- pub fn encode_fast(+ fn encode_fast(&mut self,- mtl: &MTLContext,- enc: &ComputeCommandEncoderRef,+ context: &MTLContext,+ encoder: &ComputeCommandEncoderRef,args: MatmulArguments,) -> Result<(), MTLError> {- let ps = self.get_fast_pipeline(mtl)?;- enc.set_compute_pipeline_state(ps);+ let pipeline = self.get_fast_pipeline(context)?;+ encoder.set_compute_pipeline_state(pipeline);- // Bind buffers: A (weights/matrix), x (input vector), y (output)- enc.set_buffer(0, Some(args.b), 0); // A = weights- enc.set_buffer(1, Some(args.a), 0); // x = input- enc.set_buffer(2, Some(args.d), 0); // y = output+ encoder.set_buffer(0, Some(args.b), 0);+ encoder.set_buffer(1, Some(args.a), 0);+ encoder.set_buffer(2, Some(args.d), 0);- // Scalars (as i32 to match Metal kernel)- let k = args.input_dim;- let n = args.output_dim;- let lda = args.ldb; // Weight matrix leading dimension- // For batched GEMV: stride between input/output vectors is the leading dimension- let batch_stride_x = args.lda; // stride between rows of A (input vectors)- let batch_stride_y = args.ldd; // stride between rows of D (output vectors)+ let input_dim = args.input_dim;+ let output_dim = args.output_dim;+ let weight_stride = args.ldb;+ let input_batch_stride = args.lda;+ let output_batch_stride = args.ldd;- enc.set_bytes(+ encoder.set_bytes(3,std::mem::size_of::<i32>() as u64,- &k as *const i32 as *const std::ffi::c_void,+ &input_dim as *const i32 as *const std::ffi::c_void,);- enc.set_bytes(+ encoder.set_bytes(4,std::mem::size_of::<i32>() as u64,- &n as *const i32 as *const std::ffi::c_void,+ &output_dim as *const i32 as *const std::ffi::c_void,);- enc.set_bytes(+ encoder.set_bytes(5,std::mem::size_of::<i32>() as u64,- &lda as *const i32 as *const std::ffi::c_void,+ &weight_stride as *const i32 as *const std::ffi::c_void,);- enc.set_bytes(+ encoder.set_bytes(6,std::mem::size_of::<i32>() as u64,- &batch_stride_x as *const i32 as *const std::ffi::c_void,+ &input_batch_stride as *const i32 as *const std::ffi::c_void,);- enc.set_bytes(+ encoder.set_bytes(7,std::mem::size_of::<i32>() as u64,- &batch_stride_y as *const i32 as *const std::ffi::c_void,+ &output_batch_stride as *const i32 as *const std::ffi::c_void,);- // Fast kernel: 4 rows per simdgroup, 1 simdgroup per threadgroupconst ROWS_PER_SIMDGROUP: i32 = 4;- let tg_x = ((n + ROWS_PER_SIMDGROUP - 1) / ROWS_PER_SIMDGROUP) as u64;- let tg_z = args.batch_count as u64;-- let tgs = MTLSize::new(tg_x, 1, tg_z);- let threads_per_tg = MTLSize::new(32, 1, 1); // Single simdgroup-- enc.dispatch_thread_groups(tgs, threads_per_tg);- Ok(())- }-- /// Encode using the wide GEMV kernel (16 rows per simdgroup).- /// Best for small K with large N (MLP up pattern).- pub fn encode_wide(- &mut self,- mtl: &MTLContext,- enc: &ComputeCommandEncoderRef,- args: MatmulArguments,- ) -> Result<(), MTLError> {- let ps = self.get_wide_pipeline(mtl)?;- enc.set_compute_pipeline_state(ps);-- // Same buffer layout as fast kernel- enc.set_buffer(0, Some(args.b), 0); // A = weights- enc.set_buffer(1, Some(args.a), 0); // x = input- enc.set_buffer(2, Some(args.d), 0); // y = output-- let k = args.input_dim;- let n = args.output_dim;- let lda = args.ldb;- let batch_stride_x = args.lda; // stride between rows of A- let batch_stride_y = args.ldd; // stride between rows of D-- enc.set_bytes(- 3,- std::mem::size_of::<i32>() as u64,- &k as *const i32 as *const std::ffi::c_void,- );- enc.set_bytes(- 4,- std::mem::size_of::<i32>() as u64,- &n as *const i32 as *const std::ffi::c_void,- );- enc.set_bytes(- 5,- std::mem::size_of::<i32>() as u64,- &lda as *const i32 as *const std::ffi::c_void,- );- enc.set_bytes(- 6,- std::mem::size_of::<i32>() as u64,- &batch_stride_x as *const i32 as *const std::ffi::c_void,- );- enc.set_bytes(- 7,- std::mem::size_of::<i32>() as u64,- &batch_stride_y as *const i32 as *const std::ffi::c_void,- );-- // Wide kernel: 8 rows per simdgroup- const ROWS_PER_SIMDGROUP: i32 = 8;- let tg_x = ((n + ROWS_PER_SIMDGROUP - 1) / ROWS_PER_SIMDGROUP) as u64;- let tg_z = args.batch_count as u64;+ let threadgroup_count_x =+ ((output_dim + ROWS_PER_SIMDGROUP - 1) / ROWS_PER_SIMDGROUP) as u64;+ let threadgroup_count_z = args.batch_count as u64;- let tgs = MTLSize::new(tg_x, 1, tg_z);- let threads_per_tg = MTLSize::new(32, 1, 1); // Single simdgroup+ let threadgroup_count =+ MTLSize::new(threadgroup_count_x, 1, threadgroup_count_z);+ let threads_per_threadgroup = MTLSize::new(32, 1, 1);- enc.dispatch_thread_groups(tgs, threads_per_tg);+ encoder+ .dispatch_thread_groups(threadgroup_count, threads_per_threadgroup);Ok(())}- /// Encode using the parallel GEMV kernel (8 simdgroups per threadgroup).- /// Best for small K with large N (reduces threadgroup count).- pub fn encode_parallel(+ fn encode_tiled(&mut self,- mtl: &MTLContext,- enc: &ComputeCommandEncoderRef,+ context: &MTLContext,+ encoder: &ComputeCommandEncoderRef,args: MatmulArguments,+ rows_per_threadgroup: u32,) -> Result<(), MTLError> {- let ps = self.get_parallel_pipeline(mtl)?;- enc.set_compute_pipeline_state(ps);-- // Same buffer layout as other kernels- enc.set_buffer(0, Some(args.b), 0); // A = weights- enc.set_buffer(1, Some(args.a), 0); // x = input- enc.set_buffer(2, Some(args.d), 0); // y = output-- let k = args.input_dim;- let n = args.output_dim;- let lda = args.ldb;- let batch_stride_x = args.lda; // stride between rows of A- let batch_stride_y = args.ldd; // stride between rows of D-- enc.set_bytes(- 3,- std::mem::size_of::<i32>() as u64,- &k as *const i32 as *const std::ffi::c_void,- );- enc.set_bytes(- 4,- std::mem::size_of::<i32>() as u64,- &n as *const i32 as *const std::ffi::c_void,- );- enc.set_bytes(- 5,- std::mem::size_of::<i32>() as u64,- &lda as *const i32 as *const std::ffi::c_void,- );- enc.set_bytes(- 6,- std::mem::size_of::<i32>() as u64,- &batch_stride_x as *const i32 as *const std::ffi::c_void,- );- enc.set_bytes(- 7,- std::mem::size_of::<i32>() as u64,- &batch_stride_y as *const i32 as *const std::ffi::c_void,- );-- // Parallel kernel: 32 simdgroups per TG, each computes 1 row- const SIMDGROUPS_PER_TG: i32 = 32;- let tg_x = ((n + SIMDGROUPS_PER_TG - 1) / SIMDGROUPS_PER_TG) as u64;- let tg_z = args.batch_count as u64;-- let tgs = MTLSize::new(tg_x, 1, tg_z);- let threads_per_tg =- MTLSize::new((SIMDGROUPS_PER_TG * 32) as u64, 1, 1);-- enc.dispatch_thread_groups(tgs, threads_per_tg);- Ok(())- }-- /// Encode using the legacy GEMV kernel (for fallback).- pub fn encode_legacy(- &mut self,- mtl: &MTLContext,- enc: &ComputeCommandEncoderRef,- args: MatmulArguments,- rows_per_group: u32,- ) -> Result<(), MTLError> {- let rows_per_group = rows_per_group.max(1);- let ps = self.get_legacy_pipeline(mtl, rows_per_group)?;- enc.set_compute_pipeline_state(ps);-- // Bind buffers: matrix (B), vector (A), output (D)- enc.set_buffer(0, Some(args.b), 0);- enc.set_buffer(1, Some(args.a), 0);- enc.set_buffer(2, Some(args.d), 0);-- // Scalars- let k = args.input_dim as u32;- let n = args.output_dim as u32;- let ldb = args.ldb as u32;- let lda = args.lda as u32;- let ldd = args.ldd as u32;-- enc.set_bytes(+ let rows_per_threadgroup = rows_per_threadgroup.max(1);+ let pipeline =+ self.get_tiled_pipeline(context, rows_per_threadgroup)?;+ encoder.set_compute_pipeline_state(pipeline);++ encoder.set_buffer(0, Some(args.b), 0);+ encoder.set_buffer(1, Some(args.a), 0);+ encoder.set_buffer(2, Some(args.d), 0);++ let input_dim = args.input_dim as u32;+ let output_dim = args.output_dim as u32;+ let weight_stride = args.ldb as u32;+ let input_stride = args.lda as u32;+ let output_stride = args.ldd as u32;++ encoder.set_bytes(3,std::mem::size_of::<u32>() as u64,- &k as *const u32 as *const std::ffi::c_void,+ &input_dim as *const u32 as *const std::ffi::c_void,);- enc.set_bytes(+ encoder.set_bytes(4,std::mem::size_of::<u32>() as u64,- &n as *const u32 as *const std::ffi::c_void,+ &output_dim as *const u32 as *const std::ffi::c_void,);- enc.set_bytes(+ encoder.set_bytes(5,std::mem::size_of::<u32>() as u64,- &ldb as *const u32 as *const std::ffi::c_void,+ &weight_stride as *const u32 as *const std::ffi::c_void,);- enc.set_bytes(+ encoder.set_bytes(6,std::mem::size_of::<u32>() as u64,- &lda as *const u32 as *const std::ffi::c_void,+ &input_stride as *const u32 as *const std::ffi::c_void,);- enc.set_bytes(+ encoder.set_bytes(7,std::mem::size_of::<u32>() as u64,- &ldd as *const u32 as *const std::ffi::c_void,+ &output_stride as *const u32 as *const std::ffi::c_void,);- // Threadgroup sizing- let threads_per_tg = MTLSize::new((rows_per_group * 32) as u64, 1, 1);- let tg_x = ((n + rows_per_group - 1) / rows_per_group) as u64;- let tg_z = args.batch_count as u64;- let tgs = MTLSize::new(tg_x, 1, tg_z);+ let threads_per_threadgroup =+ MTLSize::new((rows_per_threadgroup * 32) as u64, 1, 1);+ let threadgroup_count_x = ((output_dim + rows_per_threadgroup - 1)+ / rows_per_threadgroup) as u64;+ let threadgroup_count_z = args.batch_count as u64;+ let threadgroup_count =+ MTLSize::new(threadgroup_count_x, 1, threadgroup_count_z);- enc.dispatch_thread_groups(tgs, threads_per_tg);+ encoder+ .dispatch_thread_groups(threadgroup_count, threads_per_threadgroup);Ok(())}- /// Main encode entry point - automatically selects best kernel variant.pub fn encode(&mut self,- mtl: &MTLContext,- enc: &ComputeCommandEncoderRef,+ context: &MTLContext,+ encoder: &ComputeCommandEncoderRef,args: MatmulArguments,- rows_per_group: u32,+ rows_per_threadgroup: u32,) -> Result<(), MTLError> {- // Priority: parallel > wide > fast > legacy- if self.should_use_parallel(&args) {- self.encode_parallel(mtl, enc, args)- } else if self.should_use_wide(&args) {- self.encode_wide(mtl, enc, args)- } else if self.can_use_fast(&args) {- self.encode_fast(mtl, enc, args)+ if self.can_use_fast_kernel(&args) {+ self.encode_fast(context, encoder, args)} else {- self.encode_legacy(mtl, enc, args, rows_per_group)+ self.encode_tiled(context, encoder, args, rows_per_threadgroup)}}}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/kernel.rsindex 1c7663734..f2a29a5c1 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs@@ -92,10 +92,13 @@ impl MatmulKernel {enc: &ComputeCommandEncoderRef,args: MatmulArguments,) -> Result<bool, MTLError> {- // Apply GEMV when batch (M) is small or output_dim (N) is 1, and A is not transposed.+ // GEMV kernel requires:+ // - A not transposed (input vector is contiguous)+ // - B transposed (weight matrix is [N, K] row-major, each row contiguous)+ // Without transpose_b, B is [K, N] and columns are not contiguouslet m = args.batch;let n = args.output_dim;- if self.transpose_a {+ if self.transpose_a || !self.transpose_b {return Ok(false);}