Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

speed up gemv

eugene42f39032fe27
diff --git a/crates/uzu/examples/matmul_benchmark.rs b/crates/uzu/examples/matmul_benchmark.rs
index 9790c354f..4a9d38f9d 100644
--- a/crates/uzu/examples/matmul_benchmark.rs
+++ b/crates/uzu/examples/matmul_benchmark.rs
@@ -48,10 +48,12 @@ fn benchmark_metal_gemm(
warmup_iters: usize,
bench_iters: usize,
) -> f64 {
+ // A is [M, K] row-major
let a: Vec<bf16> = (0..(m * k))
.map(|i| bf16::from_f32(((i % 13) as f32) * 0.01))
.collect();
- let b: Vec<bf16> = (0..(k * n))
+ // B is [N, K] row-major (transpose_b = true, like LLM weight matrices)
+ let b: Vec<bf16> = (0..(n * k))
.map(|i| bf16::from_f32(((i % 17) as f32) * 0.02 - 0.1))
.collect();
@@ -70,7 +72,8 @@ fn benchmark_metal_gemm(
MTLResourceOptions::StorageModeShared,
);
- let mut kernel = MatmulKernel::new(ctx, DataType::BF16, false, false)
+ // transpose_b = true: B is stored as [N, K], matching LLM weight layout
+ let mut kernel = MatmulKernel::new(ctx, DataType::BF16, false, true)
.expect("kernel new");
// Warmup
@@ -89,7 +92,7 @@ fn benchmark_metal_gemm(
input_dim: k as i32,
output_dim: n as i32,
lda: k as i32,
- ldb: n as i32,
+ ldb: k as i32, // B is [N, K], stride = K
ldd: n as i32,
batch_count: 1,
},
@@ -117,7 +120,7 @@ fn benchmark_metal_gemm(
input_dim: k as i32,
output_dim: n as i32,
lda: k as i32,
- ldb: n as i32,
+ ldb: k as i32, // B is [N, K], stride = K
ldd: n as i32,
batch_count: 1,
},
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv.h b/crates/uzu/src/backends/metal/kernel/matmul/gemv.h
deleted file mode 100644
index 2b0da5b02..000000000
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv.h
+++ /dev/null
@@ -1,19 +0,0 @@
-// Declarations for GEMV kernels (Metal).
-// Implementations are in gemv.metal; names must match Rust dispatch.
-
-#pragma once
-
-#ifdef __METAL_VERSION__
-// No host-side declarations needed; compiled by Metal.
-#else
-// Host-side string literals for kernel names.
-#define GEMV_F16_ROWS2 "gemv_f16_rows2"
-#define GEMV_F16_ROWS4 "gemv_f16_rows4"
-#define GEMV_F16_ROWS8 "gemv_f16_rows8"
-
-#define GEMV_BF16_ROWS2 "gemv_bf16_rows2"
-#define GEMV_BF16_ROWS4 "gemv_bf16_rows4"
-#define GEMV_BF16_ROWS8 "gemv_bf16_rows8"
-#endif
-
-
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemv.metal
index d1ee2ea1f..5ba80b972 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv.metal
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv.metal
@@ -1,344 +1,209 @@
// SPDX-License-Identifier: MIT
-// Optimized GEMV kernels:
-// - Multi-row output per simdgroup with input vector reuse
-// - Vectorized loads (4 elements per thread)
-// - simd_sum for fast K-dimension reduction
-// - Register-only (no threadgroup memory)
-//
-// Two variants:
-// - gemv_fast_*: 4 rows per simdgroup, best for large K (≥4096)
-// - gemv_wide_*: 16 rows per simdgroup, best for small K with large N (MLP up pattern)
#include <metal_stdlib>
using namespace metal;
-// Elements loaded per thread per K iteration (4 elements vectorized)
-#define ELEMS_PER_THREAD 4
-
-// Templated GEMV implementation with configurable rows per simdgroup
-// y = A * x where A is [N, K] row-major
-template <typename T, typename AccT, int ROWS>
-METAL_FUNC void gemv_impl(
- const device T* __restrict A,
- const device T* __restrict x,
- device T* __restrict y,
- const int K,
- const int N,
- const int lda,
+template <typename T, typename AccT, int ROWS_PER_SIMDGROUP, int ELEMS_PER_THREAD>
+METAL_FUNC void gemv_multirow(
+ const device T* __restrict weights,
+ const device T* __restrict input,
+ device T* __restrict output,
+ const int input_dim,
+ const int output_dim,
+ const int weight_stride,
const int batch_idx,
- const int batch_stride_x,
- const int batch_stride_y,
+ const int input_batch_stride,
+ const int output_batch_stride,
uint row_base,
uint simd_lid) {
- // Block size in K dimension: 32 threads * 4 elements = 128
constexpr int BLOCK_K = 32 * ELEMS_PER_THREAD;
- // Position pointers for this batch
- x += batch_idx * batch_stride_x;
- y += batch_idx * batch_stride_y;
+ input += batch_idx * input_batch_stride;
+ output += batch_idx * output_batch_stride;
- // Bounds check
- if (row_base >= static_cast<uint>(N)) {
+ if (row_base >= static_cast<uint>(output_dim)) {
return;
}
- const int rows_to_compute = min(ROWS, N - static_cast<int>(row_base));
+ const int rows_to_compute = min(ROWS_PER_SIMDGROUP, output_dim - static_cast<int>(row_base));
- // Accumulators in registers
- AccT result[ROWS];
- #pragma unroll(ROWS)
- for (int r = 0; r < ROWS; ++r) {
- result[r] = AccT(0);
+ AccT accumulators[ROWS_PER_SIMDGROUP];
+ #pragma unroll(ROWS_PER_SIMDGROUP)
+ for (int r = 0; r < ROWS_PER_SIMDGROUP; ++r) {
+ accumulators[r] = AccT(0);
}
- // Input vector cache
- AccT x_cache[ELEMS_PER_THREAD];
+ AccT input_cache[ELEMS_PER_THREAD];
- // Weight row pointers
- const device T* row_ptrs[ROWS];
- #pragma unroll(ROWS)
- for (int r = 0; r < ROWS; ++r) {
- row_ptrs[r] = A + (row_base + r) * lda;
+ const device T* weight_row_ptrs[ROWS_PER_SIMDGROUP];
+ #pragma unroll(ROWS_PER_SIMDGROUP)
+ for (int r = 0; r < ROWS_PER_SIMDGROUP; ++r) {
+ weight_row_ptrs[r] = weights + (row_base + r) * weight_stride;
}
- // Main K loop
int k = 0;
- for (; k + BLOCK_K <= K; k += BLOCK_K) {
+ for (; k + BLOCK_K <= input_dim; k += BLOCK_K) {
const int thread_k = k + simd_lid * ELEMS_PER_THREAD;
- // Load input vector chunk
#pragma unroll(ELEMS_PER_THREAD)
for (int e = 0; e < ELEMS_PER_THREAD; ++e) {
- x_cache[e] = static_cast<AccT>(x[thread_k + e]);
+ input_cache[e] = static_cast<AccT>(input[thread_k + e]);
}
- // Dot product for each row
- #pragma unroll(ROWS)
- for (int r = 0; r < ROWS; ++r) {
+ #pragma unroll(ROWS_PER_SIMDGROUP)
+ for (int r = 0; r < ROWS_PER_SIMDGROUP; ++r) {
AccT dot = 0;
#pragma unroll(ELEMS_PER_THREAD)
for (int e = 0; e < ELEMS_PER_THREAD; ++e) {
- dot += x_cache[e] * static_cast<AccT>(row_ptrs[r][thread_k + e]);
+ dot += input_cache[e] * static_cast<AccT>(weight_row_ptrs[r][thread_k + e]);
}
- result[r] += dot;
+ accumulators[r] += dot;
}
}
- // Handle leftover K elements
- if (k < K) {
+ if (k < input_dim) {
const int thread_k = k + simd_lid * ELEMS_PER_THREAD;
- const int remaining = K - k;
+ const int remaining = input_dim - k;
#pragma unroll(ELEMS_PER_THREAD)
for (int e = 0; e < ELEMS_PER_THREAD; ++e) {
const int idx = simd_lid * ELEMS_PER_THREAD + e;
- x_cache[e] = (idx < remaining) ? static_cast<AccT>(x[thread_k + e]) : AccT(0);
+ input_cache[e] = (idx < remaining) ? static_cast<AccT>(input[thread_k + e]) : AccT(0);
}
- #pragma unroll(ROWS)
- for (int r = 0; r < ROWS; ++r) {
+ #pragma unroll(ROWS_PER_SIMDGROUP)
+ for (int r = 0; r < ROWS_PER_SIMDGROUP; ++r) {
AccT dot = 0;
#pragma unroll(ELEMS_PER_THREAD)
for (int e = 0; e < ELEMS_PER_THREAD; ++e) {
const int idx = simd_lid * ELEMS_PER_THREAD + e;
if (idx < remaining) {
- dot += x_cache[e] * static_cast<AccT>(row_ptrs[r][thread_k + e]);
+ dot += input_cache[e] * static_cast<AccT>(weight_row_ptrs[r][thread_k + e]);
}
}
- result[r] += dot;
+ accumulators[r] += dot;
}
}
- // Reduce across simdgroup
- #pragma unroll(ROWS)
- for (int r = 0; r < ROWS; ++r) {
- result[r] = simd_sum(result[r]);
+ #pragma unroll(ROWS_PER_SIMDGROUP)
+ for (int r = 0; r < ROWS_PER_SIMDGROUP; ++r) {
+ accumulators[r] = simd_sum(accumulators[r]);
}
- // Write output
if (simd_lid == 0) {
- #pragma unroll(ROWS)
+ #pragma unroll(ROWS_PER_SIMDGROUP)
for (int r = 0; r < rows_to_compute; ++r) {
- y[row_base + r] = static_cast<T>(result[r]);
+ output[row_base + r] = static_cast<T>(accumulators[r]);
}
}
}
-// Standard GEMV: 4 rows per simdgroup (best for large K)
-#define DECL_GEMV_FAST(NAME, TYPE, ACC) \
- [[kernel]] void NAME( \
- const device TYPE* A [[buffer(0)]], \
- const device TYPE* x [[buffer(1)]], \
- device TYPE* y [[buffer(2)]], \
- constant int& K [[buffer(3)]], \
- constant int& N [[buffer(4)]], \
- constant int& lda [[buffer(5)]], \
- constant int& batch_stride_x [[buffer(6)]], \
- constant int& batch_stride_y [[buffer(7)]], \
- uint3 tgid [[threadgroup_position_in_grid]], \
- uint simd_lid [[thread_index_in_simdgroup]]) { \
- constexpr int ROWS = 4; \
- const uint row_base = tgid.x * ROWS; \
- gemv_impl<TYPE, ACC, ROWS>( \
- A, x, y, K, N, lda, tgid.z, batch_stride_x, batch_stride_y, \
- row_base, simd_lid); \
+#define DECL_GEMV_FAST(NAME, TYPE, ACC) \
+ [[kernel]] void NAME( \
+ const device TYPE* weights [[buffer(0)]], \
+ const device TYPE* input [[buffer(1)]], \
+ device TYPE* output [[buffer(2)]], \
+ constant int& input_dim [[buffer(3)]], \
+ constant int& output_dim [[buffer(4)]], \
+ constant int& weight_stride [[buffer(5)]], \
+ constant int& input_batch_stride [[buffer(6)]], \
+ constant int& output_batch_stride [[buffer(7)]], \
+ uint3 tgid [[threadgroup_position_in_grid]], \
+ uint simd_lid [[thread_index_in_simdgroup]]) { \
+ constexpr int ROWS_PER_SIMDGROUP = 4; \
+ constexpr int ELEMS_PER_THREAD = 4; \
+ const uint row_base = tgid.x * ROWS_PER_SIMDGROUP; \
+ gemv_multirow<TYPE, ACC, ROWS_PER_SIMDGROUP, ELEMS_PER_THREAD>( \
+ weights, input, output, input_dim, output_dim, weight_stride, \
+ tgid.z, input_batch_stride, output_batch_stride, row_base, simd_lid); \
}
-// Wide GEMV: 8 rows per simdgroup (balanced for small K, large N)
-#define DECL_GEMV_WIDE(NAME, TYPE, ACC) \
- [[kernel]] void NAME( \
- const device TYPE* A [[buffer(0)]], \
- const device TYPE* x [[buffer(1)]], \
- device TYPE* y [[buffer(2)]], \
- constant int& K [[buffer(3)]], \
- constant int& N [[buffer(4)]], \
- constant int& lda [[buffer(5)]], \
- constant int& batch_stride_x [[buffer(6)]], \
- constant int& batch_stride_y [[buffer(7)]], \
- uint3 tgid [[threadgroup_position_in_grid]], \
- uint simd_lid [[thread_index_in_simdgroup]]) { \
- constexpr int ROWS = 8; \
- const uint row_base = tgid.x * ROWS; \
- gemv_impl<TYPE, ACC, ROWS>( \
- A, x, y, K, N, lda, tgid.z, batch_stride_x, batch_stride_y, \
- row_base, simd_lid); \
- }
-
-// Instantiate standard (4-row) kernels
DECL_GEMV_FAST(gemv_fast_f16, half, float)
DECL_GEMV_FAST(gemv_fast_bf16, bfloat, float)
DECL_GEMV_FAST(gemv_fast_f32, float, float)
-// Instantiate wide (8-row) kernels
-DECL_GEMV_WIDE(gemv_wide_f16, half, float)
-DECL_GEMV_WIDE(gemv_wide_bf16, bfloat, float)
-DECL_GEMV_WIDE(gemv_wide_f32, float, float)
-
-// =============================================================================
-// Parallel GEMV: Multiple simdgroups per threadgroup with shared input vector
-// Best for small K with large N (reduces threadgroup count, amortizes overhead)
-// =============================================================================
-
-template <typename T, typename AccT, int SIMDGROUPS_PER_TG, int BK>
-[[kernel, max_total_threads_per_threadgroup(SIMDGROUPS_PER_TG * 32)]]
-void gemv_parallel_impl(
- const device T* __restrict A [[buffer(0)]],
- const device T* __restrict x [[buffer(1)]],
- device T* __restrict y [[buffer(2)]],
- constant int& K [[buffer(3)]],
- constant int& N [[buffer(4)]],
- constant int& lda [[buffer(5)]],
- constant int& batch_stride_x [[buffer(6)]],
- constant int& batch_stride_y [[buffer(7)]],
- uint3 tgid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]],
- uint tid [[thread_index_in_threadgroup]]) {
-
- // Shared memory for input vector tile
- threadgroup AccT x_shared[BK];
-
- // Position pointers for this batch
- const int batch_idx = tgid.z;
- x += batch_idx * batch_stride_x;
- y += batch_idx * batch_stride_y;
-
- // Each simdgroup computes one output row
- const uint row = tgid.x * SIMDGROUPS_PER_TG + simd_gid;
- if (row >= static_cast<uint>(N)) {
+template <typename T, typename AccT, ushort ROWS_PER_THREADGROUP, ushort TILE_K>
+METAL_FUNC void gemv_tiled(
+ const device T* __restrict weights,
+ const device T* __restrict input,
+ device T* __restrict output,
+ constant uint& input_dim,
+ constant uint& output_dim,
+ constant uint& weight_stride,
+ constant uint& input_stride,
+ constant uint& output_stride,
+ uint3 tgid,
+ uint simd_gid,
+ uint simd_lid,
+ uint tid,
+ threadgroup AccT* input_tile) {
+
+ const ushort THREADS_PER_THREADGROUP = ROWS_PER_THREADGROUP * 32;
+ if (simd_gid >= ROWS_PER_THREADGROUP) {
return;
}
- const device T* row_ptr = A + row * lda;
- AccT acc = AccT(0);
-
- // Process K in tiles
- for (int k_start = 0; k_start < K; k_start += BK) {
- const int remaining = K - k_start;
- const int tile_k = remaining < BK ? remaining : BK;
-
- // Cooperative load of input vector tile into shared memory
- for (int idx = tid; idx < tile_k; idx += SIMDGROUPS_PER_TG * 32) {
- x_shared[idx] = static_cast<AccT>(x[k_start + idx]);
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // Each thread in simdgroup processes part of the tile
- for (int idx = simd_lid; idx < tile_k; idx += 32) {
- acc += static_cast<AccT>(row_ptr[k_start + idx]) * x_shared[idx];
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
- }
-
- // Reduce within simdgroup
- acc = simd_sum(acc);
-
- // Write output
- if (simd_lid == 0) {
- y[row] = static_cast<T>(acc);
- }
-}
-
-// Parallel kernels: 32 simdgroups per threadgroup, BK=1024 for maximum occupancy
-template [[host_name("gemv_parallel_f16")]] kernel void gemv_parallel_impl<half, float, 32, 1024>(
- const device half*, const device half*, device half*,
- constant int&, constant int&, constant int&, constant int&, constant int&,
- uint3, uint, uint, uint);
-template [[host_name("gemv_parallel_bf16")]] kernel void gemv_parallel_impl<bfloat, float, 32, 1024>(
- const device bfloat*, const device bfloat*, device bfloat*,
- constant int&, constant int&, constant int&, constant int&, constant int&,
- uint3, uint, uint, uint);
-template [[host_name("gemv_parallel_f32")]] kernel void gemv_parallel_impl<float, float, 32, 1024>(
- const device float*, const device float*, device float*,
- constant int&, constant int&, constant int&, constant int&, constant int&,
- uint3, uint, uint, uint);
-
-// =============================================================================
-// Legacy kernels for backwards compatibility (used by existing Rust dispatcher)
-// =============================================================================
-
-template <typename T, typename AccT, ushort ROWS_PER_TG, ushort BK>
-METAL_FUNC void gemv_impl_tiled(
- const device T* __restrict matrix [[buffer(0)]],
- const device T* __restrict vector [[buffer(1)]],
- device T* __restrict output [[buffer(2)]],
- constant uint& k [[buffer(3)]],
- constant uint& n [[buffer(4)]],
- constant uint& ldb [[buffer(5)]],
- constant uint& lda [[buffer(6)]],
- constant uint& ldd [[buffer(7)]],
- uint3 tgid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]],
- uint tid [[thread_index_in_threadgroup]],
- threadgroup AccT* vec_tile) {
- const ushort TG_THREADS = ROWS_PER_TG * 32;
- if (simd_gid >= ROWS_PER_TG) {
- return;
- }
- const uint row = tgid.x * ROWS_PER_TG + simd_gid;
- if (row >= n) {
+ const uint row = tgid.x * ROWS_PER_THREADGROUP + simd_gid;
+ if (row >= output_dim) {
return;
}
- const uint batch = tgid.z;
- const device T* row_ptr = matrix + row * ldb;
- const device T* vec_ptr = vector + batch * lda;
- device T* out_ptr = output + batch * ldd;
+ const uint batch = tgid.z;
+ const device T* weight_row = weights + row * weight_stride;
+ const device T* input_ptr = input + batch * input_stride;
+ device T* output_ptr = output + batch * output_stride;
- AccT acc = static_cast<AccT>(0);
+ AccT accumulator = AccT(0);
- for (uint k_block = 0; k_block < k; k_block += BK) {
- const uint remaining = k - k_block;
- const uint tile_elems = remaining < BK ? remaining : BK;
+ for (uint k_block = 0; k_block < input_dim; k_block += TILE_K) {
+ const uint remaining = input_dim - k_block;
+ const uint tile_size = remaining < TILE_K ? remaining : TILE_K;
- if (tid < tile_elems && tid < TG_THREADS) {
- vec_tile[tid] = static_cast<AccT>(vec_ptr[k_block + tid]);
+ if (tid < tile_size && tid < THREADS_PER_THREADGROUP) {
+ input_tile[tid] = static_cast<AccT>(input_ptr[k_block + tid]);
}
threadgroup_barrier(mem_flags::mem_threadgroup);
- for (uint idx = simd_lid; idx < tile_elems; idx += 32) {
- acc += static_cast<AccT>(row_ptr[k_block + idx]) * vec_tile[idx];
+ for (uint idx = simd_lid; idx < tile_size; idx += 32) {
+ accumulator += static_cast<AccT>(weight_row[k_block + idx]) * input_tile[idx];
}
threadgroup_barrier(mem_flags::mem_threadgroup);
}
for (ushort offset = 16; offset > 0; offset >>= 1) {
- acc += simd_shuffle_down(acc, offset);
+ accumulator += simd_shuffle_down(accumulator, offset);
}
if (simd_lid == 0) {
- out_ptr[row] = static_cast<T>(acc);
+ output_ptr[row] = static_cast<T>(accumulator);
}
}
-#define DECL_GEMV_LEGACY(NAME, TYPE, ACC, ROWS, BK) \
- [[kernel, max_total_threads_per_threadgroup(ROWS * 32)]] void NAME( \
- const device TYPE* matrix [[buffer(0)]], \
- const device TYPE* vector [[buffer(1)]], \
- device TYPE* output [[buffer(2)]], \
- constant uint& k [[buffer(3)]], \
- constant uint& n [[buffer(4)]], \
- constant uint& ldb [[buffer(5)]], \
- constant uint& lda [[buffer(6)]], \
- constant uint& ldd [[buffer(7)]], \
- uint3 tgid [[threadgroup_position_in_grid]], \
- uint simd_gid [[simdgroup_index_in_threadgroup]], \
- uint simd_lid [[thread_index_in_simdgroup]], \
- uint tid [[thread_index_in_threadgroup]]) { \
- threadgroup ACC vec_tile[BK]; \
- gemv_impl_tiled<TYPE, ACC, ROWS, BK>( \
- matrix, vector, output, k, n, ldb, lda, ldd, \
- tgid, simd_gid, simd_lid, tid, vec_tile); \
- }
-
-// Legacy half variants
-DECL_GEMV_LEGACY(gemv_f16_rows2, half, float, 2, 128)
-DECL_GEMV_LEGACY(gemv_f16_rows4, half, float, 4, 128)
-DECL_GEMV_LEGACY(gemv_f16_rows8, half, float, 8, 128)
-
-// Legacy bfloat16 variants
-DECL_GEMV_LEGACY(gemv_bf16_rows2, bfloat, float, 2, 128)
-DECL_GEMV_LEGACY(gemv_bf16_rows4, bfloat, float, 4, 128)
-DECL_GEMV_LEGACY(gemv_bf16_rows8, bfloat, float, 8, 128)
+#define DECL_GEMV_TILED(NAME, TYPE, ACC, ROWS, TILE_K) \
+ [[kernel, max_total_threads_per_threadgroup(ROWS * 32)]] void NAME( \
+ const device TYPE* weights [[buffer(0)]], \
+ const device TYPE* input [[buffer(1)]], \
+ device TYPE* output [[buffer(2)]], \
+ constant uint& input_dim [[buffer(3)]], \
+ constant uint& output_dim [[buffer(4)]], \
+ constant uint& weight_stride [[buffer(5)]], \
+ constant uint& input_stride [[buffer(6)]], \
+ constant uint& output_stride [[buffer(7)]], \
+ uint3 tgid [[threadgroup_position_in_grid]], \
+ uint simd_gid [[simdgroup_index_in_threadgroup]], \
+ uint simd_lid [[thread_index_in_simdgroup]], \
+ uint tid [[thread_index_in_threadgroup]]) { \
+ threadgroup ACC input_tile[TILE_K]; \
+ gemv_tiled<TYPE, ACC, ROWS, TILE_K>( \
+ weights, input, output, input_dim, output_dim, weight_stride, \
+ input_stride, output_stride, tgid, simd_gid, simd_lid, tid, input_tile); \
+ }
+
+DECL_GEMV_TILED(gemv_f16_rows2, half, float, 2, 128)
+DECL_GEMV_TILED(gemv_f16_rows4, half, float, 4, 128)
+DECL_GEMV_TILED(gemv_f16_rows8, half, float, 8, 128)
+
+DECL_GEMV_TILED(gemv_bf16_rows2, bfloat, float, 2, 128)
+DECL_GEMV_TILED(gemv_bf16_rows4, bfloat, float, 4, 128)
+DECL_GEMV_TILED(gemv_bf16_rows8, bfloat, float, 8, 128)
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv.rs
index f3ed8f0b1..ad43ee52a 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv.rs
@@ -12,430 +12,224 @@ use crate::{
};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-enum PipelineKind {
- Legacy {
- rows_per_group: u32,
+enum PipelineVariant {
+ Tiled {
+ rows_per_threadgroup: u32,
},
Fast,
- Wide,
- Parallel,
}
pub struct GemvKernel {
- dt: DataType,
- pipelines: HashMap<PipelineKind, MTLComputePipelineState>,
+ data_type: DataType,
+ pipelines: HashMap<PipelineVariant, MTLComputePipelineState>,
}
impl GemvKernel {
- pub fn new(dt: DataType) -> Self {
+ pub fn new(data_type: DataType) -> Self {
Self {
- dt,
+ data_type,
pipelines: HashMap::new(),
}
}
fn fast_kernel_name(&self) -> Result<String, MTLError> {
- match self.dt {
+ match self.data_type {
DataType::F16 => Ok("gemv_fast_f16".to_string()),
DataType::BF16 => Ok("gemv_fast_bf16".to_string()),
DataType::F32 => Ok("gemv_fast_f32".to_string()),
_ => Err(MTLError::Generic(format!(
- "Unsupported dtype for fast GEMV: {:?}",
- self.dt
+ "Unsupported data type for fast GEMV: {:?}",
+ self.data_type
))),
}
}
- fn wide_kernel_name(&self) -> Result<String, MTLError> {
- match self.dt {
- DataType::F16 => Ok("gemv_wide_f16".to_string()),
- DataType::BF16 => Ok("gemv_wide_bf16".to_string()),
- DataType::F32 => Ok("gemv_wide_f32".to_string()),
- _ => Err(MTLError::Generic(format!(
- "Unsupported dtype for wide GEMV: {:?}",
- self.dt
- ))),
- }
- }
-
- fn parallel_kernel_name(&self) -> Result<String, MTLError> {
- match self.dt {
- DataType::F16 => Ok("gemv_parallel_f16".to_string()),
- DataType::BF16 => Ok("gemv_parallel_bf16".to_string()),
- DataType::F32 => Ok("gemv_parallel_f32".to_string()),
- _ => Err(MTLError::Generic(format!(
- "Unsupported dtype for parallel GEMV: {:?}",
- self.dt
- ))),
- }
- }
-
- fn legacy_kernel_name(
+ fn tiled_kernel_name(
&self,
- rows: u32,
+ rows_per_threadgroup: u32,
) -> Result<String, MTLError> {
- let base = match self.dt {
+ let prefix = match self.data_type {
DataType::F16 => "gemv_f16",
DataType::BF16 => "gemv_bf16",
_ => {
return Err(MTLError::Generic(format!(
- "Unsupported dtype for legacy GEMV: {:?}",
- self.dt
+ "Unsupported data type for tiled GEMV: {:?}",
+ self.data_type
)));
},
};
- Ok(format!("{base}_rows{rows}"))
+ Ok(format!("{prefix}_rows{rows_per_threadgroup}"))
}
fn get_fast_pipeline(
&mut self,
- mtl: &MTLContext,
- ) -> Result<&MTLComputePipelineState, MTLError> {
- let key = PipelineKind::Fast;
- if !self.pipelines.contains_key(&key) {
- let name = self.fast_kernel_name()?;
- let ps = mtl.compute_pipeline_state(&name, None)?;
- self.pipelines.insert(key, ps);
- }
- Ok(self.pipelines.get(&key).unwrap())
- }
-
- fn get_wide_pipeline(
- &mut self,
- mtl: &MTLContext,
+ context: &MTLContext,
) -> Result<&MTLComputePipelineState, MTLError> {
- let key = PipelineKind::Wide;
- if !self.pipelines.contains_key(&key) {
- let name = self.wide_kernel_name()?;
- let ps = mtl.compute_pipeline_state(&name, None)?;
- self.pipelines.insert(key, ps);
+ let variant = PipelineVariant::Fast;
+ if !self.pipelines.contains_key(&variant) {
+ let kernel_name = self.fast_kernel_name()?;
+ let pipeline =
+ context.compute_pipeline_state(&kernel_name, None)?;
+ self.pipelines.insert(variant, pipeline);
}
- Ok(self.pipelines.get(&key).unwrap())
+ Ok(self.pipelines.get(&variant).unwrap())
}
- fn get_parallel_pipeline(
+ fn get_tiled_pipeline(
&mut self,
- mtl: &MTLContext,
+ context: &MTLContext,
+ rows_per_threadgroup: u32,
) -> Result<&MTLComputePipelineState, MTLError> {
- let key = PipelineKind::Parallel;
- if !self.pipelines.contains_key(&key) {
- let name = self.parallel_kernel_name()?;
- let ps = mtl.compute_pipeline_state(&name, None)?;
- self.pipelines.insert(key, ps);
- }
- Ok(self.pipelines.get(&key).unwrap())
- }
-
- fn get_legacy_pipeline(
- &mut self,
- mtl: &MTLContext,
- rows: u32,
- ) -> Result<&MTLComputePipelineState, MTLError> {
- let key = PipelineKind::Legacy {
- rows_per_group: rows,
+ let variant = PipelineVariant::Tiled {
+ rows_per_threadgroup,
};
- if !self.pipelines.contains_key(&key) {
- let name = self.legacy_kernel_name(rows)?;
- let ps = mtl.compute_pipeline_state(&name, None)?;
- self.pipelines.insert(key, ps);
+ if !self.pipelines.contains_key(&variant) {
+ let kernel_name = self.tiled_kernel_name(rows_per_threadgroup)?;
+ let pipeline =
+ context.compute_pipeline_state(&kernel_name, None)?;
+ self.pipelines.insert(variant, pipeline);
}
- Ok(self.pipelines.get(&key).unwrap())
+ Ok(self.pipelines.get(&variant).unwrap())
}
- /// Check if wide GEMV is applicable.
- /// Currently disabled as it doesn't improve MLP up pattern.
- fn should_use_wide(
- &self,
- _args: &MatmulArguments,
- ) -> bool {
- false
- }
-
- /// Check if parallel GEMV is applicable.
- /// Parallel kernel (8 simdgroups/TG) is best for small K with large N.
- fn should_use_parallel(
+ fn can_use_fast_kernel(
&self,
args: &MatmulArguments,
) -> bool {
- let k = args.input_dim;
- let n = args.output_dim;
- // Use parallel kernel when:
- // - K is small (< 4096) so threadgroup memory can hold a good chunk
- // - N is large (≥ 4096) so we benefit from reduced TG count
- // - K is aligned for vectorized loads
- k < 4096 && n >= 4096 && k % 4 == 0 && k >= 128
- }
-
- /// Check if fast GEMV is applicable.
- /// Fast kernel works best when K is large enough for vectorized loads.
- fn can_use_fast(
- &self,
- args: &MatmulArguments,
- ) -> bool {
- // Fast kernel uses BLOCK_K = 128 (32 threads * 4 elements)
- // Works well when K >= 128 and is reasonably aligned
args.input_dim >= 128 && args.input_dim % 4 == 0
}
- /// Encode using the fast optimized GEMV kernel.
- pub fn encode_fast(
+ fn encode_fast(
&mut self,
- mtl: &MTLContext,
- enc: &ComputeCommandEncoderRef,
+ context: &MTLContext,
+ encoder: &ComputeCommandEncoderRef,
args: MatmulArguments,
) -> Result<(), MTLError> {
- let ps = self.get_fast_pipeline(mtl)?;
- enc.set_compute_pipeline_state(ps);
+ let pipeline = self.get_fast_pipeline(context)?;
+ encoder.set_compute_pipeline_state(pipeline);
- // Bind buffers: A (weights/matrix), x (input vector), y (output)
- enc.set_buffer(0, Some(args.b), 0); // A = weights
- enc.set_buffer(1, Some(args.a), 0); // x = input
- enc.set_buffer(2, Some(args.d), 0); // y = output
+ encoder.set_buffer(0, Some(args.b), 0);
+ encoder.set_buffer(1, Some(args.a), 0);
+ encoder.set_buffer(2, Some(args.d), 0);
- // Scalars (as i32 to match Metal kernel)
- let k = args.input_dim;
- let n = args.output_dim;
- let lda = args.ldb; // Weight matrix leading dimension
- // For batched GEMV: stride between input/output vectors is the leading dimension
- let batch_stride_x = args.lda; // stride between rows of A (input vectors)
- let batch_stride_y = args.ldd; // stride between rows of D (output vectors)
+ let input_dim = args.input_dim;
+ let output_dim = args.output_dim;
+ let weight_stride = args.ldb;
+ let input_batch_stride = args.lda;
+ let output_batch_stride = args.ldd;
- enc.set_bytes(
+ encoder.set_bytes(
3,
std::mem::size_of::<i32>() as u64,
- &k as *const i32 as *const std::ffi::c_void,
+ &input_dim as *const i32 as *const std::ffi::c_void,
);
- enc.set_bytes(
+ encoder.set_bytes(
4,
std::mem::size_of::<i32>() as u64,
- &n as *const i32 as *const std::ffi::c_void,
+ &output_dim as *const i32 as *const std::ffi::c_void,
);
- enc.set_bytes(
+ encoder.set_bytes(
5,
std::mem::size_of::<i32>() as u64,
- &lda as *const i32 as *const std::ffi::c_void,
+ &weight_stride as *const i32 as *const std::ffi::c_void,
);
- enc.set_bytes(
+ encoder.set_bytes(
6,
std::mem::size_of::<i32>() as u64,
- &batch_stride_x as *const i32 as *const std::ffi::c_void,
+ &input_batch_stride as *const i32 as *const std::ffi::c_void,
);
- enc.set_bytes(
+ encoder.set_bytes(
7,
std::mem::size_of::<i32>() as u64,
- &batch_stride_y as *const i32 as *const std::ffi::c_void,
+ &output_batch_stride as *const i32 as *const std::ffi::c_void,
);
- // Fast kernel: 4 rows per simdgroup, 1 simdgroup per threadgroup
const ROWS_PER_SIMDGROUP: i32 = 4;
- let tg_x = ((n + ROWS_PER_SIMDGROUP - 1) / ROWS_PER_SIMDGROUP) as u64;
- let tg_z = args.batch_count as u64;
-
- let tgs = MTLSize::new(tg_x, 1, tg_z);
- let threads_per_tg = MTLSize::new(32, 1, 1); // Single simdgroup
-
- enc.dispatch_thread_groups(tgs, threads_per_tg);
- Ok(())
- }
-
- /// Encode using the wide GEMV kernel (16 rows per simdgroup).
- /// Best for small K with large N (MLP up pattern).
- pub fn encode_wide(
- &mut self,
- mtl: &MTLContext,
- enc: &ComputeCommandEncoderRef,
- args: MatmulArguments,
- ) -> Result<(), MTLError> {
- let ps = self.get_wide_pipeline(mtl)?;
- enc.set_compute_pipeline_state(ps);
-
- // Same buffer layout as fast kernel
- enc.set_buffer(0, Some(args.b), 0); // A = weights
- enc.set_buffer(1, Some(args.a), 0); // x = input
- enc.set_buffer(2, Some(args.d), 0); // y = output
-
- let k = args.input_dim;
- let n = args.output_dim;
- let lda = args.ldb;
- let batch_stride_x = args.lda; // stride between rows of A
- let batch_stride_y = args.ldd; // stride between rows of D
-
- enc.set_bytes(
- 3,
- std::mem::size_of::<i32>() as u64,
- &k as *const i32 as *const std::ffi::c_void,
- );
- enc.set_bytes(
- 4,
- std::mem::size_of::<i32>() as u64,
- &n as *const i32 as *const std::ffi::c_void,
- );
- enc.set_bytes(
- 5,
- std::mem::size_of::<i32>() as u64,
- &lda as *const i32 as *const std::ffi::c_void,
- );
- enc.set_bytes(
- 6,
- std::mem::size_of::<i32>() as u64,
- &batch_stride_x as *const i32 as *const std::ffi::c_void,
- );
- enc.set_bytes(
- 7,
- std::mem::size_of::<i32>() as u64,
- &batch_stride_y as *const i32 as *const std::ffi::c_void,
- );
-
- // Wide kernel: 8 rows per simdgroup
- const ROWS_PER_SIMDGROUP: i32 = 8;
- let tg_x = ((n + ROWS_PER_SIMDGROUP - 1) / ROWS_PER_SIMDGROUP) as u64;
- let tg_z = args.batch_count as u64;
+ let threadgroup_count_x =
+ ((output_dim + ROWS_PER_SIMDGROUP - 1) / ROWS_PER_SIMDGROUP) as u64;
+ let threadgroup_count_z = args.batch_count as u64;
- let tgs = MTLSize::new(tg_x, 1, tg_z);
- let threads_per_tg = MTLSize::new(32, 1, 1); // Single simdgroup
+ let threadgroup_count =
+ MTLSize::new(threadgroup_count_x, 1, threadgroup_count_z);
+ let threads_per_threadgroup = MTLSize::new(32, 1, 1);
- enc.dispatch_thread_groups(tgs, threads_per_tg);
+ encoder
+ .dispatch_thread_groups(threadgroup_count, threads_per_threadgroup);
Ok(())
}
- /// Encode using the parallel GEMV kernel (8 simdgroups per threadgroup).
- /// Best for small K with large N (reduces threadgroup count).
- pub fn encode_parallel(
+ fn encode_tiled(
&mut self,
- mtl: &MTLContext,
- enc: &ComputeCommandEncoderRef,
+ context: &MTLContext,
+ encoder: &ComputeCommandEncoderRef,
args: MatmulArguments,
+ rows_per_threadgroup: u32,
) -> Result<(), MTLError> {
- let ps = self.get_parallel_pipeline(mtl)?;
- enc.set_compute_pipeline_state(ps);
-
- // Same buffer layout as other kernels
- enc.set_buffer(0, Some(args.b), 0); // A = weights
- enc.set_buffer(1, Some(args.a), 0); // x = input
- enc.set_buffer(2, Some(args.d), 0); // y = output
-
- let k = args.input_dim;
- let n = args.output_dim;
- let lda = args.ldb;
- let batch_stride_x = args.lda; // stride between rows of A
- let batch_stride_y = args.ldd; // stride between rows of D
-
- enc.set_bytes(
- 3,
- std::mem::size_of::<i32>() as u64,
- &k as *const i32 as *const std::ffi::c_void,
- );
- enc.set_bytes(
- 4,
- std::mem::size_of::<i32>() as u64,
- &n as *const i32 as *const std::ffi::c_void,
- );
- enc.set_bytes(
- 5,
- std::mem::size_of::<i32>() as u64,
- &lda as *const i32 as *const std::ffi::c_void,
- );
- enc.set_bytes(
- 6,
- std::mem::size_of::<i32>() as u64,
- &batch_stride_x as *const i32 as *const std::ffi::c_void,
- );
- enc.set_bytes(
- 7,
- std::mem::size_of::<i32>() as u64,
- &batch_stride_y as *const i32 as *const std::ffi::c_void,
- );
-
- // Parallel kernel: 32 simdgroups per TG, each computes 1 row
- const SIMDGROUPS_PER_TG: i32 = 32;
- let tg_x = ((n + SIMDGROUPS_PER_TG - 1) / SIMDGROUPS_PER_TG) as u64;
- let tg_z = args.batch_count as u64;
-
- let tgs = MTLSize::new(tg_x, 1, tg_z);
- let threads_per_tg =
- MTLSize::new((SIMDGROUPS_PER_TG * 32) as u64, 1, 1);
-
- enc.dispatch_thread_groups(tgs, threads_per_tg);
- Ok(())
- }
-
- /// Encode using the legacy GEMV kernel (for fallback).
- pub fn encode_legacy(
- &mut self,
- mtl: &MTLContext,
- enc: &ComputeCommandEncoderRef,
- args: MatmulArguments,
- rows_per_group: u32,
- ) -> Result<(), MTLError> {
- let rows_per_group = rows_per_group.max(1);
- let ps = self.get_legacy_pipeline(mtl, rows_per_group)?;
- enc.set_compute_pipeline_state(ps);
-
- // Bind buffers: matrix (B), vector (A), output (D)
- enc.set_buffer(0, Some(args.b), 0);
- enc.set_buffer(1, Some(args.a), 0);
- enc.set_buffer(2, Some(args.d), 0);
-
- // Scalars
- let k = args.input_dim as u32;
- let n = args.output_dim as u32;
- let ldb = args.ldb as u32;
- let lda = args.lda as u32;
- let ldd = args.ldd as u32;
-
- enc.set_bytes(
+ let rows_per_threadgroup = rows_per_threadgroup.max(1);
+ let pipeline =
+ self.get_tiled_pipeline(context, rows_per_threadgroup)?;
+ encoder.set_compute_pipeline_state(pipeline);
+
+ encoder.set_buffer(0, Some(args.b), 0);
+ encoder.set_buffer(1, Some(args.a), 0);
+ encoder.set_buffer(2, Some(args.d), 0);
+
+ let input_dim = args.input_dim as u32;
+ let output_dim = args.output_dim as u32;
+ let weight_stride = args.ldb as u32;
+ let input_stride = args.lda as u32;
+ let output_stride = args.ldd as u32;
+
+ encoder.set_bytes(
3,
std::mem::size_of::<u32>() as u64,
- &k as *const u32 as *const std::ffi::c_void,
+ &input_dim as *const u32 as *const std::ffi::c_void,
);
- enc.set_bytes(
+ encoder.set_bytes(
4,
std::mem::size_of::<u32>() as u64,
- &n as *const u32 as *const std::ffi::c_void,
+ &output_dim as *const u32 as *const std::ffi::c_void,
);
- enc.set_bytes(
+ encoder.set_bytes(
5,
std::mem::size_of::<u32>() as u64,
- &ldb as *const u32 as *const std::ffi::c_void,
+ &weight_stride as *const u32 as *const std::ffi::c_void,
);
- enc.set_bytes(
+ encoder.set_bytes(
6,
std::mem::size_of::<u32>() as u64,
- &lda as *const u32 as *const std::ffi::c_void,
+ &input_stride as *const u32 as *const std::ffi::c_void,
);
- enc.set_bytes(
+ encoder.set_bytes(
7,
std::mem::size_of::<u32>() as u64,
- &ldd as *const u32 as *const std::ffi::c_void,
+ &output_stride as *const u32 as *const std::ffi::c_void,
);
- // Threadgroup sizing
- let threads_per_tg = MTLSize::new((rows_per_group * 32) as u64, 1, 1);
- let tg_x = ((n + rows_per_group - 1) / rows_per_group) as u64;
- let tg_z = args.batch_count as u64;
- let tgs = MTLSize::new(tg_x, 1, tg_z);
+ let threads_per_threadgroup =
+ MTLSize::new((rows_per_threadgroup * 32) as u64, 1, 1);
+ let threadgroup_count_x = ((output_dim + rows_per_threadgroup - 1)
+ / rows_per_threadgroup) as u64;
+ let threadgroup_count_z = args.batch_count as u64;
+ let threadgroup_count =
+ MTLSize::new(threadgroup_count_x, 1, threadgroup_count_z);
- enc.dispatch_thread_groups(tgs, threads_per_tg);
+ encoder
+ .dispatch_thread_groups(threadgroup_count, threads_per_threadgroup);
Ok(())
}
- /// Main encode entry point - automatically selects best kernel variant.
pub fn encode(
&mut self,
- mtl: &MTLContext,
- enc: &ComputeCommandEncoderRef,
+ context: &MTLContext,
+ encoder: &ComputeCommandEncoderRef,
args: MatmulArguments,
- rows_per_group: u32,
+ rows_per_threadgroup: u32,
) -> Result<(), MTLError> {
- // Priority: parallel > wide > fast > legacy
- if self.should_use_parallel(&args) {
- self.encode_parallel(mtl, enc, args)
- } else if self.should_use_wide(&args) {
- self.encode_wide(mtl, enc, args)
- } else if self.can_use_fast(&args) {
- self.encode_fast(mtl, enc, args)
+ if self.can_use_fast_kernel(&args) {
+ self.encode_fast(context, encoder, args)
} else {
- self.encode_legacy(mtl, enc, args, rows_per_group)
+ self.encode_tiled(context, encoder, args, rows_per_threadgroup)
}
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs
index 1c7663734..f2a29a5c1 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs
@@ -92,10 +92,13 @@ impl MatmulKernel {
enc: &ComputeCommandEncoderRef,
args: MatmulArguments,
) -> Result<bool, MTLError> {
- // Apply GEMV when batch (M) is small or output_dim (N) is 1, and A is not transposed.
+ // GEMV kernel requires:
+ // - A not transposed (input vector is contiguous)
+ // - B transposed (weight matrix is [N, K] row-major, each row contiguous)
+ // Without transpose_b, B is [K, N] and columns are not contiguous
let m = args.batch;
let n = args.output_dim;
- if self.transpose_a {
+ if self.transpose_a || !self.transpose_b {
return Ok(false);
}