A high-performance inference engine for AI models
update gemv
Eugene Bokhan74f2c5d68f56
diff --git a/crates/uzu/examples/matmul_benchmark.rs b/crates/uzu/examples/matmul_benchmark.rsindex 4a9d38f9d..4eb752a5c 100644--- a/crates/uzu/examples/matmul_benchmark.rs+++ b/crates/uzu/examples/matmul_benchmark.rs@@ -134,6 +134,96 @@ fn benchmark_metal_gemm((elapsed.as_secs_f64() * 1000.0) / bench_iters as f64}+fn benchmark_metal_gemv(+ ctx: &MTLContext,+ m: usize,+ k: usize,+ n: usize,+ warmup_iters: usize,+ bench_iters: usize,+) -> f64 {+ // GEMV requires transpose_b = true (B stored as [N, K]).+ let a: Vec<bf16> = (0..(m * k))+ .map(|i| bf16::from_f32(((i % 13) as f32) * 0.01))+ .collect();+ let b: Vec<bf16> = (0..(n * k))+ .map(|i| bf16::from_f32(((i % 17) as f32) * 0.02 - 0.1))+ .collect();++ let a_buf = ctx.device.new_buffer_with_data(+ a.as_ptr() as *const _,+ (a.len() * core::mem::size_of::<bf16>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );+ let b_buf = ctx.device.new_buffer_with_data(+ b.as_ptr() as *const _,+ (b.len() * core::mem::size_of::<bf16>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );+ let d_buf = ctx.device.new_buffer(+ (m * n * core::mem::size_of::<bf16>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );++ let mut kernel = MatmulKernel::new(ctx, DataType::BF16, false, true)+ .expect("kernel new");++ for _ in 0..warmup_iters {+ let cb = ctx.command_queue.new_command_buffer().to_owned();+ let enc = cb.new_compute_command_encoder();+ kernel+ .encode(+ ctx,+ &enc,+ MatmulArguments {+ a: &a_buf,+ b: &b_buf,+ d: &d_buf,+ batch: m as i32,+ input_dim: k as i32,+ output_dim: n as i32,+ lda: k as i32,+ ldb: k as i32,+ ldd: n as i32,+ batch_count: 1,+ },+ )+ .expect("encode");+ enc.end_encoding();+ cb.commit();+ cb.wait_until_completed();+ }++ let start = Instant::now();+ for _ in 0..bench_iters {+ let cb = ctx.command_queue.new_command_buffer().to_owned();+ let enc = cb.new_compute_command_encoder();+ kernel+ .encode(+ ctx,+ &enc,+ MatmulArguments {+ a: &a_buf,+ b: &b_buf,+ d: &d_buf,+ batch: m as i32,+ input_dim: k as i32,+ output_dim: n as i32,+ lda: k as i32,+ ldb: k as i32,+ ldd: n as i32,+ batch_count: 1,+ },+ )+ .expect("encode");+ enc.end_encoding();+ cb.commit();+ cb.wait_until_completed();+ }+ let elapsed = start.elapsed();+ (elapsed.as_secs_f64() * 1000.0) / bench_iters as f64+}+fn benchmark_mpsgraph_matmul(device: &metal::Device,m: usize,@@ -297,7 +387,11 @@ fn run_benchmark() -> BenchmarkResult {print!("[Metal] ");let _ = std::io::stdout().flush();- let metal_ms = benchmark_metal_gemm(ctx, m, k, n, warmup, iters);+ let metal_ms = if m <= 8 || n == 1 {+ benchmark_metal_gemv(ctx, m, k, n, warmup, iters)+ } else {+ benchmark_metal_gemm(ctx, m, k, n, warmup, iters)+ };print!("{:.2}ms ", metal_ms);let _ = std::io::stdout().flush();diff --git a/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs b/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rsindex c8bf57461..03ce41159 100644--- a/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs+++ b/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs@@ -9,7 +9,6 @@ use crate::{encodable_block::{EncodableBlock, EncodingParameters},forward_pass::{ArrayId, ForwardPassState},kernel::{- KernelDataType, TensorAddBias,matmul::{MatmulArguments, MatmulKernel},},},@@ -19,7 +18,6 @@ use crate::{pub struct FullPrecisionLinear {kernel: RefCell<MatmulKernel>,- bias_add_kernel: Option<TensorAddBias>,biases_buffer: Option<MTLBuffer>,weights_buffer: MTLBuffer,input_dim: usize,@@ -69,39 +67,33 @@ impl FullPrecisionLinear {let weights_buffer: MTLBuffer =unsafe { weights.mtl_buffer() }.to_owned();- let (bias_add_kernel, biases_buffer) =- match parameter_tree.leaf("biases") {- Ok(mut biases) => {- if biases.shape() != [output_dim] {- return Err(MTLError::Generic(format!(- "Bias shape mismatch: got {:?}, expected [{:?}]",- biases.shape(),- output_dim- )));- }- if biases.data_type() != precision {- return Err(MTLError::Generic(format!(- "Bias dtype mismatch: got {:?}, expected {:?}",- biases.data_type(),- precision- )));- }- let bias_add_kernel = Some(TensorAddBias::new(- mtl_context,- KernelDataType::from(precision),- )?);- let biases_buffer: MTLBuffer =- unsafe { biases.mtl_buffer() }.to_owned();- (bias_add_kernel, Some(biases_buffer))- },- Err(_) => (None, None),- };+ let biases_buffer = match parameter_tree.leaf("biases") {+ Ok(mut biases) => {+ if biases.shape() != [output_dim] {+ return Err(MTLError::Generic(format!(+ "Bias shape mismatch: got {:?}, expected [{:?}]",+ biases.shape(),+ output_dim+ )));+ }+ if biases.data_type() != precision {+ return Err(MTLError::Generic(format!(+ "Bias dtype mismatch: got {:?}, expected {:?}",+ biases.data_type(),+ precision+ )));+ }+ let biases_buffer: MTLBuffer =+ unsafe { biases.mtl_buffer() }.to_owned();+ Some(biases_buffer)+ },+ Err(_) => None,+ };let kernel = MatmulKernel::new(mtl_context, precision, false, true)?;Ok(Self {kernel: RefCell::new(kernel),- bias_add_kernel,biases_buffer,weights_buffer,input_dim,@@ -143,26 +135,22 @@ impl EncodableBlock for FullPrecisionLinear {batch_count: 1,};- self.kernel- .borrow_mut()- .encode(state.mtl_context(), encoder, args)- .expect("Failed to encode matmul kernel");+ let mut kernel = self.kernel.borrow_mut();+ if let Some(bias_buf) = &self.biases_buffer {+ kernel+ .encode_with_bias(state.mtl_context(), encoder, args, bias_buf)+ .expect("Failed to encode matmul kernel");+ } else {+ kernel+ .encode(state.mtl_context(), encoder, args)+ .expect("Failed to encode matmul kernel");+ }encoder.end_encoding();- if let (Some(bias_add), Some(bias_buf)) =- (&self.bias_add_kernel, &self.biases_buffer)- {- let total_len = batch_size * self.output_dim;- bias_add.encode_into_command_buffer(- &output_buffer,- bias_buf,- &output_buffer,- self.output_dim,- total_len,- command_buffer,- parameters.predicate,- );+ if parameters.wait_until_completed {+ command_buffer.commit();+ command_buffer.wait_until_completed();}}@@ -174,7 +162,7 @@ impl EncodableBlock for FullPrecisionLinear {&self,state: &mut ForwardPassState,encoder: &ComputeCommandEncoderRef,- parameters: &EncodingParameters,+ _parameters: &EncodingParameters,) {let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);let batch_size = state.active_suffix_length();@@ -198,24 +186,15 @@ impl EncodableBlock for FullPrecisionLinear {batch_count: 1,};- self.kernel- .borrow_mut()- .encode(state.mtl_context(), encoder, args)- .expect("Failed to encode matmul kernel");-- if let (Some(bias_add), Some(bias_buf)) =- (&self.bias_add_kernel, &self.biases_buffer)- {- let total_len = batch_size * self.output_dim;- bias_add.encode_with_encoder(- &output_buffer,- bias_buf,- &output_buffer,- self.output_dim,- total_len,- encoder,- parameters.predicate,- );+ let mut kernel = self.kernel.borrow_mut();+ if let Some(bias_buf) = &self.biases_buffer {+ kernel+ .encode_with_bias(state.mtl_context(), encoder, args, bias_buf)+ .expect("Failed to encode matmul kernel");+ } else {+ kernel+ .encode(state.mtl_context(), encoder, args)+ .expect("Failed to encode matmul kernel");}}}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemv.metalindex 70d310cef..c93b3510a 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv.metal+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv.metal@@ -143,6 +143,51 @@ DECL_GEMV_FAST(gemv_fast_f16, half, float)DECL_GEMV_FAST(gemv_fast_bf16, bfloat, float)DECL_GEMV_FAST(gemv_fast_f32, float, float)+#define DECL_GEMV_FAST_BIAS(NAME, TYPE, ACC) \+ [[kernel]] void NAME( \+ const device TYPE* weights [[buffer(0)]], \+ const device TYPE* input [[buffer(1)]], \+ const device TYPE* bias [[buffer(2)]], \+ device TYPE* output [[buffer(3)]], \+ constant int& input_dim [[buffer(4)]], \+ constant int& output_dim [[buffer(5)]], \+ constant int& weight_stride [[buffer(6)]], \+ constant int& input_batch_stride [[buffer(7)]], \+ constant int& output_batch_stride [[buffer(8)]], \+ uint3 tgid [[threadgroup_position_in_grid]], \+ uint simd_lid [[thread_index_in_simdgroup]] \+ ) { \+ constexpr int ROWS_PER_SIMDGROUP = 4; \+ constexpr int ELEMS_PER_THREAD = 4; \+ const uint row_base = tgid.x * ROWS_PER_SIMDGROUP; \+ gemv_multirow<TYPE, ACC, ROWS_PER_SIMDGROUP, ELEMS_PER_THREAD>( \+ weights, \+ input, \+ output, \+ input_dim, \+ output_dim, \+ weight_stride, \+ tgid.z, \+ input_batch_stride, \+ output_batch_stride, \+ row_base, \+ simd_lid \+ ); \+ if (simd_lid == 0 && row_base < static_cast<uint>(output_dim)) { \+ const int rows_to_compute = \+ min(ROWS_PER_SIMDGROUP, output_dim - static_cast<int>(row_base)); \+ device TYPE* out_ptr = output + tgid.z * output_batch_stride; \+ for (int r = 0; r < rows_to_compute; ++r) { \+ const uint row = row_base + static_cast<uint>(r); \+ out_ptr[row] = out_ptr[row] + bias[row]; \+ } \+ } \+ }++DECL_GEMV_FAST_BIAS(gemv_fast_f16_bias, half, float)+DECL_GEMV_FAST_BIAS(gemv_fast_bf16_bias, bfloat, float)+DECL_GEMV_FAST_BIAS(gemv_fast_f32_bias, float, float)+template <typename T, typename AccT, ushort ROWS_PER_THREADGROUP, ushort TILE_K>METAL_FUNC void gemv_tiled(const device T* __restrict weights,@@ -235,10 +280,321 @@ METAL_FUNC void gemv_tiled(); \}+#define DECL_GEMV_TILED_BIAS(NAME, TYPE, ACC, ROWS, TILE_K) \+ [[kernel, max_total_threads_per_threadgroup(ROWS * 32)]] void NAME( \+ const device TYPE* weights [[buffer(0)]], \+ const device TYPE* input [[buffer(1)]], \+ const device TYPE* bias [[buffer(2)]], \+ device TYPE* output [[buffer(3)]], \+ constant uint& input_dim [[buffer(4)]], \+ constant uint& output_dim [[buffer(5)]], \+ constant uint& weight_stride [[buffer(6)]], \+ constant uint& input_stride [[buffer(7)]], \+ constant uint& output_stride [[buffer(8)]], \+ uint3 tgid [[threadgroup_position_in_grid]], \+ uint simd_gid [[simdgroup_index_in_threadgroup]], \+ uint simd_lid [[thread_index_in_simdgroup]], \+ uint tid [[thread_index_in_threadgroup]] \+ ) { \+ threadgroup ACC input_tile[TILE_K]; \+ gemv_tiled<TYPE, ACC, ROWS, TILE_K>( \+ weights, \+ input, \+ output, \+ input_dim, \+ output_dim, \+ weight_stride, \+ input_stride, \+ output_stride, \+ tgid, \+ simd_gid, \+ simd_lid, \+ tid, \+ input_tile \+ ); \+ const uint row = tgid.x * ROWS + simd_gid; \+ if (simd_lid == 0 && simd_gid < ROWS && row < output_dim) { \+ device TYPE* out_ptr = output + tgid.z * output_stride; \+ out_ptr[row] = out_ptr[row] + bias[row]; \+ } \+ }++DECL_GEMV_TILED_BIAS(gemv_f16_rows2_bias, half, float, 2, 128)+DECL_GEMV_TILED_BIAS(gemv_f16_rows4_bias, half, float, 4, 128)+DECL_GEMV_TILED_BIAS(gemv_f16_rows8_bias, half, float, 8, 128)+DECL_GEMV_TILED_BIAS(gemv_f16_rows16_bias, half, float, 16, 128)++DECL_GEMV_TILED_BIAS(gemv_bf16_rows2_bias, bfloat, float, 2, 128)+DECL_GEMV_TILED_BIAS(gemv_bf16_rows4_bias, bfloat, float, 4, 128)+DECL_GEMV_TILED_BIAS(gemv_bf16_rows8_bias, bfloat, float, 8, 128)+DECL_GEMV_TILED_BIAS(gemv_bf16_rows16_bias, bfloat, float, 16, 128)++DECL_GEMV_TILED_BIAS(gemv_f32_rows2_bias, float, float, 2, 128)+DECL_GEMV_TILED_BIAS(gemv_f32_rows4_bias, float, float, 4, 128)+DECL_GEMV_TILED_BIAS(gemv_f32_rows8_bias, float, float, 8, 128)+DECL_GEMV_TILED_BIAS(gemv_f32_rows16_bias, float, float, 16, 128)+DECL_GEMV_TILED(gemv_f16_rows2, half, float, 2, 128)DECL_GEMV_TILED(gemv_f16_rows4, half, float, 4, 128)DECL_GEMV_TILED(gemv_f16_rows8, half, float, 8, 128)+DECL_GEMV_TILED(gemv_f16_rows16, half, float, 16, 128)DECL_GEMV_TILED(gemv_bf16_rows2, bfloat, float, 2, 128)DECL_GEMV_TILED(gemv_bf16_rows4, bfloat, float, 4, 128)DECL_GEMV_TILED(gemv_bf16_rows8, bfloat, float, 8, 128)+DECL_GEMV_TILED(gemv_bf16_rows16, bfloat, float, 16, 128)++DECL_GEMV_TILED(gemv_f32_rows2, float, float, 2, 128)+DECL_GEMV_TILED(gemv_f32_rows4, float, float, 4, 128)+DECL_GEMV_TILED(gemv_f32_rows8, float, float, 8, 128)+DECL_GEMV_TILED(gemv_f32_rows16, float, float, 16, 128)++template <typename T, typename AccT, ushort KPARTS>+METAL_FUNC void gemv_split_k_rows16_impl(+ const device T* __restrict weights,+ const device T* __restrict input,+ const device T* __restrict bias,+ device T* __restrict output,+ const int input_dim,+ const int output_dim,+ const int weight_stride,+ const int input_batch_stride,+ const int output_batch_stride,+ uint3 tgid,+ uint simd_gid,+ uint simd_lid,+ threadgroup AccT* partial_sums+) {+ constexpr ushort rows_per_threadgroup = 16;+ constexpr ushort rows_per_rowgroup = 2;+ constexpr ushort rowgroups = rows_per_threadgroup / rows_per_rowgroup;+ constexpr ushort elems_per_thread = 4;+ constexpr int block_k = 32 * elems_per_thread;++ const uint rowgroup_idx = simd_gid / KPARTS;+ const uint kpart_idx = simd_gid - rowgroup_idx * KPARTS;++ const uint batch = tgid.z;+ const uint row_base =+ tgid.x * rows_per_threadgroup + rowgroup_idx * rows_per_rowgroup;++ const bool valid_rowgroup = row_base < static_cast<uint>(output_dim);+ const uint rows_to_compute = valid_rowgroup+ ? min(static_cast<uint>(rows_per_rowgroup),+ static_cast<uint>(output_dim) - row_base)+ : 0;++ const device T* input_ptr = input + batch * input_batch_stride;+ device T* output_ptr = output + batch * output_batch_stride;++ AccT accumulators[rows_per_rowgroup] = {AccT(0), AccT(0)};++ if (valid_rowgroup) {+ const device T* weight_row_ptrs[rows_per_rowgroup];+ weight_row_ptrs[0] = weights + (row_base + 0) * weight_stride;+ weight_row_ptrs[1] = weights + (row_base + 1) * weight_stride;++ for (int k_block = static_cast<int>(kpart_idx) * block_k;+ k_block < input_dim;+ k_block += block_k * static_cast<int>(KPARTS))+ {+ AccT input_cache[elems_per_thread];++#pragma unroll(elems_per_thread)+ for (int e = 0; e < elems_per_thread; ++e) {+ const int col = k_block + static_cast<int>(simd_lid) * elems_per_thread + e;+ input_cache[e] =+ (col < input_dim) ? static_cast<AccT>(input_ptr[col]) : AccT(0);+ }++#pragma unroll(rows_per_rowgroup)+ for (int r = 0; r < rows_per_rowgroup; ++r) {+ AccT dot = 0;+#pragma unroll(elems_per_thread)+ for (int e = 0; e < elems_per_thread; ++e) {+ const int col =+ k_block + static_cast<int>(simd_lid) * elems_per_thread + e;+ if (col < input_dim) {+ dot += input_cache[e] * static_cast<AccT>(weight_row_ptrs[r][col]);+ }+ }+ accumulators[r] += dot;+ }+ }+ }++#pragma unroll(rows_per_rowgroup)+ for (int r = 0; r < rows_per_rowgroup; ++r) {+ accumulators[r] = simd_sum(accumulators[r]);+ }++ const uint partial_base =+ rowgroup_idx * (KPARTS * rows_per_rowgroup) + kpart_idx * rows_per_rowgroup;++ if (simd_lid == 0) {+ partial_sums[partial_base + 0] = accumulators[0];+ partial_sums[partial_base + 1] = accumulators[1];+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ if (kpart_idx == 0 && simd_lid == 0 && rows_to_compute > 0) {+ for (uint r = 0; r < rows_to_compute; ++r) {+ AccT sum = 0;+ for (uint kp = 0; kp < KPARTS; ++kp) {+ sum += partial_sums[rowgroup_idx * (KPARTS * rows_per_rowgroup) ++ kp * rows_per_rowgroup + r];+ }+ const uint row = row_base + r;+ if (bias != nullptr) {+ sum += static_cast<AccT>(bias[row]);+ }+ output_ptr[row] = static_cast<T>(sum);+ }+ }+}++#define DECL_GEMV_SPLIT_K_KPARTS2(NAME, TYPE, ACC) \+ [[kernel, max_total_threads_per_threadgroup(512)]] void NAME( \+ const device TYPE* weights [[buffer(0)]], \+ const device TYPE* input [[buffer(1)]], \+ device TYPE* output [[buffer(2)]], \+ constant int& input_dim [[buffer(3)]], \+ constant int& output_dim [[buffer(4)]], \+ constant int& weight_stride [[buffer(5)]], \+ constant int& input_batch_stride [[buffer(6)]], \+ constant int& output_batch_stride [[buffer(7)]], \+ uint3 tgid [[threadgroup_position_in_grid]], \+ uint simd_gid [[simdgroup_index_in_threadgroup]], \+ uint simd_lid [[thread_index_in_simdgroup]] \+ ) { \+ threadgroup ACC partial_sums[8 * 2 * 2]; \+ gemv_split_k_rows16_impl<TYPE, ACC, 2>( \+ weights, \+ input, \+ nullptr, \+ output, \+ input_dim, \+ output_dim, \+ weight_stride, \+ input_batch_stride, \+ output_batch_stride, \+ tgid, \+ simd_gid, \+ simd_lid, \+ partial_sums \+ ); \+ }++#define DECL_GEMV_SPLIT_K_KPARTS4(NAME, TYPE, ACC) \+ [[kernel, max_total_threads_per_threadgroup(1024)]] void NAME( \+ const device TYPE* weights [[buffer(0)]], \+ const device TYPE* input [[buffer(1)]], \+ device TYPE* output [[buffer(2)]], \+ constant int& input_dim [[buffer(3)]], \+ constant int& output_dim [[buffer(4)]], \+ constant int& weight_stride [[buffer(5)]], \+ constant int& input_batch_stride [[buffer(6)]], \+ constant int& output_batch_stride [[buffer(7)]], \+ uint3 tgid [[threadgroup_position_in_grid]], \+ uint simd_gid [[simdgroup_index_in_threadgroup]], \+ uint simd_lid [[thread_index_in_simdgroup]] \+ ) { \+ threadgroup ACC partial_sums[8 * 4 * 2]; \+ gemv_split_k_rows16_impl<TYPE, ACC, 4>( \+ weights, \+ input, \+ nullptr, \+ output, \+ input_dim, \+ output_dim, \+ weight_stride, \+ input_batch_stride, \+ output_batch_stride, \+ tgid, \+ simd_gid, \+ simd_lid, \+ partial_sums \+ ); \+ }++#define DECL_GEMV_SPLIT_K_KPARTS2_BIAS(NAME, TYPE, ACC) \+ [[kernel, max_total_threads_per_threadgroup(512)]] void NAME( \+ const device TYPE* weights [[buffer(0)]], \+ const device TYPE* input [[buffer(1)]], \+ const device TYPE* bias [[buffer(2)]], \+ device TYPE* output [[buffer(3)]], \+ constant int& input_dim [[buffer(4)]], \+ constant int& output_dim [[buffer(5)]], \+ constant int& weight_stride [[buffer(6)]], \+ constant int& input_batch_stride [[buffer(7)]], \+ constant int& output_batch_stride [[buffer(8)]], \+ uint3 tgid [[threadgroup_position_in_grid]], \+ uint simd_gid [[simdgroup_index_in_threadgroup]], \+ uint simd_lid [[thread_index_in_simdgroup]] \+ ) { \+ threadgroup ACC partial_sums[8 * 2 * 2]; \+ gemv_split_k_rows16_impl<TYPE, ACC, 2>( \+ weights, \+ input, \+ bias, \+ output, \+ input_dim, \+ output_dim, \+ weight_stride, \+ input_batch_stride, \+ output_batch_stride, \+ tgid, \+ simd_gid, \+ simd_lid, \+ partial_sums \+ ); \+ }++#define DECL_GEMV_SPLIT_K_KPARTS4_BIAS(NAME, TYPE, ACC) \+ [[kernel, max_total_threads_per_threadgroup(1024)]] void NAME( \+ const device TYPE* weights [[buffer(0)]], \+ const device TYPE* input [[buffer(1)]], \+ const device TYPE* bias [[buffer(2)]], \+ device TYPE* output [[buffer(3)]], \+ constant int& input_dim [[buffer(4)]], \+ constant int& output_dim [[buffer(5)]], \+ constant int& weight_stride [[buffer(6)]], \+ constant int& input_batch_stride [[buffer(7)]], \+ constant int& output_batch_stride [[buffer(8)]], \+ uint3 tgid [[threadgroup_position_in_grid]], \+ uint simd_gid [[simdgroup_index_in_threadgroup]], \+ uint simd_lid [[thread_index_in_simdgroup]] \+ ) { \+ threadgroup ACC partial_sums[8 * 4 * 2]; \+ gemv_split_k_rows16_impl<TYPE, ACC, 4>( \+ weights, \+ input, \+ bias, \+ output, \+ input_dim, \+ output_dim, \+ weight_stride, \+ input_batch_stride, \+ output_batch_stride, \+ tgid, \+ simd_gid, \+ simd_lid, \+ partial_sums \+ ); \+ }++DECL_GEMV_SPLIT_K_KPARTS2(gemv_split_k_f16_rows16_kparts2, half, float)+DECL_GEMV_SPLIT_K_KPARTS4(gemv_split_k_f16_rows16_kparts4, half, float)+DECL_GEMV_SPLIT_K_KPARTS2_BIAS(gemv_split_k_f16_rows16_kparts2_bias, half, float)+DECL_GEMV_SPLIT_K_KPARTS4_BIAS(gemv_split_k_f16_rows16_kparts4_bias, half, float)++DECL_GEMV_SPLIT_K_KPARTS2(gemv_split_k_bf16_rows16_kparts2, bfloat, float)+DECL_GEMV_SPLIT_K_KPARTS4(gemv_split_k_bf16_rows16_kparts4, bfloat, float)+DECL_GEMV_SPLIT_K_KPARTS2_BIAS(gemv_split_k_bf16_rows16_kparts2_bias, bfloat, float)+DECL_GEMV_SPLIT_K_KPARTS4_BIAS(gemv_split_k_bf16_rows16_kparts4_bias, bfloat, float)++DECL_GEMV_SPLIT_K_KPARTS2(gemv_split_k_f32_rows16_kparts2, float, float)+DECL_GEMV_SPLIT_K_KPARTS4(gemv_split_k_f32_rows16_kparts4, float, float)+DECL_GEMV_SPLIT_K_KPARTS2_BIAS(gemv_split_k_f32_rows16_kparts2_bias, float, float)+DECL_GEMV_SPLIT_K_KPARTS4_BIAS(gemv_split_k_f32_rows16_kparts4_bias, float, float)\ No newline at end of filediff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv.rsindex ad43ee52a..e7a0f28f7 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv.rs@@ -1,6 +1,7 @@use std::collections::HashMap;use metal::{+ Buffer as MTLBuffer,ComputeCommandEncoderRef, ComputePipelineState as MTLComputePipelineState,MTLSize,};@@ -13,10 +14,18 @@ use crate::{#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]enum PipelineVariant {+ Fast {+ has_bias: bool,+ },Tiled {rows_per_threadgroup: u32,+ has_bias: bool,+ },+ SplitK {+ rows_per_threadgroup: u32,+ kparts: u32,+ has_bias: bool,},- Fast,}pub struct GemvKernel {@@ -32,25 +41,37 @@ impl GemvKernel {}}- fn fast_kernel_name(&self) -> Result<String, MTLError> {- match self.data_type {- DataType::F16 => Ok("gemv_fast_f16".to_string()),- DataType::BF16 => Ok("gemv_fast_bf16".to_string()),- DataType::F32 => Ok("gemv_fast_f32".to_string()),- _ => Err(MTLError::Generic(format!(- "Unsupported data type for fast GEMV: {:?}",- self.data_type- ))),- }+ fn fast_kernel_name(+ &self,+ has_bias: bool,+ ) -> Result<String, MTLError> {+ let base = match self.data_type {+ DataType::F16 => "gemv_fast_f16",+ DataType::BF16 => "gemv_fast_bf16",+ DataType::F32 => "gemv_fast_f32",+ _ => {+ return Err(MTLError::Generic(format!(+ "Unsupported data type for fast GEMV: {:?}",+ self.data_type+ )));+ },+ };+ Ok(if has_bias {+ format!("{base}_bias")+ } else {+ base.to_string()+ })}fn tiled_kernel_name(&self,rows_per_threadgroup: u32,+ has_bias: bool,) -> Result<String, MTLError> {let prefix = match self.data_type {DataType::F16 => "gemv_f16",DataType::BF16 => "gemv_bf16",+ DataType::F32 => "gemv_f32",_ => {return Err(MTLError::Generic(format!("Unsupported data type for tiled GEMV: {:?}",@@ -58,16 +79,52 @@ impl GemvKernel {)));},};- Ok(format!("{prefix}_rows{rows_per_threadgroup}"))+ Ok(if has_bias {+ format!("{prefix}_rows{rows_per_threadgroup}_bias")+ } else {+ format!("{prefix}_rows{rows_per_threadgroup}")+ })+ }++ fn split_k_kernel_name(+ &self,+ kparts: u32,+ has_bias: bool,+ ) -> Result<String, MTLError> {+ let dtype = match self.data_type {+ DataType::F16 => "f16",+ DataType::BF16 => "bf16",+ DataType::F32 => "f32",+ _ => {+ return Err(MTLError::Generic(format!(+ "Unsupported data type for split-k GEMV: {:?}",+ self.data_type+ )));+ },+ };+ if !matches!(kparts, 2 | 4) {+ return Err(MTLError::Generic(format!(+ "Unsupported split-k kparts={kparts} (expected 2 or 4)"+ )));+ }+ let base = format!("gemv_split_k_{dtype}_rows16_kparts{kparts}");+ Ok(if has_bias {+ format!("{base}_bias")+ } else {+ base+ })}fn get_fast_pipeline(&mut self,context: &MTLContext,+ has_bias: bool,) -> Result<&MTLComputePipelineState, MTLError> {- let variant = PipelineVariant::Fast;+ let variant = PipelineVariant::Fast {+ has_bias,+ };if !self.pipelines.contains_key(&variant) {- let kernel_name = self.fast_kernel_name()?;+ let kernel_name = self.fast_kernel_name(has_bias)?;let pipeline =context.compute_pipeline_state(&kernel_name, None)?;self.pipelines.insert(variant, pipeline);@@ -79,12 +136,35 @@ impl GemvKernel {&mut self,context: &MTLContext,rows_per_threadgroup: u32,+ has_bias: bool,) -> Result<&MTLComputePipelineState, MTLError> {let variant = PipelineVariant::Tiled {rows_per_threadgroup,+ has_bias,+ };+ if !self.pipelines.contains_key(&variant) {+ let kernel_name =+ self.tiled_kernel_name(rows_per_threadgroup, has_bias)?;+ let pipeline =+ context.compute_pipeline_state(&kernel_name, None)?;+ self.pipelines.insert(variant, pipeline);+ }+ Ok(self.pipelines.get(&variant).unwrap())+ }++ fn get_split_k_pipeline(+ &mut self,+ context: &MTLContext,+ kparts: u32,+ has_bias: bool,+ ) -> Result<&MTLComputePipelineState, MTLError> {+ let variant = PipelineVariant::SplitK {+ rows_per_threadgroup: 16,+ kparts,+ has_bias,};if !self.pipelines.contains_key(&variant) {- let kernel_name = self.tiled_kernel_name(rows_per_threadgroup)?;+ let kernel_name = self.split_k_kernel_name(kparts, has_bias)?;let pipeline =context.compute_pipeline_state(&kernel_name, None)?;self.pipelines.insert(variant, pipeline);@@ -99,18 +179,49 @@ impl GemvKernel {args.input_dim >= 128 && args.input_dim % 4 == 0}+ fn split_k_kparts(+ &self,+ args: &MatmulArguments,+ ) -> Option<u32> {+ let k = args.input_dim;+ let n = args.output_dim;+ if k >= 8192 && n >= 2048 {+ return Some(4);+ }+ if k >= 4096 && n >= 2048 {+ return Some(2);+ }+ None+ }++ fn is_logits_like(+ &self,+ args: &MatmulArguments,+ ) -> bool {+ let k = args.input_dim;+ let n = args.output_dim;+ n >= 50_000 || (n >= 16_384 && n >= k * 7)+ }+fn encode_fast(&mut self,context: &MTLContext,encoder: &ComputeCommandEncoderRef,args: MatmulArguments,+ bias: Option<&MTLBuffer>,) -> Result<(), MTLError> {- let pipeline = self.get_fast_pipeline(context)?;+ let has_bias = bias.is_some();+ let pipeline = self.get_fast_pipeline(context, has_bias)?;encoder.set_compute_pipeline_state(pipeline);encoder.set_buffer(0, Some(args.b), 0);encoder.set_buffer(1, Some(args.a), 0);- encoder.set_buffer(2, Some(args.d), 0);+ if let Some(bias) = bias {+ encoder.set_buffer(2, Some(bias), 0);+ encoder.set_buffer(3, Some(args.d), 0);+ } else {+ encoder.set_buffer(2, Some(args.d), 0);+ }let input_dim = args.input_dim;let output_dim = args.output_dim;@@ -118,28 +229,29 @@ impl GemvKernel {let input_batch_stride = args.lda;let output_batch_stride = args.ldd;+ let base = if has_bias { 4 } else { 3 };encoder.set_bytes(- 3,+ base,std::mem::size_of::<i32>() as u64,&input_dim as *const i32 as *const std::ffi::c_void,);encoder.set_bytes(- 4,+ base + 1,std::mem::size_of::<i32>() as u64,&output_dim as *const i32 as *const std::ffi::c_void,);encoder.set_bytes(- 5,+ base + 2,std::mem::size_of::<i32>() as u64,&weight_stride as *const i32 as *const std::ffi::c_void,);encoder.set_bytes(- 6,+ base + 3,std::mem::size_of::<i32>() as u64,&input_batch_stride as *const i32 as *const std::ffi::c_void,);encoder.set_bytes(- 7,+ base + 4,std::mem::size_of::<i32>() as u64,&output_batch_stride as *const i32 as *const std::ffi::c_void,);@@ -164,15 +276,25 @@ impl GemvKernel {encoder: &ComputeCommandEncoderRef,args: MatmulArguments,rows_per_threadgroup: u32,+ bias: Option<&MTLBuffer>,) -> Result<(), MTLError> {let rows_per_threadgroup = rows_per_threadgroup.max(1);- let pipeline =- self.get_tiled_pipeline(context, rows_per_threadgroup)?;+ let has_bias = bias.is_some();+ let pipeline = self.get_tiled_pipeline(+ context,+ rows_per_threadgroup,+ has_bias,+ )?;encoder.set_compute_pipeline_state(pipeline);encoder.set_buffer(0, Some(args.b), 0);encoder.set_buffer(1, Some(args.a), 0);- encoder.set_buffer(2, Some(args.d), 0);+ if let Some(bias) = bias {+ encoder.set_buffer(2, Some(bias), 0);+ encoder.set_buffer(3, Some(args.d), 0);+ } else {+ encoder.set_buffer(2, Some(args.d), 0);+ }let input_dim = args.input_dim as u32;let output_dim = args.output_dim as u32;@@ -180,28 +302,29 @@ impl GemvKernel {let input_stride = args.lda as u32;let output_stride = args.ldd as u32;+ let base = if has_bias { 4 } else { 3 };encoder.set_bytes(- 3,+ base,std::mem::size_of::<u32>() as u64,&input_dim as *const u32 as *const std::ffi::c_void,);encoder.set_bytes(- 4,+ base + 1,std::mem::size_of::<u32>() as u64,&output_dim as *const u32 as *const std::ffi::c_void,);encoder.set_bytes(- 5,+ base + 2,std::mem::size_of::<u32>() as u64,&weight_stride as *const u32 as *const std::ffi::c_void,);encoder.set_bytes(- 6,+ base + 3,std::mem::size_of::<u32>() as u64,&input_stride as *const u32 as *const std::ffi::c_void,);encoder.set_bytes(- 7,+ base + 4,std::mem::size_of::<u32>() as u64,&output_stride as *const u32 as *const std::ffi::c_void,);@@ -219,17 +342,141 @@ impl GemvKernel {Ok(())}- pub fn encode(+ fn encode_split_k(+ &mut self,+ context: &MTLContext,+ encoder: &ComputeCommandEncoderRef,+ args: MatmulArguments,+ kparts: u32,+ bias: Option<&MTLBuffer>,+ ) -> Result<(), MTLError> {+ let has_bias = bias.is_some();+ let pipeline = self.get_split_k_pipeline(context, kparts, has_bias)?;+ encoder.set_compute_pipeline_state(pipeline);++ encoder.set_buffer(0, Some(args.b), 0);+ encoder.set_buffer(1, Some(args.a), 0);+ if let Some(bias) = bias {+ encoder.set_buffer(2, Some(bias), 0);+ encoder.set_buffer(3, Some(args.d), 0);+ } else {+ encoder.set_buffer(2, Some(args.d), 0);+ }++ let input_dim = args.input_dim;+ let output_dim = args.output_dim;+ let weight_stride = args.ldb;+ let input_batch_stride = args.lda;+ let output_batch_stride = args.ldd;++ let base = if has_bias { 4 } else { 3 };+ encoder.set_bytes(+ base,+ std::mem::size_of::<i32>() as u64,+ &input_dim as *const i32 as *const std::ffi::c_void,+ );+ encoder.set_bytes(+ base + 1,+ std::mem::size_of::<i32>() as u64,+ &output_dim as *const i32 as *const std::ffi::c_void,+ );+ encoder.set_bytes(+ base + 2,+ std::mem::size_of::<i32>() as u64,+ &weight_stride as *const i32 as *const std::ffi::c_void,+ );+ encoder.set_bytes(+ base + 3,+ std::mem::size_of::<i32>() as u64,+ &input_batch_stride as *const i32 as *const std::ffi::c_void,+ );+ encoder.set_bytes(+ base + 4,+ std::mem::size_of::<i32>() as u64,+ &output_batch_stride as *const i32 as *const std::ffi::c_void,+ );++ const ROWS_PER_THREADGROUP: i32 = 16;+ let threadgroup_count_x = ((output_dim + ROWS_PER_THREADGROUP - 1)+ / ROWS_PER_THREADGROUP) as u64;+ let threadgroup_count_z = args.batch_count as u64;++ let threads_per_threadgroup_x = match kparts {+ 2 => 512,+ 4 => 1024,+ _ => {+ return Err(MTLError::Generic(format!(+ "Unsupported split-k kparts={kparts} (expected 2 or 4)"+ )));+ },+ };++ let threadgroup_count =+ MTLSize::new(threadgroup_count_x, 1, threadgroup_count_z);+ let threads_per_threadgroup =+ MTLSize::new(threads_per_threadgroup_x, 1, 1);++ encoder+ .dispatch_thread_groups(threadgroup_count, threads_per_threadgroup);+ Ok(())+ }++ fn encode_inner(&mut self,context: &MTLContext,encoder: &ComputeCommandEncoderRef,args: MatmulArguments,rows_per_threadgroup: u32,+ bias: Option<&MTLBuffer>,) -> Result<(), MTLError> {+ if self.is_logits_like(&args) {+ if args.input_dim >= 8192 {+ if let Some(kparts) = self.split_k_kparts(&args) {+ return self.encode_split_k(+ context,+ encoder,+ args,+ kparts,+ bias,+ );+ }+ }+ return self.encode_tiled(context, encoder, args, 16, bias);+ }+ if let Some(kparts) = self.split_k_kparts(&args) {+ return self.encode_split_k(context, encoder, args, kparts, bias);+ }if self.can_use_fast_kernel(&args) {- self.encode_fast(context, encoder, args)+ self.encode_fast(context, encoder, args, bias)} else {- self.encode_tiled(context, encoder, args, rows_per_threadgroup)+ self.encode_tiled(context, encoder, args, rows_per_threadgroup, bias)}}++ pub fn encode(+ &mut self,+ context: &MTLContext,+ encoder: &ComputeCommandEncoderRef,+ args: MatmulArguments,+ rows_per_threadgroup: u32,+ ) -> Result<(), MTLError> {+ self.encode_inner(context, encoder, args, rows_per_threadgroup, None)+ }++ pub fn encode_with_bias(+ &mut self,+ context: &MTLContext,+ encoder: &ComputeCommandEncoderRef,+ args: MatmulArguments,+ rows_per_threadgroup: u32,+ bias: &MTLBuffer,+ ) -> Result<(), MTLError> {+ self.encode_inner(+ context,+ encoder,+ args,+ rows_per_threadgroup,+ Some(bias),+ )+ }}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv_masked.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemv_masked.metaldeleted file mode 100644index 622969fde..000000000--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv_masked.metal+++ /dev/null@@ -1,177 +0,0 @@-// Masked and gather GEMV variants. Masks/indices are honored only for output-// write; data loads remain dense.--#include <metal_stdlib>-using namespace metal;--template <typename T, typename AccT, ushort ROWS>-METAL_FUNC void gemv_masked_impl(- const device T* __restrict matrix [[buffer(0)]],- const device T* __restrict vector [[buffer(1)]],- const device uchar* __restrict out_mask [[buffer(2)]],- const device uchar* __restrict op_mask [[buffer(3)]],- device T* __restrict output [[buffer(4)]],- constant uint& k [[buffer(5)]],- constant uint& n [[buffer(6)]],- constant uint& ldb [[buffer(7)]],- constant uint& lda [[buffer(8)]],- constant uint& ldd [[buffer(9)]],- uint3 tgid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-) {- if (simd_gid >= ROWS) {- return;- }- const uint row = tgid.x * ROWS + simd_gid;- if (row >= n) {- return;- }- const uint batch = tgid.z;- const device T* row_ptr = matrix + row * ldb;- const device T* vec_ptr = vector + batch * lda;- device T* out_ptr = output + batch * ldd;-- AccT acc = static_cast<AccT>(0);- for (uint idx = simd_lid; idx < k; idx += 32) {- // If op_mask provided, zero out masked elements- const bool op_on = (op_mask == nullptr) || (op_mask[row] != 0);- if (op_on) {- acc += static_cast<AccT>(row_ptr[idx]) * static_cast<AccT>(vec_ptr[idx]);- }- }-- for (ushort offset = 16; offset > 0; offset >>= 1) {- acc += simd_shuffle_down(acc, offset);- }-- if (simd_lid == 0) {- if (out_mask == nullptr || out_mask[row] != 0) {- out_ptr[row] = static_cast<T>(acc);- }- }-}--template <typename T, typename AccT, ushort ROWS>-METAL_FUNC void gemv_gather_impl(- const device T* __restrict matrix [[buffer(0)]],- const device T* __restrict vector [[buffer(1)]],- const device uint* __restrict vec_indices [[buffer(2)]],- const device uint* __restrict mat_indices [[buffer(3)]],- device T* __restrict output [[buffer(4)]],- constant uint& k [[buffer(5)]],- constant uint& n [[buffer(6)]],- constant uint& ldb [[buffer(7)]],- constant uint& lda [[buffer(8)]],- constant uint& ldd [[buffer(9)]],- uint3 tgid [[threadgroup_position_in_grid]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-) {- if (simd_gid >= ROWS) {- return;- }- const uint row = tgid.x * ROWS + simd_gid;- if (row >= n) {- return;- }- const uint batch = tgid.z;- const device T* base_row = matrix + row * ldb;- device T* out_ptr = output + batch * ldd;-- AccT acc = static_cast<AccT>(0);- for (uint idx = simd_lid; idx < k; idx += 32) {- const uint col = mat_indices ? mat_indices[idx] : idx;- const uint vec_idx = vec_indices ? vec_indices[idx] : idx;- acc += static_cast<AccT>(base_row[col]) *- static_cast<AccT>(vector[batch * lda + vec_idx]);- }-- for (ushort offset = 16; offset > 0; offset >>= 1) {- acc += simd_shuffle_down(acc, offset);- }-- if (simd_lid == 0) {- out_ptr[row] = static_cast<T>(acc);- }-}--#define DECL_GEMV_MASKED(NAME, TYPE, ACC, ROWS) \- [[kernel, max_total_threads_per_threadgroup(ROWS * 32)]] void NAME( \- const device TYPE* matrix [[buffer(0)]], \- const device TYPE* vector [[buffer(1)]], \- const device uchar* out_mask [[buffer(2)]], \- const device uchar* op_mask [[buffer(3)]], \- device TYPE* output [[buffer(4)]], \- constant uint& k [[buffer(5)]], \- constant uint& n [[buffer(6)]], \- constant uint& ldb [[buffer(7)]], \- constant uint& lda [[buffer(8)]], \- constant uint& ldd [[buffer(9)]], \- uint3 tgid [[threadgroup_position_in_grid]], \- uint simd_gid [[simdgroup_index_in_threadgroup]], \- uint simd_lid [[thread_index_in_simdgroup]] \- ) { \- gemv_masked_impl<TYPE, ACC, ROWS>( \- matrix, \- vector, \- out_mask, \- op_mask, \- output, \- k, \- n, \- ldb, \- lda, \- ldd, \- tgid, \- simd_gid, \- simd_lid \- ); \- }--#define DECL_GEMV_GATHER(NAME, TYPE, ACC, ROWS) \- [[kernel, max_total_threads_per_threadgroup(ROWS * 32)]] void NAME( \- const device TYPE* matrix [[buffer(0)]], \- const device TYPE* vector [[buffer(1)]], \- const device uint* vec_indices [[buffer(2)]], \- const device uint* mat_indices [[buffer(3)]], \- device TYPE* output [[buffer(4)]], \- constant uint& k [[buffer(5)]], \- constant uint& n [[buffer(6)]], \- constant uint& ldb [[buffer(7)]], \- constant uint& lda [[buffer(8)]], \- constant uint& ldd [[buffer(9)]], \- uint3 tgid [[threadgroup_position_in_grid]], \- uint simd_gid [[simdgroup_index_in_threadgroup]], \- uint simd_lid [[thread_index_in_simdgroup]] \- ) { \- gemv_gather_impl<TYPE, ACC, ROWS>( \- matrix, \- vector, \- vec_indices, \- mat_indices, \- output, \- k, \- n, \- ldb, \- lda, \- ldd, \- tgid, \- simd_gid, \- simd_lid \- ); \- }--// Masked instantiations-DECL_GEMV_MASKED(gemv_masked_f16_rows4, half, float, 4)-DECL_GEMV_MASKED(gemv_masked_f16_rows8, half, float, 8)-DECL_GEMV_MASKED(gemv_masked_bf16_rows4, bfloat, float, 4)-DECL_GEMV_MASKED(gemv_masked_bf16_rows8, bfloat, float, 8)--// Gather instantiations-DECL_GEMV_GATHER(gemv_gather_f16_rows4, half, float, 4)-DECL_GEMV_GATHER(gemv_gather_f16_rows8, half, float, 8)-DECL_GEMV_GATHER(gemv_gather_bf16_rows4, bfloat, float, 4)-DECL_GEMV_GATHER(gemv_gather_bf16_rows8, bfloat, float, 8)-// Masked/gather GEMV kernels are stubbed out (not used). Keeping file empty to-// avoid shader compilation errors.diff --git a/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/kernel.rsindex 39d1c62f1..81e82a79e 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs@@ -1,6 +1,7 @@use std::collections::HashMap;use metal::{+ Buffer as MTLBuffer,ComputeCommandEncoderRef, ComputePipelineState as MTLComputePipelineState,MTLSize,};@@ -10,6 +11,7 @@ use super::{shared_types::GEMMParams, splitk::SplitKGemm,transpose::transpose_configuration,};+use super::super::{KernelDataType, TensorAddBias};use crate::{DataType,backends::metal::{MTLContext, MTLError},@@ -21,6 +23,7 @@ pub struct MatmulKernel {transpose_b: bool,gemv: Option<GemvKernel>,splitk: Option<SplitKGemm>,+ bias_add: Option<TensorAddBias>,pipelines: HashMap<PipelineKey, MTLComputePipelineState>,}@@ -69,6 +72,7 @@ impl MatmulKernel {transpose_b,gemv: None,splitk: None,+ bias_add: None,pipelines: HashMap::new(),})}@@ -86,11 +90,12 @@ impl MatmulKernel {}}- fn maybe_use_gemv(+ fn maybe_use_gemv_impl(&mut self,mtl: &MTLContext,enc: &ComputeCommandEncoderRef,args: MatmulArguments,+ bias: Option<&MTLBuffer>,) -> Result<bool, MTLError> {// GEMV kernel requires:// - A not transposed (input vector is contiguous)@@ -123,7 +128,11 @@ impl MatmulKernel {let rows = self.select_gemv_rows(n);let gemv =self.gemv.get_or_insert_with(|| GemvKernel::new(self.dt));- gemv.encode(mtl, enc, batched_args, rows)?;+ if let Some(bias) = bias {+ gemv.encode_with_bias(mtl, enc, batched_args, rows, bias)?;+ } else {+ gemv.encode(mtl, enc, batched_args, rows)?;+ }return Ok(true);}@@ -133,10 +142,33 @@ impl MatmulKernel {}let rows = self.select_gemv_rows(n);let gemv = self.gemv.get_or_insert_with(|| GemvKernel::new(self.dt));- gemv.encode(mtl, enc, args, rows)?;+ if let Some(bias) = bias {+ gemv.encode_with_bias(mtl, enc, args, rows, bias)?;+ } else {+ gemv.encode(mtl, enc, args, rows)?;+ }Ok(true)}+ fn maybe_use_gemv(+ &mut self,+ mtl: &MTLContext,+ enc: &ComputeCommandEncoderRef,+ args: MatmulArguments,+ ) -> Result<bool, MTLError> {+ self.maybe_use_gemv_impl(mtl, enc, args, None)+ }++ fn maybe_use_gemv_with_bias(+ &mut self,+ mtl: &MTLContext,+ enc: &ComputeCommandEncoderRef,+ args: MatmulArguments,+ bias: &MTLBuffer,+ ) -> Result<bool, MTLError> {+ self.maybe_use_gemv_impl(mtl, enc, args, Some(bias))+ }+fn maybe_use_splitk(&mut self,mtl: &MTLContext,@@ -367,24 +399,12 @@ impl MatmulKernel {Ok(self.pipelines.get(&key).unwrap())}- pub fn encode(+ fn encode_gemm(&mut self,mtl: &MTLContext,enc: &ComputeCommandEncoderRef,- mut args: MatmulArguments,+ args: MatmulArguments,) -> Result<(), MTLError> {- self.apply_batch_collapse(&mut args);-- // Try GEMV fast path for decode shapes.- if self.maybe_use_gemv(mtl, enc, args)? {- return Ok(());- }-- // Try Split-K for small M*N and large K.- if self.maybe_use_splitk(mtl, enc, args)? {- return Ok(());- }-let (bm, bn, bk, wm, wn) = self.select_tiles(mtl, &args);let kname = self.kernel_name(bm, bn, bk, wm, wn);@@ -409,11 +429,7 @@ impl MatmulKernel {let tiles_n = (n + bn - 1) / bn;let tiles_m = (m + bm - 1) / bm;// Swizzle: small tm -> no swizzle, otherwise simple 2-way.- let swizzle_log = if tiles_m <= 3 {- 0- } else {- 1- };+ let swizzle_log = if tiles_m <= 3 { 0 } else { 1 };let tile = 1 << swizzle_log;let tm_swizzled = (tiles_m + tile - 1) / tile;@@ -452,6 +468,83 @@ impl MatmulKernel {Ok(())}+ pub fn encode(+ &mut self,+ mtl: &MTLContext,+ enc: &ComputeCommandEncoderRef,+ mut args: MatmulArguments,+ ) -> Result<(), MTLError> {+ self.apply_batch_collapse(&mut args);++ // Try GEMV fast path for decode shapes.+ if self.maybe_use_gemv(mtl, enc, args)? {+ return Ok(());+ }++ // Try Split-K for small M*N and large K.+ if self.maybe_use_splitk(mtl, enc, args)? {+ return Ok(());+ }+ self.encode_gemm(mtl, enc, args)+ }++ pub fn encode_with_bias(+ &mut self,+ mtl: &MTLContext,+ enc: &ComputeCommandEncoderRef,+ mut args: MatmulArguments,+ bias: &MTLBuffer,+ ) -> Result<(), MTLError> {+ self.apply_batch_collapse(&mut args);++ if self.maybe_use_gemv_with_bias(mtl, enc, args, bias)? {+ return Ok(());+ }++ if self.maybe_use_splitk(mtl, enc, args)? {+ self.apply_bias_add(mtl, enc, args, bias)?;+ return Ok(());+ }++ self.encode_gemm(mtl, enc, args)?;+ self.apply_bias_add(mtl, enc, args, bias)?;+ Ok(())+ }++ fn apply_bias_add(+ &mut self,+ mtl: &MTLContext,+ enc: &ComputeCommandEncoderRef,+ args: MatmulArguments,+ bias: &MTLBuffer,+ ) -> Result<(), MTLError> {+ let m = args.batch as usize;+ let n = args.output_dim as usize;+ let batch_count = args.batch_count as usize;+ let total_len = m * n * batch_count;+ if total_len == 0 {+ return Ok(());+ }++ if self.bias_add.is_none() {+ self.bias_add = Some(TensorAddBias::new(+ mtl,+ KernelDataType::from(self.dt),+ )?);+ }+ let bias_add = self.bias_add.as_ref().unwrap();+ bias_add.encode_with_encoder(+ args.d,+ bias,+ args.d,+ n,+ total_len,+ enc,+ None,+ );+ Ok(())+ }+fn apply_batch_collapse(&self,args: &mut MatmulArguments,