Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

update gemv

Eugene Bokhan74f2c5d68f56
diff --git a/crates/uzu/examples/matmul_benchmark.rs b/crates/uzu/examples/matmul_benchmark.rs
index 4a9d38f9d..4eb752a5c 100644
--- a/crates/uzu/examples/matmul_benchmark.rs
+++ b/crates/uzu/examples/matmul_benchmark.rs
@@ -134,6 +134,96 @@ fn benchmark_metal_gemm(
(elapsed.as_secs_f64() * 1000.0) / bench_iters as f64
}
+fn benchmark_metal_gemv(
+ ctx: &MTLContext,
+ m: usize,
+ k: usize,
+ n: usize,
+ warmup_iters: usize,
+ bench_iters: usize,
+) -> f64 {
+ // GEMV requires transpose_b = true (B stored as [N, K]).
+ let a: Vec<bf16> = (0..(m * k))
+ .map(|i| bf16::from_f32(((i % 13) as f32) * 0.01))
+ .collect();
+ let b: Vec<bf16> = (0..(n * k))
+ .map(|i| bf16::from_f32(((i % 17) as f32) * 0.02 - 0.1))
+ .collect();
+
+ let a_buf = ctx.device.new_buffer_with_data(
+ a.as_ptr() as *const _,
+ (a.len() * core::mem::size_of::<bf16>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+ let b_buf = ctx.device.new_buffer_with_data(
+ b.as_ptr() as *const _,
+ (b.len() * core::mem::size_of::<bf16>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+ let d_buf = ctx.device.new_buffer(
+ (m * n * core::mem::size_of::<bf16>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+
+ let mut kernel = MatmulKernel::new(ctx, DataType::BF16, false, true)
+ .expect("kernel new");
+
+ for _ in 0..warmup_iters {
+ let cb = ctx.command_queue.new_command_buffer().to_owned();
+ let enc = cb.new_compute_command_encoder();
+ kernel
+ .encode(
+ ctx,
+ &enc,
+ MatmulArguments {
+ a: &a_buf,
+ b: &b_buf,
+ d: &d_buf,
+ batch: m as i32,
+ input_dim: k as i32,
+ output_dim: n as i32,
+ lda: k as i32,
+ ldb: k as i32,
+ ldd: n as i32,
+ batch_count: 1,
+ },
+ )
+ .expect("encode");
+ enc.end_encoding();
+ cb.commit();
+ cb.wait_until_completed();
+ }
+
+ let start = Instant::now();
+ for _ in 0..bench_iters {
+ let cb = ctx.command_queue.new_command_buffer().to_owned();
+ let enc = cb.new_compute_command_encoder();
+ kernel
+ .encode(
+ ctx,
+ &enc,
+ MatmulArguments {
+ a: &a_buf,
+ b: &b_buf,
+ d: &d_buf,
+ batch: m as i32,
+ input_dim: k as i32,
+ output_dim: n as i32,
+ lda: k as i32,
+ ldb: k as i32,
+ ldd: n as i32,
+ batch_count: 1,
+ },
+ )
+ .expect("encode");
+ enc.end_encoding();
+ cb.commit();
+ cb.wait_until_completed();
+ }
+ let elapsed = start.elapsed();
+ (elapsed.as_secs_f64() * 1000.0) / bench_iters as f64
+}
+
fn benchmark_mpsgraph_matmul(
device: &metal::Device,
m: usize,
@@ -297,7 +387,11 @@ fn run_benchmark(
) -> BenchmarkResult {
print!("[Metal] ");
let _ = std::io::stdout().flush();
- let metal_ms = benchmark_metal_gemm(ctx, m, k, n, warmup, iters);
+ let metal_ms = if m <= 8 || n == 1 {
+ benchmark_metal_gemv(ctx, m, k, n, warmup, iters)
+ } else {
+ benchmark_metal_gemm(ctx, m, k, n, warmup, iters)
+ };
print!("{:.2}ms ", metal_ms);
let _ = std::io::stdout().flush();
diff --git a/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs b/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs
index c8bf57461..03ce41159 100644
--- a/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs
@@ -9,7 +9,6 @@ use crate::{
encodable_block::{EncodableBlock, EncodingParameters},
forward_pass::{ArrayId, ForwardPassState},
kernel::{
- KernelDataType, TensorAddBias,
matmul::{MatmulArguments, MatmulKernel},
},
},
@@ -19,7 +18,6 @@ use crate::{
pub struct FullPrecisionLinear {
kernel: RefCell<MatmulKernel>,
- bias_add_kernel: Option<TensorAddBias>,
biases_buffer: Option<MTLBuffer>,
weights_buffer: MTLBuffer,
input_dim: usize,
@@ -69,39 +67,33 @@ impl FullPrecisionLinear {
let weights_buffer: MTLBuffer =
unsafe { weights.mtl_buffer() }.to_owned();
- let (bias_add_kernel, biases_buffer) =
- match parameter_tree.leaf("biases") {
- Ok(mut biases) => {
- if biases.shape() != [output_dim] {
- return Err(MTLError::Generic(format!(
- "Bias shape mismatch: got {:?}, expected [{:?}]",
- biases.shape(),
- output_dim
- )));
- }
- if biases.data_type() != precision {
- return Err(MTLError::Generic(format!(
- "Bias dtype mismatch: got {:?}, expected {:?}",
- biases.data_type(),
- precision
- )));
- }
- let bias_add_kernel = Some(TensorAddBias::new(
- mtl_context,
- KernelDataType::from(precision),
- )?);
- let biases_buffer: MTLBuffer =
- unsafe { biases.mtl_buffer() }.to_owned();
- (bias_add_kernel, Some(biases_buffer))
- },
- Err(_) => (None, None),
- };
+ let biases_buffer = match parameter_tree.leaf("biases") {
+ Ok(mut biases) => {
+ if biases.shape() != [output_dim] {
+ return Err(MTLError::Generic(format!(
+ "Bias shape mismatch: got {:?}, expected [{:?}]",
+ biases.shape(),
+ output_dim
+ )));
+ }
+ if biases.data_type() != precision {
+ return Err(MTLError::Generic(format!(
+ "Bias dtype mismatch: got {:?}, expected {:?}",
+ biases.data_type(),
+ precision
+ )));
+ }
+ let biases_buffer: MTLBuffer =
+ unsafe { biases.mtl_buffer() }.to_owned();
+ Some(biases_buffer)
+ },
+ Err(_) => None,
+ };
let kernel = MatmulKernel::new(mtl_context, precision, false, true)?;
Ok(Self {
kernel: RefCell::new(kernel),
- bias_add_kernel,
biases_buffer,
weights_buffer,
input_dim,
@@ -143,26 +135,22 @@ impl EncodableBlock for FullPrecisionLinear {
batch_count: 1,
};
- self.kernel
- .borrow_mut()
- .encode(state.mtl_context(), encoder, args)
- .expect("Failed to encode matmul kernel");
+ let mut kernel = self.kernel.borrow_mut();
+ if let Some(bias_buf) = &self.biases_buffer {
+ kernel
+ .encode_with_bias(state.mtl_context(), encoder, args, bias_buf)
+ .expect("Failed to encode matmul kernel");
+ } else {
+ kernel
+ .encode(state.mtl_context(), encoder, args)
+ .expect("Failed to encode matmul kernel");
+ }
encoder.end_encoding();
- if let (Some(bias_add), Some(bias_buf)) =
- (&self.bias_add_kernel, &self.biases_buffer)
- {
- let total_len = batch_size * self.output_dim;
- bias_add.encode_into_command_buffer(
- &output_buffer,
- bias_buf,
- &output_buffer,
- self.output_dim,
- total_len,
- command_buffer,
- parameters.predicate,
- );
+ if parameters.wait_until_completed {
+ command_buffer.commit();
+ command_buffer.wait_until_completed();
}
}
@@ -174,7 +162,7 @@ impl EncodableBlock for FullPrecisionLinear {
&self,
state: &mut ForwardPassState,
encoder: &ComputeCommandEncoderRef,
- parameters: &EncodingParameters,
+ _parameters: &EncodingParameters,
) {
let arrays = state.arrays(&[self.input_array_id, self.output_array_id]);
let batch_size = state.active_suffix_length();
@@ -198,24 +186,15 @@ impl EncodableBlock for FullPrecisionLinear {
batch_count: 1,
};
- self.kernel
- .borrow_mut()
- .encode(state.mtl_context(), encoder, args)
- .expect("Failed to encode matmul kernel");
-
- if let (Some(bias_add), Some(bias_buf)) =
- (&self.bias_add_kernel, &self.biases_buffer)
- {
- let total_len = batch_size * self.output_dim;
- bias_add.encode_with_encoder(
- &output_buffer,
- bias_buf,
- &output_buffer,
- self.output_dim,
- total_len,
- encoder,
- parameters.predicate,
- );
+ let mut kernel = self.kernel.borrow_mut();
+ if let Some(bias_buf) = &self.biases_buffer {
+ kernel
+ .encode_with_bias(state.mtl_context(), encoder, args, bias_buf)
+ .expect("Failed to encode matmul kernel");
+ } else {
+ kernel
+ .encode(state.mtl_context(), encoder, args)
+ .expect("Failed to encode matmul kernel");
}
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemv.metal
index 70d310cef..c93b3510a 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv.metal
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv.metal
@@ -143,6 +143,51 @@ DECL_GEMV_FAST(gemv_fast_f16, half, float)
DECL_GEMV_FAST(gemv_fast_bf16, bfloat, float)
DECL_GEMV_FAST(gemv_fast_f32, float, float)
+#define DECL_GEMV_FAST_BIAS(NAME, TYPE, ACC) \
+ [[kernel]] void NAME( \
+ const device TYPE* weights [[buffer(0)]], \
+ const device TYPE* input [[buffer(1)]], \
+ const device TYPE* bias [[buffer(2)]], \
+ device TYPE* output [[buffer(3)]], \
+ constant int& input_dim [[buffer(4)]], \
+ constant int& output_dim [[buffer(5)]], \
+ constant int& weight_stride [[buffer(6)]], \
+ constant int& input_batch_stride [[buffer(7)]], \
+ constant int& output_batch_stride [[buffer(8)]], \
+ uint3 tgid [[threadgroup_position_in_grid]], \
+ uint simd_lid [[thread_index_in_simdgroup]] \
+ ) { \
+ constexpr int ROWS_PER_SIMDGROUP = 4; \
+ constexpr int ELEMS_PER_THREAD = 4; \
+ const uint row_base = tgid.x * ROWS_PER_SIMDGROUP; \
+ gemv_multirow<TYPE, ACC, ROWS_PER_SIMDGROUP, ELEMS_PER_THREAD>( \
+ weights, \
+ input, \
+ output, \
+ input_dim, \
+ output_dim, \
+ weight_stride, \
+ tgid.z, \
+ input_batch_stride, \
+ output_batch_stride, \
+ row_base, \
+ simd_lid \
+ ); \
+ if (simd_lid == 0 && row_base < static_cast<uint>(output_dim)) { \
+ const int rows_to_compute = \
+ min(ROWS_PER_SIMDGROUP, output_dim - static_cast<int>(row_base)); \
+ device TYPE* out_ptr = output + tgid.z * output_batch_stride; \
+ for (int r = 0; r < rows_to_compute; ++r) { \
+ const uint row = row_base + static_cast<uint>(r); \
+ out_ptr[row] = out_ptr[row] + bias[row]; \
+ } \
+ } \
+ }
+
+DECL_GEMV_FAST_BIAS(gemv_fast_f16_bias, half, float)
+DECL_GEMV_FAST_BIAS(gemv_fast_bf16_bias, bfloat, float)
+DECL_GEMV_FAST_BIAS(gemv_fast_f32_bias, float, float)
+
template <typename T, typename AccT, ushort ROWS_PER_THREADGROUP, ushort TILE_K>
METAL_FUNC void gemv_tiled(
const device T* __restrict weights,
@@ -235,10 +280,321 @@ METAL_FUNC void gemv_tiled(
); \
}
+#define DECL_GEMV_TILED_BIAS(NAME, TYPE, ACC, ROWS, TILE_K) \
+ [[kernel, max_total_threads_per_threadgroup(ROWS * 32)]] void NAME( \
+ const device TYPE* weights [[buffer(0)]], \
+ const device TYPE* input [[buffer(1)]], \
+ const device TYPE* bias [[buffer(2)]], \
+ device TYPE* output [[buffer(3)]], \
+ constant uint& input_dim [[buffer(4)]], \
+ constant uint& output_dim [[buffer(5)]], \
+ constant uint& weight_stride [[buffer(6)]], \
+ constant uint& input_stride [[buffer(7)]], \
+ constant uint& output_stride [[buffer(8)]], \
+ uint3 tgid [[threadgroup_position_in_grid]], \
+ uint simd_gid [[simdgroup_index_in_threadgroup]], \
+ uint simd_lid [[thread_index_in_simdgroup]], \
+ uint tid [[thread_index_in_threadgroup]] \
+ ) { \
+ threadgroup ACC input_tile[TILE_K]; \
+ gemv_tiled<TYPE, ACC, ROWS, TILE_K>( \
+ weights, \
+ input, \
+ output, \
+ input_dim, \
+ output_dim, \
+ weight_stride, \
+ input_stride, \
+ output_stride, \
+ tgid, \
+ simd_gid, \
+ simd_lid, \
+ tid, \
+ input_tile \
+ ); \
+ const uint row = tgid.x * ROWS + simd_gid; \
+ if (simd_lid == 0 && simd_gid < ROWS && row < output_dim) { \
+ device TYPE* out_ptr = output + tgid.z * output_stride; \
+ out_ptr[row] = out_ptr[row] + bias[row]; \
+ } \
+ }
+
+DECL_GEMV_TILED_BIAS(gemv_f16_rows2_bias, half, float, 2, 128)
+DECL_GEMV_TILED_BIAS(gemv_f16_rows4_bias, half, float, 4, 128)
+DECL_GEMV_TILED_BIAS(gemv_f16_rows8_bias, half, float, 8, 128)
+DECL_GEMV_TILED_BIAS(gemv_f16_rows16_bias, half, float, 16, 128)
+
+DECL_GEMV_TILED_BIAS(gemv_bf16_rows2_bias, bfloat, float, 2, 128)
+DECL_GEMV_TILED_BIAS(gemv_bf16_rows4_bias, bfloat, float, 4, 128)
+DECL_GEMV_TILED_BIAS(gemv_bf16_rows8_bias, bfloat, float, 8, 128)
+DECL_GEMV_TILED_BIAS(gemv_bf16_rows16_bias, bfloat, float, 16, 128)
+
+DECL_GEMV_TILED_BIAS(gemv_f32_rows2_bias, float, float, 2, 128)
+DECL_GEMV_TILED_BIAS(gemv_f32_rows4_bias, float, float, 4, 128)
+DECL_GEMV_TILED_BIAS(gemv_f32_rows8_bias, float, float, 8, 128)
+DECL_GEMV_TILED_BIAS(gemv_f32_rows16_bias, float, float, 16, 128)
+
DECL_GEMV_TILED(gemv_f16_rows2, half, float, 2, 128)
DECL_GEMV_TILED(gemv_f16_rows4, half, float, 4, 128)
DECL_GEMV_TILED(gemv_f16_rows8, half, float, 8, 128)
+DECL_GEMV_TILED(gemv_f16_rows16, half, float, 16, 128)
DECL_GEMV_TILED(gemv_bf16_rows2, bfloat, float, 2, 128)
DECL_GEMV_TILED(gemv_bf16_rows4, bfloat, float, 4, 128)
DECL_GEMV_TILED(gemv_bf16_rows8, bfloat, float, 8, 128)
+DECL_GEMV_TILED(gemv_bf16_rows16, bfloat, float, 16, 128)
+
+DECL_GEMV_TILED(gemv_f32_rows2, float, float, 2, 128)
+DECL_GEMV_TILED(gemv_f32_rows4, float, float, 4, 128)
+DECL_GEMV_TILED(gemv_f32_rows8, float, float, 8, 128)
+DECL_GEMV_TILED(gemv_f32_rows16, float, float, 16, 128)
+
+template <typename T, typename AccT, ushort KPARTS>
+METAL_FUNC void gemv_split_k_rows16_impl(
+ const device T* __restrict weights,
+ const device T* __restrict input,
+ const device T* __restrict bias,
+ device T* __restrict output,
+ const int input_dim,
+ const int output_dim,
+ const int weight_stride,
+ const int input_batch_stride,
+ const int output_batch_stride,
+ uint3 tgid,
+ uint simd_gid,
+ uint simd_lid,
+ threadgroup AccT* partial_sums
+) {
+ constexpr ushort rows_per_threadgroup = 16;
+ constexpr ushort rows_per_rowgroup = 2;
+ constexpr ushort rowgroups = rows_per_threadgroup / rows_per_rowgroup;
+ constexpr ushort elems_per_thread = 4;
+ constexpr int block_k = 32 * elems_per_thread;
+
+ const uint rowgroup_idx = simd_gid / KPARTS;
+ const uint kpart_idx = simd_gid - rowgroup_idx * KPARTS;
+
+ const uint batch = tgid.z;
+ const uint row_base =
+ tgid.x * rows_per_threadgroup + rowgroup_idx * rows_per_rowgroup;
+
+ const bool valid_rowgroup = row_base < static_cast<uint>(output_dim);
+ const uint rows_to_compute = valid_rowgroup
+ ? min(static_cast<uint>(rows_per_rowgroup),
+ static_cast<uint>(output_dim) - row_base)
+ : 0;
+
+ const device T* input_ptr = input + batch * input_batch_stride;
+ device T* output_ptr = output + batch * output_batch_stride;
+
+ AccT accumulators[rows_per_rowgroup] = {AccT(0), AccT(0)};
+
+ if (valid_rowgroup) {
+ const device T* weight_row_ptrs[rows_per_rowgroup];
+ weight_row_ptrs[0] = weights + (row_base + 0) * weight_stride;
+ weight_row_ptrs[1] = weights + (row_base + 1) * weight_stride;
+
+ for (int k_block = static_cast<int>(kpart_idx) * block_k;
+ k_block < input_dim;
+ k_block += block_k * static_cast<int>(KPARTS))
+ {
+ AccT input_cache[elems_per_thread];
+
+#pragma unroll(elems_per_thread)
+ for (int e = 0; e < elems_per_thread; ++e) {
+ const int col = k_block + static_cast<int>(simd_lid) * elems_per_thread + e;
+ input_cache[e] =
+ (col < input_dim) ? static_cast<AccT>(input_ptr[col]) : AccT(0);
+ }
+
+#pragma unroll(rows_per_rowgroup)
+ for (int r = 0; r < rows_per_rowgroup; ++r) {
+ AccT dot = 0;
+#pragma unroll(elems_per_thread)
+ for (int e = 0; e < elems_per_thread; ++e) {
+ const int col =
+ k_block + static_cast<int>(simd_lid) * elems_per_thread + e;
+ if (col < input_dim) {
+ dot += input_cache[e] * static_cast<AccT>(weight_row_ptrs[r][col]);
+ }
+ }
+ accumulators[r] += dot;
+ }
+ }
+ }
+
+#pragma unroll(rows_per_rowgroup)
+ for (int r = 0; r < rows_per_rowgroup; ++r) {
+ accumulators[r] = simd_sum(accumulators[r]);
+ }
+
+ const uint partial_base =
+ rowgroup_idx * (KPARTS * rows_per_rowgroup) + kpart_idx * rows_per_rowgroup;
+
+ if (simd_lid == 0) {
+ partial_sums[partial_base + 0] = accumulators[0];
+ partial_sums[partial_base + 1] = accumulators[1];
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ if (kpart_idx == 0 && simd_lid == 0 && rows_to_compute > 0) {
+ for (uint r = 0; r < rows_to_compute; ++r) {
+ AccT sum = 0;
+ for (uint kp = 0; kp < KPARTS; ++kp) {
+ sum += partial_sums[rowgroup_idx * (KPARTS * rows_per_rowgroup) +
+ kp * rows_per_rowgroup + r];
+ }
+ const uint row = row_base + r;
+ if (bias != nullptr) {
+ sum += static_cast<AccT>(bias[row]);
+ }
+ output_ptr[row] = static_cast<T>(sum);
+ }
+ }
+}
+
+#define DECL_GEMV_SPLIT_K_KPARTS2(NAME, TYPE, ACC) \
+ [[kernel, max_total_threads_per_threadgroup(512)]] void NAME( \
+ const device TYPE* weights [[buffer(0)]], \
+ const device TYPE* input [[buffer(1)]], \
+ device TYPE* output [[buffer(2)]], \
+ constant int& input_dim [[buffer(3)]], \
+ constant int& output_dim [[buffer(4)]], \
+ constant int& weight_stride [[buffer(5)]], \
+ constant int& input_batch_stride [[buffer(6)]], \
+ constant int& output_batch_stride [[buffer(7)]], \
+ uint3 tgid [[threadgroup_position_in_grid]], \
+ uint simd_gid [[simdgroup_index_in_threadgroup]], \
+ uint simd_lid [[thread_index_in_simdgroup]] \
+ ) { \
+ threadgroup ACC partial_sums[8 * 2 * 2]; \
+ gemv_split_k_rows16_impl<TYPE, ACC, 2>( \
+ weights, \
+ input, \
+ nullptr, \
+ output, \
+ input_dim, \
+ output_dim, \
+ weight_stride, \
+ input_batch_stride, \
+ output_batch_stride, \
+ tgid, \
+ simd_gid, \
+ simd_lid, \
+ partial_sums \
+ ); \
+ }
+
+#define DECL_GEMV_SPLIT_K_KPARTS4(NAME, TYPE, ACC) \
+ [[kernel, max_total_threads_per_threadgroup(1024)]] void NAME( \
+ const device TYPE* weights [[buffer(0)]], \
+ const device TYPE* input [[buffer(1)]], \
+ device TYPE* output [[buffer(2)]], \
+ constant int& input_dim [[buffer(3)]], \
+ constant int& output_dim [[buffer(4)]], \
+ constant int& weight_stride [[buffer(5)]], \
+ constant int& input_batch_stride [[buffer(6)]], \
+ constant int& output_batch_stride [[buffer(7)]], \
+ uint3 tgid [[threadgroup_position_in_grid]], \
+ uint simd_gid [[simdgroup_index_in_threadgroup]], \
+ uint simd_lid [[thread_index_in_simdgroup]] \
+ ) { \
+ threadgroup ACC partial_sums[8 * 4 * 2]; \
+ gemv_split_k_rows16_impl<TYPE, ACC, 4>( \
+ weights, \
+ input, \
+ nullptr, \
+ output, \
+ input_dim, \
+ output_dim, \
+ weight_stride, \
+ input_batch_stride, \
+ output_batch_stride, \
+ tgid, \
+ simd_gid, \
+ simd_lid, \
+ partial_sums \
+ ); \
+ }
+
+#define DECL_GEMV_SPLIT_K_KPARTS2_BIAS(NAME, TYPE, ACC) \
+ [[kernel, max_total_threads_per_threadgroup(512)]] void NAME( \
+ const device TYPE* weights [[buffer(0)]], \
+ const device TYPE* input [[buffer(1)]], \
+ const device TYPE* bias [[buffer(2)]], \
+ device TYPE* output [[buffer(3)]], \
+ constant int& input_dim [[buffer(4)]], \
+ constant int& output_dim [[buffer(5)]], \
+ constant int& weight_stride [[buffer(6)]], \
+ constant int& input_batch_stride [[buffer(7)]], \
+ constant int& output_batch_stride [[buffer(8)]], \
+ uint3 tgid [[threadgroup_position_in_grid]], \
+ uint simd_gid [[simdgroup_index_in_threadgroup]], \
+ uint simd_lid [[thread_index_in_simdgroup]] \
+ ) { \
+ threadgroup ACC partial_sums[8 * 2 * 2]; \
+ gemv_split_k_rows16_impl<TYPE, ACC, 2>( \
+ weights, \
+ input, \
+ bias, \
+ output, \
+ input_dim, \
+ output_dim, \
+ weight_stride, \
+ input_batch_stride, \
+ output_batch_stride, \
+ tgid, \
+ simd_gid, \
+ simd_lid, \
+ partial_sums \
+ ); \
+ }
+
+#define DECL_GEMV_SPLIT_K_KPARTS4_BIAS(NAME, TYPE, ACC) \
+ [[kernel, max_total_threads_per_threadgroup(1024)]] void NAME( \
+ const device TYPE* weights [[buffer(0)]], \
+ const device TYPE* input [[buffer(1)]], \
+ const device TYPE* bias [[buffer(2)]], \
+ device TYPE* output [[buffer(3)]], \
+ constant int& input_dim [[buffer(4)]], \
+ constant int& output_dim [[buffer(5)]], \
+ constant int& weight_stride [[buffer(6)]], \
+ constant int& input_batch_stride [[buffer(7)]], \
+ constant int& output_batch_stride [[buffer(8)]], \
+ uint3 tgid [[threadgroup_position_in_grid]], \
+ uint simd_gid [[simdgroup_index_in_threadgroup]], \
+ uint simd_lid [[thread_index_in_simdgroup]] \
+ ) { \
+ threadgroup ACC partial_sums[8 * 4 * 2]; \
+ gemv_split_k_rows16_impl<TYPE, ACC, 4>( \
+ weights, \
+ input, \
+ bias, \
+ output, \
+ input_dim, \
+ output_dim, \
+ weight_stride, \
+ input_batch_stride, \
+ output_batch_stride, \
+ tgid, \
+ simd_gid, \
+ simd_lid, \
+ partial_sums \
+ ); \
+ }
+
+DECL_GEMV_SPLIT_K_KPARTS2(gemv_split_k_f16_rows16_kparts2, half, float)
+DECL_GEMV_SPLIT_K_KPARTS4(gemv_split_k_f16_rows16_kparts4, half, float)
+DECL_GEMV_SPLIT_K_KPARTS2_BIAS(gemv_split_k_f16_rows16_kparts2_bias, half, float)
+DECL_GEMV_SPLIT_K_KPARTS4_BIAS(gemv_split_k_f16_rows16_kparts4_bias, half, float)
+
+DECL_GEMV_SPLIT_K_KPARTS2(gemv_split_k_bf16_rows16_kparts2, bfloat, float)
+DECL_GEMV_SPLIT_K_KPARTS4(gemv_split_k_bf16_rows16_kparts4, bfloat, float)
+DECL_GEMV_SPLIT_K_KPARTS2_BIAS(gemv_split_k_bf16_rows16_kparts2_bias, bfloat, float)
+DECL_GEMV_SPLIT_K_KPARTS4_BIAS(gemv_split_k_bf16_rows16_kparts4_bias, bfloat, float)
+
+DECL_GEMV_SPLIT_K_KPARTS2(gemv_split_k_f32_rows16_kparts2, float, float)
+DECL_GEMV_SPLIT_K_KPARTS4(gemv_split_k_f32_rows16_kparts4, float, float)
+DECL_GEMV_SPLIT_K_KPARTS2_BIAS(gemv_split_k_f32_rows16_kparts2_bias, float, float)
+DECL_GEMV_SPLIT_K_KPARTS4_BIAS(gemv_split_k_f32_rows16_kparts4_bias, float, float)
\ No newline at end of file
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv.rs
index ad43ee52a..e7a0f28f7 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv.rs
@@ -1,6 +1,7 @@
use std::collections::HashMap;
use metal::{
+ Buffer as MTLBuffer,
ComputeCommandEncoderRef, ComputePipelineState as MTLComputePipelineState,
MTLSize,
};
@@ -13,10 +14,18 @@ use crate::{
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
enum PipelineVariant {
+ Fast {
+ has_bias: bool,
+ },
Tiled {
rows_per_threadgroup: u32,
+ has_bias: bool,
+ },
+ SplitK {
+ rows_per_threadgroup: u32,
+ kparts: u32,
+ has_bias: bool,
},
- Fast,
}
pub struct GemvKernel {
@@ -32,25 +41,37 @@ impl GemvKernel {
}
}
- fn fast_kernel_name(&self) -> Result<String, MTLError> {
- match self.data_type {
- DataType::F16 => Ok("gemv_fast_f16".to_string()),
- DataType::BF16 => Ok("gemv_fast_bf16".to_string()),
- DataType::F32 => Ok("gemv_fast_f32".to_string()),
- _ => Err(MTLError::Generic(format!(
- "Unsupported data type for fast GEMV: {:?}",
- self.data_type
- ))),
- }
+ fn fast_kernel_name(
+ &self,
+ has_bias: bool,
+ ) -> Result<String, MTLError> {
+ let base = match self.data_type {
+ DataType::F16 => "gemv_fast_f16",
+ DataType::BF16 => "gemv_fast_bf16",
+ DataType::F32 => "gemv_fast_f32",
+ _ => {
+ return Err(MTLError::Generic(format!(
+ "Unsupported data type for fast GEMV: {:?}",
+ self.data_type
+ )));
+ },
+ };
+ Ok(if has_bias {
+ format!("{base}_bias")
+ } else {
+ base.to_string()
+ })
}
fn tiled_kernel_name(
&self,
rows_per_threadgroup: u32,
+ has_bias: bool,
) -> Result<String, MTLError> {
let prefix = match self.data_type {
DataType::F16 => "gemv_f16",
DataType::BF16 => "gemv_bf16",
+ DataType::F32 => "gemv_f32",
_ => {
return Err(MTLError::Generic(format!(
"Unsupported data type for tiled GEMV: {:?}",
@@ -58,16 +79,52 @@ impl GemvKernel {
)));
},
};
- Ok(format!("{prefix}_rows{rows_per_threadgroup}"))
+ Ok(if has_bias {
+ format!("{prefix}_rows{rows_per_threadgroup}_bias")
+ } else {
+ format!("{prefix}_rows{rows_per_threadgroup}")
+ })
+ }
+
+ fn split_k_kernel_name(
+ &self,
+ kparts: u32,
+ has_bias: bool,
+ ) -> Result<String, MTLError> {
+ let dtype = match self.data_type {
+ DataType::F16 => "f16",
+ DataType::BF16 => "bf16",
+ DataType::F32 => "f32",
+ _ => {
+ return Err(MTLError::Generic(format!(
+ "Unsupported data type for split-k GEMV: {:?}",
+ self.data_type
+ )));
+ },
+ };
+ if !matches!(kparts, 2 | 4) {
+ return Err(MTLError::Generic(format!(
+ "Unsupported split-k kparts={kparts} (expected 2 or 4)"
+ )));
+ }
+ let base = format!("gemv_split_k_{dtype}_rows16_kparts{kparts}");
+ Ok(if has_bias {
+ format!("{base}_bias")
+ } else {
+ base
+ })
}
fn get_fast_pipeline(
&mut self,
context: &MTLContext,
+ has_bias: bool,
) -> Result<&MTLComputePipelineState, MTLError> {
- let variant = PipelineVariant::Fast;
+ let variant = PipelineVariant::Fast {
+ has_bias,
+ };
if !self.pipelines.contains_key(&variant) {
- let kernel_name = self.fast_kernel_name()?;
+ let kernel_name = self.fast_kernel_name(has_bias)?;
let pipeline =
context.compute_pipeline_state(&kernel_name, None)?;
self.pipelines.insert(variant, pipeline);
@@ -79,12 +136,35 @@ impl GemvKernel {
&mut self,
context: &MTLContext,
rows_per_threadgroup: u32,
+ has_bias: bool,
) -> Result<&MTLComputePipelineState, MTLError> {
let variant = PipelineVariant::Tiled {
rows_per_threadgroup,
+ has_bias,
+ };
+ if !self.pipelines.contains_key(&variant) {
+ let kernel_name =
+ self.tiled_kernel_name(rows_per_threadgroup, has_bias)?;
+ let pipeline =
+ context.compute_pipeline_state(&kernel_name, None)?;
+ self.pipelines.insert(variant, pipeline);
+ }
+ Ok(self.pipelines.get(&variant).unwrap())
+ }
+
+ fn get_split_k_pipeline(
+ &mut self,
+ context: &MTLContext,
+ kparts: u32,
+ has_bias: bool,
+ ) -> Result<&MTLComputePipelineState, MTLError> {
+ let variant = PipelineVariant::SplitK {
+ rows_per_threadgroup: 16,
+ kparts,
+ has_bias,
};
if !self.pipelines.contains_key(&variant) {
- let kernel_name = self.tiled_kernel_name(rows_per_threadgroup)?;
+ let kernel_name = self.split_k_kernel_name(kparts, has_bias)?;
let pipeline =
context.compute_pipeline_state(&kernel_name, None)?;
self.pipelines.insert(variant, pipeline);
@@ -99,18 +179,49 @@ impl GemvKernel {
args.input_dim >= 128 && args.input_dim % 4 == 0
}
+ fn split_k_kparts(
+ &self,
+ args: &MatmulArguments,
+ ) -> Option<u32> {
+ let k = args.input_dim;
+ let n = args.output_dim;
+ if k >= 8192 && n >= 2048 {
+ return Some(4);
+ }
+ if k >= 4096 && n >= 2048 {
+ return Some(2);
+ }
+ None
+ }
+
+ fn is_logits_like(
+ &self,
+ args: &MatmulArguments,
+ ) -> bool {
+ let k = args.input_dim;
+ let n = args.output_dim;
+ n >= 50_000 || (n >= 16_384 && n >= k * 7)
+ }
+
fn encode_fast(
&mut self,
context: &MTLContext,
encoder: &ComputeCommandEncoderRef,
args: MatmulArguments,
+ bias: Option<&MTLBuffer>,
) -> Result<(), MTLError> {
- let pipeline = self.get_fast_pipeline(context)?;
+ let has_bias = bias.is_some();
+ let pipeline = self.get_fast_pipeline(context, has_bias)?;
encoder.set_compute_pipeline_state(pipeline);
encoder.set_buffer(0, Some(args.b), 0);
encoder.set_buffer(1, Some(args.a), 0);
- encoder.set_buffer(2, Some(args.d), 0);
+ if let Some(bias) = bias {
+ encoder.set_buffer(2, Some(bias), 0);
+ encoder.set_buffer(3, Some(args.d), 0);
+ } else {
+ encoder.set_buffer(2, Some(args.d), 0);
+ }
let input_dim = args.input_dim;
let output_dim = args.output_dim;
@@ -118,28 +229,29 @@ impl GemvKernel {
let input_batch_stride = args.lda;
let output_batch_stride = args.ldd;
+ let base = if has_bias { 4 } else { 3 };
encoder.set_bytes(
- 3,
+ base,
std::mem::size_of::<i32>() as u64,
&input_dim as *const i32 as *const std::ffi::c_void,
);
encoder.set_bytes(
- 4,
+ base + 1,
std::mem::size_of::<i32>() as u64,
&output_dim as *const i32 as *const std::ffi::c_void,
);
encoder.set_bytes(
- 5,
+ base + 2,
std::mem::size_of::<i32>() as u64,
&weight_stride as *const i32 as *const std::ffi::c_void,
);
encoder.set_bytes(
- 6,
+ base + 3,
std::mem::size_of::<i32>() as u64,
&input_batch_stride as *const i32 as *const std::ffi::c_void,
);
encoder.set_bytes(
- 7,
+ base + 4,
std::mem::size_of::<i32>() as u64,
&output_batch_stride as *const i32 as *const std::ffi::c_void,
);
@@ -164,15 +276,25 @@ impl GemvKernel {
encoder: &ComputeCommandEncoderRef,
args: MatmulArguments,
rows_per_threadgroup: u32,
+ bias: Option<&MTLBuffer>,
) -> Result<(), MTLError> {
let rows_per_threadgroup = rows_per_threadgroup.max(1);
- let pipeline =
- self.get_tiled_pipeline(context, rows_per_threadgroup)?;
+ let has_bias = bias.is_some();
+ let pipeline = self.get_tiled_pipeline(
+ context,
+ rows_per_threadgroup,
+ has_bias,
+ )?;
encoder.set_compute_pipeline_state(pipeline);
encoder.set_buffer(0, Some(args.b), 0);
encoder.set_buffer(1, Some(args.a), 0);
- encoder.set_buffer(2, Some(args.d), 0);
+ if let Some(bias) = bias {
+ encoder.set_buffer(2, Some(bias), 0);
+ encoder.set_buffer(3, Some(args.d), 0);
+ } else {
+ encoder.set_buffer(2, Some(args.d), 0);
+ }
let input_dim = args.input_dim as u32;
let output_dim = args.output_dim as u32;
@@ -180,28 +302,29 @@ impl GemvKernel {
let input_stride = args.lda as u32;
let output_stride = args.ldd as u32;
+ let base = if has_bias { 4 } else { 3 };
encoder.set_bytes(
- 3,
+ base,
std::mem::size_of::<u32>() as u64,
&input_dim as *const u32 as *const std::ffi::c_void,
);
encoder.set_bytes(
- 4,
+ base + 1,
std::mem::size_of::<u32>() as u64,
&output_dim as *const u32 as *const std::ffi::c_void,
);
encoder.set_bytes(
- 5,
+ base + 2,
std::mem::size_of::<u32>() as u64,
&weight_stride as *const u32 as *const std::ffi::c_void,
);
encoder.set_bytes(
- 6,
+ base + 3,
std::mem::size_of::<u32>() as u64,
&input_stride as *const u32 as *const std::ffi::c_void,
);
encoder.set_bytes(
- 7,
+ base + 4,
std::mem::size_of::<u32>() as u64,
&output_stride as *const u32 as *const std::ffi::c_void,
);
@@ -219,17 +342,141 @@ impl GemvKernel {
Ok(())
}
- pub fn encode(
+ fn encode_split_k(
+ &mut self,
+ context: &MTLContext,
+ encoder: &ComputeCommandEncoderRef,
+ args: MatmulArguments,
+ kparts: u32,
+ bias: Option<&MTLBuffer>,
+ ) -> Result<(), MTLError> {
+ let has_bias = bias.is_some();
+ let pipeline = self.get_split_k_pipeline(context, kparts, has_bias)?;
+ encoder.set_compute_pipeline_state(pipeline);
+
+ encoder.set_buffer(0, Some(args.b), 0);
+ encoder.set_buffer(1, Some(args.a), 0);
+ if let Some(bias) = bias {
+ encoder.set_buffer(2, Some(bias), 0);
+ encoder.set_buffer(3, Some(args.d), 0);
+ } else {
+ encoder.set_buffer(2, Some(args.d), 0);
+ }
+
+ let input_dim = args.input_dim;
+ let output_dim = args.output_dim;
+ let weight_stride = args.ldb;
+ let input_batch_stride = args.lda;
+ let output_batch_stride = args.ldd;
+
+ let base = if has_bias { 4 } else { 3 };
+ encoder.set_bytes(
+ base,
+ std::mem::size_of::<i32>() as u64,
+ &input_dim as *const i32 as *const std::ffi::c_void,
+ );
+ encoder.set_bytes(
+ base + 1,
+ std::mem::size_of::<i32>() as u64,
+ &output_dim as *const i32 as *const std::ffi::c_void,
+ );
+ encoder.set_bytes(
+ base + 2,
+ std::mem::size_of::<i32>() as u64,
+ &weight_stride as *const i32 as *const std::ffi::c_void,
+ );
+ encoder.set_bytes(
+ base + 3,
+ std::mem::size_of::<i32>() as u64,
+ &input_batch_stride as *const i32 as *const std::ffi::c_void,
+ );
+ encoder.set_bytes(
+ base + 4,
+ std::mem::size_of::<i32>() as u64,
+ &output_batch_stride as *const i32 as *const std::ffi::c_void,
+ );
+
+ const ROWS_PER_THREADGROUP: i32 = 16;
+ let threadgroup_count_x = ((output_dim + ROWS_PER_THREADGROUP - 1)
+ / ROWS_PER_THREADGROUP) as u64;
+ let threadgroup_count_z = args.batch_count as u64;
+
+ let threads_per_threadgroup_x = match kparts {
+ 2 => 512,
+ 4 => 1024,
+ _ => {
+ return Err(MTLError::Generic(format!(
+ "Unsupported split-k kparts={kparts} (expected 2 or 4)"
+ )));
+ },
+ };
+
+ let threadgroup_count =
+ MTLSize::new(threadgroup_count_x, 1, threadgroup_count_z);
+ let threads_per_threadgroup =
+ MTLSize::new(threads_per_threadgroup_x, 1, 1);
+
+ encoder
+ .dispatch_thread_groups(threadgroup_count, threads_per_threadgroup);
+ Ok(())
+ }
+
+ fn encode_inner(
&mut self,
context: &MTLContext,
encoder: &ComputeCommandEncoderRef,
args: MatmulArguments,
rows_per_threadgroup: u32,
+ bias: Option<&MTLBuffer>,
) -> Result<(), MTLError> {
+ if self.is_logits_like(&args) {
+ if args.input_dim >= 8192 {
+ if let Some(kparts) = self.split_k_kparts(&args) {
+ return self.encode_split_k(
+ context,
+ encoder,
+ args,
+ kparts,
+ bias,
+ );
+ }
+ }
+ return self.encode_tiled(context, encoder, args, 16, bias);
+ }
+ if let Some(kparts) = self.split_k_kparts(&args) {
+ return self.encode_split_k(context, encoder, args, kparts, bias);
+ }
if self.can_use_fast_kernel(&args) {
- self.encode_fast(context, encoder, args)
+ self.encode_fast(context, encoder, args, bias)
} else {
- self.encode_tiled(context, encoder, args, rows_per_threadgroup)
+ self.encode_tiled(context, encoder, args, rows_per_threadgroup, bias)
}
}
+
+ pub fn encode(
+ &mut self,
+ context: &MTLContext,
+ encoder: &ComputeCommandEncoderRef,
+ args: MatmulArguments,
+ rows_per_threadgroup: u32,
+ ) -> Result<(), MTLError> {
+ self.encode_inner(context, encoder, args, rows_per_threadgroup, None)
+ }
+
+ pub fn encode_with_bias(
+ &mut self,
+ context: &MTLContext,
+ encoder: &ComputeCommandEncoderRef,
+ args: MatmulArguments,
+ rows_per_threadgroup: u32,
+ bias: &MTLBuffer,
+ ) -> Result<(), MTLError> {
+ self.encode_inner(
+ context,
+ encoder,
+ args,
+ rows_per_threadgroup,
+ Some(bias),
+ )
+ }
}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv_masked.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemv_masked.metal
deleted file mode 100644
index 622969fde..000000000
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv_masked.metal
+++ /dev/null
@@ -1,177 +0,0 @@
-// Masked and gather GEMV variants. Masks/indices are honored only for output
-// write; data loads remain dense.
-
-#include <metal_stdlib>
-using namespace metal;
-
-template <typename T, typename AccT, ushort ROWS>
-METAL_FUNC void gemv_masked_impl(
- const device T* __restrict matrix [[buffer(0)]],
- const device T* __restrict vector [[buffer(1)]],
- const device uchar* __restrict out_mask [[buffer(2)]],
- const device uchar* __restrict op_mask [[buffer(3)]],
- device T* __restrict output [[buffer(4)]],
- constant uint& k [[buffer(5)]],
- constant uint& n [[buffer(6)]],
- constant uint& ldb [[buffer(7)]],
- constant uint& lda [[buffer(8)]],
- constant uint& ldd [[buffer(9)]],
- uint3 tgid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-) {
- if (simd_gid >= ROWS) {
- return;
- }
- const uint row = tgid.x * ROWS + simd_gid;
- if (row >= n) {
- return;
- }
- const uint batch = tgid.z;
- const device T* row_ptr = matrix + row * ldb;
- const device T* vec_ptr = vector + batch * lda;
- device T* out_ptr = output + batch * ldd;
-
- AccT acc = static_cast<AccT>(0);
- for (uint idx = simd_lid; idx < k; idx += 32) {
- // If op_mask provided, zero out masked elements
- const bool op_on = (op_mask == nullptr) || (op_mask[row] != 0);
- if (op_on) {
- acc += static_cast<AccT>(row_ptr[idx]) * static_cast<AccT>(vec_ptr[idx]);
- }
- }
-
- for (ushort offset = 16; offset > 0; offset >>= 1) {
- acc += simd_shuffle_down(acc, offset);
- }
-
- if (simd_lid == 0) {
- if (out_mask == nullptr || out_mask[row] != 0) {
- out_ptr[row] = static_cast<T>(acc);
- }
- }
-}
-
-template <typename T, typename AccT, ushort ROWS>
-METAL_FUNC void gemv_gather_impl(
- const device T* __restrict matrix [[buffer(0)]],
- const device T* __restrict vector [[buffer(1)]],
- const device uint* __restrict vec_indices [[buffer(2)]],
- const device uint* __restrict mat_indices [[buffer(3)]],
- device T* __restrict output [[buffer(4)]],
- constant uint& k [[buffer(5)]],
- constant uint& n [[buffer(6)]],
- constant uint& ldb [[buffer(7)]],
- constant uint& lda [[buffer(8)]],
- constant uint& ldd [[buffer(9)]],
- uint3 tgid [[threadgroup_position_in_grid]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-) {
- if (simd_gid >= ROWS) {
- return;
- }
- const uint row = tgid.x * ROWS + simd_gid;
- if (row >= n) {
- return;
- }
- const uint batch = tgid.z;
- const device T* base_row = matrix + row * ldb;
- device T* out_ptr = output + batch * ldd;
-
- AccT acc = static_cast<AccT>(0);
- for (uint idx = simd_lid; idx < k; idx += 32) {
- const uint col = mat_indices ? mat_indices[idx] : idx;
- const uint vec_idx = vec_indices ? vec_indices[idx] : idx;
- acc += static_cast<AccT>(base_row[col]) *
- static_cast<AccT>(vector[batch * lda + vec_idx]);
- }
-
- for (ushort offset = 16; offset > 0; offset >>= 1) {
- acc += simd_shuffle_down(acc, offset);
- }
-
- if (simd_lid == 0) {
- out_ptr[row] = static_cast<T>(acc);
- }
-}
-
-#define DECL_GEMV_MASKED(NAME, TYPE, ACC, ROWS) \
- [[kernel, max_total_threads_per_threadgroup(ROWS * 32)]] void NAME( \
- const device TYPE* matrix [[buffer(0)]], \
- const device TYPE* vector [[buffer(1)]], \
- const device uchar* out_mask [[buffer(2)]], \
- const device uchar* op_mask [[buffer(3)]], \
- device TYPE* output [[buffer(4)]], \
- constant uint& k [[buffer(5)]], \
- constant uint& n [[buffer(6)]], \
- constant uint& ldb [[buffer(7)]], \
- constant uint& lda [[buffer(8)]], \
- constant uint& ldd [[buffer(9)]], \
- uint3 tgid [[threadgroup_position_in_grid]], \
- uint simd_gid [[simdgroup_index_in_threadgroup]], \
- uint simd_lid [[thread_index_in_simdgroup]] \
- ) { \
- gemv_masked_impl<TYPE, ACC, ROWS>( \
- matrix, \
- vector, \
- out_mask, \
- op_mask, \
- output, \
- k, \
- n, \
- ldb, \
- lda, \
- ldd, \
- tgid, \
- simd_gid, \
- simd_lid \
- ); \
- }
-
-#define DECL_GEMV_GATHER(NAME, TYPE, ACC, ROWS) \
- [[kernel, max_total_threads_per_threadgroup(ROWS * 32)]] void NAME( \
- const device TYPE* matrix [[buffer(0)]], \
- const device TYPE* vector [[buffer(1)]], \
- const device uint* vec_indices [[buffer(2)]], \
- const device uint* mat_indices [[buffer(3)]], \
- device TYPE* output [[buffer(4)]], \
- constant uint& k [[buffer(5)]], \
- constant uint& n [[buffer(6)]], \
- constant uint& ldb [[buffer(7)]], \
- constant uint& lda [[buffer(8)]], \
- constant uint& ldd [[buffer(9)]], \
- uint3 tgid [[threadgroup_position_in_grid]], \
- uint simd_gid [[simdgroup_index_in_threadgroup]], \
- uint simd_lid [[thread_index_in_simdgroup]] \
- ) { \
- gemv_gather_impl<TYPE, ACC, ROWS>( \
- matrix, \
- vector, \
- vec_indices, \
- mat_indices, \
- output, \
- k, \
- n, \
- ldb, \
- lda, \
- ldd, \
- tgid, \
- simd_gid, \
- simd_lid \
- ); \
- }
-
-// Masked instantiations
-DECL_GEMV_MASKED(gemv_masked_f16_rows4, half, float, 4)
-DECL_GEMV_MASKED(gemv_masked_f16_rows8, half, float, 8)
-DECL_GEMV_MASKED(gemv_masked_bf16_rows4, bfloat, float, 4)
-DECL_GEMV_MASKED(gemv_masked_bf16_rows8, bfloat, float, 8)
-
-// Gather instantiations
-DECL_GEMV_GATHER(gemv_gather_f16_rows4, half, float, 4)
-DECL_GEMV_GATHER(gemv_gather_f16_rows8, half, float, 8)
-DECL_GEMV_GATHER(gemv_gather_bf16_rows4, bfloat, float, 4)
-DECL_GEMV_GATHER(gemv_gather_bf16_rows8, bfloat, float, 8)
-// Masked/gather GEMV kernels are stubbed out (not used). Keeping file empty to
-// avoid shader compilation errors.
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs
index 39d1c62f1..81e82a79e 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs
@@ -1,6 +1,7 @@
use std::collections::HashMap;
use metal::{
+ Buffer as MTLBuffer,
ComputeCommandEncoderRef, ComputePipelineState as MTLComputePipelineState,
MTLSize,
};
@@ -10,6 +11,7 @@ use super::{
shared_types::GEMMParams, splitk::SplitKGemm,
transpose::transpose_configuration,
};
+use super::super::{KernelDataType, TensorAddBias};
use crate::{
DataType,
backends::metal::{MTLContext, MTLError},
@@ -21,6 +23,7 @@ pub struct MatmulKernel {
transpose_b: bool,
gemv: Option<GemvKernel>,
splitk: Option<SplitKGemm>,
+ bias_add: Option<TensorAddBias>,
pipelines: HashMap<PipelineKey, MTLComputePipelineState>,
}
@@ -69,6 +72,7 @@ impl MatmulKernel {
transpose_b,
gemv: None,
splitk: None,
+ bias_add: None,
pipelines: HashMap::new(),
})
}
@@ -86,11 +90,12 @@ impl MatmulKernel {
}
}
- fn maybe_use_gemv(
+ fn maybe_use_gemv_impl(
&mut self,
mtl: &MTLContext,
enc: &ComputeCommandEncoderRef,
args: MatmulArguments,
+ bias: Option<&MTLBuffer>,
) -> Result<bool, MTLError> {
// GEMV kernel requires:
// - A not transposed (input vector is contiguous)
@@ -123,7 +128,11 @@ impl MatmulKernel {
let rows = self.select_gemv_rows(n);
let gemv =
self.gemv.get_or_insert_with(|| GemvKernel::new(self.dt));
- gemv.encode(mtl, enc, batched_args, rows)?;
+ if let Some(bias) = bias {
+ gemv.encode_with_bias(mtl, enc, batched_args, rows, bias)?;
+ } else {
+ gemv.encode(mtl, enc, batched_args, rows)?;
+ }
return Ok(true);
}
@@ -133,10 +142,33 @@ impl MatmulKernel {
}
let rows = self.select_gemv_rows(n);
let gemv = self.gemv.get_or_insert_with(|| GemvKernel::new(self.dt));
- gemv.encode(mtl, enc, args, rows)?;
+ if let Some(bias) = bias {
+ gemv.encode_with_bias(mtl, enc, args, rows, bias)?;
+ } else {
+ gemv.encode(mtl, enc, args, rows)?;
+ }
Ok(true)
}
+ fn maybe_use_gemv(
+ &mut self,
+ mtl: &MTLContext,
+ enc: &ComputeCommandEncoderRef,
+ args: MatmulArguments,
+ ) -> Result<bool, MTLError> {
+ self.maybe_use_gemv_impl(mtl, enc, args, None)
+ }
+
+ fn maybe_use_gemv_with_bias(
+ &mut self,
+ mtl: &MTLContext,
+ enc: &ComputeCommandEncoderRef,
+ args: MatmulArguments,
+ bias: &MTLBuffer,
+ ) -> Result<bool, MTLError> {
+ self.maybe_use_gemv_impl(mtl, enc, args, Some(bias))
+ }
+
fn maybe_use_splitk(
&mut self,
mtl: &MTLContext,
@@ -367,24 +399,12 @@ impl MatmulKernel {
Ok(self.pipelines.get(&key).unwrap())
}
- pub fn encode(
+ fn encode_gemm(
&mut self,
mtl: &MTLContext,
enc: &ComputeCommandEncoderRef,
- mut args: MatmulArguments,
+ args: MatmulArguments,
) -> Result<(), MTLError> {
- self.apply_batch_collapse(&mut args);
-
- // Try GEMV fast path for decode shapes.
- if self.maybe_use_gemv(mtl, enc, args)? {
- return Ok(());
- }
-
- // Try Split-K for small M*N and large K.
- if self.maybe_use_splitk(mtl, enc, args)? {
- return Ok(());
- }
-
let (bm, bn, bk, wm, wn) = self.select_tiles(mtl, &args);
let kname = self.kernel_name(bm, bn, bk, wm, wn);
@@ -409,11 +429,7 @@ impl MatmulKernel {
let tiles_n = (n + bn - 1) / bn;
let tiles_m = (m + bm - 1) / bm;
// Swizzle: small tm -> no swizzle, otherwise simple 2-way.
- let swizzle_log = if tiles_m <= 3 {
- 0
- } else {
- 1
- };
+ let swizzle_log = if tiles_m <= 3 { 0 } else { 1 };
let tile = 1 << swizzle_log;
let tm_swizzled = (tiles_m + tile - 1) / tile;
@@ -452,6 +468,83 @@ impl MatmulKernel {
Ok(())
}
+ pub fn encode(
+ &mut self,
+ mtl: &MTLContext,
+ enc: &ComputeCommandEncoderRef,
+ mut args: MatmulArguments,
+ ) -> Result<(), MTLError> {
+ self.apply_batch_collapse(&mut args);
+
+ // Try GEMV fast path for decode shapes.
+ if self.maybe_use_gemv(mtl, enc, args)? {
+ return Ok(());
+ }
+
+ // Try Split-K for small M*N and large K.
+ if self.maybe_use_splitk(mtl, enc, args)? {
+ return Ok(());
+ }
+ self.encode_gemm(mtl, enc, args)
+ }
+
+ pub fn encode_with_bias(
+ &mut self,
+ mtl: &MTLContext,
+ enc: &ComputeCommandEncoderRef,
+ mut args: MatmulArguments,
+ bias: &MTLBuffer,
+ ) -> Result<(), MTLError> {
+ self.apply_batch_collapse(&mut args);
+
+ if self.maybe_use_gemv_with_bias(mtl, enc, args, bias)? {
+ return Ok(());
+ }
+
+ if self.maybe_use_splitk(mtl, enc, args)? {
+ self.apply_bias_add(mtl, enc, args, bias)?;
+ return Ok(());
+ }
+
+ self.encode_gemm(mtl, enc, args)?;
+ self.apply_bias_add(mtl, enc, args, bias)?;
+ Ok(())
+ }
+
+ fn apply_bias_add(
+ &mut self,
+ mtl: &MTLContext,
+ enc: &ComputeCommandEncoderRef,
+ args: MatmulArguments,
+ bias: &MTLBuffer,
+ ) -> Result<(), MTLError> {
+ let m = args.batch as usize;
+ let n = args.output_dim as usize;
+ let batch_count = args.batch_count as usize;
+ let total_len = m * n * batch_count;
+ if total_len == 0 {
+ return Ok(());
+ }
+
+ if self.bias_add.is_none() {
+ self.bias_add = Some(TensorAddBias::new(
+ mtl,
+ KernelDataType::from(self.dt),
+ )?);
+ }
+ let bias_add = self.bias_add.as_ref().unwrap();
+ bias_add.encode_with_encoder(
+ args.d,
+ bias,
+ args.d,
+ n,
+ total_len,
+ enc,
+ None,
+ );
+ Ok(())
+ }
+
fn apply_batch_collapse(
&self,
args: &mut MatmulArguments,