Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

wip

Eugene Bokhan560b31dc3f8c
diff --git a/Cargo.lock b/Cargo.lock
index 0c8247c5e..a8d408bb4 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -309,8 +309,11 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
name = "benchmarks"
version = "0.1.7"
dependencies = [
+ "half",
"libc",
"mach2",
+ "metal",
+ "objc2",
"serde",
"serde_json",
"sysinfo",
diff --git a/crates/benchmarks/Cargo.toml b/crates/benchmarks/Cargo.toml
index d82f672f1..6c45b5944 100644
--- a/crates/benchmarks/Cargo.toml
+++ b/crates/benchmarks/Cargo.toml
@@ -10,3 +10,6 @@ serde_json = "1.0.148"
sysinfo = "0.37.2"
libc = "0.2.178"
mach2 = "0.6.0"
+half = { version = "2.6.0", features = ["bytemuck", "num-traits"] }
+objc2 = "0.6.2"
+metal = "0.32.0"
diff --git a/crates/benchmarks/src/lib.rs b/crates/benchmarks/src/lib.rs
index 7a7c54c23..9326797ac 100644
--- a/crates/benchmarks/src/lib.rs
+++ b/crates/benchmarks/src/lib.rs
@@ -1,3 +1,5 @@
+pub mod mlp_fused_bench;
pub mod runner;
+pub use mlp_fused_bench::run_mlp_fused_benchmark;
pub use runner::*;
diff --git a/crates/benchmarks/src/mlp_fused_bench.rs b/crates/benchmarks/src/mlp_fused_bench.rs
new file mode 100644
index 000000000..d70dc18f5
--- /dev/null
+++ b/crates/benchmarks/src/mlp_fused_bench.rs
@@ -0,0 +1,411 @@
+//! MLP Fused Kernel Performance Benchmark
+//! Compares fused vs unfused MLP paths to measure speedup
+
+use std::time::{Duration, Instant};
+
+use half::f16;
+use metal::{Device, MTLResourceOptions};
+use objc2::rc::autoreleasepool;
+use uzu::{
+ DataType,
+ backends::metal::{
+ MTLContext,
+ kernel::{
+ matmul::{
+ MatmulArguments, MatmulKernel, MlpFusedGemmArguments,
+ MlpFusedGemmKernel, MlpFusedGemvArguments, MlpFusedGemvKernel,
+ },
+ mlp::MlpActivationType,
+ },
+ },
+};
+
+fn create_test_context() -> Option<MTLContext> {
+ let device = Device::system_default()?;
+ let command_queue = device.new_command_queue();
+ MTLContext::new(device, command_queue).ok()
+}
+
+struct BenchResult {
+ fused_time_us: f64,
+ unfused_time_us: f64,
+ speedup: f64,
+}
+
+impl BenchResult {
+ fn new(
+ fused: Duration,
+ unfused: Duration,
+ ) -> Self {
+ let fused_us = fused.as_secs_f64() * 1_000_000.0;
+ let unfused_us = unfused.as_secs_f64() * 1_000_000.0;
+ Self {
+ fused_time_us: fused_us,
+ unfused_time_us: unfused_us,
+ speedup: unfused_us / fused_us,
+ }
+ }
+}
+
+fn bench_gemv_fused(
+ ctx: &MTLContext,
+ k: usize,
+ hidden_dim: usize,
+ iterations: usize,
+ warmup: usize,
+) -> BenchResult {
+ // Allocate buffers
+ let input_buf = ctx.device.new_buffer(
+ (k * std::mem::size_of::<f16>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+ let weights_buf = ctx.device.new_buffer(
+ (2 * hidden_dim * k * std::mem::size_of::<f16>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+ let output_buf = ctx.device.new_buffer(
+ (hidden_dim * std::mem::size_of::<f16>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+ let fused_up_buf = ctx.device.new_buffer(
+ (2 * hidden_dim * std::mem::size_of::<f16>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+
+ let mut fused_kernel =
+ MlpFusedGemvKernel::new(DataType::F16).expect("fused kernel");
+ let mut matmul_kernel =
+ MatmulKernel::new(ctx, DataType::F16, false, true).expect("matmul");
+
+ // Warmup fused
+ for _ in 0..warmup {
+ let cb = ctx.command_queue.new_command_buffer().to_owned();
+ let enc = cb.new_compute_command_encoder();
+ let args = MlpFusedGemvArguments {
+ weights: &weights_buf,
+ input: &input_buf,
+ input_offset: 0,
+ output: &output_buf,
+ input_dim: k as i32,
+ hidden_dim: hidden_dim as i32,
+ weights_ld: k as i32,
+ batch_count: 1,
+ activation: MlpActivationType::SiLU,
+ };
+ fused_kernel.encode(ctx, &enc, &args).unwrap();
+ enc.end_encoding();
+ cb.commit();
+ cb.wait_until_completed();
+ }
+
+ // Benchmark fused
+ let fused_start = Instant::now();
+ for _ in 0..iterations {
+ let cb = ctx.command_queue.new_command_buffer().to_owned();
+ let enc = cb.new_compute_command_encoder();
+ let args = MlpFusedGemvArguments {
+ weights: &weights_buf,
+ input: &input_buf,
+ input_offset: 0,
+ output: &output_buf,
+ input_dim: k as i32,
+ hidden_dim: hidden_dim as i32,
+ weights_ld: k as i32,
+ batch_count: 1,
+ activation: MlpActivationType::SiLU,
+ };
+ fused_kernel.encode(ctx, &enc, &args).unwrap();
+ enc.end_encoding();
+ cb.commit();
+ cb.wait_until_completed();
+ }
+ let fused_time = fused_start.elapsed() / iterations as u32;
+
+ // Warmup unfused (matmul only - simplified comparison)
+ for _ in 0..warmup {
+ let cb = ctx.command_queue.new_command_buffer().to_owned();
+ let enc = cb.new_compute_command_encoder();
+ matmul_kernel
+ .encode(
+ ctx,
+ &enc,
+ MatmulArguments {
+ a: &input_buf,
+ a_offset: 0,
+ b: &weights_buf,
+ c: None,
+ d: &fused_up_buf,
+ batch: 1,
+ input_dim: k as i32,
+ output_dim: (2 * hidden_dim) as i32,
+ lda: k as i32,
+ ldb: k as i32,
+ ldd: (2 * hidden_dim) as i32,
+ batch_count: 1,
+ alpha: 1.0,
+ beta: 0.0,
+ },
+ )
+ .unwrap();
+ enc.end_encoding();
+ cb.commit();
+ cb.wait_until_completed();
+ }
+
+ // Benchmark unfused (matmul + separate activation would add more time)
+ let unfused_start = Instant::now();
+ for _ in 0..iterations {
+ let cb = ctx.command_queue.new_command_buffer().to_owned();
+ let enc = cb.new_compute_command_encoder();
+ matmul_kernel
+ .encode(
+ ctx,
+ &enc,
+ MatmulArguments {
+ a: &input_buf,
+ a_offset: 0,
+ b: &weights_buf,
+ c: None,
+ d: &fused_up_buf,
+ batch: 1,
+ input_dim: k as i32,
+ output_dim: (2 * hidden_dim) as i32,
+ lda: k as i32,
+ ldb: k as i32,
+ ldd: (2 * hidden_dim) as i32,
+ batch_count: 1,
+ alpha: 1.0,
+ beta: 0.0,
+ },
+ )
+ .unwrap();
+ enc.end_encoding();
+ cb.commit();
+ cb.wait_until_completed();
+ }
+ let unfused_time = unfused_start.elapsed() / iterations as u32;
+
+ BenchResult::new(fused_time, unfused_time)
+}
+
+fn bench_gemm_fused(
+ ctx: &MTLContext,
+ m: usize,
+ k: usize,
+ hidden_dim: usize,
+ iterations: usize,
+ warmup: usize,
+) -> BenchResult {
+ let input_buf = ctx.device.new_buffer(
+ (m * k * std::mem::size_of::<f16>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+ let weights_buf = ctx.device.new_buffer(
+ (2 * hidden_dim * k * std::mem::size_of::<f16>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+ let output_buf = ctx.device.new_buffer(
+ (m * hidden_dim * std::mem::size_of::<f16>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+ let fused_up_buf = ctx.device.new_buffer(
+ (m * 2 * hidden_dim * std::mem::size_of::<f16>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+
+ let mut fused_kernel =
+ MlpFusedGemmKernel::new(DataType::F16, true).expect("fused kernel");
+ let mut matmul_kernel =
+ MatmulKernel::new(ctx, DataType::F16, false, true).expect("matmul");
+
+ // Warmup fused
+ for _ in 0..warmup {
+ let cb = ctx.command_queue.new_command_buffer().to_owned();
+ let enc = cb.new_compute_command_encoder();
+ let args = MlpFusedGemmArguments {
+ input: &input_buf,
+ input_offset: 0,
+ weights: &weights_buf,
+ output: &output_buf,
+ batch: m as i32,
+ input_dim: k as i32,
+ hidden_dim: hidden_dim as i32,
+ lda: k as i32,
+ ldb: k as i32,
+ ldd: hidden_dim as i32,
+ activation: MlpActivationType::SiLU,
+ };
+ fused_kernel.encode(ctx, &enc, &args).unwrap();
+ enc.end_encoding();
+ cb.commit();
+ cb.wait_until_completed();
+ }
+
+ let fused_start = Instant::now();
+ for _ in 0..iterations {
+ let cb = ctx.command_queue.new_command_buffer().to_owned();
+ let enc = cb.new_compute_command_encoder();
+ let args = MlpFusedGemmArguments {
+ input: &input_buf,
+ input_offset: 0,
+ weights: &weights_buf,
+ output: &output_buf,
+ batch: m as i32,
+ input_dim: k as i32,
+ hidden_dim: hidden_dim as i32,
+ lda: k as i32,
+ ldb: k as i32,
+ ldd: hidden_dim as i32,
+ activation: MlpActivationType::SiLU,
+ };
+ fused_kernel.encode(ctx, &enc, &args).unwrap();
+ enc.end_encoding();
+ cb.commit();
+ cb.wait_until_completed();
+ }
+ let fused_time = fused_start.elapsed() / iterations as u32;
+
+ // Warmup unfused
+ for _ in 0..warmup {
+ let cb = ctx.command_queue.new_command_buffer().to_owned();
+ let enc = cb.new_compute_command_encoder();
+ matmul_kernel
+ .encode(
+ ctx,
+ &enc,
+ MatmulArguments {
+ a: &input_buf,
+ a_offset: 0,
+ b: &weights_buf,
+ c: None,
+ d: &fused_up_buf,
+ batch: m as i32,
+ input_dim: k as i32,
+ output_dim: (2 * hidden_dim) as i32,
+ lda: k as i32,
+ ldb: k as i32,
+ ldd: (2 * hidden_dim) as i32,
+ batch_count: 1,
+ alpha: 1.0,
+ beta: 0.0,
+ },
+ )
+ .unwrap();
+ enc.end_encoding();
+ cb.commit();
+ cb.wait_until_completed();
+ }
+
+ let unfused_start = Instant::now();
+ for _ in 0..iterations {
+ let cb = ctx.command_queue.new_command_buffer().to_owned();
+ let enc = cb.new_compute_command_encoder();
+ matmul_kernel
+ .encode(
+ ctx,
+ &enc,
+ MatmulArguments {
+ a: &input_buf,
+ a_offset: 0,
+ b: &weights_buf,
+ c: None,
+ d: &fused_up_buf,
+ batch: m as i32,
+ input_dim: k as i32,
+ output_dim: (2 * hidden_dim) as i32,
+ lda: k as i32,
+ ldb: k as i32,
+ ldd: (2 * hidden_dim) as i32,
+ batch_count: 1,
+ alpha: 1.0,
+ beta: 0.0,
+ },
+ )
+ .unwrap();
+ enc.end_encoding();
+ cb.commit();
+ cb.wait_until_completed();
+ }
+ let unfused_time = unfused_start.elapsed() / iterations as u32;
+
+ BenchResult::new(fused_time, unfused_time)
+}
+
+pub fn run_mlp_fused_benchmark() {
+ autoreleasepool(|_| {
+ let ctx = match create_test_context() {
+ Some(c) => c,
+ None => {
+ eprintln!("No Metal device available");
+ return;
+ },
+ };
+
+ println!("\n=== MLP Fused Kernel Benchmark ===\n");
+ println!(
+ "{:<40} {:>12} {:>12} {:>10}",
+ "Configuration", "Fused (µs)", "Unfused (µs)", "Speedup"
+ );
+ println!("{}", "-".repeat(76));
+
+ let iterations = 100;
+ let warmup = 10;
+
+ // Decode configurations (M=1)
+ for (k, hidden_dim) in [(2048, 4096), (4096, 11008), (8192, 22016)] {
+ let result =
+ bench_gemv_fused(&ctx, k, hidden_dim, iterations, warmup);
+ println!("GEMV decode K={} H={}", k, hidden_dim);
+ println!(
+ "{:<40} {:>12.2} {:>12.2} {:>9.2}x",
+ format!(" M=1, K={}, H={}", k, hidden_dim),
+ result.fused_time_us,
+ result.unfused_time_us,
+ result.speedup
+ );
+ }
+
+ println!();
+
+ // Prefill configurations (M>1)
+ for (m, k, hidden_dim) in
+ [(32, 2048, 4096), (128, 2048, 4096), (512, 4096, 11008)]
+ {
+ let result =
+ bench_gemm_fused(&ctx, m, k, hidden_dim, iterations, warmup);
+ println!(
+ "{:<40} {:>12.2} {:>12.2} {:>9.2}x",
+ format!("GEMM M={}, K={}, H={}", m, k, hidden_dim),
+ result.fused_time_us,
+ result.unfused_time_us,
+ result.speedup
+ );
+ }
+
+ println!("\nNote: Unfused time excludes activation kernel overhead.");
+ println!(
+ " Actual unfused path would be slower due to activation dispatch.\n"
+ );
+ });
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_benchmark_runs() {
+ // Just verify the benchmark doesn't crash
+ autoreleasepool(|_| {
+ if let Some(ctx) = create_test_context() {
+ let _ = bench_gemv_fused(&ctx, 512, 256, 5, 2);
+ }
+ });
+ }
+
+ #[test]
+ #[ignore] // Run with: cargo test --package benchmarks -- run_full_benchmark --ignored --nocapture
+ fn run_full_benchmark() {
+ run_mlp_fused_benchmark();
+ }
+}
diff --git a/crates/uzu/src/backends/metal/encodable_block/mlp.rs b/crates/uzu/src/backends/metal/encodable_block/mlp.rs
index e02038a5c..76ed0bb31 100644
--- a/crates/uzu/src/backends/metal/encodable_block/mlp.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/mlp.rs
@@ -1,14 +1,28 @@
//! MLP block encodable.
-use metal::{CommandBufferRef, ComputeCommandEncoderRef};
+use std::{cell::RefCell, rc::Rc};
+
+use metal::{Buffer as MTLBuffer, CommandBufferRef, ComputeCommandEncoderRef};
use super::{EncodableBlock, EncodingParameters};
use crate::{
- Array,
+ Array, DataType,
backends::metal::{
+ MTLContext,
forward_pass::{ArrayId, ForwardPassState},
- kernel::mlp::MlpGateActMulEncodable,
+ kernel::{
+ matmul::{
+ MlpFusedGemmArguments, MlpFusedGemmKernel,
+ MlpFusedGemvArguments, MlpFusedGemvKernel,
+ },
+ mlp::{MlpActivationType, MlpGateActMulEncodable},
+ quant_matmul::{
+ MlpFusedQmmArguments, MlpFusedQmmKernel, MlpFusedQmvArguments,
+ MlpFusedQmvKernel,
+ },
+ },
},
+ config::Activation,
};
pub struct MlpBlock {
@@ -103,3 +117,279 @@ impl EncodableBlock for MlpBlock {
self.down.encode_with_shared_encoder(state, encoder, params);
}
}
+
+/// MLP Fused Up Projection Variant
+/// Selects between full precision and quantized fused kernels
+pub enum MlpFusedUpKernel {
+ FullPrecision {
+ gemv: RefCell<MlpFusedGemvKernel>,
+ gemm: RefCell<MlpFusedGemmKernel>,
+ },
+ Quantized {
+ qmv: MlpFusedQmvKernel,
+ qmm: MlpFusedQmmKernel,
+ },
+}
+
+/// MLP block with fused up projection and activation
+/// Combines up * activation(gate) into a single kernel call
+pub struct MlpFusedBlock {
+ context: Rc<MTLContext>,
+ fused_up: MlpFusedUpKernel,
+ down: Box<dyn EncodableBlock>,
+ weights_buffer: MTLBuffer,
+ scales_buffer: Option<MTLBuffer>,
+ zero_points_or_biases_buffer: Option<MTLBuffer>,
+ input_dim: usize,
+ hidden_dim: usize,
+ activation: MlpActivationType,
+ input_array_id: ArrayId,
+ hidden_array_id: ArrayId,
+}
+
+impl MlpFusedBlock {
+ /// Create MLP fused block with full precision weights
+ pub fn new_full_precision(
+ context: Rc<MTLContext>,
+ data_type: DataType,
+ weights_buffer: MTLBuffer,
+ input_dim: usize,
+ hidden_dim: usize,
+ activation: &Activation,
+ down: Box<dyn EncodableBlock>,
+ input_array_id: ArrayId,
+ hidden_array_id: ArrayId,
+ ) -> Result<Self, crate::backends::metal::MTLError> {
+ let gemv = MlpFusedGemvKernel::new(data_type)?;
+ let gemm = MlpFusedGemmKernel::new(data_type, true)?; // Weights transposed
+
+ Ok(Self {
+ context,
+ fused_up: MlpFusedUpKernel::FullPrecision {
+ gemv: RefCell::new(gemv),
+ gemm: RefCell::new(gemm),
+ },
+ down,
+ weights_buffer,
+ scales_buffer: None,
+ zero_points_or_biases_buffer: None,
+ input_dim,
+ hidden_dim,
+ activation: MlpActivationType::from(activation),
+ input_array_id,
+ hidden_array_id,
+ })
+ }
+
+ /// Create MLP fused block with quantized weights
+ pub fn new_quantized(
+ context: Rc<MTLContext>,
+ data_type: DataType,
+ weights_buffer: MTLBuffer,
+ scales_buffer: MTLBuffer,
+ zero_points_or_biases_buffer: MTLBuffer,
+ input_dim: usize,
+ hidden_dim: usize,
+ group_size: usize,
+ mode: crate::config::QuantizationMode,
+ quantization_type: crate::backends::metal::kernel::quant_matmul::QuantizationType,
+ activation: &Activation,
+ down: Box<dyn EncodableBlock>,
+ input_array_id: ArrayId,
+ hidden_array_id: ArrayId,
+ ) -> Result<Self, crate::backends::metal::MTLError> {
+ let qmv = MlpFusedQmvKernel::new(
+ &context,
+ data_type,
+ group_size,
+ mode,
+ quantization_type,
+ )
+ .map_err(|e| {
+ crate::backends::metal::MTLError::Generic(format!("{:?}", e))
+ })?;
+
+ let qmm = MlpFusedQmmKernel::new(
+ &context,
+ data_type,
+ group_size,
+ mode,
+ quantization_type,
+ )
+ .map_err(|e| {
+ crate::backends::metal::MTLError::Generic(format!("{:?}", e))
+ })?;
+
+ Ok(Self {
+ context,
+ fused_up: MlpFusedUpKernel::Quantized {
+ qmv,
+ qmm,
+ },
+ down,
+ weights_buffer,
+ scales_buffer: Some(scales_buffer),
+ zero_points_or_biases_buffer: Some(zero_points_or_biases_buffer),
+ input_dim,
+ hidden_dim,
+ activation: MlpActivationType::from(activation),
+ input_array_id,
+ hidden_array_id,
+ })
+ }
+
+ fn encode_fused_up(
+ &self,
+ encoder: &ComputeCommandEncoderRef,
+ input: &MTLBuffer,
+ input_offset: u64,
+ output: &MTLBuffer,
+ batch: i32,
+ ) {
+ let is_decode = batch == 1;
+
+ match &self.fused_up {
+ MlpFusedUpKernel::FullPrecision {
+ gemv,
+ gemm,
+ } => {
+ if is_decode {
+ let args = MlpFusedGemvArguments {
+ weights: &self.weights_buffer,
+ input,
+ input_offset,
+ output,
+ input_dim: self.input_dim as i32,
+ hidden_dim: self.hidden_dim as i32,
+ weights_ld: self.input_dim as i32,
+ batch_count: batch,
+ activation: self.activation,
+ };
+ gemv.borrow_mut()
+ .encode(&self.context, encoder, &args)
+ .expect("Failed to encode MLP fused GEMV");
+ } else {
+ let args = MlpFusedGemmArguments {
+ input,
+ input_offset,
+ weights: &self.weights_buffer,
+ output,
+ batch,
+ input_dim: self.input_dim as i32,
+ hidden_dim: self.hidden_dim as i32,
+ lda: self.input_dim as i32,
+ ldb: self.input_dim as i32,
+ ldd: self.hidden_dim as i32,
+ activation: self.activation,
+ };
+ gemm.borrow_mut()
+ .encode(&self.context, encoder, &args)
+ .expect("Failed to encode MLP fused GEMM");
+ }
+ },
+ MlpFusedUpKernel::Quantized {
+ qmv,
+ qmm,
+ } => {
+ let scales = self.scales_buffer.as_ref().unwrap();
+ let zp_or_biases =
+ self.zero_points_or_biases_buffer.as_ref().unwrap();
+
+ if is_decode {
+ let args = MlpFusedQmvArguments {
+ weights: &self.weights_buffer,
+ scales,
+ zero_points_or_biases: zp_or_biases,
+ input,
+ input_offset,
+ output,
+ input_dim: self.input_dim as i32,
+ hidden_dim: self.hidden_dim as i32,
+ batch_count: batch,
+ };
+ qmv.encode(encoder, &args)
+ .expect("Failed to encode MLP fused QMV");
+ } else {
+ let args = MlpFusedQmmArguments {
+ weights: &self.weights_buffer,
+ scales,
+ zero_points_or_biases: zp_or_biases,
+ input,
+ input_offset,
+ output,
+ batch,
+ input_dim: self.input_dim as i32,
+ hidden_dim: self.hidden_dim as i32,
+ };
+ qmm.encode(encoder, &args)
+ .expect("Failed to encode MLP fused QMM");
+ }
+ },
+ }
+ }
+}
+
+impl EncodableBlock for MlpFusedBlock {
+ fn encode(
+ &self,
+ state: &mut ForwardPassState,
+ command_buffer: &CommandBufferRef,
+ params: &EncodingParameters,
+ ) {
+ if self.supports_shared_encoder() {
+ let encoder = command_buffer.new_compute_command_encoder();
+ self.encode_with_shared_encoder(state, &encoder, params);
+ encoder.end_encoding();
+ } else {
+ // Fused up + activation
+ {
+ let arrays =
+ state.arrays(&[self.input_array_id, self.hidden_array_id]);
+ let mut input = arrays[0].borrow_mut();
+ let mut hidden = arrays[1].borrow_mut();
+ let batch = input.shape()[0] as i32;
+ let input_buf = unsafe { input.mtl_buffer() };
+ let hidden_buf = unsafe { hidden.mtl_buffer() };
+
+ let encoder = command_buffer.new_compute_command_encoder();
+ self.encode_fused_up(&encoder, input_buf, 0, hidden_buf, batch);
+ encoder.end_encoding();
+ }
+
+ // Down
+ self.down.encode(state, command_buffer, params);
+ }
+
+ if params.wait_until_completed {
+ command_buffer.commit();
+ command_buffer.wait_until_completed();
+ }
+ }
+
+ fn supports_shared_encoder(&self) -> bool {
+ self.down.supports_shared_encoder()
+ }
+
+ fn encode_with_shared_encoder(
+ &self,
+ state: &mut ForwardPassState,
+ encoder: &ComputeCommandEncoderRef,
+ params: &EncodingParameters,
+ ) {
+ // Fused up + activation
+ let arrays = state.arrays(&[self.input_array_id, self.hidden_array_id]);
+ let mut input = arrays[0].borrow_mut();
+ let mut hidden = arrays[1].borrow_mut();
+ let batch = input.shape()[0] as i32;
+ let input_buf = unsafe { input.mtl_buffer() };
+ let hidden_buf = unsafe { hidden.mtl_buffer() };
+
+ self.encode_fused_up(encoder, input_buf, 0, hidden_buf, batch);
+
+ drop(input);
+ drop(hidden);
+
+ // Down
+ self.down.encode_with_shared_encoder(state, encoder, params);
+ }
+}
diff --git a/crates/uzu/src/backends/metal/encodable_block/mod.rs b/crates/uzu/src/backends/metal/encodable_block/mod.rs
index 0c651bb14..199583445 100644
--- a/crates/uzu/src/backends/metal/encodable_block/mod.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/mod.rs
@@ -44,7 +44,7 @@ pub use encoding_parameters::EncodingParameters;
pub use layer::LayerExecutables;
pub use linear::{FullPrecisionLinear, QuantizedLinear};
pub(crate) use mamba_mixer::MambaMixer;
-pub use mlp::MlpBlock;
+pub use mlp::{MlpBlock, MlpFusedBlock, MlpFusedUpKernel};
pub use moe_block::{MoeBlock, SharedMoeWeights};
pub use normalization::{
LayerNorm, Normalization, NormalizationError, QKNorm, RMSNorm,
diff --git a/crates/uzu/src/backends/metal/kernel/common/mlp_epilogue.h b/crates/uzu/src/backends/metal/kernel/common/mlp_epilogue.h
new file mode 100644
index 000000000..f7cc9376b
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/common/mlp_epilogue.h
@@ -0,0 +1,76 @@
+#pragma once
+
+#include <metal_stdlib>
+using namespace metal;
+
+// MLP Fusion Function Constants
+// These control whether the kernel computes paired up/gate projections
+// and applies activation fusion (SwiGLU/GeGLU style).
+//
+// Index allocation:
+// 50 - MLP_FUSED (bool): enable paired computation + fused activation
+// 51 - MLP_HIDDEN_DIM (uint): offset to gate weights (= hidden_dim)
+// 52 - MLP_ACTIVATION (uint): 0=SiLU, 1=GELU
+
+constant bool MLP_FUSED [[function_constant(50)]];
+constant uint MLP_HIDDEN_DIM [[function_constant(51)]];
+constant uint MLP_ACTIVATION [[function_constant(52)]];
+
+// Activation type enum for readability
+enum MlpActivationType : uint {
+ MLP_ACT_SILU = 0,
+ MLP_ACT_GELU = 1,
+};
+
+// SiLU activation: x * sigmoid(x)
+template <typename T>
+inline T mlp_silu(T x) {
+ float xf = float(x);
+ float result = xf / (1.0f + exp(-xf));
+ return T(result);
+}
+
+// GELU approximation (tanh-based)
+template <typename T>
+inline T mlp_gelu(T x) {
+ constexpr float k0 = 0.044715f;
+ constexpr float k1 = 0.7978845608f; // sqrt(2/pi)
+ float xf = float(x);
+ float t = k1 * (xf + k0 * xf * xf * xf);
+ float result = 0.5f * xf * (1.0f + metal::precise::tanh(t));
+ return T(result);
+}
+
+// Apply activation based on MLP_ACTIVATION constant
+template <typename T>
+inline T mlp_activate(T x) {
+ if (MLP_ACTIVATION == MLP_ACT_SILU) {
+ return mlp_silu(x);
+ }
+ return mlp_gelu(x);
+}
+
+// Fused MLP epilogue: up * activation(gate)
+// Used in SwiGLU (SiLU) and GeGLU (GELU) style MLPs
+template <typename T>
+inline T mlp_fused_epilogue(T up, T gate) {
+ T activated_gate = mlp_activate(gate);
+ return T(float(up) * float(activated_gate));
+}
+
+// Fused MLP epilogue with float accumulators
+inline float mlp_fused_epilogue_f32(float up, float gate) {
+ float activated_gate;
+ if (MLP_ACTIVATION == MLP_ACT_SILU) {
+ activated_gate = up / (1.0f + exp(-gate));
+ // Note: SwiGLU is gate_act * up, not up_act * gate
+ // Correcting: activated_gate = gate / (1.0f + exp(-gate))
+ activated_gate = gate / (1.0f + exp(-gate));
+ } else {
+ constexpr float k0 = 0.044715f;
+ constexpr float k1 = 0.7978845608f;
+ float t = k1 * (gate + k0 * gate * gate * gate);
+ activated_gate = 0.5f * gate * (1.0f + metal::precise::tanh(t));
+ }
+ return up * activated_gate;
+}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv.rs
index e96a0ef11..db5ee62da 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv.rs
@@ -5,7 +5,7 @@ use metal::{
MTLSize,
};
-use super::arguments::MatmulArguments;
+use super::{super::mlp::MlpActivationType, arguments::MatmulArguments};
use crate::{
DataType,
backends::metal::{MTLContext, MTLError},
@@ -458,3 +458,215 @@ impl GemvKernel {
)
}
}
+
+/// MLP Fused GEMV configuration
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+struct MlpFusedGemvConfiguration {
+ threadgroup_rows: u32,
+ threadgroup_cols: u32,
+ threads_per_simdgroup_row: u32,
+ threads_per_simdgroup_col: u32,
+ elements_per_thread_row: u32,
+ elements_per_thread_col: u32,
+ activation: MlpActivationType,
+}
+
+impl MlpFusedGemvConfiguration {
+ fn output_elements_per_threadgroup(&self) -> u32 {
+ self.threadgroup_rows
+ * self.threads_per_simdgroup_row
+ * self.elements_per_thread_row
+ }
+
+ fn threads_per_threadgroup(&self) -> MTLSize {
+ MTLSize::new(
+ 32,
+ self.threadgroup_cols as u64,
+ self.threadgroup_rows as u64,
+ )
+ }
+}
+
+fn mlp_fused_gemv_kernel_name(
+ data_type: DataType,
+ config: &MlpFusedGemvConfiguration,
+) -> Result<String, MTLError> {
+ let dtype_name = match data_type {
+ DataType::F16 => "float16",
+ DataType::BF16 => "bfloat16",
+ DataType::F32 => "float32",
+ _ => {
+ return Err(MTLError::Generic(format!(
+ "Unsupported data type for MLP fused GEMV: {:?}",
+ data_type
+ )));
+ },
+ };
+
+ Ok(format!(
+ "gemv_mlp_fused_{}_bm{}_bn{}_sm{}_sn{}_tm{}_tn{}",
+ dtype_name,
+ config.threadgroup_rows,
+ config.threadgroup_cols,
+ config.threads_per_simdgroup_row,
+ config.threads_per_simdgroup_col,
+ config.elements_per_thread_row,
+ config.elements_per_thread_col,
+ ))
+}
+
+fn select_mlp_fused_gemv_configuration(
+ hidden_dim: i32,
+ activation: MlpActivationType,
+) -> MlpFusedGemvConfiguration {
+ // Select configuration based on hidden dimension
+ let threadgroup_rows = if hidden_dim >= 4096 {
+ 8
+ } else {
+ 4
+ };
+
+ MlpFusedGemvConfiguration {
+ threadgroup_rows,
+ threadgroup_cols: 1,
+ threads_per_simdgroup_row: 1,
+ threads_per_simdgroup_col: 32,
+ elements_per_thread_row: 4,
+ elements_per_thread_col: 4,
+ activation,
+ }
+}
+
+/// Arguments for MLP fused GEMV
+#[derive(Debug)]
+pub struct MlpFusedGemvArguments<'a> {
+ /// Weight matrix [2*hidden_dim, input_dim] (up and gate concatenated)
+ pub weights: &'a MTLBuffer,
+ /// Input vector [input_dim]
+ pub input: &'a MTLBuffer,
+ /// Input byte offset
+ pub input_offset: u64,
+ /// Output vector [hidden_dim]
+ pub output: &'a MTLBuffer,
+ /// Input dimension (K)
+ pub input_dim: i32,
+ /// Hidden dimension (output size, half of weight rows)
+ pub hidden_dim: i32,
+ /// Weight matrix leading dimension
+ pub weights_ld: i32,
+ /// Batch count
+ pub batch_count: i32,
+ /// Activation type
+ pub activation: MlpActivationType,
+}
+
+/// MLP Fused GEMV Kernel
+/// Computes paired up/gate projections with fused activation: out = up * activation(gate)
+pub struct MlpFusedGemvKernel {
+ data_type: DataType,
+ pipelines: HashMap<MlpFusedGemvConfiguration, ComputePipelineState>,
+}
+
+impl MlpFusedGemvKernel {
+ pub fn new(data_type: DataType) -> Result<Self, MTLError> {
+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)
+ {
+ return Err(MTLError::Generic(format!(
+ "Unsupported dtype for MLP fused GEMV: {:?}",
+ data_type
+ )));
+ }
+ Ok(Self {
+ data_type,
+ pipelines: HashMap::new(),
+ })
+ }
+
+ fn get_pipeline(
+ &mut self,
+ context: &MTLContext,
+ config: MlpFusedGemvConfiguration,
+ ) -> Result<&ComputePipelineState, MTLError> {
+ if !self.pipelines.contains_key(&config) {
+ let kernel_name =
+ mlp_fused_gemv_kernel_name(self.data_type, &config)?;
+
+ // Create function constant values for MLP activation
+ let fcv = metal::FunctionConstantValues::new();
+ let activation_val = config.activation as u32;
+ fcv.set_constant_value_at_index(
+ &activation_val as *const u32 as *const _,
+ metal::MTLDataType::UInt,
+ 52, // MLP_ACTIVATION index
+ );
+
+ let pipeline =
+ context.compute_pipeline_state(&kernel_name, Some(&fcv))?;
+ self.pipelines.insert(config, pipeline);
+ }
+ Ok(self.pipelines.get(&config).unwrap())
+ }
+
+ pub fn encode(
+ &mut self,
+ context: &MTLContext,
+ encoder: &ComputeCommandEncoderRef,
+ args: &MlpFusedGemvArguments,
+ ) -> Result<(), MTLError> {
+ let config = select_mlp_fused_gemv_configuration(
+ args.hidden_dim,
+ args.activation,
+ );
+ let pipeline = self.get_pipeline(context, config)?;
+
+ encoder.set_compute_pipeline_state(pipeline);
+ encoder.set_buffer(0, Some(args.weights), 0);
+ encoder.set_buffer(1, Some(args.input), args.input_offset);
+ encoder.set_buffer(3, Some(args.output), 0);
+
+ encoder.set_bytes(
+ 4,
+ std::mem::size_of::<i32>() as u64,
+ &args.input_dim as *const i32 as *const std::ffi::c_void,
+ );
+ encoder.set_bytes(
+ 5,
+ std::mem::size_of::<i32>() as u64,
+ &args.hidden_dim as *const i32 as *const std::ffi::c_void,
+ );
+ encoder.set_bytes(
+ 6,
+ std::mem::size_of::<i32>() as u64,
+ &args.weights_ld as *const i32 as *const std::ffi::c_void,
+ );
+
+ // Batch strides
+ let vector_batch_stride = args.input_dim as i64;
+ let matrix_batch_stride = 0i64; // Weights not batched
+ encoder.set_bytes(
+ 11,
+ std::mem::size_of::<i64>() as u64,
+ &vector_batch_stride as *const i64 as *const std::ffi::c_void,
+ );
+ encoder.set_bytes(
+ 12,
+ std::mem::size_of::<i64>() as u64,
+ &matrix_batch_stride as *const i64 as *const std::ffi::c_void,
+ );
+
+ let output_elements_per_threadgroup =
+ config.output_elements_per_threadgroup();
+ let threadgroup_count_x =
+ ((args.hidden_dim as u32 + output_elements_per_threadgroup - 1)
+ / output_elements_per_threadgroup) as u64;
+ let threadgroup_count_z = args.batch_count.max(1) as u64;
+
+ let threadgroup_count =
+ MTLSize::new(threadgroup_count_x, 1, threadgroup_count_z);
+ let threads_per_threadgroup = config.threads_per_threadgroup();
+
+ encoder
+ .dispatch_thread_groups(threadgroup_count, threads_per_threadgroup);
+ Ok(())
+ }
+}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal
index 1c5bf124d..d79659f90 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal
@@ -2,6 +2,7 @@
#include <metal_stdlib>
#include "../../common/utils.h"
+#include "../../common/mlp_epilogue.h"
#include "../steel/utils.h"
@@ -864,3 +865,264 @@ instantiate_gemv_t_bs_blocks(float32, float);
instantiate_gemv_t_bs_blocks(float16, half);
instantiate_gemv_t_bs_blocks(bfloat16, bfloat16_t);
instantiate_gemv_t_bs_blocks(complex64, complex64_t); // clang-format on
+
+///////////////////////////////////////////////////////////////////////////////
+/// MLP Fused GEMV - Paired Up/Gate Computation with Activation Fusion
+///////////////////////////////////////////////////////////////////////////////
+
+// MLP fused GEMV kernel for decode path (M=1).
+// Computes paired up and gate projections, then applies: out = up * activation(gate)
+// Output size is hidden_dim (half of the weight matrix columns).
+template <
+ typename T,
+ const int BM, /* Threadgroup rows (in simdgroups) */
+ const int BN, /* Threadgroup cols (in simdgroups) */
+ const int SM, /* Simdgroup rows (in threads) */
+ const int SN, /* Simdgroup cols (in threads) */
+ const int TM, /* Thread rows (in elements) */
+ const int TN> /* Thread cols (in elements) */
+struct GEMVMlpFusedKernel {
+ using AccT = float;
+
+ MTL_CONST int threadsM = BM * SM;
+ MTL_CONST int threadsN = BN * SN;
+ MTL_CONST int blockM = threadsM * TM;
+ MTL_CONST int blockN = threadsN * TN;
+
+ static_assert(SM * SN == 32, "simdgroup can only have 32 threads");
+
+ MTL_CONST short tgp_mem_size = BN > 1 ? 2 * BN * (blockM + TM) : 0;
+ MTL_CONST bool needs_tgp_reduction = BN > 1;
+
+ template <typename U = T>
+ static METAL_FUNC void
+ load_unsafe(const device T* src, thread U dst[TN], const int src_offset = 0) {
+ MTL_PRAGMA_UNROLL
+ for (int tn = 0; tn < TN; tn++) {
+ dst[tn] = static_cast<U>(src[src_offset + tn]);
+ }
+ }
+
+ template <typename U = T>
+ static METAL_FUNC void load_safe(
+ const device T* src,
+ thread U dst[TN],
+ const int src_offset = 0,
+ const int src_size = TN) {
+ if (src_offset + TN <= src_size) {
+ MTL_PRAGMA_UNROLL
+ for (int tn = 0; tn < TN; tn++) {
+ dst[tn] = static_cast<U>(src[src_offset + tn]);
+ }
+ } else {
+ MTL_PRAGMA_UNROLL
+ for (int tn = 0; tn < TN; tn++) {
+ dst[tn] = src_offset + tn < src_size
+ ? static_cast<U>(src[src_offset + tn])
+ : U(0);
+ }
+ }
+ }
+
+ static METAL_FUNC void run(
+ const device T* mat [[buffer(0)]],
+ const device T* in_vec [[buffer(1)]],
+ device T* out_vec [[buffer(3)]],
+ const int in_vec_size,
+ const int hidden_dim, // Output size (half of weight matrix rows)
+ const int matrix_ld,
+ threadgroup AccT* tgp_memory,
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint3 lid [[thread_position_in_threadgroup]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]) {
+ (void)lid;
+
+ // Thread local accumulation results for up and gate
+ thread AccT result_up[TM] = {0};
+ thread AccT result_gate[TM] = {0};
+ thread T inter_up[TN];
+ thread T inter_gate[TN];
+ thread AccT v_coeff[TN];
+
+ const int thrM = SN != 32 ? simd_lid / SN : 0;
+ const int thrN = SN != 32 ? simd_lid % SN : int(simd_lid);
+
+ const int sgN = BN != 1 ? (simd_gid % BN) : 0;
+ const int simdM = BN != 1 ? SM * (simd_gid / BN) : int(SM * simd_gid);
+ const int simdN = BN != 1 ? SN * (simd_gid % BN) : 0;
+
+ int bm = (simdM + thrM) * TM;
+ int bn = (simdN + thrN) * TN;
+
+ // Block position - outputs to hidden_dim elements
+ int out_row = tid.x * blockM + bm;
+
+ if (out_row >= hidden_dim)
+ return;
+
+ out_row = out_row + TM <= hidden_dim ? out_row : hidden_dim - TM;
+
+ // Pointers to up and gate rows in the weight matrix
+ // Weight layout: [up_0, up_1, ..., up_H-1, gate_0, gate_1, ..., gate_H-1]
+ const device T* mat_up = mat + out_row * matrix_ld;
+ const device T* mat_gate = mat + (out_row + hidden_dim) * matrix_ld;
+
+ constexpr const uniform<int> loop_stride = make_uniform(blockN);
+ const uniform<int> in_size = make_uniform(in_vec_size);
+ const uniform<int> n_iter = in_size / loop_stride;
+ const uniform<int> last_iter = loop_stride * n_iter;
+ const uniform<int> leftover = in_size - last_iter;
+
+ // Main loop - compute both up and gate dot products
+ for (int i = 0; i < n_iter; ++i) {
+ load_unsafe<AccT>(in_vec, v_coeff, bn);
+
+ int mat_offset = 0;
+ MTL_PRAGMA_UNROLL
+ for (int tm = 0; tm < TM; tm++) {
+ load_unsafe(mat_up, inter_up, mat_offset + bn);
+ load_unsafe(mat_gate, inter_gate, mat_offset + bn);
+
+ MTL_PRAGMA_UNROLL
+ for (int tn = 0; tn < TN; tn++) {
+ result_up[tm] += inter_up[tn] * v_coeff[tn];
+ result_gate[tm] += inter_gate[tn] * v_coeff[tn];
+ }
+
+ mat_offset += matrix_ld;
+ }
+
+ bn += blockN;
+ }
+
+ // Handle leftover
+ if (leftover > 0) {
+ load_safe<AccT>(in_vec, v_coeff, bn, in_vec_size);
+
+ MTL_PRAGMA_UNROLL
+ for (int tm = 0; tm < TM; tm++) {
+ load_safe(&mat_up[tm * matrix_ld], inter_up, bn, in_vec_size);
+ load_safe(&mat_gate[tm * matrix_ld], inter_gate, bn, in_vec_size);
+
+ MTL_PRAGMA_UNROLL
+ for (int tn = 0; tn < TN; tn++) {
+ result_up[tm] += inter_up[tn] * v_coeff[tn];
+ result_gate[tm] += inter_gate[tn] * v_coeff[tn];
+ }
+ }
+ }
+
+ // Simdgroup reduction for both up and gate
+ MTL_PRAGMA_UNROLL
+ for (int tm = 0; tm < TM; tm++) {
+ MTL_PRAGMA_UNROLL
+ for (ushort sn = (SN / 2); sn >= 1; sn >>= 1) {
+ result_up[tm] += simd_shuffle_down(result_up[tm], sn);
+ result_gate[tm] += simd_shuffle_down(result_gate[tm], sn);
+ }
+ }
+
+ // Threadgroup reduction if needed
+ if (needs_tgp_reduction) {
+ threadgroup AccT* tgp_results_up = tgp_memory + sgN * (blockM + TM) + bm;
+ threadgroup AccT* tgp_results_gate = tgp_memory + BN * (blockM + TM) + sgN * (blockM + TM) + bm;
+
+ if (thrN == 0) {
+ MTL_PRAGMA_UNROLL
+ for (int tm = 0; tm < TM; tm++) {
+ tgp_results_up[tm] = result_up[tm];
+ tgp_results_gate[tm] = result_gate[tm];
+ }
+
+ threadgroup_barrier(mem_flags::mem_none);
+
+ if (sgN == 0) {
+ MTL_PRAGMA_UNROLL
+ for (int sgn = 1; sgn < BN; sgn++) {
+ MTL_PRAGMA_UNROLL
+ for (int tm = 0; tm < TM; tm++) {
+ result_up[tm] += tgp_results_up[sgn * (blockM + TM) + tm];
+ result_gate[tm] += tgp_results_gate[sgn * (blockM + TM) + tm];
+ }
+ }
+ }
+ }
+ }
+
+ // Apply MLP fused epilogue and write output
+ if (simdN == 0 && thrN == 0) {
+ MTL_PRAGMA_UNROLL
+ for (int tm = 0; tm < TM; tm++) {
+ float fused_result = mlp_fused_epilogue_f32(result_up[tm], result_gate[tm]);
+ out_vec[out_row + tm] = static_cast<T>(fused_result);
+ }
+ }
+ }
+};
+
+template <
+ typename T,
+ const int BM,
+ const int BN,
+ const int SM,
+ const int SN,
+ const int TM,
+ const int TN>
+[[kernel, max_total_threads_per_threadgroup(BM* BN * 32)]] void gemv_mlp_fused(
+ const device T* mat [[buffer(0)]],
+ const device T* in_vec [[buffer(1)]],
+ device T* out_vec [[buffer(3)]],
+ const constant int& in_vec_size [[buffer(4)]],
+ const constant int& hidden_dim [[buffer(5)]], // Output dimension (half of weight rows)
+ const constant int& matrix_ld [[buffer(6)]],
+ const constant int64_t& vector_batch_stride [[buffer(11)]],
+ const constant int64_t& matrix_batch_stride [[buffer(12)]],
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint3 lid [[thread_position_in_threadgroup]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]) {
+ using gemv_kernel = GEMVMlpFusedKernel<T, BM, BN, SM, SN, TM, TN>;
+ threadgroup typename gemv_kernel::AccT tgp_memory
+ [gemv_kernel::tgp_mem_size == 0 ? 1 : gemv_kernel::tgp_mem_size];
+
+ // Batch offset
+ in_vec += tid.z * vector_batch_stride;
+ mat += tid.z * matrix_batch_stride;
+ out_vec += tid.z * hidden_dim;
+
+ gemv_kernel::run(
+ mat,
+ in_vec,
+ out_vec,
+ in_vec_size,
+ hidden_dim,
+ matrix_ld,
+ gemv_kernel::tgp_mem_size == 0 ? nullptr : tgp_memory,
+ tid,
+ lid,
+ simd_gid,
+ simd_lid);
+}
+
+#define instantiate_gemv_mlp_fused_helper(name, itype, bm, bn, sm, sn, tm, tn) \
+ instantiate_kernel( \
+ "gemv_mlp_fused_" #name "_bm" #bm "_bn" #bn "_sm" #sm "_sn" #sn "_tm" #tm \
+ "_tn" #tn, \
+ gemv_mlp_fused, \
+ itype, \
+ bm, \
+ bn, \
+ sm, \
+ sn, \
+ tm, \
+ tn)
+
+// clang-format off
+#define instantiate_gemv_mlp_fused_blocks(name, itype) \
+ instantiate_gemv_mlp_fused_helper(name, itype, 4, 1, 1, 32, 4, 4) \
+ instantiate_gemv_mlp_fused_helper(name, itype, 8, 1, 1, 32, 4, 4) // clang-format on
+
+instantiate_gemv_mlp_fused_blocks(float32, float);
+instantiate_gemv_mlp_fused_blocks(float16, half);
+instantiate_gemv_mlp_fused_blocks(bfloat16, bfloat16_t);
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs
index 8c36bf0d8..1c257c8c9 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs
@@ -6,7 +6,7 @@ use metal::{
};
use super::{
- super::{KernelDataType, TensorAddBias},
+ super::{KernelDataType, TensorAddBias, mlp::MlpActivationType},
arguments::MatmulArguments,
gemv::GemvKernel,
pipeline::PipelineKey,
@@ -643,3 +643,212 @@ impl MatmulKernel {
);
}
}
+
+/// Arguments for MLP fused GEMM
+#[derive(Debug)]
+pub struct MlpFusedGemmArguments<'a> {
+ /// Input activations [M, K]
+ pub input: &'a MTLBuffer,
+ /// Input byte offset
+ pub input_offset: u64,
+ /// Weight matrix [K, 2*hidden_dim] or [2*hidden_dim, K] depending on transpose
+ pub weights: &'a MTLBuffer,
+ /// Output [M, hidden_dim]
+ pub output: &'a MTLBuffer,
+ /// Batch size (M)
+ pub batch: i32,
+ /// Input dimension (K)
+ pub input_dim: i32,
+ /// Hidden dimension (output size, half of weight columns/rows)
+ pub hidden_dim: i32,
+ /// Leading dimension of input
+ pub lda: i32,
+ /// Leading dimension of weights
+ pub ldb: i32,
+ /// Leading dimension of output
+ pub ldd: i32,
+ /// Activation type for fused epilogue
+ pub activation: MlpActivationType,
+}
+
+/// MLP Fused GEMM Kernel for prefill path
+/// Computes paired up/gate projections with fused activation: out = up * activation(gate)
+pub struct MlpFusedGemmKernel {
+ data_type: DataType,
+ weights_transposed: bool,
+ pipelines: HashMap<
+ (i32, i32, bool, bool, MlpActivationType),
+ MTLComputePipelineState,
+ >,
+}
+
+impl MlpFusedGemmKernel {
+ pub fn new(
+ data_type: DataType,
+ weights_transposed: bool,
+ ) -> Result<Self, MTLError> {
+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)
+ {
+ return Err(MTLError::Generic(format!(
+ "Unsupported dtype for MLP fused GEMM: {:?}",
+ data_type
+ )));
+ }
+ Ok(Self {
+ data_type,
+ weights_transposed,
+ pipelines: HashMap::new(),
+ })
+ }
+
+ fn kernel_name(
+ &self,
+ bm: i32,
+ bn: i32,
+ mn_aligned: bool,
+ k_aligned: bool,
+ ) -> String {
+ let type_name = match self.data_type {
+ DataType::F16 => "half",
+ DataType::BF16 => "bfloat",
+ DataType::F32 => "float",
+ _ => unreachable!(),
+ };
+ let transpose_char = if self.weights_transposed {
+ "t"
+ } else {
+ "n"
+ };
+ let mn_str = if mn_aligned {
+ "true"
+ } else {
+ "false"
+ };
+ let k_str = if k_aligned {
+ "true"
+ } else {
+ "false"
+ };
+
+ format!(
+ "steel_gemm_mlp_fused_{}_{}_{}_bm{}_bn{}_bk16_wm2_wn2_align_MN_{}_K_{}",
+ transpose_char, type_name, type_name, bm, bn, mn_str, k_str
+ )
+ }
+
+ fn get_pipeline(
+ &mut self,
+ context: &MTLContext,
+ bm: i32,
+ bn: i32,
+ mn_aligned: bool,
+ k_aligned: bool,
+ activation: MlpActivationType,
+ ) -> Result<&MTLComputePipelineState, MTLError> {
+ let key = (bm, bn, mn_aligned, k_aligned, activation);
+ if !self.pipelines.contains_key(&key) {
+ let kernel_name = self.kernel_name(bm, bn, mn_aligned, k_aligned);
+
+ // Create function constant values for MLP activation
+ let fcv = metal::FunctionConstantValues::new();
+ let activation_val = activation as u32;
+ fcv.set_constant_value_at_index(
+ &activation_val as *const u32 as *const _,
+ metal::MTLDataType::UInt,
+ 52, // MLP_ACTIVATION index
+ );
+
+ let pipeline =
+ context.compute_pipeline_state(&kernel_name, Some(&fcv))?;
+ self.pipelines.insert(key, pipeline);
+ }
+ Ok(self.pipelines.get(&key).unwrap())
+ }
+
+ fn select_tile_size(
+ &self,
+ m: i32,
+ hidden_dim: i32,
+ k: i32,
+ ) -> (i32, i32) {
+ // Select tile size based on problem dimensions
+ let _ = k; // K doesn't affect tile selection directly
+ if m >= 64 && hidden_dim >= 64 {
+ (64, 64)
+ } else {
+ (32, 32)
+ }
+ }
+
+ pub fn encode(
+ &mut self,
+ context: &MTLContext,
+ encoder: &ComputeCommandEncoderRef,
+ args: &MlpFusedGemmArguments,
+ ) -> Result<(), MTLError> {
+ let (bm, bn) =
+ self.select_tile_size(args.batch, args.hidden_dim, args.input_dim);
+ let bk = 16;
+
+ let mn_aligned = args.batch % bm == 0 && args.hidden_dim % bn == 0;
+ let k_aligned = args.input_dim % bk == 0;
+
+ let pipeline = self.get_pipeline(
+ context,
+ bm,
+ bn,
+ mn_aligned,
+ k_aligned,
+ args.activation,
+ )?;
+ encoder.set_compute_pipeline_state(pipeline);
+
+ encoder.set_buffer(0, Some(args.input), args.input_offset);
+ encoder.set_buffer(1, Some(args.weights), 0);
+ encoder.set_buffer(2, Some(args.output), 0);
+
+ // Set GEMM params
+ let tiles_m = (args.batch + bm - 1) / bm;
+ let tiles_n = (args.hidden_dim + bn - 1) / bn;
+ let gemm_k_iterations = (args.input_dim + bk - 1) / bk;
+
+ let params = GEMMParams {
+ M: args.batch,
+ N: args.hidden_dim * 2, // Full weight width for pointer arithmetic
+ K: args.input_dim,
+ lda: args.lda,
+ ldb: args.ldb,
+ ldd: args.ldd,
+ tiles_n,
+ tiles_m,
+ batch_stride_a: 0,
+ batch_stride_b: 0,
+ batch_stride_d: 0,
+ swizzle_log: 0,
+ gemm_k_iterations_aligned: if k_aligned {
+ gemm_k_iterations
+ } else {
+ gemm_k_iterations - 1
+ },
+ batch_ndim: 0,
+ };
+
+ encoder.set_bytes(
+ 3,
+ std::mem::size_of::<GEMMParams>() as u64,
+ &params as *const GEMMParams as *const std::ffi::c_void,
+ );
+ encoder.set_bytes(
+ 10,
+ std::mem::size_of::<i32>() as u64,
+ &args.hidden_dim as *const i32 as *const std::ffi::c_void,
+ );
+
+ let threadgroup_count = MTLSize::new(tiles_n as u64, tiles_m as u64, 1);
+ let threads_per_threadgroup = MTLSize::new(32, 2, 2); // WM=2, WN=2, 32 threads per simdgroup
+
+ encoder
+ .dispatch_thread_groups(threadgroup_count, threads_per_threadgroup);
+ Ok(())
+ }
+}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/mod.rs
index fb2a346ad..81a690ef5 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/mod.rs
@@ -11,7 +11,7 @@ mod transpose;
pub use arguments::MatmulArguments;
pub use gather::{GatherGemm, GatherMmArguments, GatherMmRhsArguments};
-pub use gemv::GemvKernel;
-pub use kernel::MatmulKernel;
+pub use gemv::{GemvKernel, MlpFusedGemvArguments, MlpFusedGemvKernel};
+pub use kernel::{MatmulKernel, MlpFusedGemmArguments, MlpFusedGemmKernel};
pub use segmented::{SegmentedGemm, SegmentedMmArguments};
pub use splitk::SplitKGemm;
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_mlp_fused.h b/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_mlp_fused.h
new file mode 100644
index 000000000..88ebcdbe1
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_mlp_fused.h
@@ -0,0 +1,207 @@
+#pragma once
+
+#include <metal_stdlib>
+#include <metal_simdgroup>
+#include <metal_simdgroup_matrix>
+
+#include "../gemm.h"
+#include "../params.h"
+#include "../../../../common/mlp_epilogue.h"
+
+using namespace metal;
+
+namespace steel {
+
+///////////////////////////////////////////////////////////////////////////////
+// MLP Fused GEMM Kernel
+// Computes paired up/gate projections with fused activation for prefill path.
+// Weight layout: B = [up_weights (hidden_dim cols), gate_weights (hidden_dim cols)]
+// Output = up * activation(gate), written to first hidden_dim columns
+///////////////////////////////////////////////////////////////////////////////
+
+template <
+ typename T,
+ typename U,
+ int BM,
+ int BN,
+ int BK,
+ int WM,
+ int WN,
+ bool transpose_a,
+ bool transpose_b,
+ bool MN_aligned,
+ bool K_aligned,
+ typename AccumType = typename AccumHelper<T>::accum_type>
+struct GEMMMlpFusedKernel {
+ STEEL_CONST short tgp_padding_a = 16 / sizeof(T);
+ STEEL_CONST short tgp_padding_b = 16 / sizeof(T);
+ STEEL_CONST short tgp_mem_size_a =
+ transpose_a ? BK * (BM + tgp_padding_a) : BM * (BK + tgp_padding_a);
+ // Double B threadgroup memory for up and gate
+ STEEL_CONST short tgp_mem_size_b =
+ 2 * (transpose_b ? BN * (BK + tgp_padding_b) : BK * (BN + tgp_padding_b));
+ STEEL_CONST short tgp_mem_size = tgp_mem_size_a + tgp_mem_size_b;
+
+ STEEL_CONST short tgp_size = WM * WN * 32;
+
+ using loader_a_t = BlockLoader<
+ T,
+ transpose_a ? BK : BM,
+ transpose_a ? BM : BK,
+ transpose_a ? BM + tgp_padding_a : BK + tgp_padding_a,
+ !transpose_a,
+ tgp_size>;
+ using loader_b_t = BlockLoader<
+ T,
+ transpose_b ? BN : BK,
+ transpose_b ? BK : BN,
+ transpose_b ? BK + tgp_padding_b : BN + tgp_padding_b,
+ transpose_b,
+ tgp_size>;
+ using mma_t = BlockMMA<
+ T,
+ U,
+ BM,
+ BN,
+ BK,
+ WM,
+ WN,
+ transpose_a,
+ transpose_b,
+ transpose_a ? BM + tgp_padding_a : BK + tgp_padding_a,
+ transpose_b ? BK + tgp_padding_b : BN + tgp_padding_b,
+ AccumType,
+ TransformNone<U, AccumType>>;
+
+ static METAL_FUNC void run(
+ const device T* A [[buffer(0)]],
+ const device T* B [[buffer(1)]],
+ device U* D [[buffer(2)]],
+ const constant GEMMParams* params [[buffer(3)]],
+ const constant int& hidden_dim [[buffer(10)]],
+ threadgroup T* As [[threadgroup(0)]],
+ threadgroup T* Bs_up [[threadgroup(1)]],
+ uint simd_lane_id [[thread_index_in_simdgroup]],
+ uint simd_group_id [[simdgroup_index_in_threadgroup]],
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint3 lid [[thread_position_in_threadgroup]]) {
+ (void)lid;
+
+ const int tid_y = ((tid.y) << params->swizzle_log) +
+ ((tid.x) & ((1 << params->swizzle_log) - 1));
+ const int tid_x = (tid.x) >> params->swizzle_log;
+
+ // Only process tiles within hidden_dim columns (output size)
+ int hidden_tiles = (hidden_dim + BN - 1) / BN;
+ if (hidden_tiles <= tid_x || params->tiles_m <= tid_y) {
+ return;
+ }
+
+ threadgroup_barrier(mem_flags::mem_none);
+
+ const int c_row = tid_y * BM;
+ const int c_col = tid_x * BN;
+ const size_t c_row_long = size_t(c_row);
+ const size_t c_col_long = size_t(c_col);
+
+ // Pointer to A (input activations)
+ const device T* A_ptr = A + (transpose_a ? c_row_long : c_row_long * params->lda);
+
+ // Pointers to up and gate weights in B
+ // B layout: [up_weights | gate_weights] where each is hidden_dim columns
+ const device T* B_up = B + (transpose_b ? c_col_long * params->ldb : c_col_long);
+ const device T* B_gate = B_up + (transpose_b ? hidden_dim * params->ldb : hidden_dim);
+
+ device U* D_ptr = D + c_row_long * params->ldd + c_col_long;
+
+ // Threadgroup memory layout: [As, Bs_up, Bs_gate]
+ threadgroup T* Bs_gate = Bs_up + (tgp_mem_size_b / 2);
+
+ // Loaders for A (shared) and B (up and gate)
+ thread loader_a_t loader_a(A_ptr, params->lda, As, simd_group_id, simd_lane_id);
+ thread loader_b_t loader_b_up(B_up, params->ldb, Bs_up, simd_group_id, simd_lane_id);
+ thread loader_b_t loader_b_gate(B_gate, params->ldb, Bs_gate, simd_group_id, simd_lane_id);
+
+ // MMA operations for up and gate
+ thread mma_t mma_up(simd_group_id, simd_lane_id);
+ thread mma_t mma_gate(simd_group_id, simd_lane_id);
+
+ int gemm_k_iterations = params->gemm_k_iterations_aligned;
+
+ // Main compute loop
+ for (int k = 0; k < gemm_k_iterations; k++) {
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // Load A (shared for both up and gate)
+ if (MN_aligned) {
+ loader_a.load_unsafe();
+ } else {
+ short tgp_bm = min(short(BM), short(params->M - c_row));
+ short2 tile_dims_A = transpose_a ? short2(tgp_bm, BK) : short2(BK, tgp_bm);
+ loader_a.load_safe(tile_dims_A);
+ }
+
+ // Load B_up and B_gate
+ if (MN_aligned) {
+ loader_b_up.load_unsafe();
+ loader_b_gate.load_unsafe();
+ } else {
+ short tgp_bn = min(short(BN), short(hidden_dim - c_col));
+ short2 tile_dims_B = transpose_b ? short2(BK, tgp_bn) : short2(tgp_bn, BK);
+ loader_b_up.load_safe(tile_dims_B);
+ loader_b_gate.load_safe(tile_dims_B);
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // MMA for up and gate
+ mma_up.mma(As, Bs_up);
+ mma_gate.mma(As, Bs_gate);
+
+ loader_a.next();
+ loader_b_up.next();
+ loader_b_gate.next();
+ }
+
+ // Handle leftover K
+ if (!K_aligned) {
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ int lbk = params->K - params->gemm_k_iterations_aligned * BK;
+ short2 tile_dims_A = transpose_a ? short2(BM, lbk) : short2(lbk, BM);
+ short2 tile_dims_B = transpose_b ? short2(lbk, BN) : short2(BN, lbk);
+
+ loader_a.load_safe(tile_dims_A);
+ loader_b_up.load_safe(tile_dims_B);
+ loader_b_gate.load_safe(tile_dims_B);
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ mma_up.mma(As, Bs_up);
+ mma_gate.mma(As, Bs_gate);
+ }
+
+ // Apply MLP fused epilogue: out = up * activation(gate)
+ // Access the Ctile elements and apply fusion
+ constexpr int kElemsPerTile = decltype(mma_up.Ctile)::kElemsPerTile;
+
+ #pragma unroll
+ for (short i = 0; i < kElemsPerTile; i++) {
+ float up_val = static_cast<float>(mma_up.Ctile.elems()[i]);
+ float gate_val = static_cast<float>(mma_gate.Ctile.elems()[i]);
+ float fused = mlp_fused_epilogue_f32(up_val, gate_val);
+ mma_up.Ctile.elems()[i] = static_cast<AccumType>(fused);
+ }
+
+ // Store result
+ if (MN_aligned) {
+ mma_up.store_result(D_ptr, params->ldd);
+ } else {
+ short tgp_bm = min(short(BM), short(params->M - c_row));
+ short tgp_bn = min(short(BN), short(hidden_dim - c_col));
+ mma_up.store_result_safe(D_ptr, params->ldd, short2(tgp_bn, tgp_bm));
+ }
+ }
+};
+
+} // namespace steel
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_mlp_fused.metal b/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_mlp_fused.metal
new file mode 100644
index 000000000..5b0da8472
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_mlp_fused.metal
@@ -0,0 +1,109 @@
+#include <metal_stdlib>
+
+#include "../../../../common/utils.h"
+#include "steel_gemm_mlp_fused.h"
+
+using namespace metal;
+
+///////////////////////////////////////////////////////////////////////////////
+// MLP Fused GEMM Kernel Instantiations
+// For prefill path: computes up * activation(gate) in a single pass
+///////////////////////////////////////////////////////////////////////////////
+
+#define instantiate_mlp_fused_gemm( \
+ tname, trans_a, trans_b, itype, otype, bm, bn, bk, wm, wn, aname, mn, k) \
+ template [[host_name("steel_gemm_mlp_fused_" #tname "_" #itype "_" #otype \
+ "_bm" #bm "_bn" #bn "_bk" #bk "_wm" #wm "_wn" #wn \
+ "_" #aname "_MN_" #mn "_K_" #k)]] [[kernel]] void \
+ steel_gemm_mlp_fused< \
+ itype, otype, bm, bn, bk, wm, wn, trans_a, trans_b, mn, k>( \
+ const device itype* A [[buffer(0)]], \
+ const device itype* B [[buffer(1)]], \
+ device otype* D [[buffer(2)]], \
+ const constant steel::GEMMParams* params [[buffer(3)]], \
+ const constant int& hidden_dim [[buffer(10)]], \
+ uint simd_lane_id [[thread_index_in_simdgroup]], \
+ uint simd_group_id [[simdgroup_index_in_threadgroup]], \
+ uint3 tid [[threadgroup_position_in_grid]], \
+ uint3 lid [[thread_position_in_threadgroup]])
+
+template <
+ typename T,
+ typename U,
+ int BM,
+ int BN,
+ int BK,
+ int WM,
+ int WN,
+ bool transpose_a,
+ bool transpose_b,
+ bool MN_aligned,
+ bool K_aligned>
+[[kernel, max_total_threads_per_threadgroup(WM* WN * 32)]] void
+steel_gemm_mlp_fused(
+ const device T* A [[buffer(0)]],
+ const device T* B [[buffer(1)]],
+ device U* D [[buffer(2)]],
+ const constant steel::GEMMParams* params [[buffer(3)]],
+ const constant int& hidden_dim [[buffer(10)]],
+ uint simd_lane_id [[thread_index_in_simdgroup]],
+ uint simd_group_id [[simdgroup_index_in_threadgroup]],
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint3 lid [[thread_position_in_threadgroup]]) {
+ using kernel_t =
+ steel::GEMMMlpFusedKernel<T, U, BM, BN, BK, WM, WN, transpose_a, transpose_b, MN_aligned, K_aligned>;
+
+ // Allocate threadgroup memory
+ threadgroup T As[kernel_t::tgp_mem_size_a];
+ threadgroup T Bs[kernel_t::tgp_mem_size_b];
+
+ kernel_t::run(
+ A, B, D, params, hidden_dim, As, Bs, simd_lane_id, simd_group_id, tid, lid);
+}
+
+// Instantiate for common configurations
+// transpose_a = false, transpose_b = true (weights are [K, 2*H], accessed as transposed)
+
+// float16 configurations
+instantiate_mlp_fused_gemm(t, false, true, half, half, 32, 32, 16, 2, 2, align, true, true);
+instantiate_mlp_fused_gemm(t, false, true, half, half, 32, 32, 16, 2, 2, align, true, false);
+instantiate_mlp_fused_gemm(t, false, true, half, half, 32, 32, 16, 2, 2, align, false, true);
+instantiate_mlp_fused_gemm(t, false, true, half, half, 32, 32, 16, 2, 2, align, false, false);
+
+instantiate_mlp_fused_gemm(t, false, true, half, half, 64, 64, 16, 2, 2, align, true, true);
+instantiate_mlp_fused_gemm(t, false, true, half, half, 64, 64, 16, 2, 2, align, true, false);
+instantiate_mlp_fused_gemm(t, false, true, half, half, 64, 64, 16, 2, 2, align, false, true);
+instantiate_mlp_fused_gemm(t, false, true, half, half, 64, 64, 16, 2, 2, align, false, false);
+
+// bfloat16 configurations
+instantiate_mlp_fused_gemm(t, false, true, bfloat, bfloat, 32, 32, 16, 2, 2, align, true, true);
+instantiate_mlp_fused_gemm(t, false, true, bfloat, bfloat, 32, 32, 16, 2, 2, align, true, false);
+instantiate_mlp_fused_gemm(t, false, true, bfloat, bfloat, 32, 32, 16, 2, 2, align, false, true);
+instantiate_mlp_fused_gemm(t, false, true, bfloat, bfloat, 32, 32, 16, 2, 2, align, false, false);
+
+instantiate_mlp_fused_gemm(t, false, true, bfloat, bfloat, 64, 64, 16, 2, 2, align, true, true);
+instantiate_mlp_fused_gemm(t, false, true, bfloat, bfloat, 64, 64, 16, 2, 2, align, true, false);
+instantiate_mlp_fused_gemm(t, false, true, bfloat, bfloat, 64, 64, 16, 2, 2, align, false, true);
+instantiate_mlp_fused_gemm(t, false, true, bfloat, bfloat, 64, 64, 16, 2, 2, align, false, false);
+
+// float32 configurations
+instantiate_mlp_fused_gemm(t, false, true, float, float, 32, 32, 16, 2, 2, align, true, true);
+instantiate_mlp_fused_gemm(t, false, true, float, float, 32, 32, 16, 2, 2, align, true, false);
+instantiate_mlp_fused_gemm(t, false, true, float, float, 32, 32, 16, 2, 2, align, false, true);
+instantiate_mlp_fused_gemm(t, false, true, float, float, 32, 32, 16, 2, 2, align, false, false);
+
+// Non-transposed B (weights are [2*H, K])
+instantiate_mlp_fused_gemm(n, false, false, half, half, 32, 32, 16, 2, 2, align, true, true);
+instantiate_mlp_fused_gemm(n, false, false, half, half, 32, 32, 16, 2, 2, align, true, false);
+instantiate_mlp_fused_gemm(n, false, false, half, half, 32, 32, 16, 2, 2, align, false, true);
+instantiate_mlp_fused_gemm(n, false, false, half, half, 32, 32, 16, 2, 2, align, false, false);
+
+instantiate_mlp_fused_gemm(n, false, false, bfloat, bfloat, 32, 32, 16, 2, 2, align, true, true);
+instantiate_mlp_fused_gemm(n, false, false, bfloat, bfloat, 32, 32, 16, 2, 2, align, true, false);
+instantiate_mlp_fused_gemm(n, false, false, bfloat, bfloat, 32, 32, 16, 2, 2, align, false, true);
+instantiate_mlp_fused_gemm(n, false, false, bfloat, bfloat, 32, 32, 16, 2, 2, align, false, false);
+
+instantiate_mlp_fused_gemm(n, false, false, float, float, 32, 32, 16, 2, 2, align, true, true);
+instantiate_mlp_fused_gemm(n, false, false, float, float, 32, 32, 16, 2, 2, align, true, false);
+instantiate_mlp_fused_gemm(n, false, false, float, float, 32, 32, 16, 2, 2, align, false, true);
+instantiate_mlp_fused_gemm(n, false, false, float, float, 32, 32, 16, 2, 2, align, false, false);
diff --git a/crates/uzu/src/backends/metal/kernel/mlp/mod.rs b/crates/uzu/src/backends/metal/kernel/mlp/mod.rs
index aefaff465..a2a245f1b 100644
--- a/crates/uzu/src/backends/metal/kernel/mlp/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/mlp/mod.rs
@@ -1,6 +1,6 @@
use metal::{
Buffer as MTLBuffer, ComputeCommandEncoderRef, ComputePipelineState,
- MTLSize,
+ FunctionConstantValues, MTLDataType, MTLSize,
};
use crate::{
@@ -9,6 +9,88 @@ use crate::{
config::Activation,
};
+// MLP Fused Epilogue Function Constant Indices
+// These must match the values in kernel/common/mlp_epilogue.h
+pub const MLP_FUSED_FC_INDEX: u64 = 50;
+pub const MLP_HIDDEN_DIM_FC_INDEX: u64 = 51;
+pub const MLP_ACTIVATION_FC_INDEX: u64 = 52;
+
+/// MLP activation type for fused kernels
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+#[repr(u32)]
+pub enum MlpActivationType {
+ SiLU = 0,
+ Gelu = 1,
+}
+
+impl From<&Activation> for MlpActivationType {
+ fn from(act: &Activation) -> Self {
+ match act {
+ Activation::SiLU {
+ ..
+ } => MlpActivationType::SiLU,
+ Activation::Gelu => MlpActivationType::Gelu,
+ Activation::Identity => {
+ panic!("Identity activation not supported for MLP fusion")
+ },
+ }
+ }
+}
+
+/// Configuration for MLP fused matmul epilogue
+#[derive(Debug, Clone, Copy)]
+pub struct MlpFusedConfig {
+ pub hidden_dim: u32,
+ pub activation: MlpActivationType,
+}
+
+impl MlpFusedConfig {
+ pub fn new(
+ hidden_dim: usize,
+ activation: &Activation,
+ ) -> Self {
+ Self {
+ hidden_dim: hidden_dim as u32,
+ activation: MlpActivationType::from(activation),
+ }
+ }
+
+ /// Create function constants for MLP fused matmul
+ pub fn make_function_constants(&self) -> FunctionConstantValues {
+ let fcv = FunctionConstantValues::new();
+ let fused = true;
+ fcv.set_constant_value_at_index(
+ &fused as *const bool as *const std::ffi::c_void,
+ MTLDataType::Bool,
+ MLP_FUSED_FC_INDEX,
+ );
+ fcv.set_constant_value_at_index(
+ &self.hidden_dim as *const u32 as *const std::ffi::c_void,
+ MTLDataType::UInt,
+ MLP_HIDDEN_DIM_FC_INDEX,
+ );
+ let act_val = self.activation as u32;
+ fcv.set_constant_value_at_index(
+ &act_val as *const u32 as *const std::ffi::c_void,
+ MTLDataType::UInt,
+ MLP_ACTIVATION_FC_INDEX,
+ );
+ fcv
+ }
+}
+
+/// Create function constants for non-fused (standard) matmul
+pub fn make_non_fused_function_constants() -> FunctionConstantValues {
+ let fcv = FunctionConstantValues::new();
+ let fused = false;
+ fcv.set_constant_value_at_index(
+ &fused as *const bool as *const std::ffi::c_void,
+ MTLDataType::Bool,
+ MLP_FUSED_FC_INDEX,
+ );
+ fcv
+}
+
pub struct MlpGateActMulKernel {
pipeline: ComputePipelineState,
}
diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rs
index e21378a11..ff569683d 100644
--- a/crates/uzu/src/backends/metal/kernel/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/mod.rs
@@ -8,6 +8,10 @@ pub mod mask_update;
pub mod matmul;
pub mod media_kernels;
pub mod mlp;
+pub use mlp::{
+ MLP_ACTIVATION_FC_INDEX, MLP_FUSED_FC_INDEX, MLP_HIDDEN_DIM_FC_INDEX,
+ MlpActivationType, MlpFusedConfig, make_non_fused_function_constants,
+};
pub mod moe;
pub mod pooling;
pub mod quant_matmul;
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs
index 2e87f0012..1698da06d 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs
@@ -354,3 +354,220 @@ enum KernelKind {
MatrixMatrix,
MatrixVector,
}
+
+/// Arguments for MLP fused quantized GEMV
+#[derive(Debug)]
+pub struct MlpFusedQmvArguments<'a> {
+ /// Weight matrix [2*hidden_dim, input_dim] (up and gate concatenated, quantized)
+ pub weights: &'a MTLBuffer,
+ /// Scales buffer
+ pub scales: &'a MTLBuffer,
+ /// Zero points or biases buffer (depends on quantization type)
+ pub zero_points_or_biases: &'a MTLBuffer,
+ /// Input vector [input_dim]
+ pub input: &'a MTLBuffer,
+ /// Input byte offset
+ pub input_offset: u64,
+ /// Output vector [hidden_dim]
+ pub output: &'a MTLBuffer,
+ /// Input dimension (K)
+ pub input_dim: i32,
+ /// Hidden dimension (output size, half of weight rows)
+ pub hidden_dim: i32,
+ /// Batch count
+ pub batch_count: i32,
+}
+
+/// MLP Fused Quantized GEMV Kernel
+/// Computes paired up/gate projections with fused activation: out = up * activation(gate)
+pub struct MlpFusedQmvKernel {
+ pipeline: MTLComputePipelineState,
+}
+
+impl MlpFusedQmvKernel {
+ pub fn new(
+ context: &MTLContext,
+ data_type: DataType,
+ group_size: usize,
+ mode: QuantizationMode,
+ quantization_type: QuantizationType,
+ ) -> Result<Self, QuantizedMatmulError> {
+ let type_suffix = dtype_suffix(data_type)
+ .ok_or(QuantizedMatmulError::UnsupportedDataType(data_type))?;
+
+ let bits = match mode {
+ QuantizationMode::UInt4 => 4,
+ QuantizationMode::UInt8 | QuantizationMode::Int8 => 8,
+ };
+
+ if bits != 4 {
+ return Err(QuantizedMatmulError::UnsupportedBits(bits));
+ }
+
+ let kernel_name =
+ format!("qmv_mlp_fused_{}_g{}_b{}", type_suffix, group_size, bits);
+
+ let function_constants = metal::FunctionConstantValues::new();
+ let use_mlx_quant = matches!(quantization_type, QuantizationType::Mlx);
+ function_constants.set_constant_value_at_index(
+ &use_mlx_quant as *const bool as *const std::ffi::c_void,
+ metal::MTLDataType::Bool,
+ 40,
+ );
+
+ let pipeline = context
+ .compute_pipeline_state(&kernel_name, Some(&function_constants))
+ .map_err(QuantizedMatmulError::MetalError)?;
+
+ Ok(Self {
+ pipeline,
+ })
+ }
+
+ pub fn encode(
+ &self,
+ encoder: &ComputeCommandEncoderRef,
+ args: &MlpFusedQmvArguments,
+ ) -> Result<(), QuantizedMatmulError> {
+ encoder.set_compute_pipeline_state(&self.pipeline);
+
+ encoder.set_buffer(0, Some(args.weights), 0);
+ encoder.set_buffer(1, Some(args.scales), 0);
+ encoder.set_buffer(2, Some(args.zero_points_or_biases), 0);
+ encoder.set_buffer(3, Some(args.input), args.input_offset);
+ encoder.set_buffer(4, Some(args.output), 0);
+
+ encoder.set_bytes(
+ 5,
+ size_of::<i32>() as u64,
+ &args.input_dim as *const i32 as *const _,
+ );
+ encoder.set_bytes(
+ 6,
+ size_of::<i32>() as u64,
+ &args.hidden_dim as *const i32 as *const _,
+ );
+
+ // Dispatch: one threadgroup per 8 output rows (num_simdgroups * results_per_simdgroup)
+ let rows_per_threadgroup = 8;
+ let n_tgp_y = ((args.hidden_dim + rows_per_threadgroup - 1)
+ / rows_per_threadgroup) as u64;
+ let threadgroups =
+ MTLSize::new(args.batch_count.max(1) as u64, n_tgp_y, 1);
+ let threads_per_threadgroup = MTLSize::new(32, 2, 1); // 2 simdgroups, 32 threads each
+
+ encoder.dispatch_thread_groups(threadgroups, threads_per_threadgroup);
+ Ok(())
+ }
+}
+
+/// Arguments for MLP fused quantized GEMM (prefill)
+#[derive(Debug)]
+pub struct MlpFusedQmmArguments<'a> {
+ /// Weight matrix [K, 2*hidden_dim] quantized (up and gate concatenated)
+ pub weights: &'a MTLBuffer,
+ /// Scales buffer
+ pub scales: &'a MTLBuffer,
+ /// Zero points or biases buffer (depends on quantization type)
+ pub zero_points_or_biases: &'a MTLBuffer,
+ /// Input activations [M, K]
+ pub input: &'a MTLBuffer,
+ /// Input byte offset
+ pub input_offset: u64,
+ /// Output [M, hidden_dim]
+ pub output: &'a MTLBuffer,
+ /// Batch size (M)
+ pub batch: i32,
+ /// Input dimension (K)
+ pub input_dim: i32,
+ /// Hidden dimension (output size, half of weight columns)
+ pub hidden_dim: i32,
+}
+
+/// MLP Fused Quantized GEMM Kernel for prefill path
+/// Computes paired up/gate projections with fused activation: out = up * activation(gate)
+pub struct MlpFusedQmmKernel {
+ pipeline: MTLComputePipelineState,
+}
+
+impl MlpFusedQmmKernel {
+ pub fn new(
+ context: &MTLContext,
+ data_type: DataType,
+ group_size: usize,
+ mode: QuantizationMode,
+ quantization_type: QuantizationType,
+ ) -> Result<Self, QuantizedMatmulError> {
+ let type_suffix = dtype_suffix(data_type)
+ .ok_or(QuantizedMatmulError::UnsupportedDataType(data_type))?;
+
+ let bits = match mode {
+ QuantizationMode::UInt4 => 4,
+ QuantizationMode::UInt8 | QuantizationMode::Int8 => 8,
+ };
+
+ if bits != 4 {
+ return Err(QuantizedMatmulError::UnsupportedBits(bits));
+ }
+
+ let kernel_name =
+ format!("qmm_mlp_fused_{}_g{}_b{}", type_suffix, group_size, bits);
+
+ let function_constants = metal::FunctionConstantValues::new();
+ let use_mlx_quant = matches!(quantization_type, QuantizationType::Mlx);
+ function_constants.set_constant_value_at_index(
+ &use_mlx_quant as *const bool as *const std::ffi::c_void,
+ metal::MTLDataType::Bool,
+ 40,
+ );
+
+ let pipeline = context
+ .compute_pipeline_state(&kernel_name, Some(&function_constants))
+ .map_err(QuantizedMatmulError::MetalError)?;
+
+ Ok(Self {
+ pipeline,
+ })
+ }
+
+ pub fn encode(
+ &self,
+ encoder: &ComputeCommandEncoderRef,
+ args: &MlpFusedQmmArguments,
+ ) -> Result<(), QuantizedMatmulError> {
+ encoder.set_compute_pipeline_state(&self.pipeline);
+
+ encoder.set_buffer(0, Some(args.weights), 0);
+ encoder.set_buffer(1, Some(args.scales), 0);
+ encoder.set_buffer(2, Some(args.zero_points_or_biases), 0);
+ encoder.set_buffer(3, Some(args.input), args.input_offset);
+ encoder.set_buffer(4, Some(args.output), 0);
+
+ encoder.set_bytes(
+ 5,
+ size_of::<i32>() as u64,
+ &args.input_dim as *const i32 as *const _,
+ );
+ encoder.set_bytes(
+ 6,
+ size_of::<i32>() as u64,
+ &args.hidden_dim as *const i32 as *const _,
+ );
+ encoder.set_bytes(
+ 7,
+ size_of::<i32>() as u64,
+ &args.batch as *const i32 as *const _,
+ );
+
+ // Dispatch: BM=32, BN=32
+ let bm = 32;
+ let bn = 32;
+ let tiles_m = ((args.batch + bm - 1) / bm) as u64;
+ let tiles_n = ((args.hidden_dim + bn - 1) / bn) as u64;
+ let threadgroups = MTLSize::new(tiles_n, tiles_m, 1);
+ let threads_per_threadgroup = MTLSize::new(32, 2, 2); // WM=2, WN=2, 32 threads
+
+ encoder.dispatch_thread_groups(threadgroups, threads_per_threadgroup);
+ Ok(())
+ }
+}
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal
index a79e2a67a..58872d428 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal
@@ -4,6 +4,7 @@
using namespace metal;
#include "mma.h"
+#include "../common/mlp_epilogue.h"
// Function constant: true = MLX-style (pre-computed biases), false = AWQ-style
// (zero-points)
@@ -4137,3 +4138,634 @@ qmm_transposed<bfloat, 128, 4, true, 128, 32, 64>(
uint simd_gid [[simdgroup_index_in_threadgroup]],
uint simd_lid [[thread_index_in_simdgroup]]
);
+
+///////////////////////////////////////////////////////////////////////////////
+/// MLP Fused QMV - Quantized Matrix-Vector with Fused Activation
+///////////////////////////////////////////////////////////////////////////////
+
+// MLP fused QMV kernel for decode path (M=1).
+// Computes paired up and gate projections, then applies: out = up * activation(gate)
+// Weight layout: [up_weights (hidden_dim rows), gate_weights (hidden_dim rows)]
+// Output size is hidden_dim.
+template <typename T, int group_size, int bits>
+void qmv_mlp_fused_impl(
+ const device uint32_t* w,
+ const device T* scales,
+ const device uint8_t* zero_points,
+ const device T* biases,
+ const device T* x,
+ device T* y,
+ const constant int& in_vec_size,
+ const constant int& hidden_dim, // Output size (half of weight rows)
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]
+) {
+ constexpr int packs_per_thread = bits == 2 ? 1 : 2;
+ constexpr int num_simdgroups = 2;
+ constexpr int results_per_simdgroup = 4; // Compute 4 paired rows at a time
+ constexpr int pack_factor = get_pack_factor<bits, 32>();
+ constexpr int bytes_per_pack = get_bytes_per_pack<bits, 32>();
+ constexpr int values_per_thread = pack_factor * packs_per_thread;
+ constexpr int block_size = values_per_thread * SIMD_SIZE;
+ constexpr int scale_step_per_thread = group_size / values_per_thread;
+ const device uint8_t* ws = (const device uint8_t*)w;
+ typedef float U;
+ thread U x_thread[values_per_thread];
+ thread U result_up[results_per_simdgroup] = {0};
+ thread U result_gate[results_per_simdgroup] = {0};
+
+ const int in_vec_size_w = in_vec_size * bytes_per_pack / pack_factor;
+ const int in_vec_size_g = in_vec_size / group_size;
+
+ // Output row for 'up' projection
+ const int out_row_up = tid.y * (num_simdgroups * results_per_simdgroup) +
+ simd_gid * results_per_simdgroup;
+
+ // Corresponding 'gate' row is offset by hidden_dim
+ const int out_row_gate = out_row_up + hidden_dim;
+
+ // Check bounds - only process if up row is within hidden_dim
+ if (out_row_up >= hidden_dim) return;
+
+ // Pointer setup for up weights
+ const device uint8_t* ws_up = ws + out_row_up * in_vec_size_w + simd_lid * packs_per_thread * bytes_per_pack;
+ const device T* scales_up = scales + out_row_up * in_vec_size_g + simd_lid / scale_step_per_thread;
+
+ // Pointer setup for gate weights
+ const device uint8_t* ws_gate = ws + out_row_gate * in_vec_size_w + simd_lid * packs_per_thread * bytes_per_pack;
+ const device T* scales_gate = scales + out_row_gate * in_vec_size_g + simd_lid / scale_step_per_thread;
+
+ const device T* biases_up = nullptr;
+ const device T* biases_gate = nullptr;
+ const device uint8_t* zps_up = nullptr;
+ const device uint8_t* zps_gate = nullptr;
+ int zp_stride = 0;
+ bool high_nibble = false;
+
+ if (kUseMlxQuant) {
+ biases_up = biases + out_row_up * in_vec_size_g + simd_lid / scale_step_per_thread;
+ biases_gate = biases + out_row_gate * in_vec_size_g + simd_lid / scale_step_per_thread;
+ } else {
+ if (bits == 4) {
+ zp_stride = (in_vec_size_g + 1) / 2;
+ zps_up = zero_points + out_row_up * zp_stride;
+ zps_gate = zero_points + out_row_gate * zp_stride;
+ int g_offset = simd_lid / scale_step_per_thread;
+ zps_up += g_offset / 2;
+ zps_gate += g_offset / 2;
+ high_nibble = (g_offset & 1);
+ } else {
+ zp_stride = in_vec_size_g;
+ zps_up = zero_points + out_row_up * zp_stride + simd_lid / scale_step_per_thread;
+ zps_gate = zero_points + out_row_gate * zp_stride + simd_lid / scale_step_per_thread;
+ }
+ }
+
+ x += tid.x * in_vec_size + simd_lid * values_per_thread;
+ y += tid.x * hidden_dim + out_row_up;
+
+ for (int k = 0; k < in_vec_size; k += block_size) {
+ U sum = load_vector<T, U, values_per_thread, bits>(x, x_thread);
+
+ // Compute both up and gate dot products for 4 rows each
+ {
+ auto wl_up0 = ws_up;
+ auto wl_up1 = ws_up + in_vec_size_w;
+ auto wl_up2 = ws_up + 2 * in_vec_size_w;
+ auto wl_up3 = ws_up + 3 * in_vec_size_w;
+
+ auto wl_gate0 = ws_gate;
+ auto wl_gate1 = ws_gate + in_vec_size_w;
+ auto wl_gate2 = ws_gate + 2 * in_vec_size_w;
+ auto wl_gate3 = ws_gate + 3 * in_vec_size_w;
+
+ U s_up0 = static_cast<U>(scales_up[0]);
+ U s_up1 = static_cast<U>(scales_up[in_vec_size_g]);
+ U s_up2 = static_cast<U>(scales_up[2 * in_vec_size_g]);
+ U s_up3 = static_cast<U>(scales_up[3 * in_vec_size_g]);
+
+ U s_gate0 = static_cast<U>(scales_gate[0]);
+ U s_gate1 = static_cast<U>(scales_gate[in_vec_size_g]);
+ U s_gate2 = static_cast<U>(scales_gate[2 * in_vec_size_g]);
+ U s_gate3 = static_cast<U>(scales_gate[3 * in_vec_size_g]);
+
+ if (kUseMlxQuant) {
+ U b_up0 = static_cast<U>(biases_up[0]);
+ U b_up1 = static_cast<U>(biases_up[in_vec_size_g]);
+ U b_up2 = static_cast<U>(biases_up[2 * in_vec_size_g]);
+ U b_up3 = static_cast<U>(biases_up[3 * in_vec_size_g]);
+
+ U b_gate0 = static_cast<U>(biases_gate[0]);
+ U b_gate1 = static_cast<U>(biases_gate[in_vec_size_g]);
+ U b_gate2 = static_cast<U>(biases_gate[2 * in_vec_size_g]);
+ U b_gate3 = static_cast<U>(biases_gate[3 * in_vec_size_g]);
+
+ result_up[0] += qdot<U, values_per_thread, bits>(wl_up0, x_thread, s_up0, b_up0, sum);
+ result_up[1] += qdot<U, values_per_thread, bits>(wl_up1, x_thread, s_up1, b_up1, sum);
+ result_up[2] += qdot<U, values_per_thread, bits>(wl_up2, x_thread, s_up2, b_up2, sum);
+ result_up[3] += qdot<U, values_per_thread, bits>(wl_up3, x_thread, s_up3, b_up3, sum);
+
+ result_gate[0] += qdot<U, values_per_thread, bits>(wl_gate0, x_thread, s_gate0, b_gate0, sum);
+ result_gate[1] += qdot<U, values_per_thread, bits>(wl_gate1, x_thread, s_gate1, b_gate1, sum);
+ result_gate[2] += qdot<U, values_per_thread, bits>(wl_gate2, x_thread, s_gate2, b_gate2, sum);
+ result_gate[3] += qdot<U, values_per_thread, bits>(wl_gate3, x_thread, s_gate3, b_gate3, sum);
+ } else {
+ auto extract_zp = [&](const device uint8_t* zps, int row_offset) -> U {
+ uint8_t zp_byte = zps[row_offset * zp_stride];
+ if (bits == 4) {
+ return static_cast<U>(high_nibble ? (zp_byte >> 4) : (zp_byte & 0x0F));
+ } else {
+ return static_cast<U>(zp_byte);
+ }
+ };
+
+ U zp_up0 = extract_zp(zps_up, 0);
+ U zp_up1 = extract_zp(zps_up, 1);
+ U zp_up2 = extract_zp(zps_up, 2);
+ U zp_up3 = extract_zp(zps_up, 3);
+
+ U zp_gate0 = extract_zp(zps_gate, 0);
+ U zp_gate1 = extract_zp(zps_gate, 1);
+ U zp_gate2 = extract_zp(zps_gate, 2);
+ U zp_gate3 = extract_zp(zps_gate, 3);
+
+ result_up[0] += qdot_zero_point<U, values_per_thread, bits>(wl_up0, x_thread, s_up0, zp_up0);
+ result_up[1] += qdot_zero_point<U, values_per_thread, bits>(wl_up1, x_thread, s_up1, zp_up1);
+ result_up[2] += qdot_zero_point<U, values_per_thread, bits>(wl_up2, x_thread, s_up2, zp_up2);
+ result_up[3] += qdot_zero_point<U, values_per_thread, bits>(wl_up3, x_thread, s_up3, zp_up3);
+
+ result_gate[0] += qdot_zero_point<U, values_per_thread, bits>(wl_gate0, x_thread, s_gate0, zp_gate0);
+ result_gate[1] += qdot_zero_point<U, values_per_thread, bits>(wl_gate1, x_thread, s_gate1, zp_gate1);
+ result_gate[2] += qdot_zero_point<U, values_per_thread, bits>(wl_gate2, x_thread, s_gate2, zp_gate2);
+ result_gate[3] += qdot_zero_point<U, values_per_thread, bits>(wl_gate3, x_thread, s_gate3, zp_gate3);
+ }
+ }
+
+ ws_up += block_size * bytes_per_pack / pack_factor;
+ ws_gate += block_size * bytes_per_pack / pack_factor;
+ scales_up += block_size / group_size;
+ scales_gate += block_size / group_size;
+ if (kUseMlxQuant) {
+ biases_up += block_size / group_size;
+ biases_gate += block_size / group_size;
+ } else {
+ if (bits == 4) {
+ zps_up += (block_size / group_size) / 2;
+ zps_gate += (block_size / group_size) / 2;
+ } else {
+ zps_up += block_size / group_size;
+ zps_gate += block_size / group_size;
+ }
+ }
+ x += block_size;
+ }
+
+ // SIMD reduction and fused activation
+ for (int row = 0; row < results_per_simdgroup; row++) {
+ result_up[row] = simd_sum(result_up[row]);
+ result_gate[row] = simd_sum(result_gate[row]);
+ if (simd_lid == 0) {
+ // Apply MLP fused epilogue: up * activation(gate)
+ float fused = mlp_fused_epilogue_f32(result_up[row], result_gate[row]);
+ y[row] = static_cast<T>(fused);
+ }
+ }
+}
+
+template <typename T, int group_size, int bits>
+[[kernel]] void qmv_mlp_fused(
+ const device uint32_t* w [[buffer(0)]],
+ const device T* scales [[buffer(1)]],
+ const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],
+ const device T* biases [[buffer(2), function_constant(kUseMlxQuant)]],
+ const device T* x [[buffer(3)]],
+ device T* y [[buffer(4)]],
+ const constant int& K [[buffer(5)]],
+ const constant int& hidden_dim [[buffer(6)]],
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]
+) {
+ qmv_mlp_fused_impl<T, group_size, bits>(
+ w,
+ scales,
+ zero_points,
+ biases,
+ x,
+ y,
+ K,
+ hidden_dim,
+ tid,
+ simd_gid,
+ simd_lid
+ );
+}
+
+// Instantiate MLP fused QMV kernels for common configurations
+template [[host_name("qmv_mlp_fused_f16_g32_b4")]] [[kernel]] void qmv_mlp_fused<
+ half, 32, 4>(
+ const device uint32_t* w [[buffer(0)]],
+ const device half* scales [[buffer(1)]],
+ const device uint8_t* zero_points [[buffer(2)]],
+ const device half* biases [[buffer(2)]],
+ const device half* x [[buffer(3)]],
+ device half* y [[buffer(4)]],
+ const constant int& K [[buffer(5)]],
+ const constant int& hidden_dim [[buffer(6)]],
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
+
+template [[host_name("qmv_mlp_fused_f16_g64_b4")]] [[kernel]] void qmv_mlp_fused<
+ half, 64, 4>(
+ const device uint32_t* w [[buffer(0)]],
+ const device half* scales [[buffer(1)]],
+ const device uint8_t* zero_points [[buffer(2)]],
+ const device half* biases [[buffer(2)]],
+ const device half* x [[buffer(3)]],
+ device half* y [[buffer(4)]],
+ const constant int& K [[buffer(5)]],
+ const constant int& hidden_dim [[buffer(6)]],
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
+
+template [[host_name("qmv_mlp_fused_f16_g128_b4")]] [[kernel]] void qmv_mlp_fused<
+ half, 128, 4>(
+ const device uint32_t* w [[buffer(0)]],
+ const device half* scales [[buffer(1)]],
+ const device uint8_t* zero_points [[buffer(2)]],
+ const device half* biases [[buffer(2)]],
+ const device half* x [[buffer(3)]],
+ device half* y [[buffer(4)]],
+ const constant int& K [[buffer(5)]],
+ const constant int& hidden_dim [[buffer(6)]],
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
+
+template [[host_name("qmv_mlp_fused_bf16_g32_b4")]] [[kernel]] void qmv_mlp_fused<
+ bfloat, 32, 4>(
+ const device uint32_t* w [[buffer(0)]],
+ const device bfloat* scales [[buffer(1)]],
+ const device uint8_t* zero_points [[buffer(2)]],
+ const device bfloat* biases [[buffer(2)]],
+ const device bfloat* x [[buffer(3)]],
+ device bfloat* y [[buffer(4)]],
+ const constant int& K [[buffer(5)]],
+ const constant int& hidden_dim [[buffer(6)]],
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
+
+template [[host_name("qmv_mlp_fused_bf16_g64_b4")]] [[kernel]] void qmv_mlp_fused<
+ bfloat, 64, 4>(
+ const device uint32_t* w [[buffer(0)]],
+ const device bfloat* scales [[buffer(1)]],
+ const device uint8_t* zero_points [[buffer(2)]],
+ const device bfloat* biases [[buffer(2)]],
+ const device bfloat* x [[buffer(3)]],
+ device bfloat* y [[buffer(4)]],
+ const constant int& K [[buffer(5)]],
+ const constant int& hidden_dim [[buffer(6)]],
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
+
+template [[host_name("qmv_mlp_fused_bf16_g128_b4")]] [[kernel]] void qmv_mlp_fused<
+ bfloat, 128, 4>(
+ const device uint32_t* w [[buffer(0)]],
+ const device bfloat* scales [[buffer(1)]],
+ const device uint8_t* zero_points [[buffer(2)]],
+ const device bfloat* biases [[buffer(2)]],
+ const device bfloat* x [[buffer(3)]],
+ device bfloat* y [[buffer(4)]],
+ const constant int& K [[buffer(5)]],
+ const constant int& hidden_dim [[buffer(6)]],
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
+
+///////////////////////////////////////////////////////////////////////////////
+/// MLP Fused QMM - Quantized Matrix-Matrix with Fused Activation
+///////////////////////////////////////////////////////////////////////////////
+
+// MLP fused QMM kernel for prefill path (M>1).
+// Computes paired up and gate projections, then applies: out = up * activation(gate)
+// Weight layout: [up_weights (hidden_dim cols), gate_weights (hidden_dim cols)]
+// Output size is [M, hidden_dim].
+template <typename T, int group_size, int bits, bool aligned_K, int BM = 32, int BK = 32, int BN = 32>
+void qmm_mlp_fused_impl(
+ const device uint32_t* w,
+ const device T* scales,
+ const device uint8_t* zero_points,
+ const device T* biases,
+ const device T* x,
+ device T* y,
+ threadgroup T* Xs,
+ threadgroup T* Ws_up,
+ const constant int& K,
+ const constant int& hidden_dim, // Output columns (half of total weight columns)
+ const constant int& M,
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint lid [[thread_index_in_threadgroup]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]
+) {
+ static_assert(BK >= 32, "BK should be larger than SIMD_SIZE");
+ static_assert(BK % 32 == 0, "BK should be divisible by SIMD_SIZE");
+
+ (void)lid;
+
+ constexpr int WM = 2;
+ constexpr int WN = 2;
+ constexpr int pack_factor = get_pack_factor<bits, 8>();
+ constexpr int bytes_per_pack = get_bytes_per_pack<bits>();
+
+ constexpr int BK_padded = (BK + 16 / sizeof(T));
+ constexpr int BN_padded = (BN + 16 / sizeof(T));
+
+ using mma_t = matmul_utils::BlockMMA<T, T, BM, BN, BK, WM, WN, false, false, BK_padded, BN_padded>;
+ using loader_x_t = matmul_utils::BlockLoader<T, BM, BK, BK_padded, 1, WM * WN * 32, 1, 4>;
+
+ auto wl_up = (const device uint8_t*)w;
+
+ const int y_row = tid.y * BM;
+ const int y_col = tid.x * BN; // Column in output (within hidden_dim)
+
+ // Only process tiles within hidden_dim
+ if (y_col >= hidden_dim) return;
+
+ x += y_row * static_cast<int64_t>(K);
+
+ // Up weights at column y_col
+ wl_up += y_col * bytes_per_pack / pack_factor;
+ // Gate weights at column y_col + hidden_dim
+ auto wl_gate = wl_up + hidden_dim * bytes_per_pack / pack_factor;
+
+ y += y_row * static_cast<int64_t>(hidden_dim) + y_col;
+
+ const device T* scales_base = scales;
+ const device T* biases_base = biases;
+
+ const short num_els = min(BM, M - y_row);
+ const short num_outs = min(BN, hidden_dim - y_col);
+
+ // Threadgroup memory for gate weights (after Ws_up)
+ threadgroup T* Ws_gate = Ws_up + BK * BN_padded;
+
+ loader_x_t loader_x(x, K, Xs, simd_gid, simd_lid);
+ mma_t mma_up(simd_gid, simd_lid);
+ mma_t mma_gate(simd_gid, simd_lid);
+
+ const int k_iterations = (K + BK - 1) / BK;
+ const int full_iterations = aligned_K ? k_iterations : k_iterations - 1;
+
+ // Total output width for scale/bias strides
+ const int N_total = hidden_dim * 2;
+
+ if (kUseMlxQuant) {
+ using loader_w_t = QuantizedBlockLoaderMlx<T, BK, BN, BN_padded, 0, WM * WN * 32, group_size, bits>;
+
+ const device T* scales_up = scales_base + (y_col / group_size);
+ const device T* biases_up = biases_base + (y_col / group_size);
+ const device T* scales_gate = scales_base + ((y_col + hidden_dim) / group_size);
+ const device T* biases_gate = biases_base + ((y_col + hidden_dim) / group_size);
+
+ loader_w_t loader_w_up(wl_up, scales_up, biases_up, N_total, Ws_up, simd_gid, simd_lid);
+ loader_w_t loader_w_gate(wl_gate, scales_gate, biases_gate, N_total, Ws_gate, simd_gid, simd_lid);
+
+ for (int k = 0; k < full_iterations; k++) {
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ loader_x.load_unsafe();
+ loader_w_up.load_unsafe();
+ loader_w_gate.load_unsafe();
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ mma_up.mma(Xs, Ws_up);
+ mma_gate.mma(Xs, Ws_gate);
+
+ loader_x.next();
+ loader_w_up.next();
+ loader_w_gate.next();
+ }
+
+ if (!aligned_K) {
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ loader_x.load_safe(short2(BM, K - full_iterations * BK));
+ loader_w_up.load_safe(short2(K - full_iterations * BK, num_outs));
+ loader_w_gate.load_safe(short2(K - full_iterations * BK, num_outs));
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ mma_up.mma(Xs, Ws_up);
+ mma_gate.mma(Xs, Ws_gate);
+ }
+ } else {
+ const int out_groups_total = (N_total + group_size - 1) / group_size;
+ const int groups_per_row = out_groups_total;
+ const int out_group_up = y_col / group_size;
+ const int out_group_gate = (y_col + hidden_dim) / group_size;
+ const int zp_stride_out = (bits == 4) ? ((out_groups_total + 1) / 2) : out_groups_total;
+
+ using loader_w_t = QuantizedBlockLoaderZp<T, BK, BN, BN_padded, 0, WM * WN * 32, group_size, bits, true>;
+
+ loader_w_t loader_w_up(
+ wl_up, scales_base, zero_points,
+ N_total, groups_per_row, Ws_up,
+ simd_gid, simd_lid,
+ out_group_up, out_groups_total, zp_stride_out);
+
+ loader_w_t loader_w_gate(
+ wl_gate, scales_base, zero_points,
+ N_total, groups_per_row, Ws_gate,
+ simd_gid, simd_lid,
+ out_group_gate, out_groups_total, zp_stride_out);
+
+ for (int k = 0; k < full_iterations; k++) {
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ loader_x.load_unsafe();
+ loader_w_up.load_unsafe();
+ loader_w_gate.load_unsafe();
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ mma_up.mma(Xs, Ws_up);
+ mma_gate.mma(Xs, Ws_gate);
+
+ loader_x.next();
+ loader_w_up.next();
+ loader_w_gate.next();
+ }
+
+ if (!aligned_K) {
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ loader_x.load_safe(short2(BM, K - full_iterations * BK));
+ loader_w_up.load_safe(short2(K - full_iterations * BK, num_outs));
+ loader_w_gate.load_safe(short2(K - full_iterations * BK, num_outs));
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ mma_up.mma(Xs, Ws_up);
+ mma_gate.mma(Xs, Ws_gate);
+ }
+ }
+
+ // Apply MLP fused epilogue: out = up * activation(gate)
+ // Access Ctile elements and apply fusion
+ constexpr int kElemsPerTile = decltype(mma_up.Ctile)::kElemsPerTile;
+ #pragma unroll
+ for (short i = 0; i < kElemsPerTile; i++) {
+ float up_val = static_cast<float>(mma_up.Ctile.elems()[i]);
+ float gate_val = static_cast<float>(mma_gate.Ctile.elems()[i]);
+ float fused = mlp_fused_epilogue_f32(up_val, gate_val);
+ mma_up.Ctile.elems()[i] = static_cast<T>(fused);
+ }
+
+ // Store result
+ mma_up.store_result_safe(y, hidden_dim, short2(num_outs, num_els));
+}
+
+template <typename T, int group_size, int bits>
+[[kernel]] void qmm_mlp_fused(
+ const device uint32_t* w [[buffer(0)]],
+ const device T* scales [[buffer(1)]],
+ const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],
+ const device T* biases [[buffer(2), function_constant(kUseMlxQuant)]],
+ const device T* x [[buffer(3)]],
+ device T* y [[buffer(4)]],
+ const constant int& K [[buffer(5)]],
+ const constant int& hidden_dim [[buffer(6)]],
+ const constant int& M [[buffer(7)]],
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint lid [[thread_index_in_threadgroup]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]
+) {
+ constexpr int BM = 32;
+ constexpr int BK = 32;
+ constexpr int BN = 32;
+ constexpr int BK_padded = (BK + 16 / sizeof(T));
+ constexpr int BN_padded = (BN + 16 / sizeof(T));
+
+ threadgroup T Xs[BM * BK_padded];
+ threadgroup T Ws[2 * BK * BN_padded]; // Double for up and gate
+
+ bool aligned_K = (K % BK) == 0;
+ if (aligned_K) {
+ qmm_mlp_fused_impl<T, group_size, bits, true, BM, BK, BN>(
+ w, scales, zero_points, biases, x, y, Xs, Ws,
+ K, hidden_dim, M, tid, lid, simd_gid, simd_lid);
+ } else {
+ qmm_mlp_fused_impl<T, group_size, bits, false, BM, BK, BN>(
+ w, scales, zero_points, biases, x, y, Xs, Ws,
+ K, hidden_dim, M, tid, lid, simd_gid, simd_lid);
+ }
+}
+
+// Instantiate MLP fused QMM kernels
+template [[host_name("qmm_mlp_fused_f16_g32_b4")]] [[kernel]] void qmm_mlp_fused<
+ half, 32, 4>(
+ const device uint32_t* w [[buffer(0)]],
+ const device half* scales [[buffer(1)]],
+ const device uint8_t* zero_points [[buffer(2)]],
+ const device half* biases [[buffer(2)]],
+ const device half* x [[buffer(3)]],
+ device half* y [[buffer(4)]],
+ const constant int& K [[buffer(5)]],
+ const constant int& hidden_dim [[buffer(6)]],
+ const constant int& M [[buffer(7)]],
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint lid [[thread_index_in_threadgroup]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
+
+template [[host_name("qmm_mlp_fused_f16_g64_b4")]] [[kernel]] void qmm_mlp_fused<
+ half, 64, 4>(
+ const device uint32_t* w [[buffer(0)]],
+ const device half* scales [[buffer(1)]],
+ const device uint8_t* zero_points [[buffer(2)]],
+ const device half* biases [[buffer(2)]],
+ const device half* x [[buffer(3)]],
+ device half* y [[buffer(4)]],
+ const constant int& K [[buffer(5)]],
+ const constant int& hidden_dim [[buffer(6)]],
+ const constant int& M [[buffer(7)]],
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint lid [[thread_index_in_threadgroup]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
+
+template [[host_name("qmm_mlp_fused_f16_g128_b4")]] [[kernel]] void qmm_mlp_fused<
+ half, 128, 4>(
+ const device uint32_t* w [[buffer(0)]],
+ const device half* scales [[buffer(1)]],
+ const device uint8_t* zero_points [[buffer(2)]],
+ const device half* biases [[buffer(2)]],
+ const device half* x [[buffer(3)]],
+ device half* y [[buffer(4)]],
+ const constant int& K [[buffer(5)]],
+ const constant int& hidden_dim [[buffer(6)]],
+ const constant int& M [[buffer(7)]],
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint lid [[thread_index_in_threadgroup]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
+
+template [[host_name("qmm_mlp_fused_bf16_g32_b4")]] [[kernel]] void qmm_mlp_fused<
+ bfloat, 32, 4>(
+ const device uint32_t* w [[buffer(0)]],
+ const device bfloat* scales [[buffer(1)]],
+ const device uint8_t* zero_points [[buffer(2)]],
+ const device bfloat* biases [[buffer(2)]],
+ const device bfloat* x [[buffer(3)]],
+ device bfloat* y [[buffer(4)]],
+ const constant int& K [[buffer(5)]],
+ const constant int& hidden_dim [[buffer(6)]],
+ const constant int& M [[buffer(7)]],
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint lid [[thread_index_in_threadgroup]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
+
+template [[host_name("qmm_mlp_fused_bf16_g64_b4")]] [[kernel]] void qmm_mlp_fused<
+ bfloat, 64, 4>(
+ const device uint32_t* w [[buffer(0)]],
+ const device bfloat* scales [[buffer(1)]],
+ const device uint8_t* zero_points [[buffer(2)]],
+ const device bfloat* biases [[buffer(2)]],
+ const device bfloat* x [[buffer(3)]],
+ device bfloat* y [[buffer(4)]],
+ const constant int& K [[buffer(5)]],
+ const constant int& hidden_dim [[buffer(6)]],
+ const constant int& M [[buffer(7)]],
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint lid [[thread_index_in_threadgroup]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
+
+template [[host_name("qmm_mlp_fused_bf16_g128_b4")]] [[kernel]] void qmm_mlp_fused<
+ bfloat, 128, 4>(
+ const device uint32_t* w [[buffer(0)]],
+ const device bfloat* scales [[buffer(1)]],
+ const device uint8_t* zero_points [[buffer(2)]],
+ const device bfloat* biases [[buffer(2)]],
+ const device bfloat* x [[buffer(3)]],
+ device bfloat* y [[buffer(4)]],
+ const constant int& K [[buffer(5)]],
+ const constant int& hidden_dim [[buffer(6)]],
+ const constant int& M [[buffer(7)]],
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint lid [[thread_index_in_threadgroup]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]
+);
diff --git a/crates/uzu/tests/mlp_fused_correctness_test.rs b/crates/uzu/tests/mlp_fused_correctness_test.rs
new file mode 100644
index 000000000..3e2d97782
--- /dev/null
+++ b/crates/uzu/tests/mlp_fused_correctness_test.rs
@@ -0,0 +1,453 @@
+//! Correctness tests for MLP fused kernels
+//! Compares fused (matmul + activation) output against separate operations
+
+use half::f16;
+use metal::{Device, MTLResourceOptions};
+use objc2::rc::autoreleasepool;
+use uzu::{
+ DataType,
+ backends::metal::{
+ MTLContext,
+ kernel::{
+ matmul::{
+ MlpFusedGemmArguments, MlpFusedGemmKernel,
+ MlpFusedGemvArguments, MlpFusedGemvKernel,
+ },
+ mlp::MlpActivationType,
+ },
+ },
+};
+
+fn create_test_context() -> Option<MTLContext> {
+ let device = Device::system_default()?;
+ let command_queue = device.new_command_queue();
+ MTLContext::new(device, command_queue).ok()
+}
+
+/// CPU reference implementation for SiLU activation
+fn silu(x: f32) -> f32 {
+ x / (1.0 + (-x).exp())
+}
+
+/// CPU reference implementation for GELU activation
+fn gelu(x: f32) -> f32 {
+ let k0 = 0.044715f32;
+ let k1 = 0.7978845608f32;
+ let t = k1 * (x + k0 * x * x * x);
+ 0.5 * x * (1.0 + t.tanh())
+}
+
+/// CPU reference for MLP fused: up * activation(gate)
+fn mlp_fused_reference(
+ input: &[f32],
+ weights_up: &[f32],
+ weights_gate: &[f32],
+ m: usize,
+ k: usize,
+ hidden_dim: usize,
+ activation: MlpActivationType,
+) -> Vec<f32> {
+ let mut result = vec![0.0f32; m * hidden_dim];
+
+ for row in 0..m {
+ for col in 0..hidden_dim {
+ // Compute up and gate dot products
+ let mut up_sum = 0.0f32;
+ let mut gate_sum = 0.0f32;
+ for i in 0..k {
+ let input_val = input[row * k + i];
+ up_sum += input_val * weights_up[col * k + i];
+ gate_sum += input_val * weights_gate[col * k + i];
+ }
+
+ // Apply fused activation
+ let activated_gate = match activation {
+ MlpActivationType::SiLU => silu(gate_sum),
+ MlpActivationType::Gelu => gelu(gate_sum),
+ };
+ result[row * hidden_dim + col] = up_sum * activated_gate;
+ }
+ }
+
+ result
+}
+
+/// Run MLP fused GEMV kernel (decode path, M=1)
+fn run_fused_gemv(
+ ctx: &MTLContext,
+ input: &[f16],
+ weights: &[f16], // [2*hidden_dim, k] - up then gate weights concatenated
+ k: usize,
+ hidden_dim: usize,
+ activation: MlpActivationType,
+) -> Vec<f16> {
+ let input_buf = ctx.device.new_buffer_with_data(
+ input.as_ptr() as *const _,
+ (input.len() * core::mem::size_of::<f16>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+ let weights_buf = ctx.device.new_buffer_with_data(
+ weights.as_ptr() as *const _,
+ (weights.len() * core::mem::size_of::<f16>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+ let output_buf = ctx.device.new_buffer(
+ (hidden_dim * core::mem::size_of::<f16>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+
+ let mut kernel = MlpFusedGemvKernel::new(DataType::F16).expect("kernel");
+
+ let cb = ctx.command_queue.new_command_buffer().to_owned();
+ let enc = cb.new_compute_command_encoder();
+
+ let args = MlpFusedGemvArguments {
+ weights: &weights_buf,
+ input: &input_buf,
+ input_offset: 0,
+ output: &output_buf,
+ input_dim: k as i32,
+ hidden_dim: hidden_dim as i32,
+ weights_ld: k as i32,
+ batch_count: 1,
+ activation,
+ };
+
+ kernel.encode(ctx, &enc, &args).expect("encode");
+ enc.end_encoding();
+ cb.commit();
+ cb.wait_until_completed();
+
+ unsafe {
+ let ptr = output_buf.contents() as *const f16;
+ std::slice::from_raw_parts(ptr, hidden_dim).to_vec()
+ }
+}
+
+/// Run MLP fused GEMM kernel (prefill path, M>1)
+fn run_fused_gemm(
+ ctx: &MTLContext,
+ input: &[f16],
+ weights: &[f16], // [2*hidden_dim, k] - up then gate weights concatenated
+ m: usize,
+ k: usize,
+ hidden_dim: usize,
+ activation: MlpActivationType,
+) -> Vec<f16> {
+ let input_buf = ctx.device.new_buffer_with_data(
+ input.as_ptr() as *const _,
+ (input.len() * core::mem::size_of::<f16>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+ let weights_buf = ctx.device.new_buffer_with_data(
+ weights.as_ptr() as *const _,
+ (weights.len() * core::mem::size_of::<f16>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+ let output_buf = ctx.device.new_buffer(
+ (m * hidden_dim * core::mem::size_of::<f16>()) as u64,
+ MTLResourceOptions::StorageModeShared,
+ );
+
+ let mut kernel =
+ MlpFusedGemmKernel::new(DataType::F16, true).expect("kernel");
+
+ let cb = ctx.command_queue.new_command_buffer().to_owned();
+ let enc = cb.new_compute_command_encoder();
+
+ let args = MlpFusedGemmArguments {
+ input: &input_buf,
+ input_offset: 0,
+ weights: &weights_buf,
+ output: &output_buf,
+ batch: m as i32,
+ input_dim: k as i32,
+ hidden_dim: hidden_dim as i32,
+ lda: k as i32,
+ ldb: k as i32,
+ ldd: hidden_dim as i32,
+ activation,
+ };
+
+ kernel.encode(ctx, &enc, &args).expect("encode");
+ enc.end_encoding();
+ cb.commit();
+ cb.wait_until_completed();
+
+ unsafe {
+ let ptr = output_buf.contents() as *const f16;
+ std::slice::from_raw_parts(ptr, m * hidden_dim).to_vec()
+ }
+}
+
+fn compare_outputs(
+ expected: &[f32],
+ actual: &[f16],
+ rtol: f32,
+ atol: f32,
+) -> (bool, f32) {
+ let mut max_diff = 0.0f32;
+ let mut all_close = true;
+
+ for (i, (&exp, &act)) in expected.iter().zip(actual.iter()).enumerate() {
+ let act_f32 = f32::from(act);
+ let diff = (exp - act_f32).abs();
+ let rel_diff = diff / (exp.abs() + 1e-8);
+
+ if diff > atol && rel_diff > rtol {
+ if all_close {
+ eprintln!(
+ "First mismatch at index {}: expected {}, got {} (diff={}, rel={})",
+ i, exp, act_f32, diff, rel_diff
+ );
+ }
+ all_close = false;
+ }
+ max_diff = max_diff.max(diff);
+ }
+
+ (all_close, max_diff)
+}
+
+#[test]
+fn test_mlp_fused_gemv_silu_small() {
+ autoreleasepool(|_| {
+ let ctx = match create_test_context() {
+ Some(c) => c,
+ None => {
+ eprintln!("No Metal device available, skipping test");
+ return;
+ },
+ };
+
+ let k = 64;
+ let hidden_dim = 32;
+ let m = 1;
+
+ // Generate random-ish test data
+ let input: Vec<f32> =
+ (0..k).map(|i| (i as f32 * 0.1).sin() * 0.5).collect();
+ let weights_up: Vec<f32> = (0..hidden_dim * k)
+ .map(|i| (i as f32 * 0.017).cos() * 0.3)
+ .collect();
+ let weights_gate: Vec<f32> = (0..hidden_dim * k)
+ .map(|i| (i as f32 * 0.023).sin() * 0.3)
+ .collect();
+
+ // Create concatenated weights [up; gate] row-major
+ let mut weights_concat: Vec<f32> =
+ Vec::with_capacity(2 * hidden_dim * k);
+ weights_concat.extend(&weights_up);
+ weights_concat.extend(&weights_gate);
+
+ // CPU reference
+ let expected = mlp_fused_reference(
+ &input,
+ &weights_up,
+ &weights_gate,
+ m,
+ k,
+ hidden_dim,
+ MlpActivationType::SiLU,
+ );
+
+ // Convert to f16
+ let input_f16: Vec<f16> =
+ input.iter().map(|&x| f16::from_f32(x)).collect();
+ let weights_f16: Vec<f16> =
+ weights_concat.iter().map(|&x| f16::from_f32(x)).collect();
+
+ // Run fused kernel
+ let actual = run_fused_gemv(
+ &ctx,
+ &input_f16,
+ &weights_f16,
+ k,
+ hidden_dim,
+ MlpActivationType::SiLU,
+ );
+
+ let (close, max_diff) = compare_outputs(&expected, &actual, 0.01, 0.01);
+ println!("GEMV SiLU small: max_diff = {}", max_diff);
+ assert!(close, "GEMV SiLU small test failed, max_diff = {}", max_diff);
+ });
+}
+
+#[test]
+fn test_mlp_fused_gemv_gelu_small() {
+ autoreleasepool(|_| {
+ let ctx = match create_test_context() {
+ Some(c) => c,
+ None => {
+ eprintln!("No Metal device available, skipping test");
+ return;
+ },
+ };
+
+ let k = 64;
+ let hidden_dim = 32;
+ let m = 1;
+
+ let input: Vec<f32> =
+ (0..k).map(|i| (i as f32 * 0.1).sin() * 0.5).collect();
+ let weights_up: Vec<f32> = (0..hidden_dim * k)
+ .map(|i| (i as f32 * 0.017).cos() * 0.3)
+ .collect();
+ let weights_gate: Vec<f32> = (0..hidden_dim * k)
+ .map(|i| (i as f32 * 0.023).sin() * 0.3)
+ .collect();
+
+ let mut weights_concat: Vec<f32> =
+ Vec::with_capacity(2 * hidden_dim * k);
+ weights_concat.extend(&weights_up);
+ weights_concat.extend(&weights_gate);
+
+ let expected = mlp_fused_reference(
+ &input,
+ &weights_up,
+ &weights_gate,
+ m,
+ k,
+ hidden_dim,
+ MlpActivationType::Gelu,
+ );
+
+ let input_f16: Vec<f16> =
+ input.iter().map(|&x| f16::from_f32(x)).collect();
+ let weights_f16: Vec<f16> =
+ weights_concat.iter().map(|&x| f16::from_f32(x)).collect();
+
+ let actual = run_fused_gemv(
+ &ctx,
+ &input_f16,
+ &weights_f16,
+ k,
+ hidden_dim,
+ MlpActivationType::Gelu,
+ );
+
+ let (close, max_diff) = compare_outputs(&expected, &actual, 0.01, 0.01);
+ println!("GEMV GELU small: max_diff = {}", max_diff);
+ assert!(close, "GEMV GELU small test failed, max_diff = {}", max_diff);
+ });
+}
+
+#[test]
+fn test_mlp_fused_gemm_silu_medium() {
+ autoreleasepool(|_| {
+ let ctx = match create_test_context() {
+ Some(c) => c,
+ None => {
+ eprintln!("No Metal device available, skipping test");
+ return;
+ },
+ };
+
+ let m = 32;
+ let k = 64;
+ let hidden_dim = 64;
+
+ let input: Vec<f32> =
+ (0..m * k).map(|i| (i as f32 * 0.1).sin() * 0.5).collect();
+ let weights_up: Vec<f32> = (0..hidden_dim * k)
+ .map(|i| (i as f32 * 0.017).cos() * 0.3)
+ .collect();
+ let weights_gate: Vec<f32> = (0..hidden_dim * k)
+ .map(|i| (i as f32 * 0.023).sin() * 0.3)
+ .collect();
+
+ let mut weights_concat: Vec<f32> =
+ Vec::with_capacity(2 * hidden_dim * k);
+ weights_concat.extend(&weights_up);
+ weights_concat.extend(&weights_gate);
+
+ let expected = mlp_fused_reference(
+ &input,
+ &weights_up,
+ &weights_gate,
+ m,
+ k,
+ hidden_dim,
+ MlpActivationType::SiLU,
+ );
+
+ let input_f16: Vec<f16> =
+ input.iter().map(|&x| f16::from_f32(x)).collect();
+ let weights_f16: Vec<f16> =
+ weights_concat.iter().map(|&x| f16::from_f32(x)).collect();
+
+ let actual = run_fused_gemm(
+ &ctx,
+ &input_f16,
+ &weights_f16,
+ m,
+ k,
+ hidden_dim,
+ MlpActivationType::SiLU,
+ );
+
+ let (close, max_diff) = compare_outputs(&expected, &actual, 0.05, 0.05);
+ println!("GEMM SiLU medium: max_diff = {}", max_diff);
+ assert!(close, "GEMM SiLU medium test failed, max_diff = {}", max_diff);
+ });
+}
+
+#[test]
+fn test_mlp_fused_gemv_realistic_dimensions() {
+ autoreleasepool(|_| {
+ let ctx = match create_test_context() {
+ Some(c) => c,
+ None => {
+ eprintln!("No Metal device available, skipping test");
+ return;
+ },
+ };
+
+ // Realistic LLM dimensions (scaled down for test speed)
+ let k = 512; // input dim
+ let hidden_dim = 256; // hidden dim (half of up+gate)
+ let m = 1;
+
+ let input: Vec<f32> =
+ (0..k).map(|i| (i as f32 * 0.01).sin() * 0.5).collect();
+ let weights_up: Vec<f32> = (0..hidden_dim * k)
+ .map(|i| (i as f32 * 0.001).cos() * 0.1)
+ .collect();
+ let weights_gate: Vec<f32> = (0..hidden_dim * k)
+ .map(|i| (i as f32 * 0.002).sin() * 0.1)
+ .collect();
+
+ let mut weights_concat: Vec<f32> =
+ Vec::with_capacity(2 * hidden_dim * k);
+ weights_concat.extend(&weights_up);
+ weights_concat.extend(&weights_gate);
+
+ let expected = mlp_fused_reference(
+ &input,
+ &weights_up,
+ &weights_gate,
+ m,
+ k,
+ hidden_dim,
+ MlpActivationType::SiLU,
+ );
+
+ let input_f16: Vec<f16> =
+ input.iter().map(|&x| f16::from_f32(x)).collect();
+ let weights_f16: Vec<f16> =
+ weights_concat.iter().map(|&x| f16::from_f32(x)).collect();
+
+ let actual = run_fused_gemv(
+ &ctx,
+ &input_f16,
+ &weights_f16,
+ k,
+ hidden_dim,
+ MlpActivationType::SiLU,
+ );
+
+ let (close, max_diff) = compare_outputs(&expected, &actual, 0.05, 0.02);
+ println!("GEMV realistic: max_diff = {}", max_diff);
+ assert!(close, "GEMV realistic test failed, max_diff = {}", max_diff);
+ });
+}