A high-performance inference engine for AI models
wip
Eugene Bokhan560b31dc3f8c
diff --git a/Cargo.lock b/Cargo.lockindex 0c8247c5e..a8d408bb4 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -309,8 +309,11 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"name = "benchmarks"version = "0.1.7"dependencies = [+ "half","libc","mach2",+ "metal",+ "objc2","serde","serde_json","sysinfo",diff --git a/crates/benchmarks/Cargo.toml b/crates/benchmarks/Cargo.tomlindex d82f672f1..6c45b5944 100644--- a/crates/benchmarks/Cargo.toml+++ b/crates/benchmarks/Cargo.toml@@ -10,3 +10,6 @@ serde_json = "1.0.148"sysinfo = "0.37.2"libc = "0.2.178"mach2 = "0.6.0"+half = { version = "2.6.0", features = ["bytemuck", "num-traits"] }+objc2 = "0.6.2"+metal = "0.32.0"diff --git a/crates/benchmarks/src/lib.rs b/crates/benchmarks/src/lib.rsindex 7a7c54c23..9326797ac 100644--- a/crates/benchmarks/src/lib.rs+++ b/crates/benchmarks/src/lib.rs@@ -1,3 +1,5 @@+pub mod mlp_fused_bench;pub mod runner;+pub use mlp_fused_bench::run_mlp_fused_benchmark;pub use runner::*;diff --git a/crates/benchmarks/src/mlp_fused_bench.rs b/crates/benchmarks/src/mlp_fused_bench.rsnew file mode 100644index 000000000..d70dc18f5--- /dev/null+++ b/crates/benchmarks/src/mlp_fused_bench.rs@@ -0,0 +1,411 @@+//! MLP Fused Kernel Performance Benchmark+//! Compares fused vs unfused MLP paths to measure speedup++use std::time::{Duration, Instant};++use half::f16;+use metal::{Device, MTLResourceOptions};+use objc2::rc::autoreleasepool;+use uzu::{+ DataType,+ backends::metal::{+ MTLContext,+ kernel::{+ matmul::{+ MatmulArguments, MatmulKernel, MlpFusedGemmArguments,+ MlpFusedGemmKernel, MlpFusedGemvArguments, MlpFusedGemvKernel,+ },+ mlp::MlpActivationType,+ },+ },+};++fn create_test_context() -> Option<MTLContext> {+ let device = Device::system_default()?;+ let command_queue = device.new_command_queue();+ MTLContext::new(device, command_queue).ok()+}++struct BenchResult {+ fused_time_us: f64,+ unfused_time_us: f64,+ speedup: f64,+}++impl BenchResult {+ fn new(+ fused: Duration,+ unfused: Duration,+ ) -> Self {+ let fused_us = fused.as_secs_f64() * 1_000_000.0;+ let unfused_us = unfused.as_secs_f64() * 1_000_000.0;+ Self {+ fused_time_us: fused_us,+ unfused_time_us: unfused_us,+ speedup: unfused_us / fused_us,+ }+ }+}++fn bench_gemv_fused(+ ctx: &MTLContext,+ k: usize,+ hidden_dim: usize,+ iterations: usize,+ warmup: usize,+) -> BenchResult {+ // Allocate buffers+ let input_buf = ctx.device.new_buffer(+ (k * std::mem::size_of::<f16>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );+ let weights_buf = ctx.device.new_buffer(+ (2 * hidden_dim * k * std::mem::size_of::<f16>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );+ let output_buf = ctx.device.new_buffer(+ (hidden_dim * std::mem::size_of::<f16>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );+ let fused_up_buf = ctx.device.new_buffer(+ (2 * hidden_dim * std::mem::size_of::<f16>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );++ let mut fused_kernel =+ MlpFusedGemvKernel::new(DataType::F16).expect("fused kernel");+ let mut matmul_kernel =+ MatmulKernel::new(ctx, DataType::F16, false, true).expect("matmul");++ // Warmup fused+ for _ in 0..warmup {+ let cb = ctx.command_queue.new_command_buffer().to_owned();+ let enc = cb.new_compute_command_encoder();+ let args = MlpFusedGemvArguments {+ weights: &weights_buf,+ input: &input_buf,+ input_offset: 0,+ output: &output_buf,+ input_dim: k as i32,+ hidden_dim: hidden_dim as i32,+ weights_ld: k as i32,+ batch_count: 1,+ activation: MlpActivationType::SiLU,+ };+ fused_kernel.encode(ctx, &enc, &args).unwrap();+ enc.end_encoding();+ cb.commit();+ cb.wait_until_completed();+ }++ // Benchmark fused+ let fused_start = Instant::now();+ for _ in 0..iterations {+ let cb = ctx.command_queue.new_command_buffer().to_owned();+ let enc = cb.new_compute_command_encoder();+ let args = MlpFusedGemvArguments {+ weights: &weights_buf,+ input: &input_buf,+ input_offset: 0,+ output: &output_buf,+ input_dim: k as i32,+ hidden_dim: hidden_dim as i32,+ weights_ld: k as i32,+ batch_count: 1,+ activation: MlpActivationType::SiLU,+ };+ fused_kernel.encode(ctx, &enc, &args).unwrap();+ enc.end_encoding();+ cb.commit();+ cb.wait_until_completed();+ }+ let fused_time = fused_start.elapsed() / iterations as u32;++ // Warmup unfused (matmul only - simplified comparison)+ for _ in 0..warmup {+ let cb = ctx.command_queue.new_command_buffer().to_owned();+ let enc = cb.new_compute_command_encoder();+ matmul_kernel+ .encode(+ ctx,+ &enc,+ MatmulArguments {+ a: &input_buf,+ a_offset: 0,+ b: &weights_buf,+ c: None,+ d: &fused_up_buf,+ batch: 1,+ input_dim: k as i32,+ output_dim: (2 * hidden_dim) as i32,+ lda: k as i32,+ ldb: k as i32,+ ldd: (2 * hidden_dim) as i32,+ batch_count: 1,+ alpha: 1.0,+ beta: 0.0,+ },+ )+ .unwrap();+ enc.end_encoding();+ cb.commit();+ cb.wait_until_completed();+ }++ // Benchmark unfused (matmul + separate activation would add more time)+ let unfused_start = Instant::now();+ for _ in 0..iterations {+ let cb = ctx.command_queue.new_command_buffer().to_owned();+ let enc = cb.new_compute_command_encoder();+ matmul_kernel+ .encode(+ ctx,+ &enc,+ MatmulArguments {+ a: &input_buf,+ a_offset: 0,+ b: &weights_buf,+ c: None,+ d: &fused_up_buf,+ batch: 1,+ input_dim: k as i32,+ output_dim: (2 * hidden_dim) as i32,+ lda: k as i32,+ ldb: k as i32,+ ldd: (2 * hidden_dim) as i32,+ batch_count: 1,+ alpha: 1.0,+ beta: 0.0,+ },+ )+ .unwrap();+ enc.end_encoding();+ cb.commit();+ cb.wait_until_completed();+ }+ let unfused_time = unfused_start.elapsed() / iterations as u32;++ BenchResult::new(fused_time, unfused_time)+}++fn bench_gemm_fused(+ ctx: &MTLContext,+ m: usize,+ k: usize,+ hidden_dim: usize,+ iterations: usize,+ warmup: usize,+) -> BenchResult {+ let input_buf = ctx.device.new_buffer(+ (m * k * std::mem::size_of::<f16>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );+ let weights_buf = ctx.device.new_buffer(+ (2 * hidden_dim * k * std::mem::size_of::<f16>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );+ let output_buf = ctx.device.new_buffer(+ (m * hidden_dim * std::mem::size_of::<f16>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );+ let fused_up_buf = ctx.device.new_buffer(+ (m * 2 * hidden_dim * std::mem::size_of::<f16>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );++ let mut fused_kernel =+ MlpFusedGemmKernel::new(DataType::F16, true).expect("fused kernel");+ let mut matmul_kernel =+ MatmulKernel::new(ctx, DataType::F16, false, true).expect("matmul");++ // Warmup fused+ for _ in 0..warmup {+ let cb = ctx.command_queue.new_command_buffer().to_owned();+ let enc = cb.new_compute_command_encoder();+ let args = MlpFusedGemmArguments {+ input: &input_buf,+ input_offset: 0,+ weights: &weights_buf,+ output: &output_buf,+ batch: m as i32,+ input_dim: k as i32,+ hidden_dim: hidden_dim as i32,+ lda: k as i32,+ ldb: k as i32,+ ldd: hidden_dim as i32,+ activation: MlpActivationType::SiLU,+ };+ fused_kernel.encode(ctx, &enc, &args).unwrap();+ enc.end_encoding();+ cb.commit();+ cb.wait_until_completed();+ }++ let fused_start = Instant::now();+ for _ in 0..iterations {+ let cb = ctx.command_queue.new_command_buffer().to_owned();+ let enc = cb.new_compute_command_encoder();+ let args = MlpFusedGemmArguments {+ input: &input_buf,+ input_offset: 0,+ weights: &weights_buf,+ output: &output_buf,+ batch: m as i32,+ input_dim: k as i32,+ hidden_dim: hidden_dim as i32,+ lda: k as i32,+ ldb: k as i32,+ ldd: hidden_dim as i32,+ activation: MlpActivationType::SiLU,+ };+ fused_kernel.encode(ctx, &enc, &args).unwrap();+ enc.end_encoding();+ cb.commit();+ cb.wait_until_completed();+ }+ let fused_time = fused_start.elapsed() / iterations as u32;++ // Warmup unfused+ for _ in 0..warmup {+ let cb = ctx.command_queue.new_command_buffer().to_owned();+ let enc = cb.new_compute_command_encoder();+ matmul_kernel+ .encode(+ ctx,+ &enc,+ MatmulArguments {+ a: &input_buf,+ a_offset: 0,+ b: &weights_buf,+ c: None,+ d: &fused_up_buf,+ batch: m as i32,+ input_dim: k as i32,+ output_dim: (2 * hidden_dim) as i32,+ lda: k as i32,+ ldb: k as i32,+ ldd: (2 * hidden_dim) as i32,+ batch_count: 1,+ alpha: 1.0,+ beta: 0.0,+ },+ )+ .unwrap();+ enc.end_encoding();+ cb.commit();+ cb.wait_until_completed();+ }++ let unfused_start = Instant::now();+ for _ in 0..iterations {+ let cb = ctx.command_queue.new_command_buffer().to_owned();+ let enc = cb.new_compute_command_encoder();+ matmul_kernel+ .encode(+ ctx,+ &enc,+ MatmulArguments {+ a: &input_buf,+ a_offset: 0,+ b: &weights_buf,+ c: None,+ d: &fused_up_buf,+ batch: m as i32,+ input_dim: k as i32,+ output_dim: (2 * hidden_dim) as i32,+ lda: k as i32,+ ldb: k as i32,+ ldd: (2 * hidden_dim) as i32,+ batch_count: 1,+ alpha: 1.0,+ beta: 0.0,+ },+ )+ .unwrap();+ enc.end_encoding();+ cb.commit();+ cb.wait_until_completed();+ }+ let unfused_time = unfused_start.elapsed() / iterations as u32;++ BenchResult::new(fused_time, unfused_time)+}++pub fn run_mlp_fused_benchmark() {+ autoreleasepool(|_| {+ let ctx = match create_test_context() {+ Some(c) => c,+ None => {+ eprintln!("No Metal device available");+ return;+ },+ };++ println!("\n=== MLP Fused Kernel Benchmark ===\n");+ println!(+ "{:<40} {:>12} {:>12} {:>10}",+ "Configuration", "Fused (µs)", "Unfused (µs)", "Speedup"+ );+ println!("{}", "-".repeat(76));++ let iterations = 100;+ let warmup = 10;++ // Decode configurations (M=1)+ for (k, hidden_dim) in [(2048, 4096), (4096, 11008), (8192, 22016)] {+ let result =+ bench_gemv_fused(&ctx, k, hidden_dim, iterations, warmup);+ println!("GEMV decode K={} H={}", k, hidden_dim);+ println!(+ "{:<40} {:>12.2} {:>12.2} {:>9.2}x",+ format!(" M=1, K={}, H={}", k, hidden_dim),+ result.fused_time_us,+ result.unfused_time_us,+ result.speedup+ );+ }++ println!();++ // Prefill configurations (M>1)+ for (m, k, hidden_dim) in+ [(32, 2048, 4096), (128, 2048, 4096), (512, 4096, 11008)]+ {+ let result =+ bench_gemm_fused(&ctx, m, k, hidden_dim, iterations, warmup);+ println!(+ "{:<40} {:>12.2} {:>12.2} {:>9.2}x",+ format!("GEMM M={}, K={}, H={}", m, k, hidden_dim),+ result.fused_time_us,+ result.unfused_time_us,+ result.speedup+ );+ }++ println!("\nNote: Unfused time excludes activation kernel overhead.");+ println!(+ " Actual unfused path would be slower due to activation dispatch.\n"+ );+ });+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn test_benchmark_runs() {+ // Just verify the benchmark doesn't crash+ autoreleasepool(|_| {+ if let Some(ctx) = create_test_context() {+ let _ = bench_gemv_fused(&ctx, 512, 256, 5, 2);+ }+ });+ }++ #[test]+ #[ignore] // Run with: cargo test --package benchmarks -- run_full_benchmark --ignored --nocapture+ fn run_full_benchmark() {+ run_mlp_fused_benchmark();+ }+}diff --git a/crates/uzu/src/backends/metal/encodable_block/mlp.rs b/crates/uzu/src/backends/metal/encodable_block/mlp.rsindex e02038a5c..76ed0bb31 100644--- a/crates/uzu/src/backends/metal/encodable_block/mlp.rs+++ b/crates/uzu/src/backends/metal/encodable_block/mlp.rs@@ -1,14 +1,28 @@//! MLP block encodable.-use metal::{CommandBufferRef, ComputeCommandEncoderRef};+use std::{cell::RefCell, rc::Rc};++use metal::{Buffer as MTLBuffer, CommandBufferRef, ComputeCommandEncoderRef};use super::{EncodableBlock, EncodingParameters};use crate::{- Array,+ Array, DataType,backends::metal::{+ MTLContext,forward_pass::{ArrayId, ForwardPassState},- kernel::mlp::MlpGateActMulEncodable,+ kernel::{+ matmul::{+ MlpFusedGemmArguments, MlpFusedGemmKernel,+ MlpFusedGemvArguments, MlpFusedGemvKernel,+ },+ mlp::{MlpActivationType, MlpGateActMulEncodable},+ quant_matmul::{+ MlpFusedQmmArguments, MlpFusedQmmKernel, MlpFusedQmvArguments,+ MlpFusedQmvKernel,+ },+ },},+ config::Activation,};pub struct MlpBlock {@@ -103,3 +117,279 @@ impl EncodableBlock for MlpBlock {self.down.encode_with_shared_encoder(state, encoder, params);}}++/// MLP Fused Up Projection Variant+/// Selects between full precision and quantized fused kernels+pub enum MlpFusedUpKernel {+ FullPrecision {+ gemv: RefCell<MlpFusedGemvKernel>,+ gemm: RefCell<MlpFusedGemmKernel>,+ },+ Quantized {+ qmv: MlpFusedQmvKernel,+ qmm: MlpFusedQmmKernel,+ },+}++/// MLP block with fused up projection and activation+/// Combines up * activation(gate) into a single kernel call+pub struct MlpFusedBlock {+ context: Rc<MTLContext>,+ fused_up: MlpFusedUpKernel,+ down: Box<dyn EncodableBlock>,+ weights_buffer: MTLBuffer,+ scales_buffer: Option<MTLBuffer>,+ zero_points_or_biases_buffer: Option<MTLBuffer>,+ input_dim: usize,+ hidden_dim: usize,+ activation: MlpActivationType,+ input_array_id: ArrayId,+ hidden_array_id: ArrayId,+}++impl MlpFusedBlock {+ /// Create MLP fused block with full precision weights+ pub fn new_full_precision(+ context: Rc<MTLContext>,+ data_type: DataType,+ weights_buffer: MTLBuffer,+ input_dim: usize,+ hidden_dim: usize,+ activation: &Activation,+ down: Box<dyn EncodableBlock>,+ input_array_id: ArrayId,+ hidden_array_id: ArrayId,+ ) -> Result<Self, crate::backends::metal::MTLError> {+ let gemv = MlpFusedGemvKernel::new(data_type)?;+ let gemm = MlpFusedGemmKernel::new(data_type, true)?; // Weights transposed++ Ok(Self {+ context,+ fused_up: MlpFusedUpKernel::FullPrecision {+ gemv: RefCell::new(gemv),+ gemm: RefCell::new(gemm),+ },+ down,+ weights_buffer,+ scales_buffer: None,+ zero_points_or_biases_buffer: None,+ input_dim,+ hidden_dim,+ activation: MlpActivationType::from(activation),+ input_array_id,+ hidden_array_id,+ })+ }++ /// Create MLP fused block with quantized weights+ pub fn new_quantized(+ context: Rc<MTLContext>,+ data_type: DataType,+ weights_buffer: MTLBuffer,+ scales_buffer: MTLBuffer,+ zero_points_or_biases_buffer: MTLBuffer,+ input_dim: usize,+ hidden_dim: usize,+ group_size: usize,+ mode: crate::config::QuantizationMode,+ quantization_type: crate::backends::metal::kernel::quant_matmul::QuantizationType,+ activation: &Activation,+ down: Box<dyn EncodableBlock>,+ input_array_id: ArrayId,+ hidden_array_id: ArrayId,+ ) -> Result<Self, crate::backends::metal::MTLError> {+ let qmv = MlpFusedQmvKernel::new(+ &context,+ data_type,+ group_size,+ mode,+ quantization_type,+ )+ .map_err(|e| {+ crate::backends::metal::MTLError::Generic(format!("{:?}", e))+ })?;++ let qmm = MlpFusedQmmKernel::new(+ &context,+ data_type,+ group_size,+ mode,+ quantization_type,+ )+ .map_err(|e| {+ crate::backends::metal::MTLError::Generic(format!("{:?}", e))+ })?;++ Ok(Self {+ context,+ fused_up: MlpFusedUpKernel::Quantized {+ qmv,+ qmm,+ },+ down,+ weights_buffer,+ scales_buffer: Some(scales_buffer),+ zero_points_or_biases_buffer: Some(zero_points_or_biases_buffer),+ input_dim,+ hidden_dim,+ activation: MlpActivationType::from(activation),+ input_array_id,+ hidden_array_id,+ })+ }++ fn encode_fused_up(+ &self,+ encoder: &ComputeCommandEncoderRef,+ input: &MTLBuffer,+ input_offset: u64,+ output: &MTLBuffer,+ batch: i32,+ ) {+ let is_decode = batch == 1;++ match &self.fused_up {+ MlpFusedUpKernel::FullPrecision {+ gemv,+ gemm,+ } => {+ if is_decode {+ let args = MlpFusedGemvArguments {+ weights: &self.weights_buffer,+ input,+ input_offset,+ output,+ input_dim: self.input_dim as i32,+ hidden_dim: self.hidden_dim as i32,+ weights_ld: self.input_dim as i32,+ batch_count: batch,+ activation: self.activation,+ };+ gemv.borrow_mut()+ .encode(&self.context, encoder, &args)+ .expect("Failed to encode MLP fused GEMV");+ } else {+ let args = MlpFusedGemmArguments {+ input,+ input_offset,+ weights: &self.weights_buffer,+ output,+ batch,+ input_dim: self.input_dim as i32,+ hidden_dim: self.hidden_dim as i32,+ lda: self.input_dim as i32,+ ldb: self.input_dim as i32,+ ldd: self.hidden_dim as i32,+ activation: self.activation,+ };+ gemm.borrow_mut()+ .encode(&self.context, encoder, &args)+ .expect("Failed to encode MLP fused GEMM");+ }+ },+ MlpFusedUpKernel::Quantized {+ qmv,+ qmm,+ } => {+ let scales = self.scales_buffer.as_ref().unwrap();+ let zp_or_biases =+ self.zero_points_or_biases_buffer.as_ref().unwrap();++ if is_decode {+ let args = MlpFusedQmvArguments {+ weights: &self.weights_buffer,+ scales,+ zero_points_or_biases: zp_or_biases,+ input,+ input_offset,+ output,+ input_dim: self.input_dim as i32,+ hidden_dim: self.hidden_dim as i32,+ batch_count: batch,+ };+ qmv.encode(encoder, &args)+ .expect("Failed to encode MLP fused QMV");+ } else {+ let args = MlpFusedQmmArguments {+ weights: &self.weights_buffer,+ scales,+ zero_points_or_biases: zp_or_biases,+ input,+ input_offset,+ output,+ batch,+ input_dim: self.input_dim as i32,+ hidden_dim: self.hidden_dim as i32,+ };+ qmm.encode(encoder, &args)+ .expect("Failed to encode MLP fused QMM");+ }+ },+ }+ }+}++impl EncodableBlock for MlpFusedBlock {+ fn encode(+ &self,+ state: &mut ForwardPassState,+ command_buffer: &CommandBufferRef,+ params: &EncodingParameters,+ ) {+ if self.supports_shared_encoder() {+ let encoder = command_buffer.new_compute_command_encoder();+ self.encode_with_shared_encoder(state, &encoder, params);+ encoder.end_encoding();+ } else {+ // Fused up + activation+ {+ let arrays =+ state.arrays(&[self.input_array_id, self.hidden_array_id]);+ let mut input = arrays[0].borrow_mut();+ let mut hidden = arrays[1].borrow_mut();+ let batch = input.shape()[0] as i32;+ let input_buf = unsafe { input.mtl_buffer() };+ let hidden_buf = unsafe { hidden.mtl_buffer() };++ let encoder = command_buffer.new_compute_command_encoder();+ self.encode_fused_up(&encoder, input_buf, 0, hidden_buf, batch);+ encoder.end_encoding();+ }++ // Down+ self.down.encode(state, command_buffer, params);+ }++ if params.wait_until_completed {+ command_buffer.commit();+ command_buffer.wait_until_completed();+ }+ }++ fn supports_shared_encoder(&self) -> bool {+ self.down.supports_shared_encoder()+ }++ fn encode_with_shared_encoder(+ &self,+ state: &mut ForwardPassState,+ encoder: &ComputeCommandEncoderRef,+ params: &EncodingParameters,+ ) {+ // Fused up + activation+ let arrays = state.arrays(&[self.input_array_id, self.hidden_array_id]);+ let mut input = arrays[0].borrow_mut();+ let mut hidden = arrays[1].borrow_mut();+ let batch = input.shape()[0] as i32;+ let input_buf = unsafe { input.mtl_buffer() };+ let hidden_buf = unsafe { hidden.mtl_buffer() };++ self.encode_fused_up(encoder, input_buf, 0, hidden_buf, batch);++ drop(input);+ drop(hidden);++ // Down+ self.down.encode_with_shared_encoder(state, encoder, params);+ }+}diff --git a/crates/uzu/src/backends/metal/encodable_block/mod.rs b/crates/uzu/src/backends/metal/encodable_block/mod.rsindex 0c651bb14..199583445 100644--- a/crates/uzu/src/backends/metal/encodable_block/mod.rs+++ b/crates/uzu/src/backends/metal/encodable_block/mod.rs@@ -44,7 +44,7 @@ pub use encoding_parameters::EncodingParameters;pub use layer::LayerExecutables;pub use linear::{FullPrecisionLinear, QuantizedLinear};pub(crate) use mamba_mixer::MambaMixer;-pub use mlp::MlpBlock;+pub use mlp::{MlpBlock, MlpFusedBlock, MlpFusedUpKernel};pub use moe_block::{MoeBlock, SharedMoeWeights};pub use normalization::{LayerNorm, Normalization, NormalizationError, QKNorm, RMSNorm,diff --git a/crates/uzu/src/backends/metal/kernel/common/mlp_epilogue.h b/crates/uzu/src/backends/metal/kernel/common/mlp_epilogue.hnew file mode 100644index 000000000..f7cc9376b--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/common/mlp_epilogue.h@@ -0,0 +1,76 @@+#pragma once++#include <metal_stdlib>+using namespace metal;++// MLP Fusion Function Constants+// These control whether the kernel computes paired up/gate projections+// and applies activation fusion (SwiGLU/GeGLU style).+//+// Index allocation:+// 50 - MLP_FUSED (bool): enable paired computation + fused activation+// 51 - MLP_HIDDEN_DIM (uint): offset to gate weights (= hidden_dim)+// 52 - MLP_ACTIVATION (uint): 0=SiLU, 1=GELU++constant bool MLP_FUSED [[function_constant(50)]];+constant uint MLP_HIDDEN_DIM [[function_constant(51)]];+constant uint MLP_ACTIVATION [[function_constant(52)]];++// Activation type enum for readability+enum MlpActivationType : uint {+ MLP_ACT_SILU = 0,+ MLP_ACT_GELU = 1,+};++// SiLU activation: x * sigmoid(x)+template <typename T>+inline T mlp_silu(T x) {+ float xf = float(x);+ float result = xf / (1.0f + exp(-xf));+ return T(result);+}++// GELU approximation (tanh-based)+template <typename T>+inline T mlp_gelu(T x) {+ constexpr float k0 = 0.044715f;+ constexpr float k1 = 0.7978845608f; // sqrt(2/pi)+ float xf = float(x);+ float t = k1 * (xf + k0 * xf * xf * xf);+ float result = 0.5f * xf * (1.0f + metal::precise::tanh(t));+ return T(result);+}++// Apply activation based on MLP_ACTIVATION constant+template <typename T>+inline T mlp_activate(T x) {+ if (MLP_ACTIVATION == MLP_ACT_SILU) {+ return mlp_silu(x);+ }+ return mlp_gelu(x);+}++// Fused MLP epilogue: up * activation(gate)+// Used in SwiGLU (SiLU) and GeGLU (GELU) style MLPs+template <typename T>+inline T mlp_fused_epilogue(T up, T gate) {+ T activated_gate = mlp_activate(gate);+ return T(float(up) * float(activated_gate));+}++// Fused MLP epilogue with float accumulators+inline float mlp_fused_epilogue_f32(float up, float gate) {+ float activated_gate;+ if (MLP_ACTIVATION == MLP_ACT_SILU) {+ activated_gate = up / (1.0f + exp(-gate));+ // Note: SwiGLU is gate_act * up, not up_act * gate+ // Correcting: activated_gate = gate / (1.0f + exp(-gate))+ activated_gate = gate / (1.0f + exp(-gate));+ } else {+ constexpr float k0 = 0.044715f;+ constexpr float k1 = 0.7978845608f;+ float t = k1 * (gate + k0 * gate * gate * gate);+ activated_gate = 0.5f * gate * (1.0f + metal::precise::tanh(t));+ }+ return up * activated_gate;+}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv.rsindex e96a0ef11..db5ee62da 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv.rs@@ -5,7 +5,7 @@ use metal::{MTLSize,};-use super::arguments::MatmulArguments;+use super::{super::mlp::MlpActivationType, arguments::MatmulArguments};use crate::{DataType,backends::metal::{MTLContext, MTLError},@@ -458,3 +458,215 @@ impl GemvKernel {)}}++/// MLP Fused GEMV configuration+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+struct MlpFusedGemvConfiguration {+ threadgroup_rows: u32,+ threadgroup_cols: u32,+ threads_per_simdgroup_row: u32,+ threads_per_simdgroup_col: u32,+ elements_per_thread_row: u32,+ elements_per_thread_col: u32,+ activation: MlpActivationType,+}++impl MlpFusedGemvConfiguration {+ fn output_elements_per_threadgroup(&self) -> u32 {+ self.threadgroup_rows+ * self.threads_per_simdgroup_row+ * self.elements_per_thread_row+ }++ fn threads_per_threadgroup(&self) -> MTLSize {+ MTLSize::new(+ 32,+ self.threadgroup_cols as u64,+ self.threadgroup_rows as u64,+ )+ }+}++fn mlp_fused_gemv_kernel_name(+ data_type: DataType,+ config: &MlpFusedGemvConfiguration,+) -> Result<String, MTLError> {+ let dtype_name = match data_type {+ DataType::F16 => "float16",+ DataType::BF16 => "bfloat16",+ DataType::F32 => "float32",+ _ => {+ return Err(MTLError::Generic(format!(+ "Unsupported data type for MLP fused GEMV: {:?}",+ data_type+ )));+ },+ };++ Ok(format!(+ "gemv_mlp_fused_{}_bm{}_bn{}_sm{}_sn{}_tm{}_tn{}",+ dtype_name,+ config.threadgroup_rows,+ config.threadgroup_cols,+ config.threads_per_simdgroup_row,+ config.threads_per_simdgroup_col,+ config.elements_per_thread_row,+ config.elements_per_thread_col,+ ))+}++fn select_mlp_fused_gemv_configuration(+ hidden_dim: i32,+ activation: MlpActivationType,+) -> MlpFusedGemvConfiguration {+ // Select configuration based on hidden dimension+ let threadgroup_rows = if hidden_dim >= 4096 {+ 8+ } else {+ 4+ };++ MlpFusedGemvConfiguration {+ threadgroup_rows,+ threadgroup_cols: 1,+ threads_per_simdgroup_row: 1,+ threads_per_simdgroup_col: 32,+ elements_per_thread_row: 4,+ elements_per_thread_col: 4,+ activation,+ }+}++/// Arguments for MLP fused GEMV+#[derive(Debug)]+pub struct MlpFusedGemvArguments<'a> {+ /// Weight matrix [2*hidden_dim, input_dim] (up and gate concatenated)+ pub weights: &'a MTLBuffer,+ /// Input vector [input_dim]+ pub input: &'a MTLBuffer,+ /// Input byte offset+ pub input_offset: u64,+ /// Output vector [hidden_dim]+ pub output: &'a MTLBuffer,+ /// Input dimension (K)+ pub input_dim: i32,+ /// Hidden dimension (output size, half of weight rows)+ pub hidden_dim: i32,+ /// Weight matrix leading dimension+ pub weights_ld: i32,+ /// Batch count+ pub batch_count: i32,+ /// Activation type+ pub activation: MlpActivationType,+}++/// MLP Fused GEMV Kernel+/// Computes paired up/gate projections with fused activation: out = up * activation(gate)+pub struct MlpFusedGemvKernel {+ data_type: DataType,+ pipelines: HashMap<MlpFusedGemvConfiguration, ComputePipelineState>,+}++impl MlpFusedGemvKernel {+ pub fn new(data_type: DataType) -> Result<Self, MTLError> {+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)+ {+ return Err(MTLError::Generic(format!(+ "Unsupported dtype for MLP fused GEMV: {:?}",+ data_type+ )));+ }+ Ok(Self {+ data_type,+ pipelines: HashMap::new(),+ })+ }++ fn get_pipeline(+ &mut self,+ context: &MTLContext,+ config: MlpFusedGemvConfiguration,+ ) -> Result<&ComputePipelineState, MTLError> {+ if !self.pipelines.contains_key(&config) {+ let kernel_name =+ mlp_fused_gemv_kernel_name(self.data_type, &config)?;++ // Create function constant values for MLP activation+ let fcv = metal::FunctionConstantValues::new();+ let activation_val = config.activation as u32;+ fcv.set_constant_value_at_index(+ &activation_val as *const u32 as *const _,+ metal::MTLDataType::UInt,+ 52, // MLP_ACTIVATION index+ );++ let pipeline =+ context.compute_pipeline_state(&kernel_name, Some(&fcv))?;+ self.pipelines.insert(config, pipeline);+ }+ Ok(self.pipelines.get(&config).unwrap())+ }++ pub fn encode(+ &mut self,+ context: &MTLContext,+ encoder: &ComputeCommandEncoderRef,+ args: &MlpFusedGemvArguments,+ ) -> Result<(), MTLError> {+ let config = select_mlp_fused_gemv_configuration(+ args.hidden_dim,+ args.activation,+ );+ let pipeline = self.get_pipeline(context, config)?;++ encoder.set_compute_pipeline_state(pipeline);+ encoder.set_buffer(0, Some(args.weights), 0);+ encoder.set_buffer(1, Some(args.input), args.input_offset);+ encoder.set_buffer(3, Some(args.output), 0);++ encoder.set_bytes(+ 4,+ std::mem::size_of::<i32>() as u64,+ &args.input_dim as *const i32 as *const std::ffi::c_void,+ );+ encoder.set_bytes(+ 5,+ std::mem::size_of::<i32>() as u64,+ &args.hidden_dim as *const i32 as *const std::ffi::c_void,+ );+ encoder.set_bytes(+ 6,+ std::mem::size_of::<i32>() as u64,+ &args.weights_ld as *const i32 as *const std::ffi::c_void,+ );++ // Batch strides+ let vector_batch_stride = args.input_dim as i64;+ let matrix_batch_stride = 0i64; // Weights not batched+ encoder.set_bytes(+ 11,+ std::mem::size_of::<i64>() as u64,+ &vector_batch_stride as *const i64 as *const std::ffi::c_void,+ );+ encoder.set_bytes(+ 12,+ std::mem::size_of::<i64>() as u64,+ &matrix_batch_stride as *const i64 as *const std::ffi::c_void,+ );++ let output_elements_per_threadgroup =+ config.output_elements_per_threadgroup();+ let threadgroup_count_x =+ ((args.hidden_dim as u32 + output_elements_per_threadgroup - 1)+ / output_elements_per_threadgroup) as u64;+ let threadgroup_count_z = args.batch_count.max(1) as u64;++ let threadgroup_count =+ MTLSize::new(threadgroup_count_x, 1, threadgroup_count_z);+ let threads_per_threadgroup = config.threads_per_threadgroup();++ encoder+ .dispatch_thread_groups(threadgroup_count, threads_per_threadgroup);+ Ok(())+ }+}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemv/gemv.metalindex 1c5bf124d..d79659f90 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal@@ -2,6 +2,7 @@#include <metal_stdlib>#include "../../common/utils.h"+#include "../../common/mlp_epilogue.h"#include "../steel/utils.h"@@ -864,3 +865,264 @@ instantiate_gemv_t_bs_blocks(float32, float);instantiate_gemv_t_bs_blocks(float16, half);instantiate_gemv_t_bs_blocks(bfloat16, bfloat16_t);instantiate_gemv_t_bs_blocks(complex64, complex64_t); // clang-format on++///////////////////////////////////////////////////////////////////////////////+/// MLP Fused GEMV - Paired Up/Gate Computation with Activation Fusion+///////////////////////////////////////////////////////////////////////////////++// MLP fused GEMV kernel for decode path (M=1).+// Computes paired up and gate projections, then applies: out = up * activation(gate)+// Output size is hidden_dim (half of the weight matrix columns).+template <+ typename T,+ const int BM, /* Threadgroup rows (in simdgroups) */+ const int BN, /* Threadgroup cols (in simdgroups) */+ const int SM, /* Simdgroup rows (in threads) */+ const int SN, /* Simdgroup cols (in threads) */+ const int TM, /* Thread rows (in elements) */+ const int TN> /* Thread cols (in elements) */+struct GEMVMlpFusedKernel {+ using AccT = float;++ MTL_CONST int threadsM = BM * SM;+ MTL_CONST int threadsN = BN * SN;+ MTL_CONST int blockM = threadsM * TM;+ MTL_CONST int blockN = threadsN * TN;++ static_assert(SM * SN == 32, "simdgroup can only have 32 threads");++ MTL_CONST short tgp_mem_size = BN > 1 ? 2 * BN * (blockM + TM) : 0;+ MTL_CONST bool needs_tgp_reduction = BN > 1;++ template <typename U = T>+ static METAL_FUNC void+ load_unsafe(const device T* src, thread U dst[TN], const int src_offset = 0) {+ MTL_PRAGMA_UNROLL+ for (int tn = 0; tn < TN; tn++) {+ dst[tn] = static_cast<U>(src[src_offset + tn]);+ }+ }++ template <typename U = T>+ static METAL_FUNC void load_safe(+ const device T* src,+ thread U dst[TN],+ const int src_offset = 0,+ const int src_size = TN) {+ if (src_offset + TN <= src_size) {+ MTL_PRAGMA_UNROLL+ for (int tn = 0; tn < TN; tn++) {+ dst[tn] = static_cast<U>(src[src_offset + tn]);+ }+ } else {+ MTL_PRAGMA_UNROLL+ for (int tn = 0; tn < TN; tn++) {+ dst[tn] = src_offset + tn < src_size+ ? static_cast<U>(src[src_offset + tn])+ : U(0);+ }+ }+ }++ static METAL_FUNC void run(+ const device T* mat [[buffer(0)]],+ const device T* in_vec [[buffer(1)]],+ device T* out_vec [[buffer(3)]],+ const int in_vec_size,+ const int hidden_dim, // Output size (half of weight matrix rows)+ const int matrix_ld,+ threadgroup AccT* tgp_memory,+ uint3 tid [[threadgroup_position_in_grid]],+ uint3 lid [[thread_position_in_threadgroup]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]) {+ (void)lid;++ // Thread local accumulation results for up and gate+ thread AccT result_up[TM] = {0};+ thread AccT result_gate[TM] = {0};+ thread T inter_up[TN];+ thread T inter_gate[TN];+ thread AccT v_coeff[TN];++ const int thrM = SN != 32 ? simd_lid / SN : 0;+ const int thrN = SN != 32 ? simd_lid % SN : int(simd_lid);++ const int sgN = BN != 1 ? (simd_gid % BN) : 0;+ const int simdM = BN != 1 ? SM * (simd_gid / BN) : int(SM * simd_gid);+ const int simdN = BN != 1 ? SN * (simd_gid % BN) : 0;++ int bm = (simdM + thrM) * TM;+ int bn = (simdN + thrN) * TN;++ // Block position - outputs to hidden_dim elements+ int out_row = tid.x * blockM + bm;++ if (out_row >= hidden_dim)+ return;++ out_row = out_row + TM <= hidden_dim ? out_row : hidden_dim - TM;++ // Pointers to up and gate rows in the weight matrix+ // Weight layout: [up_0, up_1, ..., up_H-1, gate_0, gate_1, ..., gate_H-1]+ const device T* mat_up = mat + out_row * matrix_ld;+ const device T* mat_gate = mat + (out_row + hidden_dim) * matrix_ld;++ constexpr const uniform<int> loop_stride = make_uniform(blockN);+ const uniform<int> in_size = make_uniform(in_vec_size);+ const uniform<int> n_iter = in_size / loop_stride;+ const uniform<int> last_iter = loop_stride * n_iter;+ const uniform<int> leftover = in_size - last_iter;++ // Main loop - compute both up and gate dot products+ for (int i = 0; i < n_iter; ++i) {+ load_unsafe<AccT>(in_vec, v_coeff, bn);++ int mat_offset = 0;+ MTL_PRAGMA_UNROLL+ for (int tm = 0; tm < TM; tm++) {+ load_unsafe(mat_up, inter_up, mat_offset + bn);+ load_unsafe(mat_gate, inter_gate, mat_offset + bn);++ MTL_PRAGMA_UNROLL+ for (int tn = 0; tn < TN; tn++) {+ result_up[tm] += inter_up[tn] * v_coeff[tn];+ result_gate[tm] += inter_gate[tn] * v_coeff[tn];+ }++ mat_offset += matrix_ld;+ }++ bn += blockN;+ }++ // Handle leftover+ if (leftover > 0) {+ load_safe<AccT>(in_vec, v_coeff, bn, in_vec_size);++ MTL_PRAGMA_UNROLL+ for (int tm = 0; tm < TM; tm++) {+ load_safe(&mat_up[tm * matrix_ld], inter_up, bn, in_vec_size);+ load_safe(&mat_gate[tm * matrix_ld], inter_gate, bn, in_vec_size);++ MTL_PRAGMA_UNROLL+ for (int tn = 0; tn < TN; tn++) {+ result_up[tm] += inter_up[tn] * v_coeff[tn];+ result_gate[tm] += inter_gate[tn] * v_coeff[tn];+ }+ }+ }++ // Simdgroup reduction for both up and gate+ MTL_PRAGMA_UNROLL+ for (int tm = 0; tm < TM; tm++) {+ MTL_PRAGMA_UNROLL+ for (ushort sn = (SN / 2); sn >= 1; sn >>= 1) {+ result_up[tm] += simd_shuffle_down(result_up[tm], sn);+ result_gate[tm] += simd_shuffle_down(result_gate[tm], sn);+ }+ }++ // Threadgroup reduction if needed+ if (needs_tgp_reduction) {+ threadgroup AccT* tgp_results_up = tgp_memory + sgN * (blockM + TM) + bm;+ threadgroup AccT* tgp_results_gate = tgp_memory + BN * (blockM + TM) + sgN * (blockM + TM) + bm;++ if (thrN == 0) {+ MTL_PRAGMA_UNROLL+ for (int tm = 0; tm < TM; tm++) {+ tgp_results_up[tm] = result_up[tm];+ tgp_results_gate[tm] = result_gate[tm];+ }++ threadgroup_barrier(mem_flags::mem_none);++ if (sgN == 0) {+ MTL_PRAGMA_UNROLL+ for (int sgn = 1; sgn < BN; sgn++) {+ MTL_PRAGMA_UNROLL+ for (int tm = 0; tm < TM; tm++) {+ result_up[tm] += tgp_results_up[sgn * (blockM + TM) + tm];+ result_gate[tm] += tgp_results_gate[sgn * (blockM + TM) + tm];+ }+ }+ }+ }+ }++ // Apply MLP fused epilogue and write output+ if (simdN == 0 && thrN == 0) {+ MTL_PRAGMA_UNROLL+ for (int tm = 0; tm < TM; tm++) {+ float fused_result = mlp_fused_epilogue_f32(result_up[tm], result_gate[tm]);+ out_vec[out_row + tm] = static_cast<T>(fused_result);+ }+ }+ }+};++template <+ typename T,+ const int BM,+ const int BN,+ const int SM,+ const int SN,+ const int TM,+ const int TN>+[[kernel, max_total_threads_per_threadgroup(BM* BN * 32)]] void gemv_mlp_fused(+ const device T* mat [[buffer(0)]],+ const device T* in_vec [[buffer(1)]],+ device T* out_vec [[buffer(3)]],+ const constant int& in_vec_size [[buffer(4)]],+ const constant int& hidden_dim [[buffer(5)]], // Output dimension (half of weight rows)+ const constant int& matrix_ld [[buffer(6)]],+ const constant int64_t& vector_batch_stride [[buffer(11)]],+ const constant int64_t& matrix_batch_stride [[buffer(12)]],+ uint3 tid [[threadgroup_position_in_grid]],+ uint3 lid [[thread_position_in_threadgroup]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]) {+ using gemv_kernel = GEMVMlpFusedKernel<T, BM, BN, SM, SN, TM, TN>;+ threadgroup typename gemv_kernel::AccT tgp_memory+ [gemv_kernel::tgp_mem_size == 0 ? 1 : gemv_kernel::tgp_mem_size];++ // Batch offset+ in_vec += tid.z * vector_batch_stride;+ mat += tid.z * matrix_batch_stride;+ out_vec += tid.z * hidden_dim;++ gemv_kernel::run(+ mat,+ in_vec,+ out_vec,+ in_vec_size,+ hidden_dim,+ matrix_ld,+ gemv_kernel::tgp_mem_size == 0 ? nullptr : tgp_memory,+ tid,+ lid,+ simd_gid,+ simd_lid);+}++#define instantiate_gemv_mlp_fused_helper(name, itype, bm, bn, sm, sn, tm, tn) \+ instantiate_kernel( \+ "gemv_mlp_fused_" #name "_bm" #bm "_bn" #bn "_sm" #sm "_sn" #sn "_tm" #tm \+ "_tn" #tn, \+ gemv_mlp_fused, \+ itype, \+ bm, \+ bn, \+ sm, \+ sn, \+ tm, \+ tn)++// clang-format off+#define instantiate_gemv_mlp_fused_blocks(name, itype) \+ instantiate_gemv_mlp_fused_helper(name, itype, 4, 1, 1, 32, 4, 4) \+ instantiate_gemv_mlp_fused_helper(name, itype, 8, 1, 1, 32, 4, 4) // clang-format on++instantiate_gemv_mlp_fused_blocks(float32, float);+instantiate_gemv_mlp_fused_blocks(float16, half);+instantiate_gemv_mlp_fused_blocks(bfloat16, bfloat16_t);diff --git a/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/kernel.rsindex 8c36bf0d8..1c257c8c9 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs@@ -6,7 +6,7 @@ use metal::{};use super::{- super::{KernelDataType, TensorAddBias},+ super::{KernelDataType, TensorAddBias, mlp::MlpActivationType},arguments::MatmulArguments,gemv::GemvKernel,pipeline::PipelineKey,@@ -643,3 +643,212 @@ impl MatmulKernel {);}}++/// Arguments for MLP fused GEMM+#[derive(Debug)]+pub struct MlpFusedGemmArguments<'a> {+ /// Input activations [M, K]+ pub input: &'a MTLBuffer,+ /// Input byte offset+ pub input_offset: u64,+ /// Weight matrix [K, 2*hidden_dim] or [2*hidden_dim, K] depending on transpose+ pub weights: &'a MTLBuffer,+ /// Output [M, hidden_dim]+ pub output: &'a MTLBuffer,+ /// Batch size (M)+ pub batch: i32,+ /// Input dimension (K)+ pub input_dim: i32,+ /// Hidden dimension (output size, half of weight columns/rows)+ pub hidden_dim: i32,+ /// Leading dimension of input+ pub lda: i32,+ /// Leading dimension of weights+ pub ldb: i32,+ /// Leading dimension of output+ pub ldd: i32,+ /// Activation type for fused epilogue+ pub activation: MlpActivationType,+}++/// MLP Fused GEMM Kernel for prefill path+/// Computes paired up/gate projections with fused activation: out = up * activation(gate)+pub struct MlpFusedGemmKernel {+ data_type: DataType,+ weights_transposed: bool,+ pipelines: HashMap<+ (i32, i32, bool, bool, MlpActivationType),+ MTLComputePipelineState,+ >,+}++impl MlpFusedGemmKernel {+ pub fn new(+ data_type: DataType,+ weights_transposed: bool,+ ) -> Result<Self, MTLError> {+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)+ {+ return Err(MTLError::Generic(format!(+ "Unsupported dtype for MLP fused GEMM: {:?}",+ data_type+ )));+ }+ Ok(Self {+ data_type,+ weights_transposed,+ pipelines: HashMap::new(),+ })+ }++ fn kernel_name(+ &self,+ bm: i32,+ bn: i32,+ mn_aligned: bool,+ k_aligned: bool,+ ) -> String {+ let type_name = match self.data_type {+ DataType::F16 => "half",+ DataType::BF16 => "bfloat",+ DataType::F32 => "float",+ _ => unreachable!(),+ };+ let transpose_char = if self.weights_transposed {+ "t"+ } else {+ "n"+ };+ let mn_str = if mn_aligned {+ "true"+ } else {+ "false"+ };+ let k_str = if k_aligned {+ "true"+ } else {+ "false"+ };++ format!(+ "steel_gemm_mlp_fused_{}_{}_{}_bm{}_bn{}_bk16_wm2_wn2_align_MN_{}_K_{}",+ transpose_char, type_name, type_name, bm, bn, mn_str, k_str+ )+ }++ fn get_pipeline(+ &mut self,+ context: &MTLContext,+ bm: i32,+ bn: i32,+ mn_aligned: bool,+ k_aligned: bool,+ activation: MlpActivationType,+ ) -> Result<&MTLComputePipelineState, MTLError> {+ let key = (bm, bn, mn_aligned, k_aligned, activation);+ if !self.pipelines.contains_key(&key) {+ let kernel_name = self.kernel_name(bm, bn, mn_aligned, k_aligned);++ // Create function constant values for MLP activation+ let fcv = metal::FunctionConstantValues::new();+ let activation_val = activation as u32;+ fcv.set_constant_value_at_index(+ &activation_val as *const u32 as *const _,+ metal::MTLDataType::UInt,+ 52, // MLP_ACTIVATION index+ );++ let pipeline =+ context.compute_pipeline_state(&kernel_name, Some(&fcv))?;+ self.pipelines.insert(key, pipeline);+ }+ Ok(self.pipelines.get(&key).unwrap())+ }++ fn select_tile_size(+ &self,+ m: i32,+ hidden_dim: i32,+ k: i32,+ ) -> (i32, i32) {+ // Select tile size based on problem dimensions+ let _ = k; // K doesn't affect tile selection directly+ if m >= 64 && hidden_dim >= 64 {+ (64, 64)+ } else {+ (32, 32)+ }+ }++ pub fn encode(+ &mut self,+ context: &MTLContext,+ encoder: &ComputeCommandEncoderRef,+ args: &MlpFusedGemmArguments,+ ) -> Result<(), MTLError> {+ let (bm, bn) =+ self.select_tile_size(args.batch, args.hidden_dim, args.input_dim);+ let bk = 16;++ let mn_aligned = args.batch % bm == 0 && args.hidden_dim % bn == 0;+ let k_aligned = args.input_dim % bk == 0;++ let pipeline = self.get_pipeline(+ context,+ bm,+ bn,+ mn_aligned,+ k_aligned,+ args.activation,+ )?;+ encoder.set_compute_pipeline_state(pipeline);++ encoder.set_buffer(0, Some(args.input), args.input_offset);+ encoder.set_buffer(1, Some(args.weights), 0);+ encoder.set_buffer(2, Some(args.output), 0);++ // Set GEMM params+ let tiles_m = (args.batch + bm - 1) / bm;+ let tiles_n = (args.hidden_dim + bn - 1) / bn;+ let gemm_k_iterations = (args.input_dim + bk - 1) / bk;++ let params = GEMMParams {+ M: args.batch,+ N: args.hidden_dim * 2, // Full weight width for pointer arithmetic+ K: args.input_dim,+ lda: args.lda,+ ldb: args.ldb,+ ldd: args.ldd,+ tiles_n,+ tiles_m,+ batch_stride_a: 0,+ batch_stride_b: 0,+ batch_stride_d: 0,+ swizzle_log: 0,+ gemm_k_iterations_aligned: if k_aligned {+ gemm_k_iterations+ } else {+ gemm_k_iterations - 1+ },+ batch_ndim: 0,+ };++ encoder.set_bytes(+ 3,+ std::mem::size_of::<GEMMParams>() as u64,+ ¶ms as *const GEMMParams as *const std::ffi::c_void,+ );+ encoder.set_bytes(+ 10,+ std::mem::size_of::<i32>() as u64,+ &args.hidden_dim as *const i32 as *const std::ffi::c_void,+ );++ let threadgroup_count = MTLSize::new(tiles_n as u64, tiles_m as u64, 1);+ let threads_per_threadgroup = MTLSize::new(32, 2, 2); // WM=2, WN=2, 32 threads per simdgroup++ encoder+ .dispatch_thread_groups(threadgroup_count, threads_per_threadgroup);+ Ok(())+ }+}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/mod.rsindex fb2a346ad..81a690ef5 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/mod.rs@@ -11,7 +11,7 @@ mod transpose;pub use arguments::MatmulArguments;pub use gather::{GatherGemm, GatherMmArguments, GatherMmRhsArguments};-pub use gemv::GemvKernel;-pub use kernel::MatmulKernel;+pub use gemv::{GemvKernel, MlpFusedGemvArguments, MlpFusedGemvKernel};+pub use kernel::{MatmulKernel, MlpFusedGemmArguments, MlpFusedGemmKernel};pub use segmented::{SegmentedGemm, SegmentedMmArguments};pub use splitk::SplitKGemm;diff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_mlp_fused.h b/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_mlp_fused.hnew file mode 100644index 000000000..88ebcdbe1--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_mlp_fused.h@@ -0,0 +1,207 @@+#pragma once++#include <metal_stdlib>+#include <metal_simdgroup>+#include <metal_simdgroup_matrix>++#include "../gemm.h"+#include "../params.h"+#include "../../../../common/mlp_epilogue.h"++using namespace metal;++namespace steel {++///////////////////////////////////////////////////////////////////////////////+// MLP Fused GEMM Kernel+// Computes paired up/gate projections with fused activation for prefill path.+// Weight layout: B = [up_weights (hidden_dim cols), gate_weights (hidden_dim cols)]+// Output = up * activation(gate), written to first hidden_dim columns+///////////////////////////////////////////////////////////////////////////////++template <+ typename T,+ typename U,+ int BM,+ int BN,+ int BK,+ int WM,+ int WN,+ bool transpose_a,+ bool transpose_b,+ bool MN_aligned,+ bool K_aligned,+ typename AccumType = typename AccumHelper<T>::accum_type>+struct GEMMMlpFusedKernel {+ STEEL_CONST short tgp_padding_a = 16 / sizeof(T);+ STEEL_CONST short tgp_padding_b = 16 / sizeof(T);+ STEEL_CONST short tgp_mem_size_a =+ transpose_a ? BK * (BM + tgp_padding_a) : BM * (BK + tgp_padding_a);+ // Double B threadgroup memory for up and gate+ STEEL_CONST short tgp_mem_size_b =+ 2 * (transpose_b ? BN * (BK + tgp_padding_b) : BK * (BN + tgp_padding_b));+ STEEL_CONST short tgp_mem_size = tgp_mem_size_a + tgp_mem_size_b;++ STEEL_CONST short tgp_size = WM * WN * 32;++ using loader_a_t = BlockLoader<+ T,+ transpose_a ? BK : BM,+ transpose_a ? BM : BK,+ transpose_a ? BM + tgp_padding_a : BK + tgp_padding_a,+ !transpose_a,+ tgp_size>;+ using loader_b_t = BlockLoader<+ T,+ transpose_b ? BN : BK,+ transpose_b ? BK : BN,+ transpose_b ? BK + tgp_padding_b : BN + tgp_padding_b,+ transpose_b,+ tgp_size>;+ using mma_t = BlockMMA<+ T,+ U,+ BM,+ BN,+ BK,+ WM,+ WN,+ transpose_a,+ transpose_b,+ transpose_a ? BM + tgp_padding_a : BK + tgp_padding_a,+ transpose_b ? BK + tgp_padding_b : BN + tgp_padding_b,+ AccumType,+ TransformNone<U, AccumType>>;++ static METAL_FUNC void run(+ const device T* A [[buffer(0)]],+ const device T* B [[buffer(1)]],+ device U* D [[buffer(2)]],+ const constant GEMMParams* params [[buffer(3)]],+ const constant int& hidden_dim [[buffer(10)]],+ threadgroup T* As [[threadgroup(0)]],+ threadgroup T* Bs_up [[threadgroup(1)]],+ uint simd_lane_id [[thread_index_in_simdgroup]],+ uint simd_group_id [[simdgroup_index_in_threadgroup]],+ uint3 tid [[threadgroup_position_in_grid]],+ uint3 lid [[thread_position_in_threadgroup]]) {+ (void)lid;++ const int tid_y = ((tid.y) << params->swizzle_log) ++ ((tid.x) & ((1 << params->swizzle_log) - 1));+ const int tid_x = (tid.x) >> params->swizzle_log;++ // Only process tiles within hidden_dim columns (output size)+ int hidden_tiles = (hidden_dim + BN - 1) / BN;+ if (hidden_tiles <= tid_x || params->tiles_m <= tid_y) {+ return;+ }++ threadgroup_barrier(mem_flags::mem_none);++ const int c_row = tid_y * BM;+ const int c_col = tid_x * BN;+ const size_t c_row_long = size_t(c_row);+ const size_t c_col_long = size_t(c_col);++ // Pointer to A (input activations)+ const device T* A_ptr = A + (transpose_a ? c_row_long : c_row_long * params->lda);++ // Pointers to up and gate weights in B+ // B layout: [up_weights | gate_weights] where each is hidden_dim columns+ const device T* B_up = B + (transpose_b ? c_col_long * params->ldb : c_col_long);+ const device T* B_gate = B_up + (transpose_b ? hidden_dim * params->ldb : hidden_dim);++ device U* D_ptr = D + c_row_long * params->ldd + c_col_long;++ // Threadgroup memory layout: [As, Bs_up, Bs_gate]+ threadgroup T* Bs_gate = Bs_up + (tgp_mem_size_b / 2);++ // Loaders for A (shared) and B (up and gate)+ thread loader_a_t loader_a(A_ptr, params->lda, As, simd_group_id, simd_lane_id);+ thread loader_b_t loader_b_up(B_up, params->ldb, Bs_up, simd_group_id, simd_lane_id);+ thread loader_b_t loader_b_gate(B_gate, params->ldb, Bs_gate, simd_group_id, simd_lane_id);++ // MMA operations for up and gate+ thread mma_t mma_up(simd_group_id, simd_lane_id);+ thread mma_t mma_gate(simd_group_id, simd_lane_id);++ int gemm_k_iterations = params->gemm_k_iterations_aligned;++ // Main compute loop+ for (int k = 0; k < gemm_k_iterations; k++) {+ threadgroup_barrier(mem_flags::mem_threadgroup);++ // Load A (shared for both up and gate)+ if (MN_aligned) {+ loader_a.load_unsafe();+ } else {+ short tgp_bm = min(short(BM), short(params->M - c_row));+ short2 tile_dims_A = transpose_a ? short2(tgp_bm, BK) : short2(BK, tgp_bm);+ loader_a.load_safe(tile_dims_A);+ }++ // Load B_up and B_gate+ if (MN_aligned) {+ loader_b_up.load_unsafe();+ loader_b_gate.load_unsafe();+ } else {+ short tgp_bn = min(short(BN), short(hidden_dim - c_col));+ short2 tile_dims_B = transpose_b ? short2(BK, tgp_bn) : short2(tgp_bn, BK);+ loader_b_up.load_safe(tile_dims_B);+ loader_b_gate.load_safe(tile_dims_B);+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ // MMA for up and gate+ mma_up.mma(As, Bs_up);+ mma_gate.mma(As, Bs_gate);++ loader_a.next();+ loader_b_up.next();+ loader_b_gate.next();+ }++ // Handle leftover K+ if (!K_aligned) {+ threadgroup_barrier(mem_flags::mem_threadgroup);++ int lbk = params->K - params->gemm_k_iterations_aligned * BK;+ short2 tile_dims_A = transpose_a ? short2(BM, lbk) : short2(lbk, BM);+ short2 tile_dims_B = transpose_b ? short2(lbk, BN) : short2(BN, lbk);++ loader_a.load_safe(tile_dims_A);+ loader_b_up.load_safe(tile_dims_B);+ loader_b_gate.load_safe(tile_dims_B);++ threadgroup_barrier(mem_flags::mem_threadgroup);++ mma_up.mma(As, Bs_up);+ mma_gate.mma(As, Bs_gate);+ }++ // Apply MLP fused epilogue: out = up * activation(gate)+ // Access the Ctile elements and apply fusion+ constexpr int kElemsPerTile = decltype(mma_up.Ctile)::kElemsPerTile;++ #pragma unroll+ for (short i = 0; i < kElemsPerTile; i++) {+ float up_val = static_cast<float>(mma_up.Ctile.elems()[i]);+ float gate_val = static_cast<float>(mma_gate.Ctile.elems()[i]);+ float fused = mlp_fused_epilogue_f32(up_val, gate_val);+ mma_up.Ctile.elems()[i] = static_cast<AccumType>(fused);+ }++ // Store result+ if (MN_aligned) {+ mma_up.store_result(D_ptr, params->ldd);+ } else {+ short tgp_bm = min(short(BM), short(params->M - c_row));+ short tgp_bn = min(short(BN), short(hidden_dim - c_col));+ mma_up.store_result_safe(D_ptr, params->ldd, short2(tgp_bn, tgp_bm));+ }+ }+};++} // namespace steeldiff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_mlp_fused.metal b/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_mlp_fused.metalnew file mode 100644index 000000000..5b0da8472--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_mlp_fused.metal@@ -0,0 +1,109 @@+#include <metal_stdlib>++#include "../../../../common/utils.h"+#include "steel_gemm_mlp_fused.h"++using namespace metal;++///////////////////////////////////////////////////////////////////////////////+// MLP Fused GEMM Kernel Instantiations+// For prefill path: computes up * activation(gate) in a single pass+///////////////////////////////////////////////////////////////////////////////++#define instantiate_mlp_fused_gemm( \+ tname, trans_a, trans_b, itype, otype, bm, bn, bk, wm, wn, aname, mn, k) \+ template [[host_name("steel_gemm_mlp_fused_" #tname "_" #itype "_" #otype \+ "_bm" #bm "_bn" #bn "_bk" #bk "_wm" #wm "_wn" #wn \+ "_" #aname "_MN_" #mn "_K_" #k)]] [[kernel]] void \+ steel_gemm_mlp_fused< \+ itype, otype, bm, bn, bk, wm, wn, trans_a, trans_b, mn, k>( \+ const device itype* A [[buffer(0)]], \+ const device itype* B [[buffer(1)]], \+ device otype* D [[buffer(2)]], \+ const constant steel::GEMMParams* params [[buffer(3)]], \+ const constant int& hidden_dim [[buffer(10)]], \+ uint simd_lane_id [[thread_index_in_simdgroup]], \+ uint simd_group_id [[simdgroup_index_in_threadgroup]], \+ uint3 tid [[threadgroup_position_in_grid]], \+ uint3 lid [[thread_position_in_threadgroup]])++template <+ typename T,+ typename U,+ int BM,+ int BN,+ int BK,+ int WM,+ int WN,+ bool transpose_a,+ bool transpose_b,+ bool MN_aligned,+ bool K_aligned>+[[kernel, max_total_threads_per_threadgroup(WM* WN * 32)]] void+steel_gemm_mlp_fused(+ const device T* A [[buffer(0)]],+ const device T* B [[buffer(1)]],+ device U* D [[buffer(2)]],+ const constant steel::GEMMParams* params [[buffer(3)]],+ const constant int& hidden_dim [[buffer(10)]],+ uint simd_lane_id [[thread_index_in_simdgroup]],+ uint simd_group_id [[simdgroup_index_in_threadgroup]],+ uint3 tid [[threadgroup_position_in_grid]],+ uint3 lid [[thread_position_in_threadgroup]]) {+ using kernel_t =+ steel::GEMMMlpFusedKernel<T, U, BM, BN, BK, WM, WN, transpose_a, transpose_b, MN_aligned, K_aligned>;++ // Allocate threadgroup memory+ threadgroup T As[kernel_t::tgp_mem_size_a];+ threadgroup T Bs[kernel_t::tgp_mem_size_b];++ kernel_t::run(+ A, B, D, params, hidden_dim, As, Bs, simd_lane_id, simd_group_id, tid, lid);+}++// Instantiate for common configurations+// transpose_a = false, transpose_b = true (weights are [K, 2*H], accessed as transposed)++// float16 configurations+instantiate_mlp_fused_gemm(t, false, true, half, half, 32, 32, 16, 2, 2, align, true, true);+instantiate_mlp_fused_gemm(t, false, true, half, half, 32, 32, 16, 2, 2, align, true, false);+instantiate_mlp_fused_gemm(t, false, true, half, half, 32, 32, 16, 2, 2, align, false, true);+instantiate_mlp_fused_gemm(t, false, true, half, half, 32, 32, 16, 2, 2, align, false, false);++instantiate_mlp_fused_gemm(t, false, true, half, half, 64, 64, 16, 2, 2, align, true, true);+instantiate_mlp_fused_gemm(t, false, true, half, half, 64, 64, 16, 2, 2, align, true, false);+instantiate_mlp_fused_gemm(t, false, true, half, half, 64, 64, 16, 2, 2, align, false, true);+instantiate_mlp_fused_gemm(t, false, true, half, half, 64, 64, 16, 2, 2, align, false, false);++// bfloat16 configurations+instantiate_mlp_fused_gemm(t, false, true, bfloat, bfloat, 32, 32, 16, 2, 2, align, true, true);+instantiate_mlp_fused_gemm(t, false, true, bfloat, bfloat, 32, 32, 16, 2, 2, align, true, false);+instantiate_mlp_fused_gemm(t, false, true, bfloat, bfloat, 32, 32, 16, 2, 2, align, false, true);+instantiate_mlp_fused_gemm(t, false, true, bfloat, bfloat, 32, 32, 16, 2, 2, align, false, false);++instantiate_mlp_fused_gemm(t, false, true, bfloat, bfloat, 64, 64, 16, 2, 2, align, true, true);+instantiate_mlp_fused_gemm(t, false, true, bfloat, bfloat, 64, 64, 16, 2, 2, align, true, false);+instantiate_mlp_fused_gemm(t, false, true, bfloat, bfloat, 64, 64, 16, 2, 2, align, false, true);+instantiate_mlp_fused_gemm(t, false, true, bfloat, bfloat, 64, 64, 16, 2, 2, align, false, false);++// float32 configurations+instantiate_mlp_fused_gemm(t, false, true, float, float, 32, 32, 16, 2, 2, align, true, true);+instantiate_mlp_fused_gemm(t, false, true, float, float, 32, 32, 16, 2, 2, align, true, false);+instantiate_mlp_fused_gemm(t, false, true, float, float, 32, 32, 16, 2, 2, align, false, true);+instantiate_mlp_fused_gemm(t, false, true, float, float, 32, 32, 16, 2, 2, align, false, false);++// Non-transposed B (weights are [2*H, K])+instantiate_mlp_fused_gemm(n, false, false, half, half, 32, 32, 16, 2, 2, align, true, true);+instantiate_mlp_fused_gemm(n, false, false, half, half, 32, 32, 16, 2, 2, align, true, false);+instantiate_mlp_fused_gemm(n, false, false, half, half, 32, 32, 16, 2, 2, align, false, true);+instantiate_mlp_fused_gemm(n, false, false, half, half, 32, 32, 16, 2, 2, align, false, false);++instantiate_mlp_fused_gemm(n, false, false, bfloat, bfloat, 32, 32, 16, 2, 2, align, true, true);+instantiate_mlp_fused_gemm(n, false, false, bfloat, bfloat, 32, 32, 16, 2, 2, align, true, false);+instantiate_mlp_fused_gemm(n, false, false, bfloat, bfloat, 32, 32, 16, 2, 2, align, false, true);+instantiate_mlp_fused_gemm(n, false, false, bfloat, bfloat, 32, 32, 16, 2, 2, align, false, false);++instantiate_mlp_fused_gemm(n, false, false, float, float, 32, 32, 16, 2, 2, align, true, true);+instantiate_mlp_fused_gemm(n, false, false, float, float, 32, 32, 16, 2, 2, align, true, false);+instantiate_mlp_fused_gemm(n, false, false, float, float, 32, 32, 16, 2, 2, align, false, true);+instantiate_mlp_fused_gemm(n, false, false, float, float, 32, 32, 16, 2, 2, align, false, false);diff --git a/crates/uzu/src/backends/metal/kernel/mlp/mod.rs b/crates/uzu/src/backends/metal/kernel/mlp/mod.rsindex aefaff465..a2a245f1b 100644--- a/crates/uzu/src/backends/metal/kernel/mlp/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/mlp/mod.rs@@ -1,6 +1,6 @@use metal::{Buffer as MTLBuffer, ComputeCommandEncoderRef, ComputePipelineState,- MTLSize,+ FunctionConstantValues, MTLDataType, MTLSize,};use crate::{@@ -9,6 +9,88 @@ use crate::{config::Activation,};+// MLP Fused Epilogue Function Constant Indices+// These must match the values in kernel/common/mlp_epilogue.h+pub const MLP_FUSED_FC_INDEX: u64 = 50;+pub const MLP_HIDDEN_DIM_FC_INDEX: u64 = 51;+pub const MLP_ACTIVATION_FC_INDEX: u64 = 52;++/// MLP activation type for fused kernels+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+#[repr(u32)]+pub enum MlpActivationType {+ SiLU = 0,+ Gelu = 1,+}++impl From<&Activation> for MlpActivationType {+ fn from(act: &Activation) -> Self {+ match act {+ Activation::SiLU {+ ..+ } => MlpActivationType::SiLU,+ Activation::Gelu => MlpActivationType::Gelu,+ Activation::Identity => {+ panic!("Identity activation not supported for MLP fusion")+ },+ }+ }+}++/// Configuration for MLP fused matmul epilogue+#[derive(Debug, Clone, Copy)]+pub struct MlpFusedConfig {+ pub hidden_dim: u32,+ pub activation: MlpActivationType,+}++impl MlpFusedConfig {+ pub fn new(+ hidden_dim: usize,+ activation: &Activation,+ ) -> Self {+ Self {+ hidden_dim: hidden_dim as u32,+ activation: MlpActivationType::from(activation),+ }+ }++ /// Create function constants for MLP fused matmul+ pub fn make_function_constants(&self) -> FunctionConstantValues {+ let fcv = FunctionConstantValues::new();+ let fused = true;+ fcv.set_constant_value_at_index(+ &fused as *const bool as *const std::ffi::c_void,+ MTLDataType::Bool,+ MLP_FUSED_FC_INDEX,+ );+ fcv.set_constant_value_at_index(+ &self.hidden_dim as *const u32 as *const std::ffi::c_void,+ MTLDataType::UInt,+ MLP_HIDDEN_DIM_FC_INDEX,+ );+ let act_val = self.activation as u32;+ fcv.set_constant_value_at_index(+ &act_val as *const u32 as *const std::ffi::c_void,+ MTLDataType::UInt,+ MLP_ACTIVATION_FC_INDEX,+ );+ fcv+ }+}++/// Create function constants for non-fused (standard) matmul+pub fn make_non_fused_function_constants() -> FunctionConstantValues {+ let fcv = FunctionConstantValues::new();+ let fused = false;+ fcv.set_constant_value_at_index(+ &fused as *const bool as *const std::ffi::c_void,+ MTLDataType::Bool,+ MLP_FUSED_FC_INDEX,+ );+ fcv+}+pub struct MlpGateActMulKernel {pipeline: ComputePipelineState,}diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rsindex e21378a11..ff569683d 100644--- a/crates/uzu/src/backends/metal/kernel/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/mod.rs@@ -8,6 +8,10 @@ pub mod mask_update;pub mod matmul;pub mod media_kernels;pub mod mlp;+pub use mlp::{+ MLP_ACTIVATION_FC_INDEX, MLP_FUSED_FC_INDEX, MLP_HIDDEN_DIM_FC_INDEX,+ MlpActivationType, MlpFusedConfig, make_non_fused_function_constants,+};pub mod moe;pub mod pooling;pub mod quant_matmul;diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rsindex 2e87f0012..1698da06d 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs@@ -354,3 +354,220 @@ enum KernelKind {MatrixMatrix,MatrixVector,}++/// Arguments for MLP fused quantized GEMV+#[derive(Debug)]+pub struct MlpFusedQmvArguments<'a> {+ /// Weight matrix [2*hidden_dim, input_dim] (up and gate concatenated, quantized)+ pub weights: &'a MTLBuffer,+ /// Scales buffer+ pub scales: &'a MTLBuffer,+ /// Zero points or biases buffer (depends on quantization type)+ pub zero_points_or_biases: &'a MTLBuffer,+ /// Input vector [input_dim]+ pub input: &'a MTLBuffer,+ /// Input byte offset+ pub input_offset: u64,+ /// Output vector [hidden_dim]+ pub output: &'a MTLBuffer,+ /// Input dimension (K)+ pub input_dim: i32,+ /// Hidden dimension (output size, half of weight rows)+ pub hidden_dim: i32,+ /// Batch count+ pub batch_count: i32,+}++/// MLP Fused Quantized GEMV Kernel+/// Computes paired up/gate projections with fused activation: out = up * activation(gate)+pub struct MlpFusedQmvKernel {+ pipeline: MTLComputePipelineState,+}++impl MlpFusedQmvKernel {+ pub fn new(+ context: &MTLContext,+ data_type: DataType,+ group_size: usize,+ mode: QuantizationMode,+ quantization_type: QuantizationType,+ ) -> Result<Self, QuantizedMatmulError> {+ let type_suffix = dtype_suffix(data_type)+ .ok_or(QuantizedMatmulError::UnsupportedDataType(data_type))?;++ let bits = match mode {+ QuantizationMode::UInt4 => 4,+ QuantizationMode::UInt8 | QuantizationMode::Int8 => 8,+ };++ if bits != 4 {+ return Err(QuantizedMatmulError::UnsupportedBits(bits));+ }++ let kernel_name =+ format!("qmv_mlp_fused_{}_g{}_b{}", type_suffix, group_size, bits);++ let function_constants = metal::FunctionConstantValues::new();+ let use_mlx_quant = matches!(quantization_type, QuantizationType::Mlx);+ function_constants.set_constant_value_at_index(+ &use_mlx_quant as *const bool as *const std::ffi::c_void,+ metal::MTLDataType::Bool,+ 40,+ );++ let pipeline = context+ .compute_pipeline_state(&kernel_name, Some(&function_constants))+ .map_err(QuantizedMatmulError::MetalError)?;++ Ok(Self {+ pipeline,+ })+ }++ pub fn encode(+ &self,+ encoder: &ComputeCommandEncoderRef,+ args: &MlpFusedQmvArguments,+ ) -> Result<(), QuantizedMatmulError> {+ encoder.set_compute_pipeline_state(&self.pipeline);++ encoder.set_buffer(0, Some(args.weights), 0);+ encoder.set_buffer(1, Some(args.scales), 0);+ encoder.set_buffer(2, Some(args.zero_points_or_biases), 0);+ encoder.set_buffer(3, Some(args.input), args.input_offset);+ encoder.set_buffer(4, Some(args.output), 0);++ encoder.set_bytes(+ 5,+ size_of::<i32>() as u64,+ &args.input_dim as *const i32 as *const _,+ );+ encoder.set_bytes(+ 6,+ size_of::<i32>() as u64,+ &args.hidden_dim as *const i32 as *const _,+ );++ // Dispatch: one threadgroup per 8 output rows (num_simdgroups * results_per_simdgroup)+ let rows_per_threadgroup = 8;+ let n_tgp_y = ((args.hidden_dim + rows_per_threadgroup - 1)+ / rows_per_threadgroup) as u64;+ let threadgroups =+ MTLSize::new(args.batch_count.max(1) as u64, n_tgp_y, 1);+ let threads_per_threadgroup = MTLSize::new(32, 2, 1); // 2 simdgroups, 32 threads each++ encoder.dispatch_thread_groups(threadgroups, threads_per_threadgroup);+ Ok(())+ }+}++/// Arguments for MLP fused quantized GEMM (prefill)+#[derive(Debug)]+pub struct MlpFusedQmmArguments<'a> {+ /// Weight matrix [K, 2*hidden_dim] quantized (up and gate concatenated)+ pub weights: &'a MTLBuffer,+ /// Scales buffer+ pub scales: &'a MTLBuffer,+ /// Zero points or biases buffer (depends on quantization type)+ pub zero_points_or_biases: &'a MTLBuffer,+ /// Input activations [M, K]+ pub input: &'a MTLBuffer,+ /// Input byte offset+ pub input_offset: u64,+ /// Output [M, hidden_dim]+ pub output: &'a MTLBuffer,+ /// Batch size (M)+ pub batch: i32,+ /// Input dimension (K)+ pub input_dim: i32,+ /// Hidden dimension (output size, half of weight columns)+ pub hidden_dim: i32,+}++/// MLP Fused Quantized GEMM Kernel for prefill path+/// Computes paired up/gate projections with fused activation: out = up * activation(gate)+pub struct MlpFusedQmmKernel {+ pipeline: MTLComputePipelineState,+}++impl MlpFusedQmmKernel {+ pub fn new(+ context: &MTLContext,+ data_type: DataType,+ group_size: usize,+ mode: QuantizationMode,+ quantization_type: QuantizationType,+ ) -> Result<Self, QuantizedMatmulError> {+ let type_suffix = dtype_suffix(data_type)+ .ok_or(QuantizedMatmulError::UnsupportedDataType(data_type))?;++ let bits = match mode {+ QuantizationMode::UInt4 => 4,+ QuantizationMode::UInt8 | QuantizationMode::Int8 => 8,+ };++ if bits != 4 {+ return Err(QuantizedMatmulError::UnsupportedBits(bits));+ }++ let kernel_name =+ format!("qmm_mlp_fused_{}_g{}_b{}", type_suffix, group_size, bits);++ let function_constants = metal::FunctionConstantValues::new();+ let use_mlx_quant = matches!(quantization_type, QuantizationType::Mlx);+ function_constants.set_constant_value_at_index(+ &use_mlx_quant as *const bool as *const std::ffi::c_void,+ metal::MTLDataType::Bool,+ 40,+ );++ let pipeline = context+ .compute_pipeline_state(&kernel_name, Some(&function_constants))+ .map_err(QuantizedMatmulError::MetalError)?;++ Ok(Self {+ pipeline,+ })+ }++ pub fn encode(+ &self,+ encoder: &ComputeCommandEncoderRef,+ args: &MlpFusedQmmArguments,+ ) -> Result<(), QuantizedMatmulError> {+ encoder.set_compute_pipeline_state(&self.pipeline);++ encoder.set_buffer(0, Some(args.weights), 0);+ encoder.set_buffer(1, Some(args.scales), 0);+ encoder.set_buffer(2, Some(args.zero_points_or_biases), 0);+ encoder.set_buffer(3, Some(args.input), args.input_offset);+ encoder.set_buffer(4, Some(args.output), 0);++ encoder.set_bytes(+ 5,+ size_of::<i32>() as u64,+ &args.input_dim as *const i32 as *const _,+ );+ encoder.set_bytes(+ 6,+ size_of::<i32>() as u64,+ &args.hidden_dim as *const i32 as *const _,+ );+ encoder.set_bytes(+ 7,+ size_of::<i32>() as u64,+ &args.batch as *const i32 as *const _,+ );++ // Dispatch: BM=32, BN=32+ let bm = 32;+ let bn = 32;+ let tiles_m = ((args.batch + bm - 1) / bm) as u64;+ let tiles_n = ((args.hidden_dim + bn - 1) / bn) as u64;+ let threadgroups = MTLSize::new(tiles_n, tiles_m, 1);+ let threads_per_threadgroup = MTLSize::new(32, 2, 2); // WM=2, WN=2, 32 threads++ encoder.dispatch_thread_groups(threadgroups, threads_per_threadgroup);+ Ok(())+ }+}diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metalindex a79e2a67a..58872d428 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal@@ -4,6 +4,7 @@using namespace metal;#include "mma.h"+#include "../common/mlp_epilogue.h"// Function constant: true = MLX-style (pre-computed biases), false = AWQ-style// (zero-points)@@ -4137,3 +4138,634 @@ qmm_transposed<bfloat, 128, 4, true, 128, 32, 64>(uint simd_gid [[simdgroup_index_in_threadgroup]],uint simd_lid [[thread_index_in_simdgroup]]);++///////////////////////////////////////////////////////////////////////////////+/// MLP Fused QMV - Quantized Matrix-Vector with Fused Activation+///////////////////////////////////////////////////////////////////////////////++// MLP fused QMV kernel for decode path (M=1).+// Computes paired up and gate projections, then applies: out = up * activation(gate)+// Weight layout: [up_weights (hidden_dim rows), gate_weights (hidden_dim rows)]+// Output size is hidden_dim.+template <typename T, int group_size, int bits>+void qmv_mlp_fused_impl(+ const device uint32_t* w,+ const device T* scales,+ const device uint8_t* zero_points,+ const device T* biases,+ const device T* x,+ device T* y,+ const constant int& in_vec_size,+ const constant int& hidden_dim, // Output size (half of weight rows)+ uint3 tid [[threadgroup_position_in_grid]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]+) {+ constexpr int packs_per_thread = bits == 2 ? 1 : 2;+ constexpr int num_simdgroups = 2;+ constexpr int results_per_simdgroup = 4; // Compute 4 paired rows at a time+ constexpr int pack_factor = get_pack_factor<bits, 32>();+ constexpr int bytes_per_pack = get_bytes_per_pack<bits, 32>();+ constexpr int values_per_thread = pack_factor * packs_per_thread;+ constexpr int block_size = values_per_thread * SIMD_SIZE;+ constexpr int scale_step_per_thread = group_size / values_per_thread;+ const device uint8_t* ws = (const device uint8_t*)w;+ typedef float U;+ thread U x_thread[values_per_thread];+ thread U result_up[results_per_simdgroup] = {0};+ thread U result_gate[results_per_simdgroup] = {0};++ const int in_vec_size_w = in_vec_size * bytes_per_pack / pack_factor;+ const int in_vec_size_g = in_vec_size / group_size;++ // Output row for 'up' projection+ const int out_row_up = tid.y * (num_simdgroups * results_per_simdgroup) ++ simd_gid * results_per_simdgroup;++ // Corresponding 'gate' row is offset by hidden_dim+ const int out_row_gate = out_row_up + hidden_dim;++ // Check bounds - only process if up row is within hidden_dim+ if (out_row_up >= hidden_dim) return;++ // Pointer setup for up weights+ const device uint8_t* ws_up = ws + out_row_up * in_vec_size_w + simd_lid * packs_per_thread * bytes_per_pack;+ const device T* scales_up = scales + out_row_up * in_vec_size_g + simd_lid / scale_step_per_thread;++ // Pointer setup for gate weights+ const device uint8_t* ws_gate = ws + out_row_gate * in_vec_size_w + simd_lid * packs_per_thread * bytes_per_pack;+ const device T* scales_gate = scales + out_row_gate * in_vec_size_g + simd_lid / scale_step_per_thread;++ const device T* biases_up = nullptr;+ const device T* biases_gate = nullptr;+ const device uint8_t* zps_up = nullptr;+ const device uint8_t* zps_gate = nullptr;+ int zp_stride = 0;+ bool high_nibble = false;++ if (kUseMlxQuant) {+ biases_up = biases + out_row_up * in_vec_size_g + simd_lid / scale_step_per_thread;+ biases_gate = biases + out_row_gate * in_vec_size_g + simd_lid / scale_step_per_thread;+ } else {+ if (bits == 4) {+ zp_stride = (in_vec_size_g + 1) / 2;+ zps_up = zero_points + out_row_up * zp_stride;+ zps_gate = zero_points + out_row_gate * zp_stride;+ int g_offset = simd_lid / scale_step_per_thread;+ zps_up += g_offset / 2;+ zps_gate += g_offset / 2;+ high_nibble = (g_offset & 1);+ } else {+ zp_stride = in_vec_size_g;+ zps_up = zero_points + out_row_up * zp_stride + simd_lid / scale_step_per_thread;+ zps_gate = zero_points + out_row_gate * zp_stride + simd_lid / scale_step_per_thread;+ }+ }++ x += tid.x * in_vec_size + simd_lid * values_per_thread;+ y += tid.x * hidden_dim + out_row_up;++ for (int k = 0; k < in_vec_size; k += block_size) {+ U sum = load_vector<T, U, values_per_thread, bits>(x, x_thread);++ // Compute both up and gate dot products for 4 rows each+ {+ auto wl_up0 = ws_up;+ auto wl_up1 = ws_up + in_vec_size_w;+ auto wl_up2 = ws_up + 2 * in_vec_size_w;+ auto wl_up3 = ws_up + 3 * in_vec_size_w;++ auto wl_gate0 = ws_gate;+ auto wl_gate1 = ws_gate + in_vec_size_w;+ auto wl_gate2 = ws_gate + 2 * in_vec_size_w;+ auto wl_gate3 = ws_gate + 3 * in_vec_size_w;++ U s_up0 = static_cast<U>(scales_up[0]);+ U s_up1 = static_cast<U>(scales_up[in_vec_size_g]);+ U s_up2 = static_cast<U>(scales_up[2 * in_vec_size_g]);+ U s_up3 = static_cast<U>(scales_up[3 * in_vec_size_g]);++ U s_gate0 = static_cast<U>(scales_gate[0]);+ U s_gate1 = static_cast<U>(scales_gate[in_vec_size_g]);+ U s_gate2 = static_cast<U>(scales_gate[2 * in_vec_size_g]);+ U s_gate3 = static_cast<U>(scales_gate[3 * in_vec_size_g]);++ if (kUseMlxQuant) {+ U b_up0 = static_cast<U>(biases_up[0]);+ U b_up1 = static_cast<U>(biases_up[in_vec_size_g]);+ U b_up2 = static_cast<U>(biases_up[2 * in_vec_size_g]);+ U b_up3 = static_cast<U>(biases_up[3 * in_vec_size_g]);++ U b_gate0 = static_cast<U>(biases_gate[0]);+ U b_gate1 = static_cast<U>(biases_gate[in_vec_size_g]);+ U b_gate2 = static_cast<U>(biases_gate[2 * in_vec_size_g]);+ U b_gate3 = static_cast<U>(biases_gate[3 * in_vec_size_g]);++ result_up[0] += qdot<U, values_per_thread, bits>(wl_up0, x_thread, s_up0, b_up0, sum);+ result_up[1] += qdot<U, values_per_thread, bits>(wl_up1, x_thread, s_up1, b_up1, sum);+ result_up[2] += qdot<U, values_per_thread, bits>(wl_up2, x_thread, s_up2, b_up2, sum);+ result_up[3] += qdot<U, values_per_thread, bits>(wl_up3, x_thread, s_up3, b_up3, sum);++ result_gate[0] += qdot<U, values_per_thread, bits>(wl_gate0, x_thread, s_gate0, b_gate0, sum);+ result_gate[1] += qdot<U, values_per_thread, bits>(wl_gate1, x_thread, s_gate1, b_gate1, sum);+ result_gate[2] += qdot<U, values_per_thread, bits>(wl_gate2, x_thread, s_gate2, b_gate2, sum);+ result_gate[3] += qdot<U, values_per_thread, bits>(wl_gate3, x_thread, s_gate3, b_gate3, sum);+ } else {+ auto extract_zp = [&](const device uint8_t* zps, int row_offset) -> U {+ uint8_t zp_byte = zps[row_offset * zp_stride];+ if (bits == 4) {+ return static_cast<U>(high_nibble ? (zp_byte >> 4) : (zp_byte & 0x0F));+ } else {+ return static_cast<U>(zp_byte);+ }+ };++ U zp_up0 = extract_zp(zps_up, 0);+ U zp_up1 = extract_zp(zps_up, 1);+ U zp_up2 = extract_zp(zps_up, 2);+ U zp_up3 = extract_zp(zps_up, 3);++ U zp_gate0 = extract_zp(zps_gate, 0);+ U zp_gate1 = extract_zp(zps_gate, 1);+ U zp_gate2 = extract_zp(zps_gate, 2);+ U zp_gate3 = extract_zp(zps_gate, 3);++ result_up[0] += qdot_zero_point<U, values_per_thread, bits>(wl_up0, x_thread, s_up0, zp_up0);+ result_up[1] += qdot_zero_point<U, values_per_thread, bits>(wl_up1, x_thread, s_up1, zp_up1);+ result_up[2] += qdot_zero_point<U, values_per_thread, bits>(wl_up2, x_thread, s_up2, zp_up2);+ result_up[3] += qdot_zero_point<U, values_per_thread, bits>(wl_up3, x_thread, s_up3, zp_up3);++ result_gate[0] += qdot_zero_point<U, values_per_thread, bits>(wl_gate0, x_thread, s_gate0, zp_gate0);+ result_gate[1] += qdot_zero_point<U, values_per_thread, bits>(wl_gate1, x_thread, s_gate1, zp_gate1);+ result_gate[2] += qdot_zero_point<U, values_per_thread, bits>(wl_gate2, x_thread, s_gate2, zp_gate2);+ result_gate[3] += qdot_zero_point<U, values_per_thread, bits>(wl_gate3, x_thread, s_gate3, zp_gate3);+ }+ }++ ws_up += block_size * bytes_per_pack / pack_factor;+ ws_gate += block_size * bytes_per_pack / pack_factor;+ scales_up += block_size / group_size;+ scales_gate += block_size / group_size;+ if (kUseMlxQuant) {+ biases_up += block_size / group_size;+ biases_gate += block_size / group_size;+ } else {+ if (bits == 4) {+ zps_up += (block_size / group_size) / 2;+ zps_gate += (block_size / group_size) / 2;+ } else {+ zps_up += block_size / group_size;+ zps_gate += block_size / group_size;+ }+ }+ x += block_size;+ }++ // SIMD reduction and fused activation+ for (int row = 0; row < results_per_simdgroup; row++) {+ result_up[row] = simd_sum(result_up[row]);+ result_gate[row] = simd_sum(result_gate[row]);+ if (simd_lid == 0) {+ // Apply MLP fused epilogue: up * activation(gate)+ float fused = mlp_fused_epilogue_f32(result_up[row], result_gate[row]);+ y[row] = static_cast<T>(fused);+ }+ }+}++template <typename T, int group_size, int bits>+[[kernel]] void qmv_mlp_fused(+ const device uint32_t* w [[buffer(0)]],+ const device T* scales [[buffer(1)]],+ const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],+ const device T* biases [[buffer(2), function_constant(kUseMlxQuant)]],+ const device T* x [[buffer(3)]],+ device T* y [[buffer(4)]],+ const constant int& K [[buffer(5)]],+ const constant int& hidden_dim [[buffer(6)]],+ uint3 tid [[threadgroup_position_in_grid]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]+) {+ qmv_mlp_fused_impl<T, group_size, bits>(+ w,+ scales,+ zero_points,+ biases,+ x,+ y,+ K,+ hidden_dim,+ tid,+ simd_gid,+ simd_lid+ );+}++// Instantiate MLP fused QMV kernels for common configurations+template [[host_name("qmv_mlp_fused_f16_g32_b4")]] [[kernel]] void qmv_mlp_fused<+ half, 32, 4>(+ const device uint32_t* w [[buffer(0)]],+ const device half* scales [[buffer(1)]],+ const device uint8_t* zero_points [[buffer(2)]],+ const device half* biases [[buffer(2)]],+ const device half* x [[buffer(3)]],+ device half* y [[buffer(4)]],+ const constant int& K [[buffer(5)]],+ const constant int& hidden_dim [[buffer(6)]],+ uint3 tid [[threadgroup_position_in_grid]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]+);++template [[host_name("qmv_mlp_fused_f16_g64_b4")]] [[kernel]] void qmv_mlp_fused<+ half, 64, 4>(+ const device uint32_t* w [[buffer(0)]],+ const device half* scales [[buffer(1)]],+ const device uint8_t* zero_points [[buffer(2)]],+ const device half* biases [[buffer(2)]],+ const device half* x [[buffer(3)]],+ device half* y [[buffer(4)]],+ const constant int& K [[buffer(5)]],+ const constant int& hidden_dim [[buffer(6)]],+ uint3 tid [[threadgroup_position_in_grid]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]+);++template [[host_name("qmv_mlp_fused_f16_g128_b4")]] [[kernel]] void qmv_mlp_fused<+ half, 128, 4>(+ const device uint32_t* w [[buffer(0)]],+ const device half* scales [[buffer(1)]],+ const device uint8_t* zero_points [[buffer(2)]],+ const device half* biases [[buffer(2)]],+ const device half* x [[buffer(3)]],+ device half* y [[buffer(4)]],+ const constant int& K [[buffer(5)]],+ const constant int& hidden_dim [[buffer(6)]],+ uint3 tid [[threadgroup_position_in_grid]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]+);++template [[host_name("qmv_mlp_fused_bf16_g32_b4")]] [[kernel]] void qmv_mlp_fused<+ bfloat, 32, 4>(+ const device uint32_t* w [[buffer(0)]],+ const device bfloat* scales [[buffer(1)]],+ const device uint8_t* zero_points [[buffer(2)]],+ const device bfloat* biases [[buffer(2)]],+ const device bfloat* x [[buffer(3)]],+ device bfloat* y [[buffer(4)]],+ const constant int& K [[buffer(5)]],+ const constant int& hidden_dim [[buffer(6)]],+ uint3 tid [[threadgroup_position_in_grid]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]+);++template [[host_name("qmv_mlp_fused_bf16_g64_b4")]] [[kernel]] void qmv_mlp_fused<+ bfloat, 64, 4>(+ const device uint32_t* w [[buffer(0)]],+ const device bfloat* scales [[buffer(1)]],+ const device uint8_t* zero_points [[buffer(2)]],+ const device bfloat* biases [[buffer(2)]],+ const device bfloat* x [[buffer(3)]],+ device bfloat* y [[buffer(4)]],+ const constant int& K [[buffer(5)]],+ const constant int& hidden_dim [[buffer(6)]],+ uint3 tid [[threadgroup_position_in_grid]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]+);++template [[host_name("qmv_mlp_fused_bf16_g128_b4")]] [[kernel]] void qmv_mlp_fused<+ bfloat, 128, 4>(+ const device uint32_t* w [[buffer(0)]],+ const device bfloat* scales [[buffer(1)]],+ const device uint8_t* zero_points [[buffer(2)]],+ const device bfloat* biases [[buffer(2)]],+ const device bfloat* x [[buffer(3)]],+ device bfloat* y [[buffer(4)]],+ const constant int& K [[buffer(5)]],+ const constant int& hidden_dim [[buffer(6)]],+ uint3 tid [[threadgroup_position_in_grid]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]+);++///////////////////////////////////////////////////////////////////////////////+/// MLP Fused QMM - Quantized Matrix-Matrix with Fused Activation+///////////////////////////////////////////////////////////////////////////////++// MLP fused QMM kernel for prefill path (M>1).+// Computes paired up and gate projections, then applies: out = up * activation(gate)+// Weight layout: [up_weights (hidden_dim cols), gate_weights (hidden_dim cols)]+// Output size is [M, hidden_dim].+template <typename T, int group_size, int bits, bool aligned_K, int BM = 32, int BK = 32, int BN = 32>+void qmm_mlp_fused_impl(+ const device uint32_t* w,+ const device T* scales,+ const device uint8_t* zero_points,+ const device T* biases,+ const device T* x,+ device T* y,+ threadgroup T* Xs,+ threadgroup T* Ws_up,+ const constant int& K,+ const constant int& hidden_dim, // Output columns (half of total weight columns)+ const constant int& M,+ uint3 tid [[threadgroup_position_in_grid]],+ uint lid [[thread_index_in_threadgroup]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]+) {+ static_assert(BK >= 32, "BK should be larger than SIMD_SIZE");+ static_assert(BK % 32 == 0, "BK should be divisible by SIMD_SIZE");++ (void)lid;++ constexpr int WM = 2;+ constexpr int WN = 2;+ constexpr int pack_factor = get_pack_factor<bits, 8>();+ constexpr int bytes_per_pack = get_bytes_per_pack<bits>();++ constexpr int BK_padded = (BK + 16 / sizeof(T));+ constexpr int BN_padded = (BN + 16 / sizeof(T));++ using mma_t = matmul_utils::BlockMMA<T, T, BM, BN, BK, WM, WN, false, false, BK_padded, BN_padded>;+ using loader_x_t = matmul_utils::BlockLoader<T, BM, BK, BK_padded, 1, WM * WN * 32, 1, 4>;++ auto wl_up = (const device uint8_t*)w;++ const int y_row = tid.y * BM;+ const int y_col = tid.x * BN; // Column in output (within hidden_dim)++ // Only process tiles within hidden_dim+ if (y_col >= hidden_dim) return;++ x += y_row * static_cast<int64_t>(K);++ // Up weights at column y_col+ wl_up += y_col * bytes_per_pack / pack_factor;+ // Gate weights at column y_col + hidden_dim+ auto wl_gate = wl_up + hidden_dim * bytes_per_pack / pack_factor;++ y += y_row * static_cast<int64_t>(hidden_dim) + y_col;++ const device T* scales_base = scales;+ const device T* biases_base = biases;++ const short num_els = min(BM, M - y_row);+ const short num_outs = min(BN, hidden_dim - y_col);++ // Threadgroup memory for gate weights (after Ws_up)+ threadgroup T* Ws_gate = Ws_up + BK * BN_padded;++ loader_x_t loader_x(x, K, Xs, simd_gid, simd_lid);+ mma_t mma_up(simd_gid, simd_lid);+ mma_t mma_gate(simd_gid, simd_lid);++ const int k_iterations = (K + BK - 1) / BK;+ const int full_iterations = aligned_K ? k_iterations : k_iterations - 1;++ // Total output width for scale/bias strides+ const int N_total = hidden_dim * 2;++ if (kUseMlxQuant) {+ using loader_w_t = QuantizedBlockLoaderMlx<T, BK, BN, BN_padded, 0, WM * WN * 32, group_size, bits>;++ const device T* scales_up = scales_base + (y_col / group_size);+ const device T* biases_up = biases_base + (y_col / group_size);+ const device T* scales_gate = scales_base + ((y_col + hidden_dim) / group_size);+ const device T* biases_gate = biases_base + ((y_col + hidden_dim) / group_size);++ loader_w_t loader_w_up(wl_up, scales_up, biases_up, N_total, Ws_up, simd_gid, simd_lid);+ loader_w_t loader_w_gate(wl_gate, scales_gate, biases_gate, N_total, Ws_gate, simd_gid, simd_lid);++ for (int k = 0; k < full_iterations; k++) {+ threadgroup_barrier(mem_flags::mem_threadgroup);+ loader_x.load_unsafe();+ loader_w_up.load_unsafe();+ loader_w_gate.load_unsafe();+ threadgroup_barrier(mem_flags::mem_threadgroup);++ mma_up.mma(Xs, Ws_up);+ mma_gate.mma(Xs, Ws_gate);++ loader_x.next();+ loader_w_up.next();+ loader_w_gate.next();+ }++ if (!aligned_K) {+ threadgroup_barrier(mem_flags::mem_threadgroup);+ loader_x.load_safe(short2(BM, K - full_iterations * BK));+ loader_w_up.load_safe(short2(K - full_iterations * BK, num_outs));+ loader_w_gate.load_safe(short2(K - full_iterations * BK, num_outs));+ threadgroup_barrier(mem_flags::mem_threadgroup);++ mma_up.mma(Xs, Ws_up);+ mma_gate.mma(Xs, Ws_gate);+ }+ } else {+ const int out_groups_total = (N_total + group_size - 1) / group_size;+ const int groups_per_row = out_groups_total;+ const int out_group_up = y_col / group_size;+ const int out_group_gate = (y_col + hidden_dim) / group_size;+ const int zp_stride_out = (bits == 4) ? ((out_groups_total + 1) / 2) : out_groups_total;++ using loader_w_t = QuantizedBlockLoaderZp<T, BK, BN, BN_padded, 0, WM * WN * 32, group_size, bits, true>;++ loader_w_t loader_w_up(+ wl_up, scales_base, zero_points,+ N_total, groups_per_row, Ws_up,+ simd_gid, simd_lid,+ out_group_up, out_groups_total, zp_stride_out);++ loader_w_t loader_w_gate(+ wl_gate, scales_base, zero_points,+ N_total, groups_per_row, Ws_gate,+ simd_gid, simd_lid,+ out_group_gate, out_groups_total, zp_stride_out);++ for (int k = 0; k < full_iterations; k++) {+ threadgroup_barrier(mem_flags::mem_threadgroup);+ loader_x.load_unsafe();+ loader_w_up.load_unsafe();+ loader_w_gate.load_unsafe();+ threadgroup_barrier(mem_flags::mem_threadgroup);++ mma_up.mma(Xs, Ws_up);+ mma_gate.mma(Xs, Ws_gate);++ loader_x.next();+ loader_w_up.next();+ loader_w_gate.next();+ }++ if (!aligned_K) {+ threadgroup_barrier(mem_flags::mem_threadgroup);+ loader_x.load_safe(short2(BM, K - full_iterations * BK));+ loader_w_up.load_safe(short2(K - full_iterations * BK, num_outs));+ loader_w_gate.load_safe(short2(K - full_iterations * BK, num_outs));+ threadgroup_barrier(mem_flags::mem_threadgroup);++ mma_up.mma(Xs, Ws_up);+ mma_gate.mma(Xs, Ws_gate);+ }+ }++ // Apply MLP fused epilogue: out = up * activation(gate)+ // Access Ctile elements and apply fusion+ constexpr int kElemsPerTile = decltype(mma_up.Ctile)::kElemsPerTile;+ #pragma unroll+ for (short i = 0; i < kElemsPerTile; i++) {+ float up_val = static_cast<float>(mma_up.Ctile.elems()[i]);+ float gate_val = static_cast<float>(mma_gate.Ctile.elems()[i]);+ float fused = mlp_fused_epilogue_f32(up_val, gate_val);+ mma_up.Ctile.elems()[i] = static_cast<T>(fused);+ }++ // Store result+ mma_up.store_result_safe(y, hidden_dim, short2(num_outs, num_els));+}++template <typename T, int group_size, int bits>+[[kernel]] void qmm_mlp_fused(+ const device uint32_t* w [[buffer(0)]],+ const device T* scales [[buffer(1)]],+ const device uint8_t* zero_points [[buffer(2), function_constant(kUseZeroPoints)]],+ const device T* biases [[buffer(2), function_constant(kUseMlxQuant)]],+ const device T* x [[buffer(3)]],+ device T* y [[buffer(4)]],+ const constant int& K [[buffer(5)]],+ const constant int& hidden_dim [[buffer(6)]],+ const constant int& M [[buffer(7)]],+ uint3 tid [[threadgroup_position_in_grid]],+ uint lid [[thread_index_in_threadgroup]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]+) {+ constexpr int BM = 32;+ constexpr int BK = 32;+ constexpr int BN = 32;+ constexpr int BK_padded = (BK + 16 / sizeof(T));+ constexpr int BN_padded = (BN + 16 / sizeof(T));++ threadgroup T Xs[BM * BK_padded];+ threadgroup T Ws[2 * BK * BN_padded]; // Double for up and gate++ bool aligned_K = (K % BK) == 0;+ if (aligned_K) {+ qmm_mlp_fused_impl<T, group_size, bits, true, BM, BK, BN>(+ w, scales, zero_points, biases, x, y, Xs, Ws,+ K, hidden_dim, M, tid, lid, simd_gid, simd_lid);+ } else {+ qmm_mlp_fused_impl<T, group_size, bits, false, BM, BK, BN>(+ w, scales, zero_points, biases, x, y, Xs, Ws,+ K, hidden_dim, M, tid, lid, simd_gid, simd_lid);+ }+}++// Instantiate MLP fused QMM kernels+template [[host_name("qmm_mlp_fused_f16_g32_b4")]] [[kernel]] void qmm_mlp_fused<+ half, 32, 4>(+ const device uint32_t* w [[buffer(0)]],+ const device half* scales [[buffer(1)]],+ const device uint8_t* zero_points [[buffer(2)]],+ const device half* biases [[buffer(2)]],+ const device half* x [[buffer(3)]],+ device half* y [[buffer(4)]],+ const constant int& K [[buffer(5)]],+ const constant int& hidden_dim [[buffer(6)]],+ const constant int& M [[buffer(7)]],+ uint3 tid [[threadgroup_position_in_grid]],+ uint lid [[thread_index_in_threadgroup]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]+);++template [[host_name("qmm_mlp_fused_f16_g64_b4")]] [[kernel]] void qmm_mlp_fused<+ half, 64, 4>(+ const device uint32_t* w [[buffer(0)]],+ const device half* scales [[buffer(1)]],+ const device uint8_t* zero_points [[buffer(2)]],+ const device half* biases [[buffer(2)]],+ const device half* x [[buffer(3)]],+ device half* y [[buffer(4)]],+ const constant int& K [[buffer(5)]],+ const constant int& hidden_dim [[buffer(6)]],+ const constant int& M [[buffer(7)]],+ uint3 tid [[threadgroup_position_in_grid]],+ uint lid [[thread_index_in_threadgroup]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]+);++template [[host_name("qmm_mlp_fused_f16_g128_b4")]] [[kernel]] void qmm_mlp_fused<+ half, 128, 4>(+ const device uint32_t* w [[buffer(0)]],+ const device half* scales [[buffer(1)]],+ const device uint8_t* zero_points [[buffer(2)]],+ const device half* biases [[buffer(2)]],+ const device half* x [[buffer(3)]],+ device half* y [[buffer(4)]],+ const constant int& K [[buffer(5)]],+ const constant int& hidden_dim [[buffer(6)]],+ const constant int& M [[buffer(7)]],+ uint3 tid [[threadgroup_position_in_grid]],+ uint lid [[thread_index_in_threadgroup]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]+);++template [[host_name("qmm_mlp_fused_bf16_g32_b4")]] [[kernel]] void qmm_mlp_fused<+ bfloat, 32, 4>(+ const device uint32_t* w [[buffer(0)]],+ const device bfloat* scales [[buffer(1)]],+ const device uint8_t* zero_points [[buffer(2)]],+ const device bfloat* biases [[buffer(2)]],+ const device bfloat* x [[buffer(3)]],+ device bfloat* y [[buffer(4)]],+ const constant int& K [[buffer(5)]],+ const constant int& hidden_dim [[buffer(6)]],+ const constant int& M [[buffer(7)]],+ uint3 tid [[threadgroup_position_in_grid]],+ uint lid [[thread_index_in_threadgroup]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]+);++template [[host_name("qmm_mlp_fused_bf16_g64_b4")]] [[kernel]] void qmm_mlp_fused<+ bfloat, 64, 4>(+ const device uint32_t* w [[buffer(0)]],+ const device bfloat* scales [[buffer(1)]],+ const device uint8_t* zero_points [[buffer(2)]],+ const device bfloat* biases [[buffer(2)]],+ const device bfloat* x [[buffer(3)]],+ device bfloat* y [[buffer(4)]],+ const constant int& K [[buffer(5)]],+ const constant int& hidden_dim [[buffer(6)]],+ const constant int& M [[buffer(7)]],+ uint3 tid [[threadgroup_position_in_grid]],+ uint lid [[thread_index_in_threadgroup]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]+);++template [[host_name("qmm_mlp_fused_bf16_g128_b4")]] [[kernel]] void qmm_mlp_fused<+ bfloat, 128, 4>(+ const device uint32_t* w [[buffer(0)]],+ const device bfloat* scales [[buffer(1)]],+ const device uint8_t* zero_points [[buffer(2)]],+ const device bfloat* biases [[buffer(2)]],+ const device bfloat* x [[buffer(3)]],+ device bfloat* y [[buffer(4)]],+ const constant int& K [[buffer(5)]],+ const constant int& hidden_dim [[buffer(6)]],+ const constant int& M [[buffer(7)]],+ uint3 tid [[threadgroup_position_in_grid]],+ uint lid [[thread_index_in_threadgroup]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]+);diff --git a/crates/uzu/tests/mlp_fused_correctness_test.rs b/crates/uzu/tests/mlp_fused_correctness_test.rsnew file mode 100644index 000000000..3e2d97782--- /dev/null+++ b/crates/uzu/tests/mlp_fused_correctness_test.rs@@ -0,0 +1,453 @@+//! Correctness tests for MLP fused kernels+//! Compares fused (matmul + activation) output against separate operations++use half::f16;+use metal::{Device, MTLResourceOptions};+use objc2::rc::autoreleasepool;+use uzu::{+ DataType,+ backends::metal::{+ MTLContext,+ kernel::{+ matmul::{+ MlpFusedGemmArguments, MlpFusedGemmKernel,+ MlpFusedGemvArguments, MlpFusedGemvKernel,+ },+ mlp::MlpActivationType,+ },+ },+};++fn create_test_context() -> Option<MTLContext> {+ let device = Device::system_default()?;+ let command_queue = device.new_command_queue();+ MTLContext::new(device, command_queue).ok()+}++/// CPU reference implementation for SiLU activation+fn silu(x: f32) -> f32 {+ x / (1.0 + (-x).exp())+}++/// CPU reference implementation for GELU activation+fn gelu(x: f32) -> f32 {+ let k0 = 0.044715f32;+ let k1 = 0.7978845608f32;+ let t = k1 * (x + k0 * x * x * x);+ 0.5 * x * (1.0 + t.tanh())+}++/// CPU reference for MLP fused: up * activation(gate)+fn mlp_fused_reference(+ input: &[f32],+ weights_up: &[f32],+ weights_gate: &[f32],+ m: usize,+ k: usize,+ hidden_dim: usize,+ activation: MlpActivationType,+) -> Vec<f32> {+ let mut result = vec![0.0f32; m * hidden_dim];++ for row in 0..m {+ for col in 0..hidden_dim {+ // Compute up and gate dot products+ let mut up_sum = 0.0f32;+ let mut gate_sum = 0.0f32;+ for i in 0..k {+ let input_val = input[row * k + i];+ up_sum += input_val * weights_up[col * k + i];+ gate_sum += input_val * weights_gate[col * k + i];+ }++ // Apply fused activation+ let activated_gate = match activation {+ MlpActivationType::SiLU => silu(gate_sum),+ MlpActivationType::Gelu => gelu(gate_sum),+ };+ result[row * hidden_dim + col] = up_sum * activated_gate;+ }+ }++ result+}++/// Run MLP fused GEMV kernel (decode path, M=1)+fn run_fused_gemv(+ ctx: &MTLContext,+ input: &[f16],+ weights: &[f16], // [2*hidden_dim, k] - up then gate weights concatenated+ k: usize,+ hidden_dim: usize,+ activation: MlpActivationType,+) -> Vec<f16> {+ let input_buf = ctx.device.new_buffer_with_data(+ input.as_ptr() as *const _,+ (input.len() * core::mem::size_of::<f16>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );+ let weights_buf = ctx.device.new_buffer_with_data(+ weights.as_ptr() as *const _,+ (weights.len() * core::mem::size_of::<f16>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );+ let output_buf = ctx.device.new_buffer(+ (hidden_dim * core::mem::size_of::<f16>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );++ let mut kernel = MlpFusedGemvKernel::new(DataType::F16).expect("kernel");++ let cb = ctx.command_queue.new_command_buffer().to_owned();+ let enc = cb.new_compute_command_encoder();++ let args = MlpFusedGemvArguments {+ weights: &weights_buf,+ input: &input_buf,+ input_offset: 0,+ output: &output_buf,+ input_dim: k as i32,+ hidden_dim: hidden_dim as i32,+ weights_ld: k as i32,+ batch_count: 1,+ activation,+ };++ kernel.encode(ctx, &enc, &args).expect("encode");+ enc.end_encoding();+ cb.commit();+ cb.wait_until_completed();++ unsafe {+ let ptr = output_buf.contents() as *const f16;+ std::slice::from_raw_parts(ptr, hidden_dim).to_vec()+ }+}++/// Run MLP fused GEMM kernel (prefill path, M>1)+fn run_fused_gemm(+ ctx: &MTLContext,+ input: &[f16],+ weights: &[f16], // [2*hidden_dim, k] - up then gate weights concatenated+ m: usize,+ k: usize,+ hidden_dim: usize,+ activation: MlpActivationType,+) -> Vec<f16> {+ let input_buf = ctx.device.new_buffer_with_data(+ input.as_ptr() as *const _,+ (input.len() * core::mem::size_of::<f16>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );+ let weights_buf = ctx.device.new_buffer_with_data(+ weights.as_ptr() as *const _,+ (weights.len() * core::mem::size_of::<f16>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );+ let output_buf = ctx.device.new_buffer(+ (m * hidden_dim * core::mem::size_of::<f16>()) as u64,+ MTLResourceOptions::StorageModeShared,+ );++ let mut kernel =+ MlpFusedGemmKernel::new(DataType::F16, true).expect("kernel");++ let cb = ctx.command_queue.new_command_buffer().to_owned();+ let enc = cb.new_compute_command_encoder();++ let args = MlpFusedGemmArguments {+ input: &input_buf,+ input_offset: 0,+ weights: &weights_buf,+ output: &output_buf,+ batch: m as i32,+ input_dim: k as i32,+ hidden_dim: hidden_dim as i32,+ lda: k as i32,+ ldb: k as i32,+ ldd: hidden_dim as i32,+ activation,+ };++ kernel.encode(ctx, &enc, &args).expect("encode");+ enc.end_encoding();+ cb.commit();+ cb.wait_until_completed();++ unsafe {+ let ptr = output_buf.contents() as *const f16;+ std::slice::from_raw_parts(ptr, m * hidden_dim).to_vec()+ }+}++fn compare_outputs(+ expected: &[f32],+ actual: &[f16],+ rtol: f32,+ atol: f32,+) -> (bool, f32) {+ let mut max_diff = 0.0f32;+ let mut all_close = true;++ for (i, (&exp, &act)) in expected.iter().zip(actual.iter()).enumerate() {+ let act_f32 = f32::from(act);+ let diff = (exp - act_f32).abs();+ let rel_diff = diff / (exp.abs() + 1e-8);++ if diff > atol && rel_diff > rtol {+ if all_close {+ eprintln!(+ "First mismatch at index {}: expected {}, got {} (diff={}, rel={})",+ i, exp, act_f32, diff, rel_diff+ );+ }+ all_close = false;+ }+ max_diff = max_diff.max(diff);+ }++ (all_close, max_diff)+}++#[test]+fn test_mlp_fused_gemv_silu_small() {+ autoreleasepool(|_| {+ let ctx = match create_test_context() {+ Some(c) => c,+ None => {+ eprintln!("No Metal device available, skipping test");+ return;+ },+ };++ let k = 64;+ let hidden_dim = 32;+ let m = 1;++ // Generate random-ish test data+ let input: Vec<f32> =+ (0..k).map(|i| (i as f32 * 0.1).sin() * 0.5).collect();+ let weights_up: Vec<f32> = (0..hidden_dim * k)+ .map(|i| (i as f32 * 0.017).cos() * 0.3)+ .collect();+ let weights_gate: Vec<f32> = (0..hidden_dim * k)+ .map(|i| (i as f32 * 0.023).sin() * 0.3)+ .collect();++ // Create concatenated weights [up; gate] row-major+ let mut weights_concat: Vec<f32> =+ Vec::with_capacity(2 * hidden_dim * k);+ weights_concat.extend(&weights_up);+ weights_concat.extend(&weights_gate);++ // CPU reference+ let expected = mlp_fused_reference(+ &input,+ &weights_up,+ &weights_gate,+ m,+ k,+ hidden_dim,+ MlpActivationType::SiLU,+ );++ // Convert to f16+ let input_f16: Vec<f16> =+ input.iter().map(|&x| f16::from_f32(x)).collect();+ let weights_f16: Vec<f16> =+ weights_concat.iter().map(|&x| f16::from_f32(x)).collect();++ // Run fused kernel+ let actual = run_fused_gemv(+ &ctx,+ &input_f16,+ &weights_f16,+ k,+ hidden_dim,+ MlpActivationType::SiLU,+ );++ let (close, max_diff) = compare_outputs(&expected, &actual, 0.01, 0.01);+ println!("GEMV SiLU small: max_diff = {}", max_diff);+ assert!(close, "GEMV SiLU small test failed, max_diff = {}", max_diff);+ });+}++#[test]+fn test_mlp_fused_gemv_gelu_small() {+ autoreleasepool(|_| {+ let ctx = match create_test_context() {+ Some(c) => c,+ None => {+ eprintln!("No Metal device available, skipping test");+ return;+ },+ };++ let k = 64;+ let hidden_dim = 32;+ let m = 1;++ let input: Vec<f32> =+ (0..k).map(|i| (i as f32 * 0.1).sin() * 0.5).collect();+ let weights_up: Vec<f32> = (0..hidden_dim * k)+ .map(|i| (i as f32 * 0.017).cos() * 0.3)+ .collect();+ let weights_gate: Vec<f32> = (0..hidden_dim * k)+ .map(|i| (i as f32 * 0.023).sin() * 0.3)+ .collect();++ let mut weights_concat: Vec<f32> =+ Vec::with_capacity(2 * hidden_dim * k);+ weights_concat.extend(&weights_up);+ weights_concat.extend(&weights_gate);++ let expected = mlp_fused_reference(+ &input,+ &weights_up,+ &weights_gate,+ m,+ k,+ hidden_dim,+ MlpActivationType::Gelu,+ );++ let input_f16: Vec<f16> =+ input.iter().map(|&x| f16::from_f32(x)).collect();+ let weights_f16: Vec<f16> =+ weights_concat.iter().map(|&x| f16::from_f32(x)).collect();++ let actual = run_fused_gemv(+ &ctx,+ &input_f16,+ &weights_f16,+ k,+ hidden_dim,+ MlpActivationType::Gelu,+ );++ let (close, max_diff) = compare_outputs(&expected, &actual, 0.01, 0.01);+ println!("GEMV GELU small: max_diff = {}", max_diff);+ assert!(close, "GEMV GELU small test failed, max_diff = {}", max_diff);+ });+}++#[test]+fn test_mlp_fused_gemm_silu_medium() {+ autoreleasepool(|_| {+ let ctx = match create_test_context() {+ Some(c) => c,+ None => {+ eprintln!("No Metal device available, skipping test");+ return;+ },+ };++ let m = 32;+ let k = 64;+ let hidden_dim = 64;++ let input: Vec<f32> =+ (0..m * k).map(|i| (i as f32 * 0.1).sin() * 0.5).collect();+ let weights_up: Vec<f32> = (0..hidden_dim * k)+ .map(|i| (i as f32 * 0.017).cos() * 0.3)+ .collect();+ let weights_gate: Vec<f32> = (0..hidden_dim * k)+ .map(|i| (i as f32 * 0.023).sin() * 0.3)+ .collect();++ let mut weights_concat: Vec<f32> =+ Vec::with_capacity(2 * hidden_dim * k);+ weights_concat.extend(&weights_up);+ weights_concat.extend(&weights_gate);++ let expected = mlp_fused_reference(+ &input,+ &weights_up,+ &weights_gate,+ m,+ k,+ hidden_dim,+ MlpActivationType::SiLU,+ );++ let input_f16: Vec<f16> =+ input.iter().map(|&x| f16::from_f32(x)).collect();+ let weights_f16: Vec<f16> =+ weights_concat.iter().map(|&x| f16::from_f32(x)).collect();++ let actual = run_fused_gemm(+ &ctx,+ &input_f16,+ &weights_f16,+ m,+ k,+ hidden_dim,+ MlpActivationType::SiLU,+ );++ let (close, max_diff) = compare_outputs(&expected, &actual, 0.05, 0.05);+ println!("GEMM SiLU medium: max_diff = {}", max_diff);+ assert!(close, "GEMM SiLU medium test failed, max_diff = {}", max_diff);+ });+}++#[test]+fn test_mlp_fused_gemv_realistic_dimensions() {+ autoreleasepool(|_| {+ let ctx = match create_test_context() {+ Some(c) => c,+ None => {+ eprintln!("No Metal device available, skipping test");+ return;+ },+ };++ // Realistic LLM dimensions (scaled down for test speed)+ let k = 512; // input dim+ let hidden_dim = 256; // hidden dim (half of up+gate)+ let m = 1;++ let input: Vec<f32> =+ (0..k).map(|i| (i as f32 * 0.01).sin() * 0.5).collect();+ let weights_up: Vec<f32> = (0..hidden_dim * k)+ .map(|i| (i as f32 * 0.001).cos() * 0.1)+ .collect();+ let weights_gate: Vec<f32> = (0..hidden_dim * k)+ .map(|i| (i as f32 * 0.002).sin() * 0.1)+ .collect();++ let mut weights_concat: Vec<f32> =+ Vec::with_capacity(2 * hidden_dim * k);+ weights_concat.extend(&weights_up);+ weights_concat.extend(&weights_gate);++ let expected = mlp_fused_reference(+ &input,+ &weights_up,+ &weights_gate,+ m,+ k,+ hidden_dim,+ MlpActivationType::SiLU,+ );++ let input_f16: Vec<f16> =+ input.iter().map(|&x| f16::from_f32(x)).collect();+ let weights_f16: Vec<f16> =+ weights_concat.iter().map(|&x| f16::from_f32(x)).collect();++ let actual = run_fused_gemv(+ &ctx,+ &input_f16,+ &weights_f16,+ k,+ hidden_dim,+ MlpActivationType::SiLU,+ );++ let (close, max_diff) = compare_outputs(&expected, &actual, 0.05, 0.02);+ println!("GEMV realistic: max_diff = {}", max_diff);+ assert!(close, "GEMV realistic test failed, max_diff = {}", max_diff);+ });+}