A high-performance inference engine for AI models
update bench
Eugene Bokhane66a79c53efd
diff --git a/crates/benchmarks/src/mlp_fused_bench.rs b/crates/benchmarks/src/mlp_fused_bench.rsindex d96ca425f..5759c592f 100644--- a/crates/benchmarks/src/mlp_fused_bench.rs+++ b/crates/benchmarks/src/mlp_fused_bench.rs@@ -26,313 +26,323 @@ fn create_test_context() -> Option<MTLContext> {MTLContext::new(device, command_queue).ok()}-struct BenchResult {- fused_time_us: f64,- unfused_time_us: f64,+struct BenchmarkResult {+ fused_time_microseconds: f64,+ unfused_time_microseconds: f64,speedup: f64,}-impl BenchResult {+impl BenchmarkResult {fn new(- fused: Duration,- unfused: Duration,+ fused_duration: Duration,+ unfused_duration: Duration,) -> Self {- let fused_us = fused.as_secs_f64() * 1_000_000.0;- let unfused_us = unfused.as_secs_f64() * 1_000_000.0;+ let fused_microseconds = fused_duration.as_secs_f64() * 1_000_000.0;+ let unfused_microseconds = unfused_duration.as_secs_f64() * 1_000_000.0;Self {- fused_time_us: fused_us,- unfused_time_us: unfused_us,- speedup: unfused_us / fused_us,+ fused_time_microseconds: fused_microseconds,+ unfused_time_microseconds: unfused_microseconds,+ speedup: unfused_microseconds / fused_microseconds,}}}-fn bench_gemv_fused(- ctx: &MTLContext,- k: usize,- hidden_dim: usize,- iterations: usize,- warmup: usize,-) -> BenchResult {+fn benchmark_gemv_fused(+ context: &MTLContext,+ input_dimension: usize,+ hidden_dimension: usize,+ iteration_count: usize,+ warmup_count: usize,+) -> BenchmarkResult {// Allocate buffers- let input_buf = ctx.device.new_buffer(- (k * std::mem::size_of::<f16>()) as u64,+ let input_buffer = context.device.new_buffer(+ (input_dimension * std::mem::size_of::<f16>()) as u64,MTLResourceOptions::StorageModeShared,);- let weights_buf = ctx.device.new_buffer(- (2 * hidden_dim * k * std::mem::size_of::<f16>()) as u64,+ let weights_buffer = context.device.new_buffer(+ (2 * hidden_dimension * input_dimension * std::mem::size_of::<f16>())+ as u64,MTLResourceOptions::StorageModeShared,);- let output_buf = ctx.device.new_buffer(- (hidden_dim * std::mem::size_of::<f16>()) as u64,+ let output_buffer = context.device.new_buffer(+ (hidden_dimension * std::mem::size_of::<f16>()) as u64,MTLResourceOptions::StorageModeShared,);- let fused_up_buf = ctx.device.new_buffer(- (2 * hidden_dim * std::mem::size_of::<f16>()) as u64,+ let fused_up_buffer = context.device.new_buffer(+ (2 * hidden_dimension * std::mem::size_of::<f16>()) as u64,MTLResourceOptions::StorageModeShared,);let mut fused_kernel =MlpFusedGemvKernel::new(DataType::F16).expect("fused kernel");let mut matmul_kernel =- MatmulKernel::new(ctx, DataType::F16, false, true).expect("matmul");+ MatmulKernel::new(context, DataType::F16, false, true).expect("matmul");// Warmup fused- for _ in 0..warmup {- let cb = ctx.command_queue.new_command_buffer().to_owned();- let enc = cb.new_compute_command_encoder();- let args = MlpFusedGemvArguments {- weights: &weights_buf,- input: &input_buf,+ for _ in 0..warmup_count {+ let command_buffer =+ context.command_queue.new_command_buffer().to_owned();+ let encoder = command_buffer.new_compute_command_encoder();+ let arguments = MlpFusedGemvArguments {+ weights: &weights_buffer,+ input: &input_buffer,input_offset: 0,- output: &output_buf,- input_dim: k as i32,- hidden_dim: hidden_dim as i32,- weights_ld: k as i32,+ output: &output_buffer,+ input_dim: input_dimension as i32,+ hidden_dim: hidden_dimension as i32,+ weights_ld: input_dimension as i32,batch_count: 1,activation: MlpActivationType::SiLU,};- fused_kernel.encode(ctx, &enc, &args).unwrap();- enc.end_encoding();- cb.commit();- cb.wait_until_completed();+ fused_kernel.encode(context, &encoder, &arguments).unwrap();+ encoder.end_encoding();+ command_buffer.commit();+ command_buffer.wait_until_completed();}// Benchmark fusedlet fused_start = Instant::now();- for _ in 0..iterations {- let cb = ctx.command_queue.new_command_buffer().to_owned();- let enc = cb.new_compute_command_encoder();- let args = MlpFusedGemvArguments {- weights: &weights_buf,- input: &input_buf,+ for _ in 0..iteration_count {+ let command_buffer =+ context.command_queue.new_command_buffer().to_owned();+ let encoder = command_buffer.new_compute_command_encoder();+ let arguments = MlpFusedGemvArguments {+ weights: &weights_buffer,+ input: &input_buffer,input_offset: 0,- output: &output_buf,- input_dim: k as i32,- hidden_dim: hidden_dim as i32,- weights_ld: k as i32,+ output: &output_buffer,+ input_dim: input_dimension as i32,+ hidden_dim: hidden_dimension as i32,+ weights_ld: input_dimension as i32,batch_count: 1,activation: MlpActivationType::SiLU,};- fused_kernel.encode(ctx, &enc, &args).unwrap();- enc.end_encoding();- cb.commit();- cb.wait_until_completed();+ fused_kernel.encode(context, &encoder, &arguments).unwrap();+ encoder.end_encoding();+ command_buffer.commit();+ command_buffer.wait_until_completed();}- let fused_time = fused_start.elapsed() / iterations as u32;+ let fused_duration = fused_start.elapsed() / iteration_count as u32;// Warmup unfused (matmul only - simplified comparison)- for _ in 0..warmup {- let cb = ctx.command_queue.new_command_buffer().to_owned();- let enc = cb.new_compute_command_encoder();+ for _ in 0..warmup_count {+ let command_buffer =+ context.command_queue.new_command_buffer().to_owned();+ let encoder = command_buffer.new_compute_command_encoder();matmul_kernel.encode(- ctx,- &enc,+ context,+ &encoder,MatmulArguments {- a: &input_buf,+ a: &input_buffer,a_offset: 0,- b: &weights_buf,+ b: &weights_buffer,c: None,- d: &fused_up_buf,+ d: &fused_up_buffer,batch: 1,- input_dim: k as i32,- output_dim: (2 * hidden_dim) as i32,- lda: k as i32,- ldb: k as i32,- ldd: (2 * hidden_dim) as i32,+ input_dim: input_dimension as i32,+ output_dim: (2 * hidden_dimension) as i32,+ lda: input_dimension as i32,+ ldb: input_dimension as i32,+ ldd: (2 * hidden_dimension) as i32,batch_count: 1,alpha: 1.0,beta: 0.0,},).unwrap();- enc.end_encoding();- cb.commit();- cb.wait_until_completed();+ encoder.end_encoding();+ command_buffer.commit();+ command_buffer.wait_until_completed();}// Benchmark unfused (matmul + separate activation would add more time)let unfused_start = Instant::now();- for _ in 0..iterations {- let cb = ctx.command_queue.new_command_buffer().to_owned();- let enc = cb.new_compute_command_encoder();+ for _ in 0..iteration_count {+ let command_buffer =+ context.command_queue.new_command_buffer().to_owned();+ let encoder = command_buffer.new_compute_command_encoder();matmul_kernel.encode(- ctx,- &enc,+ context,+ &encoder,MatmulArguments {- a: &input_buf,+ a: &input_buffer,a_offset: 0,- b: &weights_buf,+ b: &weights_buffer,c: None,- d: &fused_up_buf,+ d: &fused_up_buffer,batch: 1,- input_dim: k as i32,- output_dim: (2 * hidden_dim) as i32,- lda: k as i32,- ldb: k as i32,- ldd: (2 * hidden_dim) as i32,+ input_dim: input_dimension as i32,+ output_dim: (2 * hidden_dimension) as i32,+ lda: input_dimension as i32,+ ldb: input_dimension as i32,+ ldd: (2 * hidden_dimension) as i32,batch_count: 1,alpha: 1.0,beta: 0.0,},).unwrap();- enc.end_encoding();- cb.commit();- cb.wait_until_completed();+ encoder.end_encoding();+ command_buffer.commit();+ command_buffer.wait_until_completed();}- let unfused_time = unfused_start.elapsed() / iterations as u32;+ let unfused_duration = unfused_start.elapsed() / iteration_count as u32;- BenchResult::new(fused_time, unfused_time)+ BenchmarkResult::new(fused_duration, unfused_duration)}-fn bench_gemm_fused(- ctx: &MTLContext,- m: usize,- k: usize,- hidden_dim: usize,- iterations: usize,- warmup: usize,-) -> BenchResult {- let input_buf = ctx.device.new_buffer(- (m * k * std::mem::size_of::<f16>()) as u64,+fn benchmark_gemm_fused(+ context: &MTLContext,+ batch_size: usize,+ input_dimension: usize,+ hidden_dimension: usize,+ iteration_count: usize,+ warmup_count: usize,+) -> BenchmarkResult {+ let input_buffer = context.device.new_buffer(+ (batch_size * input_dimension * std::mem::size_of::<f16>()) as u64,MTLResourceOptions::StorageModeShared,);- let weights_buf = ctx.device.new_buffer(- (2 * hidden_dim * k * std::mem::size_of::<f16>()) as u64,+ let weights_buffer = context.device.new_buffer(+ (2 * hidden_dimension * input_dimension * std::mem::size_of::<f16>())+ as u64,MTLResourceOptions::StorageModeShared,);- let output_buf = ctx.device.new_buffer(- (m * hidden_dim * std::mem::size_of::<f16>()) as u64,+ let output_buffer = context.device.new_buffer(+ (batch_size * hidden_dimension * std::mem::size_of::<f16>()) as u64,MTLResourceOptions::StorageModeShared,);- let fused_up_buf = ctx.device.new_buffer(- (m * 2 * hidden_dim * std::mem::size_of::<f16>()) as u64,+ let fused_up_buffer = context.device.new_buffer(+ (batch_size * 2 * hidden_dimension * std::mem::size_of::<f16>()) as u64,MTLResourceOptions::StorageModeShared,);let mut fused_kernel =MlpFusedGemmKernel::new(DataType::F16, true).expect("fused kernel");let mut matmul_kernel =- MatmulKernel::new(ctx, DataType::F16, false, true).expect("matmul");+ MatmulKernel::new(context, DataType::F16, false, true).expect("matmul");// Warmup fused- for _ in 0..warmup {- let cb = ctx.command_queue.new_command_buffer().to_owned();- let enc = cb.new_compute_command_encoder();- let args = MlpFusedGemmArguments {- input: &input_buf,+ for _ in 0..warmup_count {+ let command_buffer =+ context.command_queue.new_command_buffer().to_owned();+ let encoder = command_buffer.new_compute_command_encoder();+ let arguments = MlpFusedGemmArguments {+ input: &input_buffer,input_offset: 0,- weights: &weights_buf,- output: &output_buf,- batch: m as i32,- input_dim: k as i32,- hidden_dim: hidden_dim as i32,- lda: k as i32,- ldb: k as i32,- ldd: hidden_dim as i32,+ weights: &weights_buffer,+ output: &output_buffer,+ batch: batch_size as i32,+ input_dim: input_dimension as i32,+ hidden_dim: hidden_dimension as i32,+ lda: input_dimension as i32,+ ldb: input_dimension as i32,+ ldd: hidden_dimension as i32,activation: MlpActivationType::SiLU,};- fused_kernel.encode(ctx, &enc, &args).unwrap();- enc.end_encoding();- cb.commit();- cb.wait_until_completed();+ fused_kernel.encode(context, &encoder, &arguments).unwrap();+ encoder.end_encoding();+ command_buffer.commit();+ command_buffer.wait_until_completed();}let fused_start = Instant::now();- for _ in 0..iterations {- let cb = ctx.command_queue.new_command_buffer().to_owned();- let enc = cb.new_compute_command_encoder();- let args = MlpFusedGemmArguments {- input: &input_buf,+ for _ in 0..iteration_count {+ let command_buffer =+ context.command_queue.new_command_buffer().to_owned();+ let encoder = command_buffer.new_compute_command_encoder();+ let arguments = MlpFusedGemmArguments {+ input: &input_buffer,input_offset: 0,- weights: &weights_buf,- output: &output_buf,- batch: m as i32,- input_dim: k as i32,- hidden_dim: hidden_dim as i32,- lda: k as i32,- ldb: k as i32,- ldd: hidden_dim as i32,+ weights: &weights_buffer,+ output: &output_buffer,+ batch: batch_size as i32,+ input_dim: input_dimension as i32,+ hidden_dim: hidden_dimension as i32,+ lda: input_dimension as i32,+ ldb: input_dimension as i32,+ ldd: hidden_dimension as i32,activation: MlpActivationType::SiLU,};- fused_kernel.encode(ctx, &enc, &args).unwrap();- enc.end_encoding();- cb.commit();- cb.wait_until_completed();+ fused_kernel.encode(context, &encoder, &arguments).unwrap();+ encoder.end_encoding();+ command_buffer.commit();+ command_buffer.wait_until_completed();}- let fused_time = fused_start.elapsed() / iterations as u32;+ let fused_duration = fused_start.elapsed() / iteration_count as u32;// Warmup unfused- for _ in 0..warmup {- let cb = ctx.command_queue.new_command_buffer().to_owned();- let enc = cb.new_compute_command_encoder();+ for _ in 0..warmup_count {+ let command_buffer =+ context.command_queue.new_command_buffer().to_owned();+ let encoder = command_buffer.new_compute_command_encoder();matmul_kernel.encode(- ctx,- &enc,+ context,+ &encoder,MatmulArguments {- a: &input_buf,+ a: &input_buffer,a_offset: 0,- b: &weights_buf,+ b: &weights_buffer,c: None,- d: &fused_up_buf,- batch: m as i32,- input_dim: k as i32,- output_dim: (2 * hidden_dim) as i32,- lda: k as i32,- ldb: k as i32,- ldd: (2 * hidden_dim) as i32,+ d: &fused_up_buffer,+ batch: batch_size as i32,+ input_dim: input_dimension as i32,+ output_dim: (2 * hidden_dimension) as i32,+ lda: input_dimension as i32,+ ldb: input_dimension as i32,+ ldd: (2 * hidden_dimension) as i32,batch_count: 1,alpha: 1.0,beta: 0.0,},).unwrap();- enc.end_encoding();- cb.commit();- cb.wait_until_completed();+ encoder.end_encoding();+ command_buffer.commit();+ command_buffer.wait_until_completed();}let unfused_start = Instant::now();- for _ in 0..iterations {- let cb = ctx.command_queue.new_command_buffer().to_owned();- let enc = cb.new_compute_command_encoder();+ for _ in 0..iteration_count {+ let command_buffer =+ context.command_queue.new_command_buffer().to_owned();+ let encoder = command_buffer.new_compute_command_encoder();matmul_kernel.encode(- ctx,- &enc,+ context,+ &encoder,MatmulArguments {- a: &input_buf,+ a: &input_buffer,a_offset: 0,- b: &weights_buf,+ b: &weights_buffer,c: None,- d: &fused_up_buf,- batch: m as i32,- input_dim: k as i32,- output_dim: (2 * hidden_dim) as i32,- lda: k as i32,- ldb: k as i32,- ldd: (2 * hidden_dim) as i32,+ d: &fused_up_buffer,+ batch: batch_size as i32,+ input_dim: input_dimension as i32,+ output_dim: (2 * hidden_dimension) as i32,+ lda: input_dimension as i32,+ ldb: input_dimension as i32,+ ldd: (2 * hidden_dimension) as i32,batch_count: 1,alpha: 1.0,beta: 0.0,},).unwrap();- enc.end_encoding();- cb.commit();- cb.wait_until_completed();+ encoder.end_encoding();+ command_buffer.commit();+ command_buffer.wait_until_completed();}- let unfused_time = unfused_start.elapsed() / iterations as u32;+ let unfused_duration = unfused_start.elapsed() / iteration_count as u32;- BenchResult::new(fused_time, unfused_time)+ BenchmarkResult::new(fused_duration, unfused_duration)}/// MLP shapes from real model configs-/// Format: (K=model_dim, H=hidden_dim)+/// Format: (input_dimension, hidden_dimension)const MLP_SHAPES: &[(usize, usize)] = &[(896, 4864), // Qwen2.5-Coder-0.5B(1024, 3072), // Qwen3-0.6B@@ -358,79 +368,65 @@ const MLP_SHAPES: &[(usize, usize)] = &[(6144, 16384), // Codestral-22B];+/// Batch sizes to test: decode (1) and prefill sizes+const BATCH_SIZES: &[usize] = &[1, 8, 16, 32, 64, 128, 256, 512];+pub fn run_mlp_fused_benchmark() {autoreleasepool(|_| {- let ctx = match create_test_context() {- Some(c) => c,+ let context = match create_test_context() {+ Some(context) => context,None => {eprintln!("No Metal device available");return;},};- println!("\n=== MLP Fused Kernel Benchmark (Real Model Shapes) ===\n");-- let iterations = 100;- let warmup = 10;+ let iteration_count = 100;+ let warmup_count = 10;- // GEMV benchmark (decode path, M=1)- println!("=== GEMV (Decode, M=1) ===");println!(- "{:<25} {:>12} {:>12} {:>10}",- "Shape (K, H)", "Fused (µs)", "Unfused (µs)", "Speedup"+ "{:>12} {:>12} {:>12} {:>12} {:>12} {:>10}",+ "batch_size",+ "input_dim",+ "hidden_dim",+ "Fused (µs)",+ "Unfused (µs)",+ "Speedup");- println!("{}", "-".repeat(61));-- for &(k, hidden_dim) in MLP_SHAPES {- let result =- bench_gemv_fused(&ctx, k, hidden_dim, iterations, warmup);- let speedup_str = if result.speedup >= 1.0 {- format!("{:.2}x", result.speedup)- } else {- format!("{:.2}x", result.speedup)- };- println!(- "({:>4}, {:>5}) {:>12.1} {:>12.1} {:>10}",- k,- hidden_dim,- result.fused_time_us,- result.unfused_time_us,- speedup_str- );- }-- println!();-- // GEMM benchmark (prefill path, M=32)- println!("=== GEMM (Prefill, M=32) ===");- println!(- "{:<25} {:>12} {:>12} {:>10}",- "Shape (K, H)", "Fused (µs)", "Unfused (µs)", "Speedup"- );- println!("{}", "-".repeat(61));-- for &(k, hidden_dim) in MLP_SHAPES {- let result =- bench_gemm_fused(&ctx, 32, k, hidden_dim, iterations, warmup);- let speedup_str = if result.speedup >= 1.0 {- format!("{:.2}x", result.speedup)- } else {- format!("{:.2}x", result.speedup)- };- println!(- "({:>4}, {:>5}) {:>12.1} {:>12.1} {:>10}",- k,- hidden_dim,- result.fused_time_us,- result.unfused_time_us,- speedup_str- );+ println!("{}", "-".repeat(74));++ for &batch_size in BATCH_SIZES {+ for &(input_dimension, hidden_dimension) in MLP_SHAPES {+ let result = if batch_size == 1 {+ benchmark_gemv_fused(+ &context,+ input_dimension,+ hidden_dimension,+ iteration_count,+ warmup_count,+ )+ } else {+ benchmark_gemm_fused(+ &context,+ batch_size,+ input_dimension,+ hidden_dimension,+ iteration_count,+ warmup_count,+ )+ };+ let speedup_string = format!("{:.2}x", result.speedup);+ println!(+ "{:>12} {:>12} {:>12} {:>12.1} {:>12.1} {:>10}",+ batch_size,+ input_dimension,+ hidden_dimension,+ result.fused_time_microseconds,+ result.unfused_time_microseconds,+ speedup_string+ );+ }}-- println!("\nNote: Unfused time excludes activation kernel overhead.");- println!(- " Actual unfused path would be slower due to activation dispatch.\n"- );});}@@ -442,8 +438,8 @@ mod tests {fn test_benchmark_runs() {// Just verify the benchmark doesn't crashautoreleasepool(|_| {- if let Some(ctx) = create_test_context() {- let _ = bench_gemv_fused(&ctx, 512, 256, 5, 2);+ if let Some(context) = create_test_context() {+ let _ = benchmark_gemv_fused(&context, 512, 256, 5, 2);}});}diff --git a/crates/uzu/src/backends/metal/classifier_context.rs b/crates/uzu/src/backends/metal/classifier_context.rsindex 567507df0..d3909de1d 100644--- a/crates/uzu/src/backends/metal/classifier_context.rs+++ b/crates/uzu/src/backends/metal/classifier_context.rs@@ -198,7 +198,7 @@ impl ClassifierContext {})?;Ok(ClassifierLayer::new(- &mtl_context,+ mtl_context.clone(),layer_config,compilation_config.clone(),layer_index,diff --git a/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs b/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rsindex 959ed3dcc..260cd9f9a 100644--- a/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs+++ b/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs@@ -38,7 +38,7 @@ pub struct ClassifierLayer {impl ClassifierLayer {pub fn new(- mtl_context: &MTLContext,+ mtl_context: Rc<MTLContext>,layer_config: &TransformerLayerConfig,_compilation_config: Rc<CompilationConfig>,layer_index: usize,@@ -52,6 +52,7 @@ impl ClassifierLayer {rope: Rc<Box<dyn EncodableBlock>>,) -> Self {autoreleasepool(|_| {+ let ctx = &*mtl_context; // Reference for functions expecting &MTLContextlet attention_config = layer_config.mixer_config.as_attention()@@ -65,7 +66,7 @@ impl ClassifierLayer {let copy_main_to_shortcut_mixer: Box<dyn EncodableBlock> = Box::new(TensorCopy::new(- mtl_context,+ ctx,kernel_data_type,vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),)@@ -79,7 +80,7 @@ impl ClassifierLayer {if layer_loader.subtree("pre_mixer_norm").is_ok() {Some(Box::new(Normalization::new(- mtl_context,+ ctx,intermediate_data_type,norm_config.clone(),ArrayId::Main,@@ -108,7 +109,7 @@ impl ClassifierLayer {num_groups * head_dim,num_groups * head_dim,],- mtl_context,+ ctx,&layer_loader.subtree("mixer.qkv_projection").unwrap(),ArrayId::Main,ArrayId::QKV,@@ -121,7 +122,7 @@ impl ClassifierLayer {|| attention_config.key_norm_config.is_some(){match QKNorm::new(- mtl_context,+ ctx,intermediate_data_type,attention_config.query_norm_config.clone(),attention_config.key_norm_config.clone(),@@ -146,7 +147,7 @@ impl ClassifierLayer {attention_config.has_out_biases,num_heads * head_dim,[model_dim],- mtl_context,+ ctx,&layer_loader.subtree("mixer.out_projection").unwrap(),ArrayId::AttentionOutput,ArrayId::Main,@@ -159,7 +160,7 @@ impl ClassifierLayer {{Some(Box::new(Normalization::new(- mtl_context,+ ctx,intermediate_data_type,norm_config.clone(),ArrayId::Main,@@ -174,7 +175,7 @@ impl ClassifierLayer {let mixer_residual_add: Box<dyn EncodableBlock> = Box::new(TensorAddSwap::new(- mtl_context,+ ctx,kernel_data_type,vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),)@@ -183,7 +184,7 @@ impl ClassifierLayer {let copy_main_to_shortcut_mlp: Box<dyn EncodableBlock> = Box::new(TensorCopy::new(- mtl_context,+ ctx,kernel_data_type,vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),)@@ -192,7 +193,7 @@ impl ClassifierLayer {let pre_mlp_norm: Box<dyn EncodableBlock> = Box::new(Normalization::new(- mtl_context,+ ctx,intermediate_data_type,layer_config.pre_mlp_norm_config.clone(),ArrayId::Main,@@ -206,7 +207,7 @@ impl ClassifierLayer {&layer_config.mlp_config,model_dim,hidden_dim,- mtl_context,+ ctx,&layer_loader.subtree("mlp").unwrap(),).expect("Failed to create mlp block");@@ -215,7 +216,7 @@ impl ClassifierLayer {if let Some(norm_config) = &layer_config.post_mlp_norm_config {Some(Box::new(Normalization::new(- mtl_context,+ ctx,intermediate_data_type,norm_config.clone(),ArrayId::Main,@@ -230,7 +231,7 @@ impl ClassifierLayer {let attention: Box<dyn EncodableBlock> = Box::new(Attention::new(- mtl_context,+ ctx,kernel_data_type,layer_index,attention_scale,@@ -243,7 +244,7 @@ impl ClassifierLayer {let mlp_residual_add: Box<dyn EncodableBlock> = Box::new(TensorAddSwap::new(- mtl_context,+ ctx,kernel_data_type,vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),)diff --git a/crates/uzu/src/backends/metal/encodable_block/decoder.rs b/crates/uzu/src/backends/metal/encodable_block/decoder.rsindex 349f0713b..655ba2fb1 100644--- a/crates/uzu/src/backends/metal/encodable_block/decoder.rs+++ b/crates/uzu/src/backends/metal/encodable_block/decoder.rs@@ -129,7 +129,7 @@ impl Decoder {.unwrap();LayerExecutables::new(- &mtl_context,+ mtl_context.clone(),layer_config,layer_type,compilation_config.clone(),diff --git a/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs b/crates/uzu/src/backends/metal/encodable_block/layer/executables.rsindex 26b58bfe4..19fc3477f 100644--- a/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs+++ b/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs@@ -40,7 +40,7 @@ pub struct LayerExecutables {impl LayerExecutables {#[allow(clippy::too_many_arguments)]pub fn new(- mtl_context: &MTLContext,+ mtl_context: Rc<MTLContext>,layer_config: &DecoderLayerConfig,layer_type: &DecoderLayerType,compilation_config: Rc<CompilationConfig>,@@ -55,6 +55,7 @@ impl LayerExecutables {rope: Option<Rc<Box<dyn EncodableBlock>>>,) -> Self {autoreleasepool(|_| {+ let ctx = &*mtl_context; // Reference for functions expecting &MTLContextlet intermediate_data_type: DataType =match &layer_config.mixer_config {MixerConfig::Attention(attention) => attention@@ -74,7 +75,7 @@ impl LayerExecutables {let copy_main_to_shortcut: Box<dyn EncodableBlock> = Box::new(TensorCopy::new(- mtl_context,+ ctx,kernel_data_type,vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),)@@ -83,7 +84,7 @@ impl LayerExecutables {let pre_attention_norm: Box<dyn EncodableBlock> = Box::new(RMSNorm::new(- mtl_context,+ ctx,intermediate_data_type,layer_config.pre_attention_norm_config.clone(),ArrayId::Main,@@ -108,7 +109,7 @@ impl LayerExecutables {num_groups * head_dim,num_groups * head_dim,],- mtl_context,+ ctx,&decoder_layer_loader.subtree("mixer.qkv_projection").unwrap(),@@ -122,7 +123,7 @@ impl LayerExecutables {|| attention_config.key_norm_config.is_some(){match QKNorm::new(- mtl_context,+ ctx,intermediate_data_type,attention_config.query_norm_config.clone(),attention_config.key_norm_config.clone(),@@ -148,7 +149,7 @@ impl LayerExecutables {attention_config.has_out_biases,num_heads * head_dim,[model_dim],- mtl_context,+ ctx,&decoder_layer_loader.subtree("mixer.out_projection").unwrap(),@@ -159,7 +160,7 @@ impl LayerExecutables {let attention = Box::new(Attention::new(- mtl_context,+ ctx,kernel_data_type,layer_index,attention_scale,@@ -181,7 +182,7 @@ impl LayerExecutables {MixerConfig::Mamba(mamba_config) => {let mixer: Box<dyn EncodableBlock> =Box::new(MambaMixer::new(- mtl_context,+ ctx,layer_type.clone(),mamba_config.clone(),compilation_config.clone(),@@ -199,7 +200,7 @@ impl LayerExecutables {MixerConfig::ShortConv(short_conv_config) => {let mixer: Box<dyn EncodableBlock> =Box::new(ShortConvMixer::new(- mtl_context,+ ctx,layer_type.clone(),short_conv_config.clone(),compilation_config.clone(),@@ -219,7 +220,7 @@ impl LayerExecutables {{Some(Box::new(RMSNorm::new(- mtl_context,+ ctx,intermediate_data_type,norm_config.clone(),ArrayId::Main,@@ -236,7 +237,7 @@ impl LayerExecutables {let main_shortcut_add_swap: Box<dyn EncodableBlock> = Box::new(TensorAddSwap::new(- mtl_context,+ ctx,kernel_data_type,vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),)@@ -245,7 +246,7 @@ impl LayerExecutables {let pre_mlp_norm: Box<dyn EncodableBlock> = Box::new(RMSNorm::new(- mtl_context,+ ctx,intermediate_data_type,layer_config.pre_mlp_norm_config.clone(),ArrayId::Main,@@ -259,7 +260,7 @@ impl LayerExecutables {&layer_config.mlp_config,model_dim,hidden_dim,- mtl_context,+ ctx,&decoder_layer_loader.subtree("mlp").unwrap(),).expect("Failed to create mlp block");@@ -268,7 +269,7 @@ impl LayerExecutables {if let Some(norm_config) = &layer_config.post_mlp_norm_config {Some(Box::new(RMSNorm::new(- mtl_context,+ ctx,intermediate_data_type,norm_config.clone(),ArrayId::Main,diff --git a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rsindex ea855f24a..1da60d21a 100644--- a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs+++ b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs@@ -2,14 +2,16 @@ use std::rc::Rc;use super::{EncodableBlock, FullPrecisionEmbeddingLookup,- FullPrecisionEmbeddingReadout, FullPrecisionLinear, MlpBlock, MoeBlock,- QuantizedEmbeddingLookup, QuantizedEmbeddingReadout, QuantizedLinear,+ FullPrecisionEmbeddingReadout, FullPrecisionLinear, MlpBlock,+ MlpFusedBlock, MoeBlock, QuantizedEmbeddingLookup,+ QuantizedEmbeddingReadout, QuantizedLinear,};use crate::{DataType,backends::metal::{- MTLContext, MTLError, forward_pass::ArrayId,- kernel::mlp::MlpGateActMulEncodable,+ MTLContext, MTLError,+ forward_pass::ArrayId,+ kernel::{mlp::MlpGateActMulEncodable, quant_matmul::QuantizationType},},config::{DecoderConfig, EmbeddingConfig, LinearConfig, MLPConfig},parameters::ParameterTree,@@ -18,22 +20,22 @@ use crate::{pub fn linear_block<const N: usize>(config: &LinearConfig,_has_biases: bool,- input_dim: usize,- output_dims: [usize; N],+ input_dimension: usize,+ output_dimensions: [usize; N],context: &MTLContext,parameter_tree: &ParameterTree<Rc<MTLContext>>,input_array_id: ArrayId,output_array_id: ArrayId,) -> Result<Box<dyn EncodableBlock>, MTLError> {- let out_sum: usize = output_dims.iter().sum();+ let output_dimension_sum: usize = output_dimensions.iter().sum();match config {- LinearConfig::Quantized(quant_config)- | LinearConfig::MLXQuantized(quant_config) => {+ LinearConfig::Quantized(quantization_config)+ | LinearConfig::MLXQuantized(quantization_config) => {let block = QuantizedLinear::new(context,- quant_config,- input_dim,- out_sum,+ quantization_config,+ input_dimension,+ output_dimension_sum,parameter_tree,input_array_id,output_array_id,@@ -46,8 +48,8 @@ pub fn linear_block<const N: usize>(let block = FullPrecisionLinear::new(context,(*precision).into(),- input_dim,- out_sum,+ input_dimension,+ output_dimension_sum,parameter_tree,input_array_id,output_array_id,@@ -62,92 +64,220 @@ pub fn linear_block<const N: usize>(}}+/// Creates an MLP block using the unfused implementation (separate up, activation, down)pub fn mlp_block(config: &MLPConfig,- model_dim: usize,- hidden_dim: usize,+ model_dimension: usize,+ hidden_dimension: usize,context: &MTLContext,parameter_tree: &ParameterTree<Rc<MTLContext>>,) -> Result<Box<dyn EncodableBlock>, MTLError> {- if let crate::config::MLPConfig::Dense(dense) = config {- match &dense.linear_config {- LinearConfig::Quantized(quant_config)- | LinearConfig::MLXQuantized(quant_config) => {- let dtype: DataType =- dense.linear_config.activation_precision().into();-- let up = QuantizedLinear::new(- context,- quant_config,- model_dim,- 2 * hidden_dim,- ¶meter_tree- .subtree("up_projection")- .map_err(|e| MTLError::Generic(format!("{:?}", e)))?,- ArrayId::Main,- ArrayId::MlpFusedUp,- )?;+ if let crate::config::MLPConfig::Dense(dense_config) = config {+ let data_type: DataType =+ dense_config.linear_config.activation_precision().into();++ // Up projection (outputs 2*hidden_dimension for gate and up)+ let up_projection = linear_block(+ &dense_config.linear_config,+ false,+ model_dimension,+ [2 * hidden_dimension],+ context,+ ¶meter_tree+ .subtree("up_projection")+ .map_err(|error| MTLError::Generic(format!("{:?}", error)))?,+ ArrayId::Main,+ ArrayId::MlpFusedUp,+ )?;++ // Gate activation + multiply+ let gate_activation = MlpGateActMulEncodable::new(+ context,+ data_type,+ dense_config.activation.clone(),+ hidden_dimension,+ )?;++ // Down projection+ let down_projection = linear_block(+ &dense_config.linear_config,+ false,+ hidden_dimension,+ [model_dimension],+ context,+ ¶meter_tree+ .subtree("down_projection")+ .map_err(|error| MTLError::Generic(format!("{:?}", error)))?,+ ArrayId::MlpHidden,+ ArrayId::Main,+ )?;++ return Ok(Box::new(MlpBlock::new(+ up_projection,+ gate_activation,+ down_projection,+ )));+ }- let gate_op = MlpGateActMulEncodable::new(- context,- dtype,- dense.activation,- hidden_dim,- )?;+ if let crate::config::MLPConfig::MixtureOfExperts(+ mixture_of_experts_config,+ ) = config+ {+ let mixture_of_experts_block = MoeBlock::new(+ context,+ mixture_of_experts_config,+ model_dimension,+ hidden_dimension,+ parameter_tree,+ )?;+ return Ok(Box::new(mixture_of_experts_block));+ }- let down = QuantizedLinear::new(- context,- quant_config,- hidden_dim,- model_dim,- ¶meter_tree- .subtree("down_projection")- .map_err(|e| MTLError::Generic(format!("{:?}", e)))?,+ unreachable!("Unknown MLP config")+}++/// Creates an MLP block using the fused implementation+/// (up projection + activation fused into single kernel)+pub fn mlp_fused_block(+ config: &MLPConfig,+ model_dimension: usize,+ hidden_dimension: usize,+ context: Rc<MTLContext>,+ parameter_tree: &ParameterTree<Rc<MTLContext>>,+) -> Result<Box<dyn EncodableBlock>, MTLError> {+ if let crate::config::MLPConfig::Dense(dense_config) = config {+ match &dense_config.linear_config {+ LinearConfig::Quantized(quantization_config)+ | LinearConfig::MLXQuantized(quantization_config) => {+ let data_type: DataType =+ dense_config.linear_config.activation_precision().into();++ // Load up_projection weights directly for fused kernel+ let up_projection_tree =+ parameter_tree.subtree("up_projection").map_err(+ |error| MTLError::Generic(format!("{:?}", error)),+ )?;++ let mut up_projection_weights =+ up_projection_tree.leaf("weights").map_err(|error| {+ MTLError::Generic(format!(+ "Failed to load up weights: {:?}",+ error+ ))+ })?;+ let up_projection_weights_buffer =+ unsafe { up_projection_weights.mtl_buffer() }.to_owned();++ let mut up_projection_scales =+ up_projection_tree.leaf("scales").map_err(|error| {+ MTLError::Generic(format!(+ "Failed to load up scales: {:?}",+ error+ ))+ })?;+ let up_projection_scales_buffer =+ unsafe { up_projection_scales.mtl_buffer() }.to_owned();++ // Load zero_points or biases depending on quantization type+ let (+ up_projection_zero_points_or_biases_buffer,+ quantization_type,+ ) = if let Ok(mut biases) = up_projection_tree.leaf("biases") {+ (+ unsafe { biases.mtl_buffer() }.to_owned(),+ QuantizationType::Mlx,+ )+ } else if let Ok(mut zero_points) =+ up_projection_tree.leaf("zero_points")+ {+ (+ unsafe { zero_points.mtl_buffer() }.to_owned(),+ QuantizationType::ZeroPoint,+ )+ } else {+ return Err(MTLError::Generic(+ "Missing zero_points or biases for quantized up_projection"+ .to_string(),+ ));+ };++ // Create down projection as separate linear+ let down_projection = QuantizedLinear::new(+ &context,+ quantization_config,+ hidden_dimension,+ model_dimension,+ ¶meter_tree.subtree("down_projection").map_err(+ |error| MTLError::Generic(format!("{:?}", error)),+ )?,ArrayId::MlpHidden,ArrayId::Main,)?;- let enc = MlpBlock::new(Box::new(up), gate_op, Box::new(down));- return Ok(Box::new(enc));+ let fused_block = MlpFusedBlock::new_quantized(+ context,+ data_type,+ up_projection_weights_buffer,+ up_projection_scales_buffer,+ up_projection_zero_points_or_biases_buffer,+ model_dimension,+ hidden_dimension,+ quantization_config.group_size,+ quantization_config.weight_quantization_mode,+ quantization_type,+ &dense_config.activation,+ Box::new(down_projection),+ ArrayId::Main,+ ArrayId::MlpHidden,+ )?;+ return Ok(Box::new(fused_block));},LinearConfig::FullPrecision {precision,} => {- let dtype: DataType = (*precision).into();-- let up = FullPrecisionLinear::new(- context,- dtype,- model_dim,- 2 * hidden_dim,- ¶meter_tree- .subtree("up_projection")- .map_err(|e| MTLError::Generic(format!("{:?}", e)))?,+ let data_type: DataType = (*precision).into();++ // Load up_projection weights directly for fused kernel+ let up_projection_tree =+ parameter_tree.subtree("up_projection").map_err(+ |error| MTLError::Generic(format!("{:?}", error)),+ )?;++ let mut up_projection_weights =+ up_projection_tree.leaf("weights").map_err(|error| {+ MTLError::Generic(format!(+ "Failed to load up weights: {:?}",+ error+ ))+ })?;+ let up_projection_weights_buffer =+ unsafe { up_projection_weights.mtl_buffer() }.to_owned();++ // Create down projection as separate linear+ let down_projection = FullPrecisionLinear::new(+ &context,+ data_type,+ hidden_dimension,+ model_dimension,+ ¶meter_tree.subtree("down_projection").map_err(+ |error| MTLError::Generic(format!("{:?}", error)),+ )?,+ ArrayId::MlpHidden,ArrayId::Main,- ArrayId::MlpFusedUp,)?;- let gate_op = MlpGateActMulEncodable::new(+ let fused_block = MlpFusedBlock::new_full_precision(context,- dtype,- dense.activation,- hidden_dim,- )?;-- let down = FullPrecisionLinear::new(- context,- dtype,- hidden_dim,- model_dim,- ¶meter_tree- .subtree("down_projection")- .map_err(|e| MTLError::Generic(format!("{:?}", e)))?,- ArrayId::MlpHidden,+ data_type,+ up_projection_weights_buffer,+ model_dimension,+ hidden_dimension,+ &dense_config.activation,+ Box::new(down_projection),ArrayId::Main,+ ArrayId::MlpHidden,)?;-- let enc = MlpBlock::new(Box::new(up), gate_op, Box::new(down));- return Ok(Box::new(enc));+ return Ok(Box::new(fused_block));},LinearConfig::QLoRA {..@@ -159,10 +289,18 @@ pub fn mlp_block(}}- if let crate::config::MLPConfig::MixtureOfExperts(moe) = config {- let moe_block =- MoeBlock::new(context, moe, model_dim, hidden_dim, parameter_tree)?;- return Ok(Box::new(moe_block));+ if let crate::config::MLPConfig::MixtureOfExperts(+ mixture_of_experts_config,+ ) = config+ {+ let mixture_of_experts_block = MoeBlock::new(+ &context,+ mixture_of_experts_config,+ model_dimension,+ hidden_dimension,+ parameter_tree,+ )?;+ return Ok(Box::new(mixture_of_experts_block));}unreachable!("Unknown MLP config")