A high-performance inference engine for AI models
remove mlp fused (#192)
Aleksandr Golokozdaaa4852e4cf
diff --git a/crates/uzu/src/backends/metal/encodable_block/mlp.rs b/crates/uzu/src/backends/metal/encodable_block/mlp.rsindex 8de14db04..70298c6e8 100644--- a/crates/uzu/src/backends/metal/encodable_block/mlp.rs+++ b/crates/uzu/src/backends/metal/encodable_block/mlp.rs@@ -1,19 +1,11 @@//! MLP block encodable.-use std::{cell::RefCell, rc::Rc};-use super::{EncodableBlock, Metal};use crate::{- DataType,backends::metal::{- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, ProtocolObject, Retained,- kernel::{- mlp::{MlpActivationType, MlpGateActMulEncodable},- mlp_fused::{MlpFusedArguments, MlpFusedKernel},- quant_matmul::{MlpFusedQmmArguments, MlpFusedQmmKernel, MlpFusedQmvArguments, MlpFusedQmvKernel},- },+ MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, ProtocolObject, Retained,+ kernel::mlp::MlpGateActMulEncodable,},- config::Activation,encodable_block::EncodingParameters,forward_pass::state::{ArrayId, ForwardPassState},};@@ -109,238 +101,3 @@ impl EncodableBlock<Metal> for MlpBlock {self.down.encode_with_shared_encoder(state, params, encoder);}}--/// MLP Fused Up Projection Variant-/// Selects between full precision and quantized fused kernels-pub enum MlpFusedUpKernel {- FullPrecision {- kernel: RefCell<MlpFusedKernel>,- },- Quantized {- qmv: MlpFusedQmvKernel,- qmm: MlpFusedQmmKernel,- },-}--/// MLP block with fused up projection and activation-/// Combines up * activation(gate) into a single kernel call-pub struct MlpFusedBlock {- context: Rc<MTLContext>,- fused_up: MlpFusedUpKernel,- down: Box<dyn EncodableBlock<Metal>>,- weights_buffer: Retained<ProtocolObject<dyn MTLBuffer>>,- scales_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>>,- zero_points_or_biases_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>>,- input_dim: usize,- hidden_dim: usize,- activation: MlpActivationType,- input_array_id: ArrayId,- hidden_array_id: ArrayId,-}--impl MlpFusedBlock {- /// Create MLP fused block with full precision weights- pub fn new_full_precision(- context: Rc<MTLContext>,- data_type: DataType,- weights_buffer: Retained<ProtocolObject<dyn MTLBuffer>>,- input_dim: usize,- hidden_dim: usize,- activation: &Activation,- down: Box<dyn EncodableBlock<Metal>>,- input_array_id: ArrayId,- hidden_array_id: ArrayId,- ) -> Result<Self, crate::backends::metal::MTLError> {- let kernel = MlpFusedKernel::new(data_type, true)?; // Weights transposed-- Ok(Self {- context,- fused_up: MlpFusedUpKernel::FullPrecision {- kernel: RefCell::new(kernel),- },- down,- weights_buffer,- scales_buffer: None,- zero_points_or_biases_buffer: None,- input_dim,- hidden_dim,- activation: MlpActivationType::from(activation),- input_array_id,- hidden_array_id,- })- }-- /// Create MLP fused block with quantized weights- pub fn new_quantized(- context: Rc<MTLContext>,- data_type: DataType,- weights_buffer: Retained<ProtocolObject<dyn MTLBuffer>>,- scales_buffer: Retained<ProtocolObject<dyn MTLBuffer>>,- zero_points_or_biases_buffer: Retained<ProtocolObject<dyn MTLBuffer>>,- input_dim: usize,- hidden_dim: usize,- group_size: usize,- mode: crate::config::QuantizationMode,- quantization_type: crate::backends::metal::kernel::quant_matmul::QuantizationType,- activation: &Activation,- down: Box<dyn EncodableBlock<Metal>>,- input_array_id: ArrayId,- hidden_array_id: ArrayId,- ) -> Result<Self, crate::backends::metal::MTLError> {- let qmv = MlpFusedQmvKernel::new(&context, data_type, group_size, mode, quantization_type)- .map_err(|e| crate::backends::metal::MTLError::Generic(format!("{:?}", e)))?;-- let qmm = MlpFusedQmmKernel::new(&context, data_type, group_size, mode, quantization_type)- .map_err(|e| crate::backends::metal::MTLError::Generic(format!("{:?}", e)))?;-- Ok(Self {- context,- fused_up: MlpFusedUpKernel::Quantized {- qmv,- qmm,- },- down,- weights_buffer,- scales_buffer: Some(scales_buffer),- zero_points_or_biases_buffer: Some(zero_points_or_biases_buffer),- input_dim,- hidden_dim,- activation: MlpActivationType::from(activation),- input_array_id,- hidden_array_id,- })- }-- fn encode_fused_up(- &self,- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- input: &Retained<ProtocolObject<dyn MTLBuffer>>,- input_offset: u64,- output: &Retained<ProtocolObject<dyn MTLBuffer>>,- batch: i32,- ) {- match &self.fused_up {- MlpFusedUpKernel::FullPrecision {- kernel,- } => {- let args = MlpFusedArguments {- input,- input_offset,- weights: &self.weights_buffer,- output,- batch,- input_dim: self.input_dim as i32,- hidden_dim: self.hidden_dim as i32,- lda: self.input_dim as i32,- ldb: self.input_dim as i32,- ldd: self.hidden_dim as i32,- batch_count: batch,- activation: self.activation,- };- kernel.borrow_mut().encode(&self.context, encoder, &args).expect("Failed to encode MLP fused kernel");- },- MlpFusedUpKernel::Quantized {- qmv,- qmm,- } => {- let scales = self.scales_buffer.as_ref().unwrap();- let zp_or_biases = self.zero_points_or_biases_buffer.as_ref().unwrap();-- let is_decode = batch == 1;- if is_decode {- let args = MlpFusedQmvArguments {- weights: &self.weights_buffer,- scales,- zero_points_or_biases: zp_or_biases,- input,- input_offset,- output,- input_dim: self.input_dim as i32,- hidden_dim: self.hidden_dim as i32,- batch_count: batch,- };- qmv.encode(encoder, &args).expect("Failed to encode MLP fused QMV");- } else {- let args = MlpFusedQmmArguments {- weights: &self.weights_buffer,- scales,- zero_points_or_biases: zp_or_biases,- input,- input_offset,- output,- batch,- input_dim: self.input_dim as i32,- hidden_dim: self.hidden_dim as i32,- };- qmm.encode(encoder, &args).expect("Failed to encode MLP fused QMM");- }- },- }- }-}--impl EncodableBlock<Metal> for MlpFusedBlock {- fn encode(- &self,- state: &mut ForwardPassState<Metal>,- params: &EncodingParameters<Metal>,- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,- ) {- if self.supports_shared_encoder() {- let encoder =- command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");- self.encode_with_shared_encoder(state, params, &encoder);- encoder.end_encoding();- } else {- // Fused up + activation- {- let arrays = state.arrays(&[self.input_array_id, self.hidden_array_id]);- let input = arrays[0].borrow_mut();- let hidden = arrays[1].borrow_mut();- let batch = input.shape()[0] as i32;- let input_buf = input.buffer();- let hidden_buf = hidden.buffer();-- let encoder =- command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");- self.encode_fused_up(&encoder, input_buf, 0, hidden_buf, batch);- encoder.end_encoding();- }-- // Down- self.down.encode(state, params, command_buffer);- }-- if params.wait_until_completed {- command_buffer.commit();- command_buffer.wait_until_completed();- }- }-- fn supports_shared_encoder(&self) -> bool {- self.down.supports_shared_encoder()- }-- fn encode_with_shared_encoder(- &self,- state: &mut ForwardPassState<Metal>,- params: &EncodingParameters<Metal>,- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- ) {- // Fused up + activation- let arrays = state.arrays(&[self.input_array_id, self.hidden_array_id]);- let input = arrays[0].borrow_mut();- let hidden = arrays[1].borrow_mut();- let batch = input.shape()[0] as i32;- let input_buf = input.buffer();- let hidden_buf = hidden.buffer();-- self.encode_fused_up(encoder, input_buf, 0, hidden_buf, batch);-- drop(input);- drop(hidden);-- // Down- self.down.encode_with_shared_encoder(state, params, encoder);- }-}diff --git a/crates/uzu/src/backends/metal/encodable_block/mod.rs b/crates/uzu/src/backends/metal/encodable_block/mod.rsindex 487291f9a..275846b7b 100644--- a/crates/uzu/src/backends/metal/encodable_block/mod.rs+++ b/crates/uzu/src/backends/metal/encodable_block/mod.rs@@ -33,7 +33,7 @@ pub use embedding::{pub use layer::LayerExecutables;pub use linear::{FullPrecisionLinear, QuantizedLinear};pub(crate) use mamba_mixer::MambaMixer;-pub use mlp::{MlpBlock, MlpFusedBlock, MlpFusedUpKernel};+pub use mlp::MlpBlock;pub use moe_block::{MoeBlock, SharedMoeWeights};pub use normalization::{LayerNorm, Normalization, QKNorm, RMSNorm};pub use prediction_head::ClassifierPredictionHead;diff --git a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rsindex ac2bfa1c9..d8891e29f 100644--- a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs+++ b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs@@ -1,15 +1,10 @@-use std::rc::Rc;-use super::{EncodableBlock, FullPrecisionEmbeddingLookup, FullPrecisionEmbeddingReadout, FullPrecisionLinear, Metal, MlpBlock,- MlpFusedBlock, MoeBlock, QuantizedEmbeddingLookup, QuantizedEmbeddingReadout, QuantizedLinear,+ MoeBlock, QuantizedEmbeddingLookup, QuantizedEmbeddingReadout, QuantizedLinear,};use crate::{DataType,- backends::metal::{- MTLContext, MTLError,- kernel::{mlp::MlpGateActMulEncodable, quant_matmul::QuantizationType},- },+ backends::metal::{MTLContext, MTLError, kernel::mlp::MlpGateActMulEncodable},config::{DecoderConfig, EmbeddingConfig, LinearConfig, MLPConfig},forward_pass::state::ArrayId,parameters::ParameterTree,@@ -112,138 +107,6 @@ pub fn mlp_block(unreachable!("Unknown MLP config")}-/// Creates an MLP block using the fused implementation-/// (up projection + activation fused into single kernel)-pub fn mlp_fused_block(- config: &MLPConfig,- model_dimension: usize,- hidden_dimension: usize,- context: Rc<MTLContext>,- parameter_tree: &ParameterTree<MTLContext>,-) -> Result<Box<dyn EncodableBlock<Metal>>, MTLError> {- if let crate::config::MLPConfig::Dense(dense_config) = config {- match &dense_config.linear_config {- LinearConfig::Quantized(quantization_config) | LinearConfig::MLXQuantized(quantization_config) => {- let data_type: DataType = dense_config.linear_config.activation_precision().into();-- // Load up_projection weights directly for fused kernel- let up_projection_tree = parameter_tree- .subtree("up_projection")- .map_err(|error| MTLError::Generic(format!("{:?}", error)))?;-- let up_projection_weights = up_projection_tree- .leaf("weights")- .map_err(|error| MTLError::Generic(format!("Failed to load up weights: {:?}", error)))?;- let up_projection_weights_buffer = up_projection_weights.buffer().into();-- let up_projection_scales = up_projection_tree- .leaf("scales")- .map_err(|error| MTLError::Generic(format!("Failed to load up scales: {:?}", error)))?;- let up_projection_scales_buffer = up_projection_scales.buffer().into();-- // Load zero_points or biases depending on quantization type- let (up_projection_zero_points_or_biases_buffer, quantization_type) =- if let Ok(biases) = up_projection_tree.leaf("biases") {- (biases.buffer().into(), QuantizationType::Mlx)- } else if let Ok(zero_points) = up_projection_tree.leaf("zero_points") {- (zero_points.buffer().into(), QuantizationType::ZeroPoint)- } else {- return Err(MTLError::Generic(- "Missing zero_points or biases for quantized up_projection".to_string(),- ));- };-- // Create down projection as separate linear- let down_projection = QuantizedLinear::new(- &context,- quantization_config,- hidden_dimension,- model_dimension,- ¶meter_tree- .subtree("down_projection")- .map_err(|error| MTLError::Generic(format!("{:?}", error)))?,- ArrayId::MlpHidden,- ArrayId::Main,- )- .map_err(|error| MTLError::Generic(format!("{:?}", error)))?;-- let fused_block = MlpFusedBlock::new_quantized(- context,- data_type,- up_projection_weights_buffer,- up_projection_scales_buffer,- up_projection_zero_points_or_biases_buffer,- model_dimension,- hidden_dimension,- quantization_config.group_size,- quantization_config.weight_quantization_mode,- quantization_type,- &dense_config.activation,- Box::new(down_projection),- ArrayId::Main,- ArrayId::MlpHidden,- )?;- return Ok(Box::new(fused_block));- },- LinearConfig::FullPrecision {- precision,- } => {- let data_type: DataType = (*precision).into();-- // Load up_projection weights directly for fused kernel- let up_projection_tree = parameter_tree- .subtree("up_projection")- .map_err(|error| MTLError::Generic(format!("{:?}", error)))?;-- let up_projection_weights = up_projection_tree- .leaf("weights")- .map_err(|error| MTLError::Generic(format!("Failed to load up weights: {:?}", error)))?;- let up_projection_weights_buffer = up_projection_weights.buffer().into();-- // Create down projection as separate linear- let down_projection = FullPrecisionLinear::new(- &context,- data_type,- hidden_dimension,- model_dimension,- ¶meter_tree- .subtree("down_projection")- .map_err(|error| MTLError::Generic(format!("{:?}", error)))?,- ArrayId::MlpHidden,- ArrayId::Main,- )- .map_err(|error| MTLError::Generic(format!("{:?}", error)))?;-- let fused_block = MlpFusedBlock::new_full_precision(- context,- data_type,- up_projection_weights_buffer,- model_dimension,- hidden_dimension,- &dense_config.activation,- Box::new(down_projection),- ArrayId::Main,- ArrayId::MlpHidden,- )?;- return Ok(Box::new(fused_block));- },- LinearConfig::QLoRA {- ..- } => {- return Err(MTLError::Generic("QLoRA MLP not supported".to_string()));- },- }- }-- if let crate::config::MLPConfig::MixtureOfExperts(mixture_of_experts_config) = config {- let mixture_of_experts_block =- MoeBlock::new(&context, mixture_of_experts_config, model_dimension, hidden_dimension, parameter_tree)?;- return Ok(Box::new(mixture_of_experts_block));- }-- unreachable!("Unknown MLP config")-}-pub fn embed_block(config: &DecoderConfig,context: &MTLContext,diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/common/mlp_fused_arguments.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/common/mlp_fused_arguments.rsdeleted file mode 100644index da793f6e6..000000000--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/common/mlp_fused_arguments.rs+++ /dev/null@@ -1,17 +0,0 @@-use crate::backends::metal::{MTLBuffer, ProtocolObject, kernel::mlp::MlpActivationType};--#[derive(Debug)]-pub struct MlpFusedArguments<'a> {- pub input: &'a ProtocolObject<dyn MTLBuffer>,- pub input_offset: u64,- pub weights: &'a ProtocolObject<dyn MTLBuffer>,- pub output: &'a ProtocolObject<dyn MTLBuffer>,- pub batch: i32,- pub input_dim: i32,- pub hidden_dim: i32,- pub lda: i32,- pub ldb: i32,- pub ldd: i32,- pub batch_count: i32,- pub activation: MlpActivationType,-}diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/common/mod.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/common/mod.rsdeleted file mode 100644index 8816071bd..000000000--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/common/mod.rs+++ /dev/null@@ -1,3 +0,0 @@-mod mlp_fused_arguments;--pub use mlp_fused_arguments::MlpFusedArguments;diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/dispatch_descriptor.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/dispatch_descriptor.rsdeleted file mode 100644index 9cbedf896..000000000--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/dispatch_descriptor.rs+++ /dev/null@@ -1,72 +0,0 @@-use super::{common::MlpFusedArguments, gemm, gemv, split_k};-use crate::{- DataType,- backends::metal::{MTLContext, MTLError},-};--#[derive(Debug, Clone, Copy, PartialEq, Eq)]-pub enum MlpFusedKernelVariant {- Gemv,- SplitK,- Gemm,-}--impl MlpFusedKernelVariant {- pub fn as_str(&self) -> &'static str {- match self {- MlpFusedKernelVariant::Gemv => "GEMV",- MlpFusedKernelVariant::SplitK => "Split-K",- MlpFusedKernelVariant::Gemm => "GEMM",- }- }-}--#[derive(Debug, Clone)]-pub(crate) enum MlpFusedDispatchDescriptor {- Gemv(gemv::DispatchDescriptor),- SplitK(split_k::DispatchDescriptor),- Gemm(gemm::DispatchDescriptor),-}--impl MlpFusedDispatchDescriptor {- pub fn variant(&self) -> MlpFusedKernelVariant {- match self {- MlpFusedDispatchDescriptor::Gemv(_) => MlpFusedKernelVariant::Gemv,- MlpFusedDispatchDescriptor::SplitK(_) => MlpFusedKernelVariant::SplitK,- MlpFusedDispatchDescriptor::Gemm(_) => MlpFusedKernelVariant::Gemm,- }- }-}--pub(crate) fn choose_dispatch_descriptor(- context: &MTLContext,- data_type: DataType,- weights_transposed: bool,- arguments: &MlpFusedArguments,-) -> Result<MlpFusedDispatchDescriptor, MTLError> {- if weights_transposed {- if let Some(descriptor) = gemv::DispatchDescriptor::try_new(context, data_type, arguments)? {- return Ok(MlpFusedDispatchDescriptor::Gemv(descriptor));- }-- if let Some(descriptor) = split_k::DispatchDescriptor::try_new(context, data_type, arguments)? {- return Ok(MlpFusedDispatchDescriptor::SplitK(descriptor));- }- }-- Ok(MlpFusedDispatchDescriptor::Gemm(gemm::DispatchDescriptor::new(- context,- data_type,- weights_transposed,- arguments,- )?))-}--pub fn determine_kernel_variant(- context: &MTLContext,- data_type: DataType,- weights_transposed: bool,- arguments: &MlpFusedArguments,-) -> Result<MlpFusedKernelVariant, MTLError> {- choose_dispatch_descriptor(context, data_type, weights_transposed, arguments).map(|d| d.variant())-}diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/dispatch_descriptor.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/dispatch_descriptor.rsdeleted file mode 100644index b8cc84c3e..000000000--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/dispatch_descriptor.rs+++ /dev/null@@ -1,79 +0,0 @@-use super::{pipeline_configuration::PipelineConfiguration, tile_configuration::select_tile_configuration};-use crate::{- DataType,- backends::metal::{- MTLContext, MTLError, MTLSize,- kernel::{matmul::common::GEMMParams, mlp_fused::common::MlpFusedArguments},- },-};--#[derive(Debug, Clone)]-pub struct DispatchDescriptor {- pub(crate) pipeline_configuration: PipelineConfiguration,- pub(crate) params: GEMMParams,- pub(crate) hidden_dim: i32,- pub(crate) threadgroups: MTLSize,- pub(crate) threads_per_threadgroup: MTLSize,-}--impl DispatchDescriptor {- pub(crate) fn new(- _context: &MTLContext,- data_type: DataType,- weights_transposed: bool,- arguments: &MlpFusedArguments,- ) -> Result<Self, MTLError> {- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {- return Err(MTLError::Generic(format!("Unsupported dtype for MLP fused GEMM: {data_type:?}")));- }-- let tile = select_tile_configuration(arguments.batch, arguments.hidden_dim);-- let mn_aligned = arguments.batch % tile.block_rows == 0 && arguments.hidden_dim % tile.block_cols == 0;- let k_aligned = arguments.input_dim % tile.block_depth == 0;-- let pipeline_configuration = PipelineConfiguration {- tile,- weights_transposed,- mn_aligned,- k_aligned,- activation: arguments.activation,- };-- let tiles_m = (arguments.batch + tile.block_rows - 1) / tile.block_rows;- let tiles_n = (arguments.hidden_dim + tile.block_cols - 1) / tile.block_cols;- let gemm_k_iterations = (arguments.input_dim + tile.block_depth - 1) / tile.block_depth;-- let params = GEMMParams {- M: arguments.batch,- N: arguments.hidden_dim * 2,- K: arguments.input_dim,- lda: arguments.lda,- ldb: arguments.ldb,- ldd: arguments.ldd,- tiles_n,- tiles_m,- batch_stride_a: 0,- batch_stride_b: 0,- batch_stride_d: 0,- swizzle_log: 0,- gemm_k_iterations_aligned: if k_aligned {- gemm_k_iterations- } else {- gemm_k_iterations - 1- },- batch_ndim: 0,- };-- let threadgroups = MTLSize::new(tiles_n as usize, tiles_m as usize, 1);- let threads_per_threadgroup = MTLSize::new(32, 2, 2);-- Ok(Self {- pipeline_configuration,- params,- hidden_dim: arguments.hidden_dim,- threadgroups,- threads_per_threadgroup,- })- }-}diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/kernel.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/kernel.rsdeleted file mode 100644index adc539243..000000000--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/kernel.rs+++ /dev/null@@ -1,114 +0,0 @@-use std::collections::HashMap;--use super::{DispatchDescriptor, pipeline_configuration::PipelineConfiguration};-use crate::{- DataType,- backends::metal::{- ComputeEncoderSetValue, FunctionConstantValuesSetValue, MTLComputeCommandEncoder, MTLComputePipelineState,- MTLContext, MTLError, MTLFunctionConstantValues, ProtocolObject, Retained,- kernel::mlp_fused::common::MlpFusedArguments,- },-};--pub struct Kernel {- data_type: DataType,- weights_transposed: bool,- pipelines: HashMap<PipelineConfiguration, Retained<ProtocolObject<dyn MTLComputePipelineState>>>,-}--impl Kernel {- pub fn new(- data_type: DataType,- weights_transposed: bool,- ) -> Result<Self, MTLError> {- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {- return Err(MTLError::Generic(format!("Unsupported dtype for MLP fused GEMM: {data_type:?}")));- }- Ok(Self {- data_type,- weights_transposed,- pipelines: HashMap::new(),- })- }-- pub fn weights_transposed(&self) -> bool {- self.weights_transposed- }-- fn kernel_name(- &self,- configuration: &PipelineConfiguration,- ) -> String {- let type_name = match self.data_type {- DataType::F16 => "half",- DataType::BF16 => "bfloat",- DataType::F32 => "float",- _ => unreachable!(),- };- let transpose_char = if configuration.weights_transposed {- "t"- } else {- "n"- };- let mn_str = if configuration.mn_aligned {- "true"- } else {- "false"- };- let k_str = if configuration.k_aligned {- "true"- } else {- "false"- };-- format!(- "steel_gemm_mlp_fused_{}_{}_{}_bm{}_bn{}_bk16_wm2_wn2_align_MN_{}_K_{}",- transpose_char,- type_name,- type_name,- configuration.tile.block_rows,- configuration.tile.block_cols,- mn_str,- k_str- )- }-- fn get_or_compile_pipeline(- &mut self,- context: &MTLContext,- configuration: &PipelineConfiguration,- ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError> {- if !self.pipelines.contains_key(configuration) {- let kernel_name = self.kernel_name(configuration);-- let function_constants = MTLFunctionConstantValues::new();- let activation_val = configuration.activation as u32;- function_constants.set_value(&activation_val, 52);-- let pipeline_state = context.compute_pipeline_state(&kernel_name, Some(&function_constants))?;- self.pipelines.insert(configuration.clone(), pipeline_state);- }- Ok(self.pipelines.get(configuration).unwrap())- }-- pub(crate) fn encode_descriptor(- &mut self,- context: &MTLContext,- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- arguments: &MlpFusedArguments,- descriptor: &DispatchDescriptor,- ) -> Result<(), MTLError> {- let pipeline_state = self.get_or_compile_pipeline(context, &descriptor.pipeline_configuration)?;- encoder.set_compute_pipeline_state(pipeline_state);-- encoder.set_buffer(Some(arguments.input), arguments.input_offset as usize, 0);- encoder.set_buffer(Some(arguments.weights), 0, 1);- encoder.set_buffer(Some(arguments.output), 0, 2);-- encoder.set_value(&descriptor.params, 3);- encoder.set_value(&descriptor.hidden_dim, 10);-- encoder.dispatch_threadgroups(descriptor.threadgroups, descriptor.threads_per_threadgroup);- Ok(())- }-}diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/mod.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/mod.rsdeleted file mode 100644index 6b041d0fc..000000000--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/mod.rs+++ /dev/null@@ -1,7 +0,0 @@-mod dispatch_descriptor;-mod kernel;-mod pipeline_configuration;-mod tile_configuration;--pub(crate) use dispatch_descriptor::DispatchDescriptor;-pub use kernel::Kernel as GemmKernel;diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/pipeline_configuration.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/pipeline_configuration.rsdeleted file mode 100644index fdf5fa36b..000000000--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/pipeline_configuration.rs+++ /dev/null@@ -1,11 +0,0 @@-use super::tile_configuration::TileConfiguration;-use crate::backends::metal::kernel::mlp::MlpActivationType;--#[derive(Debug, Clone, PartialEq, Eq, Hash)]-pub struct PipelineConfiguration {- pub tile: TileConfiguration,- pub weights_transposed: bool,- pub mn_aligned: bool,- pub k_aligned: bool,- pub activation: MlpActivationType,-}diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/steel_gemm_mlp_fused.h b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/steel_gemm_mlp_fused.hdeleted file mode 100644index 6f914713f..000000000--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/steel_gemm_mlp_fused.h+++ /dev/null@@ -1,221 +0,0 @@-#pragma once--#include <metal_stdlib>-#include <metal_simdgroup>-#include <metal_simdgroup_matrix>--#include "../../matmul/common/steel/gemm/gemm.h"-#include "../../matmul/common/steel/gemm/params.h"-#include "../../common/mlp_epilogue.h"--using namespace metal;--namespace steel {--///////////////////////////////////////////////////////////////////////////////-// MLP Fused GEMM Kernel-// Computes paired up/gate projections with fused activation for prefill path.-// Weight layout: B = [up_weights (hidden_dim cols), gate_weights (hidden_dim-// cols)] Output = up * activation(gate), written to first hidden_dim columns-///////////////////////////////////////////////////////////////////////////////--template <- typename T,- typename U,- int BM,- int BN,- int BK,- int WM,- int WN,- bool transpose_a,- bool transpose_b,- bool MN_aligned,- bool K_aligned,- typename AccumType = typename AccumHelper<T>::accum_type>-struct GEMMMlpFusedKernel {- STEEL_CONST short tgp_padding_a = 16 / sizeof(T);- STEEL_CONST short tgp_padding_b = 16 / sizeof(T);- STEEL_CONST short tgp_mem_size_a =- transpose_a ? BK * (BM + tgp_padding_a) : BM * (BK + tgp_padding_a);- // Double B threadgroup memory for up and gate- STEEL_CONST short tgp_mem_size_b =- 2 * (transpose_b ? BN * (BK + tgp_padding_b) : BK * (BN + tgp_padding_b));- STEEL_CONST short tgp_mem_size = tgp_mem_size_a + tgp_mem_size_b;-- STEEL_CONST short tgp_size = WM * WN * 32;-- using loader_a_t = BlockLoader<- T,- transpose_a ? BK : BM,- transpose_a ? BM : BK,- transpose_a ? BM + tgp_padding_a : BK + tgp_padding_a,- !transpose_a,- tgp_size>;- using loader_b_t = BlockLoader<- T,- transpose_b ? BN : BK,- transpose_b ? BK : BN,- transpose_b ? BK + tgp_padding_b : BN + tgp_padding_b,- transpose_b,- tgp_size>;- using mma_t = BlockMMA<- T,- U,- BM,- BN,- BK,- WM,- WN,- transpose_a,- transpose_b,- transpose_a ? BM + tgp_padding_a : BK + tgp_padding_a,- transpose_b ? BK + tgp_padding_b : BN + tgp_padding_b,- AccumType,- TransformNone<U, AccumType>>;-- static METAL_FUNC void run(- const device T* A [[buffer(0)]],- const device T* B [[buffer(1)]],- device U* D [[buffer(2)]],- const constant GEMMParams* params [[buffer(3)]],- const constant int& hidden_dim [[buffer(10)]],- threadgroup T* As [[threadgroup(0)]],- threadgroup T* Bs_up [[threadgroup(1)]],- uint simd_lane_id [[thread_index_in_simdgroup]],- uint simd_group_id [[simdgroup_index_in_threadgroup]],- uint3 tid [[threadgroup_position_in_grid]],- uint3 lid [[thread_position_in_threadgroup]]- ) {- (void)lid;-- const int tid_y = ((tid.y) << params->swizzle_log) +- ((tid.x) & ((1 << params->swizzle_log) - 1));- const int tid_x = (tid.x) >> params->swizzle_log;-- // Only process tiles within hidden_dim columns (output size)- int hidden_tiles = (hidden_dim + BN - 1) / BN;- if (hidden_tiles <= tid_x || params->tiles_m <= tid_y) {- return;- }-- threadgroup_barrier(mem_flags::mem_none);-- const int c_row = tid_y * BM;- const int c_col = tid_x * BN;- const size_t c_row_long = size_t(c_row);- const size_t c_col_long = size_t(c_col);-- // Pointer to A (input activations)- const device T* A_ptr =- A + (transpose_a ? c_row_long : c_row_long * params->lda);-- // Pointers to up and gate weights in B- // B layout: [up_weights | gate_weights] where each is hidden_dim columns- const device T* B_up =- B + (transpose_b ? c_col_long * params->ldb : c_col_long);- const device T* B_gate =- B_up + (transpose_b ? hidden_dim * params->ldb : hidden_dim);-- device U* D_ptr = D + c_row_long * params->ldd + c_col_long;-- // Threadgroup memory layout: [As, Bs_up, Bs_gate]- threadgroup T* Bs_gate = Bs_up + (tgp_mem_size_b / 2);-- // Loaders for A (shared) and B (up and gate)- thread loader_a_t- loader_a(A_ptr, params->lda, As, simd_group_id, simd_lane_id);- thread loader_b_t- loader_b_up(B_up, params->ldb, Bs_up, simd_group_id, simd_lane_id);- thread loader_b_t loader_b_gate(- B_gate,- params->ldb,- Bs_gate,- simd_group_id,- simd_lane_id- );-- // MMA operations for up and gate- thread mma_t mma_up(simd_group_id, simd_lane_id);- thread mma_t mma_gate(simd_group_id, simd_lane_id);-- int gemm_k_iterations = params->gemm_k_iterations_aligned;-- // Main compute loop- for (int k = 0; k < gemm_k_iterations; k++) {- threadgroup_barrier(mem_flags::mem_threadgroup);-- // Load A (shared for both up and gate)- if (MN_aligned) {- loader_a.load_unsafe();- } else {- short tgp_bm = min(short(BM), short(params->M - c_row));- short2 tile_dims_A =- transpose_a ? short2(tgp_bm, BK) : short2(BK, tgp_bm);- loader_a.load_safe(tile_dims_A);- }-- // Load B_up and B_gate- if (MN_aligned) {- loader_b_up.load_unsafe();- loader_b_gate.load_unsafe();- } else {- short tgp_bn = min(short(BN), short(hidden_dim - c_col));- short2 tile_dims_B =- transpose_b ? short2(BK, tgp_bn) : short2(tgp_bn, BK);- loader_b_up.load_safe(tile_dims_B);- loader_b_gate.load_safe(tile_dims_B);- }-- threadgroup_barrier(mem_flags::mem_threadgroup);-- // MMA for up and gate- mma_up.mma(As, Bs_up);- mma_gate.mma(As, Bs_gate);-- loader_a.next();- loader_b_up.next();- loader_b_gate.next();- }-- // Handle leftover K- if (!K_aligned) {- threadgroup_barrier(mem_flags::mem_threadgroup);-- int lbk = params->K - params->gemm_k_iterations_aligned * BK;- short2 tile_dims_A = transpose_a ? short2(BM, lbk) : short2(lbk, BM);- short2 tile_dims_B = transpose_b ? short2(lbk, BN) : short2(BN, lbk);-- loader_a.load_safe(tile_dims_A);- loader_b_up.load_safe(tile_dims_B);- loader_b_gate.load_safe(tile_dims_B);-- threadgroup_barrier(mem_flags::mem_threadgroup);-- mma_up.mma(As, Bs_up);- mma_gate.mma(As, Bs_gate);- }-- // Apply MLP fused epilogue: out = up * activation(gate)- // Access the Ctile elements and apply fusion- constexpr int kElemsPerTile = decltype(mma_up.Ctile)::kElemsPerTile;--#pragma unroll- for (short i = 0; i < kElemsPerTile; i++) {- float up_val = static_cast<float>(mma_up.Ctile.elems()[i]);- float gate_val = static_cast<float>(mma_gate.Ctile.elems()[i]);- float fused = mlp_fused_epilogue_f32(up_val, gate_val);- mma_up.Ctile.elems()[i] = static_cast<AccumType>(fused);- }-- // Store result- if (MN_aligned) {- mma_up.store_result(D_ptr, params->ldd);- } else {- short tgp_bm = min(short(BM), short(params->M - c_row));- short tgp_bn = min(short(BN), short(hidden_dim - c_col));- mma_up.store_result_safe(D_ptr, params->ldd, short2(tgp_bn, tgp_bm));- }- }-};--} // namespace steeldiff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/steel_gemm_mlp_fused.metal b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/steel_gemm_mlp_fused.metaldeleted file mode 100644index 850b63b6c..000000000--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/steel_gemm_mlp_fused.metal+++ /dev/null@@ -1,605 +0,0 @@-#include <metal_stdlib>--#include "../../common/utils.h"-#include "steel_gemm_mlp_fused.h"--using namespace metal;--///////////////////////////////////////////////////////////////////////////////-// MLP Fused GEMM Kernel Instantiations-// For prefill path: computes up * activation(gate) in a single pass-///////////////////////////////////////////////////////////////////////////////--#define instantiate_mlp_fused_gemm( \- tname, \- trans_a, \- trans_b, \- itype, \- otype, \- bm, \- bn, \- bk, \- wm, \- wn, \- aname, \- mn, \- k \-) \- template [[host_name( \- "steel_gemm_mlp_fused_" #tname "_" #itype "_" #otype "_bm" #bm "_bn" #bn \- "_bk" #bk "_wm" #wm "_wn" #wn "_" #aname "_MN_" #mn "_K_" #k \- )]] [[kernel]] void \- steel_gemm_mlp_fused< \- itype, \- otype, \- bm, \- bn, \- bk, \- wm, \- wn, \- trans_a, \- trans_b, \- mn, \- k>( \- const device itype* A [[buffer(0)]], \- const device itype* B [[buffer(1)]], \- device otype* D [[buffer(2)]], \- const constant steel::GEMMParams* params [[buffer(3)]], \- const constant int& hidden_dim [[buffer(10)]], \- uint simd_lane_id [[thread_index_in_simdgroup]], \- uint simd_group_id [[simdgroup_index_in_threadgroup]], \- uint3 tid [[threadgroup_position_in_grid]], \- uint3 lid [[thread_position_in_threadgroup]] \- )--template <- typename T,- typename U,- int BM,- int BN,- int BK,- int WM,- int WN,- bool transpose_a,- bool transpose_b,- bool MN_aligned,- bool K_aligned>-[[kernel, max_total_threads_per_threadgroup(WM * WN * 32)]] void-steel_gemm_mlp_fused(- const device T* A [[buffer(0)]],- const device T* B [[buffer(1)]],- device U* D [[buffer(2)]],- const constant steel::GEMMParams* params [[buffer(3)]],- const constant int& hidden_dim [[buffer(10)]],- uint simd_lane_id [[thread_index_in_simdgroup]],- uint simd_group_id [[simdgroup_index_in_threadgroup]],- uint3 tid [[threadgroup_position_in_grid]],- uint3 lid [[thread_position_in_threadgroup]]-) {- using kernel_t = steel::GEMMMlpFusedKernel<- T,- U,- BM,- BN,- BK,- WM,- WN,- transpose_a,- transpose_b,- MN_aligned,- K_aligned>;-- // Allocate threadgroup memory- threadgroup T As[kernel_t::tgp_mem_size_a];- threadgroup T Bs[kernel_t::tgp_mem_size_b];-- kernel_t::run(- A,- B,- D,- params,- hidden_dim,- As,- Bs,- simd_lane_id,- simd_group_id,- tid,- lid- );-}--// Instantiate for common configurations-// transpose_a = false, transpose_b = true (weights are [K, 2*H], accessed as-// transposed)--// float16 configurations-instantiate_mlp_fused_gemm(- t,- false,- true,- half,- half,- 32,- 32,- 16,- 2,- 2,- align,- true,- true-);-instantiate_mlp_fused_gemm(- t,- false,- true,- half,- half,- 32,- 32,- 16,- 2,- 2,- align,- true,- false-);-instantiate_mlp_fused_gemm(- t,- false,- true,- half,- half,- 32,- 32,- 16,- 2,- 2,- align,- false,- true-);-instantiate_mlp_fused_gemm(- t,- false,- true,- half,- half,- 32,- 32,- 16,- 2,- 2,- align,- false,- false-);--instantiate_mlp_fused_gemm(- t,- false,- true,- half,- half,- 64,- 64,- 16,- 2,- 2,- align,- true,- true-);-instantiate_mlp_fused_gemm(- t,- false,- true,- half,- half,- 64,- 64,- 16,- 2,- 2,- align,- true,- false-);-instantiate_mlp_fused_gemm(- t,- false,- true,- half,- half,- 64,- 64,- 16,- 2,- 2,- align,- false,- true-);-instantiate_mlp_fused_gemm(- t,- false,- true,- half,- half,- 64,- 64,- 16,- 2,- 2,- align,- false,- false-);--// bfloat16 configurations-instantiate_mlp_fused_gemm(- t,- false,- true,- bfloat,- bfloat,- 32,- 32,- 16,- 2,- 2,- align,- true,- true-);-instantiate_mlp_fused_gemm(- t,- false,- true,- bfloat,- bfloat,- 32,- 32,- 16,- 2,- 2,- align,- true,- false-);-instantiate_mlp_fused_gemm(- t,- false,- true,- bfloat,- bfloat,- 32,- 32,- 16,- 2,- 2,- align,- false,- true-);-instantiate_mlp_fused_gemm(- t,- false,- true,- bfloat,- bfloat,- 32,- 32,- 16,- 2,- 2,- align,- false,- false-);--instantiate_mlp_fused_gemm(- t,- false,- true,- bfloat,- bfloat,- 64,- 64,- 16,- 2,- 2,- align,- true,- true-);-instantiate_mlp_fused_gemm(- t,- false,- true,- bfloat,- bfloat,- 64,- 64,- 16,- 2,- 2,- align,- true,- false-);-instantiate_mlp_fused_gemm(- t,- false,- true,- bfloat,- bfloat,- 64,- 64,- 16,- 2,- 2,- align,- false,- true-);-instantiate_mlp_fused_gemm(- t,- false,- true,- bfloat,- bfloat,- 64,- 64,- 16,- 2,- 2,- align,- false,- false-);--// float32 configurations-instantiate_mlp_fused_gemm(- t,- false,- true,- float,- float,- 32,- 32,- 16,- 2,- 2,- align,- true,- true-);-instantiate_mlp_fused_gemm(- t,- false,- true,- float,- float,- 32,- 32,- 16,- 2,- 2,- align,- true,- false-);-instantiate_mlp_fused_gemm(- t,- false,- true,- float,- float,- 32,- 32,- 16,- 2,- 2,- align,- false,- true-);-instantiate_mlp_fused_gemm(- t,- false,- true,- float,- float,- 32,- 32,- 16,- 2,- 2,- align,- false,- false-);--// Non-transposed B (weights are [2*H, K])-instantiate_mlp_fused_gemm(- n,- false,- false,- half,- half,- 32,- 32,- 16,- 2,- 2,- align,- true,- true-);-instantiate_mlp_fused_gemm(- n,- false,- false,- half,- half,- 32,- 32,- 16,- 2,- 2,- align,- true,- false-);-instantiate_mlp_fused_gemm(- n,- false,- false,- half,- half,- 32,- 32,- 16,- 2,- 2,- align,- false,- true-);-instantiate_mlp_fused_gemm(- n,- false,- false,- half,- half,- 32,- 32,- 16,- 2,- 2,- align,- false,- false-);--instantiate_mlp_fused_gemm(- n,- false,- false,- bfloat,- bfloat,- 32,- 32,- 16,- 2,- 2,- align,- true,- true-);-instantiate_mlp_fused_gemm(- n,- false,- false,- bfloat,- bfloat,- 32,- 32,- 16,- 2,- 2,- align,- true,- false-);-instantiate_mlp_fused_gemm(- n,- false,- false,- bfloat,- bfloat,- 32,- 32,- 16,- 2,- 2,- align,- false,- true-);-instantiate_mlp_fused_gemm(- n,- false,- false,- bfloat,- bfloat,- 32,- 32,- 16,- 2,- 2,- align,- false,- false-);--instantiate_mlp_fused_gemm(- n,- false,- false,- float,- float,- 32,- 32,- 16,- 2,- 2,- align,- true,- true-);-instantiate_mlp_fused_gemm(- n,- false,- false,- float,- float,- 32,- 32,- 16,- 2,- 2,- align,- true,- false-);-instantiate_mlp_fused_gemm(- n,- false,- false,- float,- float,- 32,- 32,- 16,- 2,- 2,- align,- false,- true-);-instantiate_mlp_fused_gemm(- n,- false,- false,- float,- float,- 32,- 32,- 16,- 2,- 2,- align,- false,- false-);diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/tile_configuration.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/tile_configuration.rsdeleted file mode 100644index 408008478..000000000--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/tile_configuration.rs+++ /dev/null@@ -1,31 +0,0 @@-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]-pub struct TileConfiguration {- pub block_rows: i32,- pub block_cols: i32,- pub block_depth: i32,-}--impl TileConfiguration {- pub fn new(- block_rows: i32,- block_cols: i32,- block_depth: i32,- ) -> Self {- Self {- block_rows,- block_cols,- block_depth,- }- }-}--pub fn select_tile_configuration(- batch: i32,- hidden_dim: i32,-) -> TileConfiguration {- if batch >= 64 && hidden_dim >= 64 {- TileConfiguration::new(64, 64, 16)- } else {- TileConfiguration::new(32, 32, 16)- }-}diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/dispatch_descriptor.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/dispatch_descriptor.rsdeleted file mode 100644index b560342ce..000000000--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/dispatch_descriptor.rs+++ /dev/null@@ -1,54 +0,0 @@-use super::pipeline_configuration::{PipelineConfiguration, select_configuration};-use crate::{- DataType,- backends::metal::{MTLContext, MTLError, MTLSize, kernel::mlp_fused::common::MlpFusedArguments},-};--#[derive(Debug, Clone)]-pub struct DispatchDescriptor {- pub(crate) pipeline_configuration: PipelineConfiguration,- pub(crate) input_dim: i32,- pub(crate) hidden_dim: i32,- pub(crate) weights_ld: i32,- pub(crate) vector_batch_stride: i64,- pub(crate) matrix_batch_stride: i64,- pub(crate) threadgroups: MTLSize,- pub(crate) threads_per_threadgroup: MTLSize,-}--impl DispatchDescriptor {- pub(crate) fn try_new(- _context: &MTLContext,- data_type: DataType,- arguments: &MlpFusedArguments,- ) -> Result<Option<Self>, MTLError> {- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {- return Err(MTLError::Generic(format!("Unsupported dtype for MLP fused GEMV: {data_type:?}")));- }-- if arguments.batch != 1 {- return Ok(None);- }-- let pipeline_configuration = select_configuration(arguments.hidden_dim, arguments.activation);-- let output_elements_per_threadgroup = pipeline_configuration.output_elements_per_threadgroup();- let threadgroup_count_x = ((arguments.hidden_dim as u32 + output_elements_per_threadgroup - 1)- / output_elements_per_threadgroup) as u64;- let threadgroup_count_z = arguments.batch_count.max(1) as u64;-- let threadgroups = MTLSize::new(threadgroup_count_x as usize, 1, threadgroup_count_z as usize);- let threads_per_threadgroup = pipeline_configuration.threads_per_threadgroup();-- Ok(Some(Self {- pipeline_configuration,- input_dim: arguments.input_dim,- hidden_dim: arguments.hidden_dim,- weights_ld: arguments.ldb,- vector_batch_stride: arguments.input_dim as i64,- matrix_batch_stride: 0,- threadgroups,- threads_per_threadgroup,- }))- }-}diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/gemv_mlp_fused.metal b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/gemv_mlp_fused.metaldeleted file mode 100644index a7a13fee1..000000000--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/gemv_mlp_fused.metal+++ /dev/null@@ -1,284 +0,0 @@-#include <metal_simdgroup>-#include <metal_stdlib>--#include "../../common/utils.h"-#include "../../common/mlp_epilogue.h"--#include "../../matmul/common/steel/utils.h"--using namespace metal;--///////////////////////////////////////////////////////////////////////////////-/// MLP Fused GEMV - Paired Up/Gate Computation with Activation Fusion-///////////////////////////////////////////////////////////////////////////////--#define MTL_CONST static constant constexpr const--// MLP fused GEMV kernel for decode path (M=1).-// Computes paired up and gate projections, then applies: out = up *-// activation(gate) Output size is hidden_dim (half of the weight matrix-// columns).-template <- typename T,- const int BM, /* Threadgroup rows (in simdgroups) */- const int BN, /* Threadgroup cols (in simdgroups) */- const int SM, /* Simdgroup rows (in threads) */- const int SN, /* Simdgroup cols (in threads) */- const int TM, /* Thread rows (in elements) */- const int TN> /* Thread cols (in elements) */-struct GEMVMlpFusedKernel {- using AccT = float;-- MTL_CONST int threadsM = BM * SM;- MTL_CONST int threadsN = BN * SN;- MTL_CONST int blockM = threadsM * TM;- MTL_CONST int blockN = threadsN * TN;-- static_assert(SM * SN == 32, "simdgroup can only have 32 threads");-- MTL_CONST short tgp_mem_size = BN > 1 ? 2 * BN*(blockM + TM) : 0;- MTL_CONST bool needs_tgp_reduction = BN > 1;-- template <typename U = T>- static METAL_FUNC void load_unsafe(- const device T* src,- thread U dst[TN],- const int src_offset = 0- ) {- MTL_PRAGMA_UNROLL- for (int tn = 0; tn < TN; tn++) {- dst[tn] = static_cast<U>(src[src_offset + tn]);- }- }-- template <typename U = T>- static METAL_FUNC void load_safe(- const device T* src,- thread U dst[TN],- const int src_offset = 0,- const int src_size = TN- ) {- if (src_offset + TN <= src_size) {- MTL_PRAGMA_UNROLL- for (int tn = 0; tn < TN; tn++) {- dst[tn] = static_cast<U>(src[src_offset + tn]);- }- } else {- MTL_PRAGMA_UNROLL- for (int tn = 0; tn < TN; tn++) {- dst[tn] = src_offset + tn < src_size- ? static_cast<U>(src[src_offset + tn])- : U(0);- }- }- }-- static METAL_FUNC void run(- const device T* mat [[buffer(0)]],- const device T* in_vec [[buffer(1)]],- device T* out_vec [[buffer(3)]],- const int in_vec_size,- const int hidden_dim, // Output size (half of weight matrix rows)- const int matrix_ld,- threadgroup AccT* tgp_memory,- uint3 tid [[threadgroup_position_in_grid]],- uint3 lid [[thread_position_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]- ) {- (void)lid;-- // Thread local accumulation results for up and gate- thread AccT result_up[TM] = {0};- thread AccT result_gate[TM] = {0};- thread T inter_up[TN];- thread T inter_gate[TN];- thread AccT v_coeff[TN];-- const int thrM = SN != 32 ? simd_lid / SN : 0;- const int thrN = SN != 32 ? simd_lid % SN : int(simd_lid);-- const int sgN = BN != 1 ? (simd_gid % BN) : 0;- const int simdM = BN != 1 ? SM * (simd_gid / BN) : int(SM * simd_gid);- const int simdN = BN != 1 ? SN * (simd_gid % BN) : 0;-- int bm = (simdM + thrM) * TM;- int bn = (simdN + thrN) * TN;-- // Block position - outputs to hidden_dim elements- int out_row = tid.x * blockM + bm;-- if (out_row >= hidden_dim)- return;-- out_row = out_row + TM <= hidden_dim ? out_row : hidden_dim - TM;-- // Pointers to up and gate rows in the weight matrix- // Weight layout: [up_0, up_1, ..., up_H-1, gate_0, gate_1, ..., gate_H-1]- const device T* mat_up = mat + out_row * matrix_ld;- const device T* mat_gate = mat + (out_row + hidden_dim) * matrix_ld;-- constexpr const uniform<int> loop_stride = make_uniform(blockN);- const uniform<int> in_size = make_uniform(in_vec_size);- const uniform<int> n_iter = in_size / loop_stride;- const uniform<int> last_iter = loop_stride * n_iter;- const uniform<int> leftover = in_size - last_iter;-- // Main loop - compute both up and gate dot products- for (int i = 0; i < n_iter; ++i) {- load_unsafe<AccT>(in_vec, v_coeff, bn);-- int mat_offset = 0;- MTL_PRAGMA_UNROLL- for (int tm = 0; tm < TM; tm++) {- load_unsafe(mat_up, inter_up, mat_offset + bn);- load_unsafe(mat_gate, inter_gate, mat_offset + bn);-- MTL_PRAGMA_UNROLL- for (int tn = 0; tn < TN; tn++) {- result_up[tm] += inter_up[tn] * v_coeff[tn];- result_gate[tm] += inter_gate[tn] * v_coeff[tn];- }-- mat_offset += matrix_ld;- }-- bn += blockN;- }-- // Handle leftover- if (leftover > 0) {- load_safe<AccT>(in_vec, v_coeff, bn, in_vec_size);-- MTL_PRAGMA_UNROLL- for (int tm = 0; tm < TM; tm++) {- load_safe(&mat_up[tm * matrix_ld], inter_up, bn, in_vec_size);- load_safe(&mat_gate[tm * matrix_ld], inter_gate, bn, in_vec_size);-- MTL_PRAGMA_UNROLL- for (int tn = 0; tn < TN; tn++) {- result_up[tm] += inter_up[tn] * v_coeff[tn];- result_gate[tm] += inter_gate[tn] * v_coeff[tn];- }- }- }-- // Simdgroup reduction for both up and gate- MTL_PRAGMA_UNROLL- for (int tm = 0; tm < TM; tm++) {- MTL_PRAGMA_UNROLL- for (ushort sn = (SN / 2); sn >= 1; sn >>= 1) {- result_up[tm] += simd_shuffle_down(result_up[tm], sn);- result_gate[tm] += simd_shuffle_down(result_gate[tm], sn);- }- }-- // Threadgroup reduction if needed- if (needs_tgp_reduction) {- threadgroup AccT* tgp_results_up = tgp_memory + sgN * (blockM + TM) + bm;- threadgroup AccT* tgp_results_gate =- tgp_memory + BN * (blockM + TM) + sgN * (blockM + TM) + bm;-- if (thrN == 0) {- MTL_PRAGMA_UNROLL- for (int tm = 0; tm < TM; tm++) {- tgp_results_up[tm] = result_up[tm];- tgp_results_gate[tm] = result_gate[tm];- }-- threadgroup_barrier(mem_flags::mem_none);-- if (sgN == 0) {- MTL_PRAGMA_UNROLL- for (int sgn = 1; sgn < BN; sgn++) {- MTL_PRAGMA_UNROLL- for (int tm = 0; tm < TM; tm++) {- result_up[tm] += tgp_results_up[sgn * (blockM + TM) + tm];- result_gate[tm] += tgp_results_gate[sgn * (blockM + TM) + tm];- }- }- }- }- }-- // Apply MLP fused epilogue and write output- if (simdN == 0 && thrN == 0) {- MTL_PRAGMA_UNROLL- for (int tm = 0; tm < TM; tm++) {- float fused_result =- mlp_fused_epilogue_f32(result_up[tm], result_gate[tm]);- out_vec[out_row + tm] = static_cast<T>(fused_result);- }- }- }-};--template <- typename T,- const int BM,- const int BN,- const int SM,- const int SN,- const int TM,- const int TN>-[[kernel, max_total_threads_per_threadgroup(BM * BN * 32)]] void gemv_mlp_fused(- const device T* mat [[buffer(0)]],- const device T* in_vec [[buffer(1)]],- device T* out_vec [[buffer(3)]],- const constant int& in_vec_size [[buffer(4)]],- const constant int& hidden_dim- [[buffer(5)]], // Output dimension (half of weight rows)- const constant int& matrix_ld [[buffer(6)]],- const constant int64_t& vector_batch_stride [[buffer(11)]],- const constant int64_t& matrix_batch_stride [[buffer(12)]],- uint3 tid [[threadgroup_position_in_grid]],- uint3 lid [[thread_position_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]-) {- using gemv_kernel = GEMVMlpFusedKernel<T, BM, BN, SM, SN, TM, TN>;- threadgroup typename gemv_kernel::AccT tgp_memory- [gemv_kernel::tgp_mem_size == 0 ? 1 : gemv_kernel::tgp_mem_size];-- // Batch offset- in_vec += tid.z * vector_batch_stride;- mat += tid.z * matrix_batch_stride;- out_vec += tid.z * hidden_dim;-- gemv_kernel::run(- mat,- in_vec,- out_vec,- in_vec_size,- hidden_dim,- matrix_ld,- gemv_kernel::tgp_mem_size == 0 ? nullptr : tgp_memory,- tid,- lid,- simd_gid,- simd_lid- );-}--#define instantiate_gemv_mlp_fused_helper(name, itype, bm, bn, sm, sn, tm, tn) \- instantiate_kernel( \- "gemv_mlp_fused_" #name "_bm" #bm "_bn" #bn "_sm" #sm "_sn" #sn \- "_tm" #tm "_tn" #tn, \- gemv_mlp_fused, \- itype, \- bm, \- bn, \- sm, \- sn, \- tm, \- tn \- )--// clang-format off-#define instantiate_gemv_mlp_fused_blocks(name, itype) \- instantiate_gemv_mlp_fused_helper(name, itype, 4, 1, 1, 32, 4, 4) \- instantiate_gemv_mlp_fused_helper(name, itype, 8, 1, 1, 32, 4, 4) // clang-format on--instantiate_gemv_mlp_fused_blocks(float32, float);-instantiate_gemv_mlp_fused_blocks(float16, half);-instantiate_gemv_mlp_fused_blocks(bfloat16, bfloat16_t);diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/kernel.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/kernel.rsdeleted file mode 100644index 3cc73b9da..000000000--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/kernel.rs+++ /dev/null@@ -1,95 +0,0 @@-use std::collections::HashMap;--use super::{DispatchDescriptor, pipeline_configuration::PipelineConfiguration};-use crate::{- DataType,- backends::metal::{- ComputeEncoderSetValue, FunctionConstantValuesSetValue, MTLComputeCommandEncoder, MTLComputePipelineState,- MTLContext, MTLError, MTLFunctionConstantValues, ProtocolObject, Retained,- kernel::mlp_fused::common::MlpFusedArguments,- },-};--fn kernel_name(- data_type: DataType,- configuration: &PipelineConfiguration,-) -> Result<String, MTLError> {- let dtype_name = match data_type {- DataType::F16 => "float16",- DataType::BF16 => "bfloat16",- DataType::F32 => "float32",- _ => {- return Err(MTLError::Generic(format!("Unsupported data type for MLP fused GEMV: {data_type:?}")));- },- };-- Ok(format!(- "gemv_mlp_fused_{}_bm{}_bn{}_sm{}_sn{}_tm{}_tn{}",- dtype_name,- configuration.threadgroup_rows,- configuration.threadgroup_cols,- configuration.threads_per_simdgroup_row,- configuration.threads_per_simdgroup_col,- configuration.elements_per_thread_row,- configuration.elements_per_thread_col,- ))-}--pub struct Kernel {- data_type: DataType,- pipelines: HashMap<PipelineConfiguration, Retained<ProtocolObject<dyn MTLComputePipelineState>>>,-}--impl Kernel {- pub fn new(data_type: DataType) -> Result<Self, MTLError> {- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {- return Err(MTLError::Generic(format!("Unsupported dtype for MLP fused GEMV: {data_type:?}")));- }- Ok(Self {- data_type,- pipelines: HashMap::new(),- })- }-- fn get_or_compile_pipeline(- &mut self,- context: &MTLContext,- configuration: &PipelineConfiguration,- ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError> {- if !self.pipelines.contains_key(configuration) {- let kernel_name = kernel_name(self.data_type, configuration)?;-- let function_constants = MTLFunctionConstantValues::new();- let activation_val = configuration.activation as u32;- function_constants.set_value(&activation_val, 52);-- let pipeline_state = context.compute_pipeline_state(&kernel_name, Some(&function_constants))?;- self.pipelines.insert(*configuration, pipeline_state);- }- Ok(self.pipelines.get(configuration).unwrap())- }-- pub(crate) fn encode_descriptor(- &mut self,- context: &MTLContext,- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- arguments: &MlpFusedArguments,- descriptor: &DispatchDescriptor,- ) -> Result<(), MTLError> {- let pipeline_state = self.get_or_compile_pipeline(context, &descriptor.pipeline_configuration)?;- encoder.set_compute_pipeline_state(pipeline_state);-- encoder.set_buffer(Some(arguments.weights), 0, 0);- encoder.set_buffer(Some(arguments.input), arguments.input_offset as usize, 1);- encoder.set_buffer(Some(arguments.output), 0, 3);-- encoder.set_value(&descriptor.input_dim, 4);- encoder.set_value(&descriptor.hidden_dim, 5);- encoder.set_value(&descriptor.weights_ld, 6);- encoder.set_value(&descriptor.vector_batch_stride, 11);- encoder.set_value(&descriptor.matrix_batch_stride, 12);-- encoder.dispatch_threadgroups(descriptor.threadgroups, descriptor.threads_per_threadgroup);- Ok(())- }-}diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/mod.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/mod.rsdeleted file mode 100644index 5ee340249..000000000--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/mod.rs+++ /dev/null@@ -1,6 +0,0 @@-mod dispatch_descriptor;-mod kernel;-mod pipeline_configuration;--pub(crate) use dispatch_descriptor::DispatchDescriptor;-pub use kernel::Kernel as GemvKernel;diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/pipeline_configuration.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/pipeline_configuration.rsdeleted file mode 100644index 91130ed54..000000000--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/pipeline_configuration.rs+++ /dev/null@@ -1,43 +0,0 @@-use crate::backends::metal::{MTLSize, kernel::mlp::MlpActivationType};--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]-pub struct PipelineConfiguration {- pub threadgroup_rows: u32,- pub threadgroup_cols: u32,- pub threads_per_simdgroup_row: u32,- pub threads_per_simdgroup_col: u32,- pub elements_per_thread_row: u32,- pub elements_per_thread_col: u32,- pub activation: MlpActivationType,-}--impl PipelineConfiguration {- pub fn output_elements_per_threadgroup(&self) -> u32 {- self.threadgroup_rows * self.threads_per_simdgroup_row * self.elements_per_thread_row- }-- pub fn threads_per_threadgroup(&self) -> MTLSize {- MTLSize::new(32, self.threadgroup_cols as usize, self.threadgroup_rows as usize)- }-}--pub fn select_configuration(- hidden_dim: i32,- activation: MlpActivationType,-) -> PipelineConfiguration {- let threadgroup_rows = if hidden_dim >= 4096 {- 8- } else {- 4- };-- PipelineConfiguration {- threadgroup_rows,- threadgroup_cols: 1,- threads_per_simdgroup_row: 1,- threads_per_simdgroup_col: 32,- elements_per_thread_row: 4,- elements_per_thread_col: 4,- activation,- }-}diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/kernel.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/kernel.rsdeleted file mode 100644index 1c3eb6ad9..000000000--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/kernel.rs+++ /dev/null@@ -1,93 +0,0 @@-use super::{- common::MlpFusedArguments,- dispatch_descriptor::{MlpFusedDispatchDescriptor, choose_dispatch_descriptor},- gemm::GemmKernel,- gemv::GemvKernel,- split_k::SplitKKernel,-};-use crate::{- DataType,- backends::metal::{MTLComputeCommandEncoder, MTLContext, MTLError, ProtocolObject},-};--pub struct MlpFusedKernel {- data_type: DataType,- weights_transposed: bool,- gemm: Option<GemmKernel>,- gemv: Option<GemvKernel>,- splitk: Option<SplitKKernel>,-}--impl MlpFusedKernel {- pub fn new(- data_type: DataType,- weights_transposed: bool,- ) -> Result<Self, MTLError> {- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {- return Err(MTLError::Generic(format!("Unsupported dtype for MlpFusedKernel: {data_type:?}")));- }-- Ok(Self {- data_type,- weights_transposed,- gemm: None,- gemv: None,- splitk: None,- })- }-- fn get_or_create_gemm(&mut self) -> Result<&mut GemmKernel, MTLError> {- if self.gemm.is_none() {- self.gemm = Some(GemmKernel::new(self.data_type, self.weights_transposed)?);- }- Ok(self.gemm.as_mut().unwrap())- }-- fn get_or_create_gemv(&mut self) -> Result<&mut GemvKernel, MTLError> {- if self.gemv.is_none() {- self.gemv = Some(GemvKernel::new(self.data_type)?);- }- Ok(self.gemv.as_mut().unwrap())- }-- fn get_or_create_splitk(&mut self) -> Result<&mut SplitKKernel, MTLError> {- if self.splitk.is_none() {- self.splitk = Some(SplitKKernel::new(self.data_type)?);- }- Ok(self.splitk.as_mut().unwrap())- }-- fn encode_dispatch_descriptor(- &mut self,- context: &MTLContext,- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- arguments: &MlpFusedArguments,- descriptor: &MlpFusedDispatchDescriptor,- ) -> Result<(), MTLError> {- match descriptor {- MlpFusedDispatchDescriptor::Gemv(descriptor) => {- let gemv = self.get_or_create_gemv()?;- gemv.encode_descriptor(context, encoder, arguments, descriptor)- },- MlpFusedDispatchDescriptor::SplitK(descriptor) => {- let splitk = self.get_or_create_splitk()?;- splitk.encode_descriptor(context, encoder, arguments, descriptor)- },- MlpFusedDispatchDescriptor::Gemm(descriptor) => {- let gemm = self.get_or_create_gemm()?;- gemm.encode_descriptor(context, encoder, arguments, descriptor)- },- }- }-- pub fn encode(- &mut self,- context: &MTLContext,- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- arguments: &MlpFusedArguments,- ) -> Result<(), MTLError> {- let descriptor = choose_dispatch_descriptor(context, self.data_type, self.weights_transposed, arguments)?;-- self.encode_dispatch_descriptor(context, encoder, arguments, &descriptor)- }-}diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/mod.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/mod.rsdeleted file mode 100644index f503655f3..000000000--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/mod.rs+++ /dev/null@@ -1,13 +0,0 @@-pub mod common;-mod dispatch_descriptor;-mod gemm;-mod gemv;-mod kernel;-mod split_k;--pub use common::MlpFusedArguments;-pub use dispatch_descriptor::{MlpFusedKernelVariant, determine_kernel_variant};-pub use gemm::GemmKernel;-pub use gemv::GemvKernel;-pub use kernel::MlpFusedKernel;-pub use split_k::SplitKKernel;diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/dispatch_descriptor.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/dispatch_descriptor.rsdeleted file mode 100644index 574efb5b9..000000000--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/dispatch_descriptor.rs+++ /dev/null@@ -1,129 +0,0 @@-use super::{pipeline_configuration::PipelineConfiguration, tile_configuration::select_tile_configuration};-use crate::{- DataType,- backends::metal::{- MTLContext, MTLError, MTLSize,- kernel::{matmul::common::GEMMSpiltKMlpFusedParams, mlp_fused::common::MlpFusedArguments},- },-};--#[derive(Debug, Clone)]-pub struct DispatchDescriptor {- pub(crate) pipeline_configuration: PipelineConfiguration,- pub(crate) params: GEMMSpiltKMlpFusedParams,- pub(crate) partition_count: i32,- pub(crate) output_elements_per_partition: i32,- pub(crate) accumulator_bytes: usize,- pub(crate) ldd: i32,- pub(crate) partial_threadgroups: MTLSize,- pub(crate) partial_threads_per_threadgroup: MTLSize,- pub(crate) accum_total_threads: MTLSize,- pub(crate) accum_threads_per_threadgroup: MTLSize,-}--impl DispatchDescriptor {- pub(crate) fn try_new(- _context: &MTLContext,- data_type: DataType,- arguments: &MlpFusedArguments,- ) -> Result<Option<Self>, MTLError> {- if !matches!(data_type, DataType::F16 | DataType::BF16) {- return Ok(None);- }-- if !should_use_splitk(arguments.batch, arguments.hidden_dim, arguments.input_dim) {- return Ok(None);- }-- let batch = arguments.batch;- let hidden_dim = arguments.hidden_dim;- let input_dim = arguments.input_dim;-- let tile = select_tile_configuration(batch, hidden_dim);- let partition_count = compute_partition_count(input_dim);- let mn_aligned = (batch % tile.tile_rows) == 0 && (hidden_dim % tile.tile_cols) == 0;- let k_aligned = (input_dim % tile.tile_depth) == 0;-- let pipeline_configuration = PipelineConfiguration {- tile,- mn_aligned,- k_aligned,- activation: arguments.activation,- };-- let tile_count_m = (batch + tile.tile_rows - 1) / tile.tile_rows;- let tile_count_n = (hidden_dim + tile.tile_cols - 1) / tile.tile_cols;-- let gemm_k_iterations = (input_dim / tile.tile_depth) / partition_count;- let k_elements_per_partition = gemm_k_iterations * tile.tile_depth;- let output_elements_per_partition = batch * hidden_dim;-- let accumulator_element_count = partition_count * output_elements_per_partition;- let accumulator_bytes = (accumulator_element_count as usize) * std::mem::size_of::<f32>();-- let params = GEMMSpiltKMlpFusedParams {- M: batch,- N: hidden_dim,- K: input_dim,- lda: arguments.lda,- ldb: arguments.ldb,- ldc: hidden_dim,- tiles_n: tile_count_n,- tiles_m: tile_count_m,- split_k_partitions: partition_count,- split_k_partition_stride: output_elements_per_partition,- split_k_partition_size: k_elements_per_partition,- gemm_k_iterations_aligned: gemm_k_iterations,- hidden_dim,- };-- let partial_threads_per_threadgroup =- MTLSize::new(32, tile.warps_per_col as usize, tile.warps_per_row as usize);- let partial_threadgroups = MTLSize::new(tile_count_n as usize, tile_count_m as usize, partition_count as usize);-- let accum_total_threads = MTLSize::new(hidden_dim as usize, batch as usize, 1);- let accum_threads_per_threadgroup = MTLSize::new(16.min(hidden_dim as usize), 16.min(batch as usize), 1);-- Ok(Some(Self {- pipeline_configuration,- params,- partition_count,- output_elements_per_partition,- accumulator_bytes,- ldd: arguments.ldd,- partial_threadgroups,- partial_threads_per_threadgroup,- accum_total_threads,- accum_threads_per_threadgroup,- }))- }-}--fn compute_partition_count(k: i32) -> i32 {- let k_tiles = k / 16;- if k_tiles < 16 {- 2- } else if k_tiles < 32 {- 4- } else if k_tiles < 64 {- 8- } else {- 16- }-}--pub fn should_use_splitk(- batch: i32,- hidden_dim: i32,- input_dim: i32,-) -> bool {- if batch != 1 {- return false;- }- if hidden_dim <= 0 || input_dim <= 0 {- return false;- }- let n_tiles = hidden_dim / 16;- let k_tiles = input_dim / 16;- n_tiles <= 32 && k_tiles >= 8-}diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/kernel.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/kernel.rsdeleted file mode 100644index 634bfe559..000000000--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/kernel.rs+++ /dev/null@@ -1,170 +0,0 @@-use std::collections::HashMap;--use super::{DispatchDescriptor, pipeline_configuration::PipelineConfiguration};-use crate::{- DataType,- backends::{- common::Context,- metal::{- ComputeEncoderSetValue, FunctionConstantValuesSetValue, MTLBuffer, MTLComputeCommandEncoder,- MTLComputePipelineState, MTLContext, MTLError, MTLFunctionConstantValues, ProtocolObject, Retained,- kernel::{mlp::MlpActivationType, mlp_fused::common::MlpFusedArguments},- },- },-};--pub struct Kernel {- data_type: DataType,- partial_pipelines: HashMap<PipelineConfiguration, Retained<ProtocolObject<dyn MTLComputePipelineState>>>,- accum_pipelines: HashMap<MlpActivationType, Retained<ProtocolObject<dyn MTLComputePipelineState>>>,- up_accumulator_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>>,- gate_accumulator_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>>,- accumulator_buffer_bytes: usize,-}--impl Kernel {- pub fn new(data_type: DataType) -> Result<Self, MTLError> {- if !matches!(data_type, DataType::F16 | DataType::BF16) {- return Err(MTLError::Generic(format!("Unsupported dtype for MLP fused Split-K: {data_type:?}")));- }- Ok(Self {- data_type,- partial_pipelines: HashMap::new(),- accum_pipelines: HashMap::new(),- up_accumulator_buffer: None,- gate_accumulator_buffer: None,- accumulator_buffer_bytes: 0,- })- }-- fn steel_type_name(&self) -> Result<&'static str, MTLError> {- match self.data_type {- DataType::F16 => Ok("float16"),- DataType::BF16 => Ok("bfloat16"),- _ => Err(MTLError::Generic(format!("Unsupported dtype for MLP Fused Split-K: {:?}", self.data_type))),- }- }-- fn partial_kernel_name(- &self,- configuration: &PipelineConfiguration,- ) -> Result<String, MTLError> {- let in_name = self.steel_type_name()?;- let mn_tag = if configuration.mn_aligned {- "taligned"- } else {- "naligned"- };- let k_tag = if configuration.k_aligned {- "taligned"- } else {- "naligned"- };- Ok(format!(- "steel_gemm_splitk_mlp_fused_nt_{}_float32_bm{}_bn{}_bk{}_wm{}_wn{}_MN_{}_K_{}",- in_name,- configuration.tile.tile_rows,- configuration.tile.tile_cols,- configuration.tile.tile_depth,- configuration.tile.warps_per_row,- configuration.tile.warps_per_col,- mn_tag,- k_tag,- ))- }-- fn accum_kernel_name(&self) -> Result<String, MTLError> {- let out_name = self.steel_type_name()?;- Ok(format!("steel_gemm_splitk_mlp_fused_accum_{}_float32", out_name))- }-- fn get_partial_pipeline(- &mut self,- context: &MTLContext,- configuration: &PipelineConfiguration,- ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError> {- if !self.partial_pipelines.contains_key(configuration) {- let kernel_name = self.partial_kernel_name(configuration)?;- let pipeline_state = context.compute_pipeline_state(&kernel_name, None)?;- self.partial_pipelines.insert(configuration.clone(), pipeline_state);- }- Ok(self.partial_pipelines.get(configuration).unwrap())- }-- fn get_accum_pipeline(- &mut self,- context: &MTLContext,- activation: MlpActivationType,- ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError> {- if !self.accum_pipelines.contains_key(&activation) {- let kernel_name = self.accum_kernel_name()?;- let function_constants = MTLFunctionConstantValues::new();- let activation_val = activation as u32;- function_constants.set_value(&activation_val, 52);- let pipeline_state = context.compute_pipeline_state(&kernel_name, Some(&function_constants))?;- self.accum_pipelines.insert(activation, pipeline_state);- }- Ok(self.accum_pipelines.get(&activation).unwrap())- }-- fn ensure_accumulator_buffers(- &mut self,- context: &MTLContext,- required_bytes: usize,- ) {- if required_bytes <= self.accumulator_buffer_bytes- && self.up_accumulator_buffer.is_some()- && self.gate_accumulator_buffer.is_some()- {- return;- }- self.up_accumulator_buffer = context.create_buffer(required_bytes).ok();- self.gate_accumulator_buffer = context.create_buffer(required_bytes).ok();- self.accumulator_buffer_bytes = required_bytes;- }-- pub(crate) fn encode_descriptor(- &mut self,- context: &MTLContext,- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,- arguments: &MlpFusedArguments,- descriptor: &DispatchDescriptor,- ) -> Result<(), MTLError> {- self.ensure_accumulator_buffers(context, descriptor.accumulator_bytes);- self.get_partial_pipeline(context, &descriptor.pipeline_configuration)?;- self.get_accum_pipeline(context, descriptor.pipeline_configuration.activation)?;-- let partial_pipeline_state = self- .partial_pipelines- .get(&descriptor.pipeline_configuration)- .expect("Partial pipeline must be initialized");- let accum_pipeline_state = self- .accum_pipelines- .get(&descriptor.pipeline_configuration.activation)- .expect("Accum pipeline must be initialized");- let up_accumulator_buffer =- self.up_accumulator_buffer.as_ref().expect("Up accumulator buffer must be initialized");- let gate_accumulator_buffer =- self.gate_accumulator_buffer.as_ref().expect("Gate accumulator buffer must be initialized");-- encoder.set_compute_pipeline_state(partial_pipeline_state);- encoder.set_buffer(Some(arguments.input), arguments.input_offset as usize, 0);- encoder.set_buffer(Some(arguments.weights), 0, 1);- encoder.set_buffer(Some(up_accumulator_buffer), 0, 2);- encoder.set_buffer(Some(gate_accumulator_buffer), 0, 3);- encoder.set_value(&descriptor.params, 4);- encoder.dispatch_threadgroups(descriptor.partial_threadgroups, descriptor.partial_threads_per_threadgroup);-- encoder.set_compute_pipeline_state(accum_pipeline_state);- encoder.set_buffer(Some(up_accumulator_buffer), 0, 0);- encoder.set_buffer(Some(gate_accumulator_buffer), 0, 1);- encoder.set_buffer(Some(arguments.output), 0, 2);- encoder.set_value(&descriptor.partition_count, 3);- encoder.set_value(&descriptor.output_elements_per_partition, 4);- encoder.set_value(&descriptor.ldd, 5);-- encoder.dispatch_threads(descriptor.accum_total_threads, descriptor.accum_threads_per_threadgroup);-- Ok(())- }-}diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/mod.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/mod.rsdeleted file mode 100644index efefcd2bd..000000000--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/mod.rs+++ /dev/null@@ -1,7 +0,0 @@-mod dispatch_descriptor;-mod kernel;-mod pipeline_configuration;-mod tile_configuration;--pub(crate) use dispatch_descriptor::DispatchDescriptor;-pub use kernel::Kernel as SplitKKernel;diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/pipeline_configuration.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/pipeline_configuration.rsdeleted file mode 100644index 4f21c8f5e..000000000--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/pipeline_configuration.rs+++ /dev/null@@ -1,10 +0,0 @@-use super::tile_configuration::TileConfiguration;-use crate::backends::metal::kernel::mlp::MlpActivationType;--#[derive(Debug, Clone, PartialEq, Eq, Hash)]-pub struct PipelineConfiguration {- pub tile: TileConfiguration,- pub mn_aligned: bool,- pub k_aligned: bool,- pub activation: MlpActivationType,-}diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/steel_gemm_splitk_mlp_fused.h b/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/steel_gemm_splitk_mlp_fused.hdeleted file mode 100644index 8c0522fc9..000000000--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/steel_gemm_splitk_mlp_fused.h+++ /dev/null@@ -1,235 +0,0 @@-#pragma once--#include <metal_stdlib>-#include <metal_simdgroup>-#include <metal_simdgroup_matrix>--#include "../../matmul/common/steel/gemm/gemm.h"-#include "matmul.h"-#include "../../common/mlp_epilogue.h"--using namespace metal;-using namespace steel;--///////////////////////////////////////////////////////////////////////////////-// MLP Fused Split-K GEMM Kernel-// Computes paired up/gate projections with fused activation using split-K.-///////////////////////////////////////////////////////////////////////////////--template <- typename T,- typename U,- int BM,- int BN,- int BK,- int WM,- int WN,- bool transpose_a,- bool transpose_b,- bool MN_aligned,- bool K_aligned>-[[kernel, max_total_threads_per_threadgroup(WM * WN * 32)]] void-gemm_splitk_mlp_fused(- const device T* A [[buffer(0)]],- const device T* B [[buffer(1)]],- device U* C [[buffer(2)]],- const constant uzu::matmul::GEMMSpiltKMlpFusedParams* params [[buffer(3)]],- uint simd_lane_id [[thread_index_in_simdgroup]],- uint simd_group_id [[simdgroup_index_in_threadgroup]],- uint3 tid [[threadgroup_position_in_grid]],- uint3 lid [[thread_position_in_threadgroup]]-) {- (void)lid;-- using gemm_kernel = GEMMKernel<- T,- U,- BM,- BN,- BK,- WM,- WN,- transpose_a,- transpose_b,- MN_aligned,- K_aligned>;- using loader_a_t = typename gemm_kernel::loader_a_t;- using loader_b_t = typename gemm_kernel::loader_b_t;- using mma_t = typename gemm_kernel::mma_t;-- threadgroup T As[gemm_kernel::tgp_mem_size_a];- threadgroup T Bs_up[gemm_kernel::tgp_mem_size_b];- threadgroup T Bs_gate[gemm_kernel::tgp_mem_size_b];-- const int tid_x = tid.x;- const int tid_y = tid.y;- const int tid_z = tid.z;-- // Only process tiles within hidden_dim columns- int hidden_tiles = (params->hidden_dim + BN - 1) / BN;- if (hidden_tiles <= tid_x || params->tiles_m <= tid_y) {- return;- }-- // Find block in A, B, C- const int c_row = tid_y * BM;- const int c_col = tid_x * BN;- const int k_start = params->split_k_partition_size * tid_z;-- const size_t c_row_long = size_t(c_row);- const size_t c_col_long = size_t(c_col);- const size_t k_start_long = size_t(k_start);-- // Pointers to A (input)- const device T* A_ptr =- A + (transpose_a ? (c_row_long + k_start_long * params->lda)- : (k_start_long + c_row_long * params->lda));-- // Pointers to B (up and gate weights)- // B layout: [up_weights | gate_weights] where each is hidden_dim columns- const device T* B_up =- B + (transpose_b ? (k_start_long + c_col_long * params->ldb)- : (c_col_long + k_start_long * params->ldb));- const device T* B_gate =- B_up +- (transpose_b ? params->hidden_dim * params->ldb : params->hidden_dim);-- // C output for partial sums (up and gate interleaved in split_k dimension)- // Layout: [up_partitions, gate_partitions] in the partition stride- device U* C_up = C + (size_t(params->split_k_partition_stride) * tid_z) +- (c_row_long * params->ldc + c_col_long);- device U* C_gate = C_up + (size_t(params->split_k_partition_stride) *- params->split_k_partitions);-- // Loaders- thread loader_a_t- loader_a(A_ptr, params->lda, As, simd_group_id, simd_lane_id);- thread loader_b_t- loader_b_up(B_up, params->ldb, Bs_up, simd_group_id, simd_lane_id);- thread loader_b_t- loader_b_gate(B_gate, params->ldb, Bs_gate, simd_group_id, simd_lane_id);-- // MMA operations- thread mma_t mma_up(simd_group_id, simd_lane_id);- thread mma_t mma_gate(simd_group_id, simd_lane_id);-- int gemm_k_iterations = params->gemm_k_iterations_aligned;-- short tgp_bm = min(BM, params->M - c_row);- short tgp_bn = min(BN, params->hidden_dim - c_col);- short leftover_bk = params->K % BK;-- // Main loop- for (int k = 0; k < gemm_k_iterations; k++) {- threadgroup_barrier(mem_flags::mem_threadgroup);-- if (MN_aligned) {- loader_a.load_unsafe();- loader_b_up.load_unsafe();- loader_b_gate.load_unsafe();- } else if (tgp_bm == BM && tgp_bn == BN) {- loader_a.load_unsafe();- loader_b_up.load_unsafe();- loader_b_gate.load_unsafe();- } else {- short2 tile_dims_A =- transpose_a ? short2(tgp_bm, BK) : short2(BK, tgp_bm);- short2 tile_dims_B =- transpose_b ? short2(BK, tgp_bn) : short2(tgp_bn, BK);- loader_a.load_safe(tile_dims_A);- loader_b_up.load_safe(tile_dims_B);- loader_b_gate.load_safe(tile_dims_B);- }-- threadgroup_barrier(mem_flags::mem_threadgroup);-- mma_up.mma(As, Bs_up);- mma_gate.mma(As, Bs_gate);-- loader_a.next();- loader_b_up.next();- loader_b_gate.next();- }-- threadgroup_barrier(mem_flags::mem_threadgroup);-- // Handle remaining K for last partition- if ((tid_z + 1) == (params->split_k_partitions)) {- int gemm_k_iter_remaining =- (params->K - (k_start + params->split_k_partition_size)) / BK;- for (int k = 0; k < gemm_k_iter_remaining; k++) {- threadgroup_barrier(mem_flags::mem_threadgroup);-- if (!K_aligned) {- short2 tile_dims_A = transpose_a ? short2(tgp_bm, leftover_bk)- : short2(leftover_bk, tgp_bm);- short2 tile_dims_B = transpose_b ? short2(leftover_bk, tgp_bn)- : short2(tgp_bn, leftover_bk);- loader_a.load_safe(tile_dims_A);- loader_b_up.load_safe(tile_dims_B);- loader_b_gate.load_safe(tile_dims_B);- } else {- loader_a.load_unsafe();- loader_b_up.load_unsafe();- loader_b_gate.load_unsafe();- }-- threadgroup_barrier(mem_flags::mem_threadgroup);-- mma_up.mma(As, Bs_up);- mma_gate.mma(As, Bs_gate);-- loader_a.next();- loader_b_up.next();- loader_b_gate.next();- }- }-- // Store partial results- if (MN_aligned || (tgp_bm == BM && tgp_bn == BN)) {- mma_up.store_result(C_up, params->ldc);- mma_gate.store_result(C_gate, params->ldc);- } else {- mma_up.store_result_safe(C_up, params->ldc, short2(tgp_bn, tgp_bm));- mma_gate.store_result_safe(C_gate, params->ldc, short2(tgp_bn, tgp_bm));- }-}--///////////////////////////////////////////////////////////////////////////////-// MLP Fused Split-K accumulation kernel-// Accumulates partial sums and applies MLP fused epilogue-///////////////////////////////////////////////////////////////////////////////--template <typename AccT, typename OutT>-[[kernel, max_total_threads_per_threadgroup(256)]] void-gemm_splitk_mlp_fused_accum(- const device AccT* C_split [[buffer(0)]],- device OutT* D [[buffer(1)]],- const constant int& k_partitions [[buffer(2)]],- const constant int& partition_stride [[buffer(3)]],- const constant int& ldd [[buffer(4)]],- uint2 gid [[thread_position_in_grid]]-) {- // Adjust pointers- D += gid.x + gid.y * size_t(ldd);- const device AccT* C_up = C_split + gid.x + gid.y * size_t(ldd);- const device AccT* C_gate = C_up + (size_t(partition_stride) * k_partitions);-- // Accumulate up and gate partial sums- AccT out_up = 0;- AccT out_gate = 0;-- size_t offset = 0;- for (int i = 0; i < k_partitions; i++) {- out_up += C_up[offset];- out_gate += C_gate[offset];- offset += partition_stride;- }-- // Apply MLP fused epilogue: out = up * activation(gate)- float fused = mlp_fused_epilogue_f32(- static_cast<float>(out_up),- static_cast<float>(out_gate)- );- D[0] = static_cast<OutT>(fused);-}diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/steel_gemm_splitk_mlp_fused.metal b/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/steel_gemm_splitk_mlp_fused.metaldeleted file mode 100644index e58783794..000000000--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/steel_gemm_splitk_mlp_fused.metal+++ /dev/null@@ -1,67 +0,0 @@-// clang-format off-#include "../../common/utils.h"-#include "steel_gemm_splitk_mlp_fused.h"--#define instantiate_gemm( \- tname, \- trans_a, \- trans_b, \- iname, \- itype, \- oname, \- otype, \- bm, \- bn, \- bk, \- wm, \- wn, \- aname, \- mn_aligned, \- kname, \- k_aligned) \- instantiate_kernel( \- "steel_gemm_splitk_mlp_fused_" #tname "_" #iname "_" #oname \- "_bm" #bm "_bn" #bn "_bk" #bk "_wm" #wm "_wn" #wn \- "_MN_" #aname "_K_" #kname, \- gemm_splitk_mlp_fused, \- itype, \- otype, \- bm, \- bn, \- bk, \- wm, \- wn, \- trans_a, \- trans_b, \- mn_aligned, \- k_aligned)--#define instantiate_gemm_aligned_helper(tname, trans_a, trans_b, iname, itype, oname, otype, bm, bn, bk, wm, wn) \- instantiate_gemm(tname, trans_a, trans_b, iname, itype, oname, otype, bm, bn, bk, wm, wn, taligned, true, taligned, true) \- instantiate_gemm(tname, trans_a, trans_b, iname, itype, oname, otype, bm, bn, bk, wm, wn, taligned, true, naligned, false) \- instantiate_gemm(tname, trans_a, trans_b, iname, itype, oname, otype, bm, bn, bk, wm, wn, naligned, false, taligned, true) \- instantiate_gemm(tname, trans_a, trans_b, iname, itype, oname, otype, bm, bn, bk, wm, wn, naligned, false, naligned, false)--// Only NT transpose (common case for MLP)-#define instantiate_gemm_transpose_helper(iname, itype, oname, otype, bm, bn, bk, wm, wn) \- instantiate_gemm_aligned_helper(nt, false, true, iname, itype, oname, otype, bm, bn, bk, wm, wn)--#define instantiate_gemm_shapes_helper(iname, itype, oname, otype) \- instantiate_gemm_transpose_helper(iname, itype, oname, otype, 16, 16, 16, 2, 2) \- instantiate_gemm_transpose_helper(iname, itype, oname, otype, 16, 32, 16, 2, 2) \- instantiate_gemm_transpose_helper(iname, itype, oname, otype, 32, 16, 16, 2, 2) \- instantiate_gemm_transpose_helper(iname, itype, oname, otype, 32, 32, 16, 2, 2)--instantiate_gemm_shapes_helper(float16, half, float32, float);-instantiate_gemm_shapes_helper(bfloat16, bfloat16_t, float32, float);-instantiate_gemm_shapes_helper(float32, float, float32, float);--#define instantiate_accum(oname, otype, aname, atype) \- instantiate_kernel( \- "steel_gemm_splitk_mlp_fused_accum_" #oname "_" #aname, \- gemm_splitk_mlp_fused_accum, atype, otype)--instantiate_accum(bfloat16, bfloat16_t, float32, float);-instantiate_accum(float16, half, float32, float);-instantiate_accum(float32, float, float32, float);-// clang-format ondiff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/tile_configuration.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/tile_configuration.rsdeleted file mode 100644index c1de7141a..000000000--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/tile_configuration.rs+++ /dev/null@@ -1,31 +0,0 @@-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]-pub struct TileConfiguration {- pub tile_rows: i32,- pub tile_cols: i32,- pub tile_depth: i32,- pub warps_per_row: i32,- pub warps_per_col: i32,-}--pub fn select_tile_configuration(- batch: i32,- hidden_dim: i32,-) -> TileConfiguration {- let tile_rows = if batch < 40 {- 16- } else {- 32- };- let tile_cols = if hidden_dim < 40 {- 16- } else {- 32- };- TileConfiguration {- tile_rows,- tile_cols,- tile_depth: 16,- warps_per_row: 2,- warps_per_col: 2,- }-}diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rsindex edf2b6d24..828bf05c5 100644--- a/crates/uzu/src/backends/metal/kernel/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/mod.rs@@ -11,7 +11,6 @@ pub use mlp::{MLP_ACTIVATION_FC_INDEX, MLP_FUSED_FC_INDEX, MLP_HIDDEN_DIM_FC_INDEX, MlpActivationType, MlpFusedConfig,make_non_fused_function_constants,};-pub mod mlp_fused;pub mod moe;pub mod quant_matmul;pub mod ssm;diff --git a/crates/uzu/tests/mlp_fused_correctness_test.rs b/crates/uzu/tests/mlp_fused_correctness_test.rsdeleted file mode 100644index 7d407a348..000000000--- a/crates/uzu/tests/mlp_fused_correctness_test.rs+++ /dev/null@@ -1,358 +0,0 @@-//! Correctness tests for MLP fused kernels-//! Compares fused (matmul + activation) output against separate operations--use bytemuck;-use half::f16;-use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue, MTLDeviceExt, MTLResourceOptions};-use objc2::rc::autoreleasepool;-use uzu::{- DataType,- backends::{- common::Context,- metal::{- MTLContext,- kernel::{- mlp::MlpActivationType,- mlp_fused::{MlpFusedArguments, MlpFusedKernel},- },- },- },-};--/// CPU reference implementation for SiLU activation-fn silu(x: f32) -> f32 {- x / (1.0 + (-x).exp())-}--/// CPU reference implementation for GELU activation-fn gelu(x: f32) -> f32 {- let k0 = 0.044715f32;- let k1 = 0.7978845608f32;- let t = k1 * (x + k0 * x * x * x);- 0.5 * x * (1.0 + t.tanh())-}--/// CPU reference for MLP fused: up * activation(gate)-fn mlp_fused_reference(- input: &[f32],- weights_up: &[f32],- weights_gate: &[f32],- m: usize,- k: usize,- hidden_dim: usize,- activation: MlpActivationType,-) -> Vec<f32> {- let mut result = vec![0.0f32; m * hidden_dim];-- for row in 0..m {- for col in 0..hidden_dim {- // Compute up and gate dot products- let mut up_sum = 0.0f32;- let mut gate_sum = 0.0f32;- for i in 0..k {- let input_val = input[row * k + i];- up_sum += input_val * weights_up[col * k + i];- gate_sum += input_val * weights_gate[col * k + i];- }-- // Apply fused activation- let activated_gate = match activation {- MlpActivationType::SiLU => silu(gate_sum),- MlpActivationType::Gelu => gelu(gate_sum),- };- result[row * hidden_dim + col] = up_sum * activated_gate;- }- }-- result-}--/// Run MLP fused kernel (decode path, M=1)-fn run_fused_gemv(- ctx: &MTLContext,- input: &[f16],- weights: &[f16], // [2*hidden_dim, k] - up then gate weights concatenated- k: usize,- hidden_dim: usize,- activation: MlpActivationType,-) -> Vec<f16> {- let input_buf = ctx- .device- .new_buffer_with_data(bytemuck::cast_slice(input), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");- let weights_buf = ctx- .device- .new_buffer_with_data(bytemuck::cast_slice(weights), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");- let output_buf = ctx- .device- .new_buffer(hidden_dim * core::mem::size_of::<f16>(), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");-- let mut kernel = MlpFusedKernel::new(DataType::F16, true).expect("kernel");-- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();- let enc = cb.new_compute_command_encoder().expect("Failed to create compute encoder");-- let args = MlpFusedArguments {- input: &input_buf,- input_offset: 0,- weights: &weights_buf,- output: &output_buf,- batch: 1,- input_dim: k as i32,- hidden_dim: hidden_dim as i32,- lda: k as i32,- ldb: k as i32,- ldd: hidden_dim as i32,- batch_count: 1,- activation,- };-- kernel.encode(ctx, &enc, &args).expect("encode");- enc.end_encoding();- cb.commit();- cb.wait_until_completed();-- unsafe {- let ptr = output_buf.contents().as_ptr() as *const f16;- std::slice::from_raw_parts(ptr, hidden_dim).to_vec()- }-}--/// Run MLP fused kernel (prefill path, M>1)-fn run_fused_gemm(- ctx: &MTLContext,- input: &[f16],- weights: &[f16], // [2*hidden_dim, k] - up then gate weights concatenated- m: usize,- k: usize,- hidden_dim: usize,- activation: MlpActivationType,-) -> Vec<f16> {- let input_buf = ctx- .device- .new_buffer_with_data(bytemuck::cast_slice(input), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");- let weights_buf = ctx- .device- .new_buffer_with_data(bytemuck::cast_slice(weights), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");- let output_buf = ctx- .device- .new_buffer(m * hidden_dim * core::mem::size_of::<f16>(), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");-- let mut kernel = MlpFusedKernel::new(DataType::F16, true).expect("kernel");-- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();- let enc = cb.new_compute_command_encoder().expect("Failed to create compute encoder");-- let args = MlpFusedArguments {- input: &input_buf,- input_offset: 0,- weights: &weights_buf,- output: &output_buf,- batch: m as i32,- input_dim: k as i32,- hidden_dim: hidden_dim as i32,- lda: k as i32,- ldb: k as i32,- ldd: hidden_dim as i32,- batch_count: 1,- activation,- };-- kernel.encode(ctx, &enc, &args).expect("encode");- enc.end_encoding();- cb.commit();- cb.wait_until_completed();-- unsafe {- let ptr = output_buf.contents().as_ptr() as *const f16;- std::slice::from_raw_parts(ptr, m * hidden_dim).to_vec()- }-}--fn compare_outputs(- expected: &[f32],- actual: &[f16],- rtol: f32,- atol: f32,-) -> (bool, f32) {- let mut max_diff = 0.0f32;- let mut all_close = true;-- for (i, (&exp, &act)) in expected.iter().zip(actual.iter()).enumerate() {- let act_f32 = f32::from(act);- let diff = (exp - act_f32).abs();- let rel_diff = diff / (exp.abs() + 1e-8);-- if diff > atol && rel_diff > rtol {- if all_close {- eprintln!(- "First mismatch at index {}: expected {}, got {} (diff={}, rel={})",- i, exp, act_f32, diff, rel_diff- );- }- all_close = false;- }- max_diff = max_diff.max(diff);- }-- (all_close, max_diff)-}--#[test]-fn test_mlp_fused_gemv_silu_small() {- autoreleasepool(|_| {- let ctx = match MTLContext::new().ok() {- Some(c) => c,- None => {- eprintln!("No Metal device available, skipping test");- return;- },- };-- let k = 64;- let hidden_dim = 32;- let m = 1;-- // Generate random-ish test data- let input: Vec<f32> = (0..k).map(|i| (i as f32 * 0.1).sin() * 0.5).collect();- let weights_up: Vec<f32> = (0..hidden_dim * k).map(|i| (i as f32 * 0.017).cos() * 0.3).collect();- let weights_gate: Vec<f32> = (0..hidden_dim * k).map(|i| (i as f32 * 0.023).sin() * 0.3).collect();-- // Create concatenated weights [up; gate] row-major- let mut weights_concat: Vec<f32> = Vec::with_capacity(2 * hidden_dim * k);- weights_concat.extend(&weights_up);- weights_concat.extend(&weights_gate);-- // CPU reference- let expected =- mlp_fused_reference(&input, &weights_up, &weights_gate, m, k, hidden_dim, MlpActivationType::SiLU);-- // Convert to f16- let input_f16: Vec<f16> = input.iter().map(|&x| f16::from_f32(x)).collect();- let weights_f16: Vec<f16> = weights_concat.iter().map(|&x| f16::from_f32(x)).collect();-- // Run fused kernel- let actual = run_fused_gemv(&ctx, &input_f16, &weights_f16, k, hidden_dim, MlpActivationType::SiLU);-- let (close, max_diff) = compare_outputs(&expected, &actual, 0.01, 0.01);- println!("GEMV SiLU small: max_diff = {}", max_diff);- assert!(close, "GEMV SiLU small test failed, max_diff = {}", max_diff);- });-}--#[test]-fn test_mlp_fused_gemv_gelu_small() {- autoreleasepool(|_| {- let ctx = match MTLContext::new().ok() {- Some(c) => c,- None => {- eprintln!("No Metal device available, skipping test");- return;- },- };-- let k = 64;- let hidden_dim = 32;- let m = 1;-- let input: Vec<f32> = (0..k).map(|i| (i as f32 * 0.1).sin() * 0.5).collect();- let weights_up: Vec<f32> = (0..hidden_dim * k).map(|i| (i as f32 * 0.017).cos() * 0.3).collect();- let weights_gate: Vec<f32> = (0..hidden_dim * k).map(|i| (i as f32 * 0.023).sin() * 0.3).collect();-- let mut weights_concat: Vec<f32> = Vec::with_capacity(2 * hidden_dim * k);- weights_concat.extend(&weights_up);- weights_concat.extend(&weights_gate);-- let expected =- mlp_fused_reference(&input, &weights_up, &weights_gate, m, k, hidden_dim, MlpActivationType::Gelu);-- let input_f16: Vec<f16> = input.iter().map(|&x| f16::from_f32(x)).collect();- let weights_f16: Vec<f16> = weights_concat.iter().map(|&x| f16::from_f32(x)).collect();-- let actual = run_fused_gemv(&ctx, &input_f16, &weights_f16, k, hidden_dim, MlpActivationType::Gelu);-- let (close, max_diff) = compare_outputs(&expected, &actual, 0.01, 0.01);- println!("GEMV GELU small: max_diff = {}", max_diff);- assert!(close, "GEMV GELU small test failed, max_diff = {}", max_diff);- });-}--#[test]-fn test_mlp_fused_gemm_silu_medium() {- autoreleasepool(|_| {- let ctx = match MTLContext::new().ok() {- Some(c) => c,- None => {- eprintln!("No Metal device available, skipping test");- return;- },- };-- let m = 32;- let k = 64;- let hidden_dim = 64;-- let input: Vec<f32> = (0..m * k).map(|i| (i as f32 * 0.1).sin() * 0.5).collect();- let weights_up: Vec<f32> = (0..hidden_dim * k).map(|i| (i as f32 * 0.017).cos() * 0.3).collect();- let weights_gate: Vec<f32> = (0..hidden_dim * k).map(|i| (i as f32 * 0.023).sin() * 0.3).collect();-- let mut weights_concat: Vec<f32> = Vec::with_capacity(2 * hidden_dim * k);- weights_concat.extend(&weights_up);- weights_concat.extend(&weights_gate);-- let expected =- mlp_fused_reference(&input, &weights_up, &weights_gate, m, k, hidden_dim, MlpActivationType::SiLU);-- let input_f16: Vec<f16> = input.iter().map(|&x| f16::from_f32(x)).collect();- let weights_f16: Vec<f16> = weights_concat.iter().map(|&x| f16::from_f32(x)).collect();-- let actual = run_fused_gemm(&ctx, &input_f16, &weights_f16, m, k, hidden_dim, MlpActivationType::SiLU);-- let (close, max_diff) = compare_outputs(&expected, &actual, 0.05, 0.05);- println!("GEMM SiLU medium: max_diff = {}", max_diff);- assert!(close, "GEMM SiLU medium test failed, max_diff = {}", max_diff);- });-}--#[test]-fn test_mlp_fused_gemv_realistic_dimensions() {- autoreleasepool(|_| {- let ctx = match MTLContext::new().ok() {- Some(c) => c,- None => {- eprintln!("No Metal device available, skipping test");- return;- },- };-- // Realistic LLM dimensions (scaled down for test speed)- let k = 512; // input dim- let hidden_dim = 256; // hidden dim (half of up+gate)- let m = 1;-- let input: Vec<f32> = (0..k).map(|i| (i as f32 * 0.01).sin() * 0.5).collect();- let weights_up: Vec<f32> = (0..hidden_dim * k).map(|i| (i as f32 * 0.001).cos() * 0.1).collect();- let weights_gate: Vec<f32> = (0..hidden_dim * k).map(|i| (i as f32 * 0.002).sin() * 0.1).collect();-- let mut weights_concat: Vec<f32> = Vec::with_capacity(2 * hidden_dim * k);- weights_concat.extend(&weights_up);- weights_concat.extend(&weights_gate);-- let expected =- mlp_fused_reference(&input, &weights_up, &weights_gate, m, k, hidden_dim, MlpActivationType::SiLU);-- let input_f16: Vec<f16> = input.iter().map(|&x| f16::from_f32(x)).collect();- let weights_f16: Vec<f16> = weights_concat.iter().map(|&x| f16::from_f32(x)).collect();-- let actual = run_fused_gemv(&ctx, &input_f16, &weights_f16, k, hidden_dim, MlpActivationType::SiLU);-- let (close, max_diff) = compare_outputs(&expected, &actual, 0.05, 0.02);- println!("GEMV realistic: max_diff = {}", max_diff);- assert!(close, "GEMV realistic test failed, max_diff = {}", max_diff);- });-}diff --git a/crates/uzu/tests/mlp_fused_performance_test.rs b/crates/uzu/tests/mlp_fused_performance_test.rsdeleted file mode 100644index 2333f1747..000000000--- a/crates/uzu/tests/mlp_fused_performance_test.rs+++ /dev/null@@ -1,379 +0,0 @@-//! MLP Fused Kernel Performance Benchmark-//! Compares fused vs unfused MLP paths to measure speedup--use std::time::{Duration, Instant};--use half::f16;-use metal::{MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue, MTLDeviceExt, MTLResourceOptions};-use objc2::rc::autoreleasepool;-use uzu::{- DataType,- backends::{- common::Context,- metal::{- MTLContext,- kernel::{- matmul::{MatmulArguments, MatmulKernel, determine_kernel_variant as determine_matmul_variant},- mlp::MlpActivationType,- mlp_fused::{- MlpFusedArguments, MlpFusedKernel, determine_kernel_variant as determine_mlp_fused_variant,- },- },- },- },-};--struct BenchmarkResult {- fused_time_microseconds: f64,- unfused_time_microseconds: f64,- speedup: f64,- fused_kernel_name: &'static str,- unfused_kernel_name: &'static str,-}--impl BenchmarkResult {- fn new(- fused_duration: Duration,- unfused_duration: Duration,- fused_kernel_name: &'static str,- unfused_kernel_name: &'static str,- ) -> Self {- let fused_microseconds = fused_duration.as_secs_f64() * 1_000_000.0;- let unfused_microseconds = unfused_duration.as_secs_f64() * 1_000_000.0;- Self {- fused_time_microseconds: fused_microseconds,- unfused_time_microseconds: unfused_microseconds,- speedup: unfused_microseconds / fused_microseconds,- fused_kernel_name,- unfused_kernel_name,- }- }-}--fn benchmark_gemv_fused(- context: &MTLContext,- input_dimension: usize,- hidden_dimension: usize,- iteration_count: usize,- warmup_count: usize,-) -> BenchmarkResult {- let input_buffer = context- .device- .new_buffer(input_dimension * std::mem::size_of::<f16>(), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");- let weights_buffer = context- .device- .new_buffer(- 2 * hidden_dimension * input_dimension * std::mem::size_of::<f16>(),- MTLResourceOptions::STORAGE_MODE_SHARED,- )- .expect("Failed to create buffer");- let output_buffer = context- .device- .new_buffer(hidden_dimension * std::mem::size_of::<f16>(), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");- let fused_up_buffer = context- .device- .new_buffer(2 * hidden_dimension * std::mem::size_of::<f16>(), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");-- let mut fused_kernel = MlpFusedKernel::new(DataType::F16, true).expect("fused kernel");- let mut matmul_kernel = MatmulKernel::new(DataType::F16).expect("matmul");-- let fused_args = MlpFusedArguments {- input: &input_buffer,- input_offset: 0,- weights: &weights_buffer,- output: &output_buffer,- batch: 1,- input_dim: input_dimension as i32,- hidden_dim: hidden_dimension as i32,- lda: input_dimension as i32,- ldb: input_dimension as i32,- ldd: hidden_dimension as i32,- batch_count: 1,- activation: MlpActivationType::SiLU,- };-- // Warmup fused- for _ in 0..warmup_count {- let command_buffer =- context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");- fused_kernel.encode(context, &encoder, &fused_args).unwrap();- encoder.end_encoding();- command_buffer.commit();- command_buffer.wait_until_completed();- }-- // Benchmark fused- let fused_start = Instant::now();- for _ in 0..iteration_count {- let command_buffer =- context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");- fused_kernel.encode(context, &encoder, &fused_args).unwrap();- encoder.end_encoding();- command_buffer.commit();- command_buffer.wait_until_completed();- }- let fused_duration = fused_start.elapsed() / iteration_count as u32;-- let matmul_args = MatmulArguments {- a: &input_buffer,- a_offset: 0,- b: &weights_buffer,- c: None,- d: &fused_up_buffer,- bias: None,- batch: 1,- input_dim: input_dimension as i32,- output_dim: (2 * hidden_dimension) as i32,- lda: input_dimension as i32,- ldb: input_dimension as i32,- ldd: (2 * hidden_dimension) as i32,- batch_count: 1,- alpha: 1.0,- beta: 0.0,- transpose_a: false,- transpose_b: true,- };-- let fused_kernel_name =- determine_mlp_fused_variant(context, DataType::F16, true, &fused_args).map(|v| v.as_str()).unwrap_or("?");-- let unfused_kernel_name =- determine_matmul_variant(context, DataType::F16, &matmul_args).map(|v| v.as_str()).unwrap_or("?");-- // Warmup unfused- for _ in 0..warmup_count {- let command_buffer =- context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");- matmul_kernel.encode(context, &encoder, matmul_args.clone()).unwrap();- encoder.end_encoding();- command_buffer.commit();- command_buffer.wait_until_completed();- }-- // Benchmark unfused- let unfused_start = Instant::now();- for _ in 0..iteration_count {- let command_buffer =- context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");- matmul_kernel.encode(context, &encoder, matmul_args.clone()).unwrap();- encoder.end_encoding();- command_buffer.commit();- command_buffer.wait_until_completed();- }- let unfused_duration = unfused_start.elapsed() / iteration_count as u32;-- BenchmarkResult::new(fused_duration, unfused_duration, fused_kernel_name, unfused_kernel_name)-}--fn benchmark_gemm_fused(- context: &MTLContext,- batch_size: usize,- input_dimension: usize,- hidden_dimension: usize,- iteration_count: usize,- warmup_count: usize,-) -> BenchmarkResult {- let input_buffer = context- .device- .new_buffer(batch_size * input_dimension * std::mem::size_of::<f16>(), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");- let weights_buffer = context- .device- .new_buffer(- 2 * hidden_dimension * input_dimension * std::mem::size_of::<f16>(),- MTLResourceOptions::STORAGE_MODE_SHARED,- )- .expect("Failed to create buffer");- let output_buffer = context- .device- .new_buffer(batch_size * hidden_dimension * std::mem::size_of::<f16>(), MTLResourceOptions::STORAGE_MODE_SHARED)- .expect("Failed to create buffer");- let fused_up_buffer = context- .device- .new_buffer(- batch_size * 2 * hidden_dimension * std::mem::size_of::<f16>(),- MTLResourceOptions::STORAGE_MODE_SHARED,- )- .expect("Failed to create buffer");-- let mut fused_kernel = MlpFusedKernel::new(DataType::F16, true).expect("fused kernel");- let mut matmul_kernel = MatmulKernel::new(DataType::F16).expect("matmul");-- let fused_args = MlpFusedArguments {- input: &input_buffer,- input_offset: 0,- weights: &weights_buffer,- output: &output_buffer,- batch: batch_size as i32,- input_dim: input_dimension as i32,- hidden_dim: hidden_dimension as i32,- lda: input_dimension as i32,- ldb: input_dimension as i32,- ldd: hidden_dimension as i32,- batch_count: 1,- activation: MlpActivationType::SiLU,- };-- // Warmup fused- for _ in 0..warmup_count {- let command_buffer =- context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");- fused_kernel.encode(context, &encoder, &fused_args).unwrap();- encoder.end_encoding();- command_buffer.commit();- command_buffer.wait_until_completed();- }-- let fused_start = Instant::now();- for _ in 0..iteration_count {- let command_buffer =- context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");- fused_kernel.encode(context, &encoder, &fused_args).unwrap();- encoder.end_encoding();- command_buffer.commit();- command_buffer.wait_until_completed();- }- let fused_duration = fused_start.elapsed() / iteration_count as u32;-- let matmul_args = MatmulArguments {- a: &input_buffer,- a_offset: 0,- b: &weights_buffer,- c: None,- d: &fused_up_buffer,- bias: None,- batch: batch_size as i32,- input_dim: input_dimension as i32,- output_dim: (2 * hidden_dimension) as i32,- lda: input_dimension as i32,- ldb: input_dimension as i32,- ldd: (2 * hidden_dimension) as i32,- batch_count: 1,- alpha: 1.0,- beta: 0.0,- transpose_a: false,- transpose_b: true,- };-- let fused_kernel_name =- determine_mlp_fused_variant(context, DataType::F16, true, &fused_args).map(|v| v.as_str()).unwrap_or("?");-- let unfused_kernel_name =- determine_matmul_variant(context, DataType::F16, &matmul_args).map(|v| v.as_str()).unwrap_or("?");-- // Warmup unfused- for _ in 0..warmup_count {- let command_buffer =- context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");- matmul_kernel.encode(context, &encoder, matmul_args.clone()).unwrap();- encoder.end_encoding();- command_buffer.commit();- command_buffer.wait_until_completed();- }-- let unfused_start = Instant::now();- for _ in 0..iteration_count {- let command_buffer =- context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");- matmul_kernel.encode(context, &encoder, matmul_args.clone()).unwrap();- encoder.end_encoding();- command_buffer.commit();- command_buffer.wait_until_completed();- }- let unfused_duration = unfused_start.elapsed() / iteration_count as u32;-- BenchmarkResult::new(fused_duration, unfused_duration, fused_kernel_name, unfused_kernel_name)-}--/// MLP shapes from real model configs-/// Format: (input_dimension, hidden_dimension)-const MLP_SHAPES: &[(usize, usize)] = &[- (896, 4864), // Qwen2.5-Coder-0.5B- (1024, 3072), // Qwen3-0.6B- (1024, 4608), // LFM2-350M- (1152, 6912), // gemma-3-1b- (1536, 6912), // LFM2-700M- (1536, 8960), // Qwen2.5-Coder-1.5B, DeepSeek-R1-Distill-Qwen-1.5B- (2048, 6144), // Qwen3-1.7B- (2048, 8192), // Llamba-1B, SmolLM2-1.7B, Llama-3.2-1B, LFM2-1.2B- (2048, 10752), // LFM2-2.6B- (2048, 11008), // Qwen2.5-Coder-3B- (2560, 9728), // Qwen3-4B- (2560, 10240), // gemma-3-4b- (3072, 8192), // Llama-3.2-3B, Llamba-3B- (3584, 18944), // Qwen2.5-Coder-7B- (4096, 12288), // Qwen3-8B- (4096, 14336), // Llamba-8B, Llama-3.1-8B- (4096, 16384), // rnj-1-instruct- (5120, 13824), // Qwen2.5-Coder-14B- (5120, 17408), // Qwen3-14B- (5120, 25600), // Qwen3-32B- (5376, 21504), // gemma-3-27b- (6144, 16384), // Codestral-22B-];--/// Batch sizes to test: decode (1) and prefill sizes-const BATCH_SIZES: &[usize] = &[1, 8, 16, 32, 64, 128, 256, 512];--#[test]-#[ignore] // Run with: cargo test --package uzu --test mlp_fused_performance_test -- --ignored --nocapture-fn mlp_fused_performance_benchmark() {- autoreleasepool(|_| {- let context = match MTLContext::new().ok() {- Some(context) => context,- None => {- eprintln!("No Metal device available");- return;- },- };-- let iteration_count = 100;- let warmup_count = 10;-- println!(- "{:>10} {:>10} {:>10} {:>10} {:>12} {:>10} {:>12} {:>10}",- "batch", "input", "hidden", "Fused", "Fused (µs)", "Unfused", "Unfused (µs)", "Speedup"- );- println!("{}", "-".repeat(98));-- for &batch_size in BATCH_SIZES {- for &(input_dimension, hidden_dimension) in MLP_SHAPES {- let result = if batch_size == 1 {- benchmark_gemv_fused(&context, input_dimension, hidden_dimension, iteration_count, warmup_count)- } else {- benchmark_gemm_fused(- &context,- batch_size,- input_dimension,- hidden_dimension,- iteration_count,- warmup_count,- )- };- let speedup_string = format!("{:.2}x", result.speedup);- println!(- "{:>10} {:>10} {:>10} {:>10} {:>12.1} {:>10} {:>12.1} {:>10}",- batch_size,- input_dimension,- hidden_dimension,- result.fused_kernel_name,- result.fused_time_microseconds,- result.unfused_kernel_name,- result.unfused_time_microseconds,- speedup_string- );- }- }- });-}