Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

remove mlp fused (#192)

Aleksandr Golokozdaaa4852e4cf
diff --git a/crates/uzu/src/backends/metal/encodable_block/mlp.rs b/crates/uzu/src/backends/metal/encodable_block/mlp.rs
index 8de14db04..70298c6e8 100644
--- a/crates/uzu/src/backends/metal/encodable_block/mlp.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/mlp.rs
@@ -1,19 +1,11 @@
//! MLP block encodable.
-use std::{cell::RefCell, rc::Rc};
-
use super::{EncodableBlock, Metal};
use crate::{
- DataType,
backends::metal::{
- MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, ProtocolObject, Retained,
- kernel::{
- mlp::{MlpActivationType, MlpGateActMulEncodable},
- mlp_fused::{MlpFusedArguments, MlpFusedKernel},
- quant_matmul::{MlpFusedQmmArguments, MlpFusedQmmKernel, MlpFusedQmvArguments, MlpFusedQmvKernel},
- },
+ MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, ProtocolObject, Retained,
+ kernel::mlp::MlpGateActMulEncodable,
},
- config::Activation,
encodable_block::EncodingParameters,
forward_pass::state::{ArrayId, ForwardPassState},
};
@@ -109,238 +101,3 @@ impl EncodableBlock<Metal> for MlpBlock {
self.down.encode_with_shared_encoder(state, params, encoder);
}
}
-
-/// MLP Fused Up Projection Variant
-/// Selects between full precision and quantized fused kernels
-pub enum MlpFusedUpKernel {
- FullPrecision {
- kernel: RefCell<MlpFusedKernel>,
- },
- Quantized {
- qmv: MlpFusedQmvKernel,
- qmm: MlpFusedQmmKernel,
- },
-}
-
-/// MLP block with fused up projection and activation
-/// Combines up * activation(gate) into a single kernel call
-pub struct MlpFusedBlock {
- context: Rc<MTLContext>,
- fused_up: MlpFusedUpKernel,
- down: Box<dyn EncodableBlock<Metal>>,
- weights_buffer: Retained<ProtocolObject<dyn MTLBuffer>>,
- scales_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>>,
- zero_points_or_biases_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>>,
- input_dim: usize,
- hidden_dim: usize,
- activation: MlpActivationType,
- input_array_id: ArrayId,
- hidden_array_id: ArrayId,
-}
-
-impl MlpFusedBlock {
- /// Create MLP fused block with full precision weights
- pub fn new_full_precision(
- context: Rc<MTLContext>,
- data_type: DataType,
- weights_buffer: Retained<ProtocolObject<dyn MTLBuffer>>,
- input_dim: usize,
- hidden_dim: usize,
- activation: &Activation,
- down: Box<dyn EncodableBlock<Metal>>,
- input_array_id: ArrayId,
- hidden_array_id: ArrayId,
- ) -> Result<Self, crate::backends::metal::MTLError> {
- let kernel = MlpFusedKernel::new(data_type, true)?; // Weights transposed
-
- Ok(Self {
- context,
- fused_up: MlpFusedUpKernel::FullPrecision {
- kernel: RefCell::new(kernel),
- },
- down,
- weights_buffer,
- scales_buffer: None,
- zero_points_or_biases_buffer: None,
- input_dim,
- hidden_dim,
- activation: MlpActivationType::from(activation),
- input_array_id,
- hidden_array_id,
- })
- }
-
- /// Create MLP fused block with quantized weights
- pub fn new_quantized(
- context: Rc<MTLContext>,
- data_type: DataType,
- weights_buffer: Retained<ProtocolObject<dyn MTLBuffer>>,
- scales_buffer: Retained<ProtocolObject<dyn MTLBuffer>>,
- zero_points_or_biases_buffer: Retained<ProtocolObject<dyn MTLBuffer>>,
- input_dim: usize,
- hidden_dim: usize,
- group_size: usize,
- mode: crate::config::QuantizationMode,
- quantization_type: crate::backends::metal::kernel::quant_matmul::QuantizationType,
- activation: &Activation,
- down: Box<dyn EncodableBlock<Metal>>,
- input_array_id: ArrayId,
- hidden_array_id: ArrayId,
- ) -> Result<Self, crate::backends::metal::MTLError> {
- let qmv = MlpFusedQmvKernel::new(&context, data_type, group_size, mode, quantization_type)
- .map_err(|e| crate::backends::metal::MTLError::Generic(format!("{:?}", e)))?;
-
- let qmm = MlpFusedQmmKernel::new(&context, data_type, group_size, mode, quantization_type)
- .map_err(|e| crate::backends::metal::MTLError::Generic(format!("{:?}", e)))?;
-
- Ok(Self {
- context,
- fused_up: MlpFusedUpKernel::Quantized {
- qmv,
- qmm,
- },
- down,
- weights_buffer,
- scales_buffer: Some(scales_buffer),
- zero_points_or_biases_buffer: Some(zero_points_or_biases_buffer),
- input_dim,
- hidden_dim,
- activation: MlpActivationType::from(activation),
- input_array_id,
- hidden_array_id,
- })
- }
-
- fn encode_fused_up(
- &self,
- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- input: &Retained<ProtocolObject<dyn MTLBuffer>>,
- input_offset: u64,
- output: &Retained<ProtocolObject<dyn MTLBuffer>>,
- batch: i32,
- ) {
- match &self.fused_up {
- MlpFusedUpKernel::FullPrecision {
- kernel,
- } => {
- let args = MlpFusedArguments {
- input,
- input_offset,
- weights: &self.weights_buffer,
- output,
- batch,
- input_dim: self.input_dim as i32,
- hidden_dim: self.hidden_dim as i32,
- lda: self.input_dim as i32,
- ldb: self.input_dim as i32,
- ldd: self.hidden_dim as i32,
- batch_count: batch,
- activation: self.activation,
- };
- kernel.borrow_mut().encode(&self.context, encoder, &args).expect("Failed to encode MLP fused kernel");
- },
- MlpFusedUpKernel::Quantized {
- qmv,
- qmm,
- } => {
- let scales = self.scales_buffer.as_ref().unwrap();
- let zp_or_biases = self.zero_points_or_biases_buffer.as_ref().unwrap();
-
- let is_decode = batch == 1;
- if is_decode {
- let args = MlpFusedQmvArguments {
- weights: &self.weights_buffer,
- scales,
- zero_points_or_biases: zp_or_biases,
- input,
- input_offset,
- output,
- input_dim: self.input_dim as i32,
- hidden_dim: self.hidden_dim as i32,
- batch_count: batch,
- };
- qmv.encode(encoder, &args).expect("Failed to encode MLP fused QMV");
- } else {
- let args = MlpFusedQmmArguments {
- weights: &self.weights_buffer,
- scales,
- zero_points_or_biases: zp_or_biases,
- input,
- input_offset,
- output,
- batch,
- input_dim: self.input_dim as i32,
- hidden_dim: self.hidden_dim as i32,
- };
- qmm.encode(encoder, &args).expect("Failed to encode MLP fused QMM");
- }
- },
- }
- }
-}
-
-impl EncodableBlock<Metal> for MlpFusedBlock {
- fn encode(
- &self,
- state: &mut ForwardPassState<Metal>,
- params: &EncodingParameters<Metal>,
- command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- ) {
- if self.supports_shared_encoder() {
- let encoder =
- command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
- self.encode_with_shared_encoder(state, params, &encoder);
- encoder.end_encoding();
- } else {
- // Fused up + activation
- {
- let arrays = state.arrays(&[self.input_array_id, self.hidden_array_id]);
- let input = arrays[0].borrow_mut();
- let hidden = arrays[1].borrow_mut();
- let batch = input.shape()[0] as i32;
- let input_buf = input.buffer();
- let hidden_buf = hidden.buffer();
-
- let encoder =
- command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
- self.encode_fused_up(&encoder, input_buf, 0, hidden_buf, batch);
- encoder.end_encoding();
- }
-
- // Down
- self.down.encode(state, params, command_buffer);
- }
-
- if params.wait_until_completed {
- command_buffer.commit();
- command_buffer.wait_until_completed();
- }
- }
-
- fn supports_shared_encoder(&self) -> bool {
- self.down.supports_shared_encoder()
- }
-
- fn encode_with_shared_encoder(
- &self,
- state: &mut ForwardPassState<Metal>,
- params: &EncodingParameters<Metal>,
- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- ) {
- // Fused up + activation
- let arrays = state.arrays(&[self.input_array_id, self.hidden_array_id]);
- let input = arrays[0].borrow_mut();
- let hidden = arrays[1].borrow_mut();
- let batch = input.shape()[0] as i32;
- let input_buf = input.buffer();
- let hidden_buf = hidden.buffer();
-
- self.encode_fused_up(encoder, input_buf, 0, hidden_buf, batch);
-
- drop(input);
- drop(hidden);
-
- // Down
- self.down.encode_with_shared_encoder(state, params, encoder);
- }
-}
diff --git a/crates/uzu/src/backends/metal/encodable_block/mod.rs b/crates/uzu/src/backends/metal/encodable_block/mod.rs
index 487291f9a..275846b7b 100644
--- a/crates/uzu/src/backends/metal/encodable_block/mod.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/mod.rs
@@ -33,7 +33,7 @@ pub use embedding::{
pub use layer::LayerExecutables;
pub use linear::{FullPrecisionLinear, QuantizedLinear};
pub(crate) use mamba_mixer::MambaMixer;
-pub use mlp::{MlpBlock, MlpFusedBlock, MlpFusedUpKernel};
+pub use mlp::MlpBlock;
pub use moe_block::{MoeBlock, SharedMoeWeights};
pub use normalization::{LayerNorm, Normalization, QKNorm, RMSNorm};
pub use prediction_head::ClassifierPredictionHead;
diff --git a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
index ac2bfa1c9..d8891e29f 100644
--- a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
@@ -1,15 +1,10 @@
-use std::rc::Rc;
-
use super::{
EncodableBlock, FullPrecisionEmbeddingLookup, FullPrecisionEmbeddingReadout, FullPrecisionLinear, Metal, MlpBlock,
- MlpFusedBlock, MoeBlock, QuantizedEmbeddingLookup, QuantizedEmbeddingReadout, QuantizedLinear,
+ MoeBlock, QuantizedEmbeddingLookup, QuantizedEmbeddingReadout, QuantizedLinear,
};
use crate::{
DataType,
- backends::metal::{
- MTLContext, MTLError,
- kernel::{mlp::MlpGateActMulEncodable, quant_matmul::QuantizationType},
- },
+ backends::metal::{MTLContext, MTLError, kernel::mlp::MlpGateActMulEncodable},
config::{DecoderConfig, EmbeddingConfig, LinearConfig, MLPConfig},
forward_pass::state::ArrayId,
parameters::ParameterTree,
@@ -112,138 +107,6 @@ pub fn mlp_block(
unreachable!("Unknown MLP config")
}
-/// Creates an MLP block using the fused implementation
-/// (up projection + activation fused into single kernel)
-pub fn mlp_fused_block(
- config: &MLPConfig,
- model_dimension: usize,
- hidden_dimension: usize,
- context: Rc<MTLContext>,
- parameter_tree: &ParameterTree<MTLContext>,
-) -> Result<Box<dyn EncodableBlock<Metal>>, MTLError> {
- if let crate::config::MLPConfig::Dense(dense_config) = config {
- match &dense_config.linear_config {
- LinearConfig::Quantized(quantization_config) | LinearConfig::MLXQuantized(quantization_config) => {
- let data_type: DataType = dense_config.linear_config.activation_precision().into();
-
- // Load up_projection weights directly for fused kernel
- let up_projection_tree = parameter_tree
- .subtree("up_projection")
- .map_err(|error| MTLError::Generic(format!("{:?}", error)))?;
-
- let up_projection_weights = up_projection_tree
- .leaf("weights")
- .map_err(|error| MTLError::Generic(format!("Failed to load up weights: {:?}", error)))?;
- let up_projection_weights_buffer = up_projection_weights.buffer().into();
-
- let up_projection_scales = up_projection_tree
- .leaf("scales")
- .map_err(|error| MTLError::Generic(format!("Failed to load up scales: {:?}", error)))?;
- let up_projection_scales_buffer = up_projection_scales.buffer().into();
-
- // Load zero_points or biases depending on quantization type
- let (up_projection_zero_points_or_biases_buffer, quantization_type) =
- if let Ok(biases) = up_projection_tree.leaf("biases") {
- (biases.buffer().into(), QuantizationType::Mlx)
- } else if let Ok(zero_points) = up_projection_tree.leaf("zero_points") {
- (zero_points.buffer().into(), QuantizationType::ZeroPoint)
- } else {
- return Err(MTLError::Generic(
- "Missing zero_points or biases for quantized up_projection".to_string(),
- ));
- };
-
- // Create down projection as separate linear
- let down_projection = QuantizedLinear::new(
- &context,
- quantization_config,
- hidden_dimension,
- model_dimension,
- &parameter_tree
- .subtree("down_projection")
- .map_err(|error| MTLError::Generic(format!("{:?}", error)))?,
- ArrayId::MlpHidden,
- ArrayId::Main,
- )
- .map_err(|error| MTLError::Generic(format!("{:?}", error)))?;
-
- let fused_block = MlpFusedBlock::new_quantized(
- context,
- data_type,
- up_projection_weights_buffer,
- up_projection_scales_buffer,
- up_projection_zero_points_or_biases_buffer,
- model_dimension,
- hidden_dimension,
- quantization_config.group_size,
- quantization_config.weight_quantization_mode,
- quantization_type,
- &dense_config.activation,
- Box::new(down_projection),
- ArrayId::Main,
- ArrayId::MlpHidden,
- )?;
- return Ok(Box::new(fused_block));
- },
- LinearConfig::FullPrecision {
- precision,
- } => {
- let data_type: DataType = (*precision).into();
-
- // Load up_projection weights directly for fused kernel
- let up_projection_tree = parameter_tree
- .subtree("up_projection")
- .map_err(|error| MTLError::Generic(format!("{:?}", error)))?;
-
- let up_projection_weights = up_projection_tree
- .leaf("weights")
- .map_err(|error| MTLError::Generic(format!("Failed to load up weights: {:?}", error)))?;
- let up_projection_weights_buffer = up_projection_weights.buffer().into();
-
- // Create down projection as separate linear
- let down_projection = FullPrecisionLinear::new(
- &context,
- data_type,
- hidden_dimension,
- model_dimension,
- &parameter_tree
- .subtree("down_projection")
- .map_err(|error| MTLError::Generic(format!("{:?}", error)))?,
- ArrayId::MlpHidden,
- ArrayId::Main,
- )
- .map_err(|error| MTLError::Generic(format!("{:?}", error)))?;
-
- let fused_block = MlpFusedBlock::new_full_precision(
- context,
- data_type,
- up_projection_weights_buffer,
- model_dimension,
- hidden_dimension,
- &dense_config.activation,
- Box::new(down_projection),
- ArrayId::Main,
- ArrayId::MlpHidden,
- )?;
- return Ok(Box::new(fused_block));
- },
- LinearConfig::QLoRA {
- ..
- } => {
- return Err(MTLError::Generic("QLoRA MLP not supported".to_string()));
- },
- }
- }
-
- if let crate::config::MLPConfig::MixtureOfExperts(mixture_of_experts_config) = config {
- let mixture_of_experts_block =
- MoeBlock::new(&context, mixture_of_experts_config, model_dimension, hidden_dimension, parameter_tree)?;
- return Ok(Box::new(mixture_of_experts_block));
- }
-
- unreachable!("Unknown MLP config")
-}
-
pub fn embed_block(
config: &DecoderConfig,
context: &MTLContext,
diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/common/mlp_fused_arguments.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/common/mlp_fused_arguments.rs
deleted file mode 100644
index da793f6e6..000000000
--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/common/mlp_fused_arguments.rs
+++ /dev/null
@@ -1,17 +0,0 @@
-use crate::backends::metal::{MTLBuffer, ProtocolObject, kernel::mlp::MlpActivationType};
-
-#[derive(Debug)]
-pub struct MlpFusedArguments<'a> {
- pub input: &'a ProtocolObject<dyn MTLBuffer>,
- pub input_offset: u64,
- pub weights: &'a ProtocolObject<dyn MTLBuffer>,
- pub output: &'a ProtocolObject<dyn MTLBuffer>,
- pub batch: i32,
- pub input_dim: i32,
- pub hidden_dim: i32,
- pub lda: i32,
- pub ldb: i32,
- pub ldd: i32,
- pub batch_count: i32,
- pub activation: MlpActivationType,
-}
diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/common/mod.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/common/mod.rs
deleted file mode 100644
index 8816071bd..000000000
--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/common/mod.rs
+++ /dev/null
@@ -1,3 +0,0 @@
-mod mlp_fused_arguments;
-
-pub use mlp_fused_arguments::MlpFusedArguments;
diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/dispatch_descriptor.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/dispatch_descriptor.rs
deleted file mode 100644
index 9cbedf896..000000000
--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/dispatch_descriptor.rs
+++ /dev/null
@@ -1,72 +0,0 @@
-use super::{common::MlpFusedArguments, gemm, gemv, split_k};
-use crate::{
- DataType,
- backends::metal::{MTLContext, MTLError},
-};
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum MlpFusedKernelVariant {
- Gemv,
- SplitK,
- Gemm,
-}
-
-impl MlpFusedKernelVariant {
- pub fn as_str(&self) -> &'static str {
- match self {
- MlpFusedKernelVariant::Gemv => "GEMV",
- MlpFusedKernelVariant::SplitK => "Split-K",
- MlpFusedKernelVariant::Gemm => "GEMM",
- }
- }
-}
-
-#[derive(Debug, Clone)]
-pub(crate) enum MlpFusedDispatchDescriptor {
- Gemv(gemv::DispatchDescriptor),
- SplitK(split_k::DispatchDescriptor),
- Gemm(gemm::DispatchDescriptor),
-}
-
-impl MlpFusedDispatchDescriptor {
- pub fn variant(&self) -> MlpFusedKernelVariant {
- match self {
- MlpFusedDispatchDescriptor::Gemv(_) => MlpFusedKernelVariant::Gemv,
- MlpFusedDispatchDescriptor::SplitK(_) => MlpFusedKernelVariant::SplitK,
- MlpFusedDispatchDescriptor::Gemm(_) => MlpFusedKernelVariant::Gemm,
- }
- }
-}
-
-pub(crate) fn choose_dispatch_descriptor(
- context: &MTLContext,
- data_type: DataType,
- weights_transposed: bool,
- arguments: &MlpFusedArguments,
-) -> Result<MlpFusedDispatchDescriptor, MTLError> {
- if weights_transposed {
- if let Some(descriptor) = gemv::DispatchDescriptor::try_new(context, data_type, arguments)? {
- return Ok(MlpFusedDispatchDescriptor::Gemv(descriptor));
- }
-
- if let Some(descriptor) = split_k::DispatchDescriptor::try_new(context, data_type, arguments)? {
- return Ok(MlpFusedDispatchDescriptor::SplitK(descriptor));
- }
- }
-
- Ok(MlpFusedDispatchDescriptor::Gemm(gemm::DispatchDescriptor::new(
- context,
- data_type,
- weights_transposed,
- arguments,
- )?))
-}
-
-pub fn determine_kernel_variant(
- context: &MTLContext,
- data_type: DataType,
- weights_transposed: bool,
- arguments: &MlpFusedArguments,
-) -> Result<MlpFusedKernelVariant, MTLError> {
- choose_dispatch_descriptor(context, data_type, weights_transposed, arguments).map(|d| d.variant())
-}
diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/dispatch_descriptor.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/dispatch_descriptor.rs
deleted file mode 100644
index b8cc84c3e..000000000
--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/dispatch_descriptor.rs
+++ /dev/null
@@ -1,79 +0,0 @@
-use super::{pipeline_configuration::PipelineConfiguration, tile_configuration::select_tile_configuration};
-use crate::{
- DataType,
- backends::metal::{
- MTLContext, MTLError, MTLSize,
- kernel::{matmul::common::GEMMParams, mlp_fused::common::MlpFusedArguments},
- },
-};
-
-#[derive(Debug, Clone)]
-pub struct DispatchDescriptor {
- pub(crate) pipeline_configuration: PipelineConfiguration,
- pub(crate) params: GEMMParams,
- pub(crate) hidden_dim: i32,
- pub(crate) threadgroups: MTLSize,
- pub(crate) threads_per_threadgroup: MTLSize,
-}
-
-impl DispatchDescriptor {
- pub(crate) fn new(
- _context: &MTLContext,
- data_type: DataType,
- weights_transposed: bool,
- arguments: &MlpFusedArguments,
- ) -> Result<Self, MTLError> {
- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
- return Err(MTLError::Generic(format!("Unsupported dtype for MLP fused GEMM: {data_type:?}")));
- }
-
- let tile = select_tile_configuration(arguments.batch, arguments.hidden_dim);
-
- let mn_aligned = arguments.batch % tile.block_rows == 0 && arguments.hidden_dim % tile.block_cols == 0;
- let k_aligned = arguments.input_dim % tile.block_depth == 0;
-
- let pipeline_configuration = PipelineConfiguration {
- tile,
- weights_transposed,
- mn_aligned,
- k_aligned,
- activation: arguments.activation,
- };
-
- let tiles_m = (arguments.batch + tile.block_rows - 1) / tile.block_rows;
- let tiles_n = (arguments.hidden_dim + tile.block_cols - 1) / tile.block_cols;
- let gemm_k_iterations = (arguments.input_dim + tile.block_depth - 1) / tile.block_depth;
-
- let params = GEMMParams {
- M: arguments.batch,
- N: arguments.hidden_dim * 2,
- K: arguments.input_dim,
- lda: arguments.lda,
- ldb: arguments.ldb,
- ldd: arguments.ldd,
- tiles_n,
- tiles_m,
- batch_stride_a: 0,
- batch_stride_b: 0,
- batch_stride_d: 0,
- swizzle_log: 0,
- gemm_k_iterations_aligned: if k_aligned {
- gemm_k_iterations
- } else {
- gemm_k_iterations - 1
- },
- batch_ndim: 0,
- };
-
- let threadgroups = MTLSize::new(tiles_n as usize, tiles_m as usize, 1);
- let threads_per_threadgroup = MTLSize::new(32, 2, 2);
-
- Ok(Self {
- pipeline_configuration,
- params,
- hidden_dim: arguments.hidden_dim,
- threadgroups,
- threads_per_threadgroup,
- })
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/kernel.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/kernel.rs
deleted file mode 100644
index adc539243..000000000
--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/kernel.rs
+++ /dev/null
@@ -1,114 +0,0 @@
-use std::collections::HashMap;
-
-use super::{DispatchDescriptor, pipeline_configuration::PipelineConfiguration};
-use crate::{
- DataType,
- backends::metal::{
- ComputeEncoderSetValue, FunctionConstantValuesSetValue, MTLComputeCommandEncoder, MTLComputePipelineState,
- MTLContext, MTLError, MTLFunctionConstantValues, ProtocolObject, Retained,
- kernel::mlp_fused::common::MlpFusedArguments,
- },
-};
-
-pub struct Kernel {
- data_type: DataType,
- weights_transposed: bool,
- pipelines: HashMap<PipelineConfiguration, Retained<ProtocolObject<dyn MTLComputePipelineState>>>,
-}
-
-impl Kernel {
- pub fn new(
- data_type: DataType,
- weights_transposed: bool,
- ) -> Result<Self, MTLError> {
- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
- return Err(MTLError::Generic(format!("Unsupported dtype for MLP fused GEMM: {data_type:?}")));
- }
- Ok(Self {
- data_type,
- weights_transposed,
- pipelines: HashMap::new(),
- })
- }
-
- pub fn weights_transposed(&self) -> bool {
- self.weights_transposed
- }
-
- fn kernel_name(
- &self,
- configuration: &PipelineConfiguration,
- ) -> String {
- let type_name = match self.data_type {
- DataType::F16 => "half",
- DataType::BF16 => "bfloat",
- DataType::F32 => "float",
- _ => unreachable!(),
- };
- let transpose_char = if configuration.weights_transposed {
- "t"
- } else {
- "n"
- };
- let mn_str = if configuration.mn_aligned {
- "true"
- } else {
- "false"
- };
- let k_str = if configuration.k_aligned {
- "true"
- } else {
- "false"
- };
-
- format!(
- "steel_gemm_mlp_fused_{}_{}_{}_bm{}_bn{}_bk16_wm2_wn2_align_MN_{}_K_{}",
- transpose_char,
- type_name,
- type_name,
- configuration.tile.block_rows,
- configuration.tile.block_cols,
- mn_str,
- k_str
- )
- }
-
- fn get_or_compile_pipeline(
- &mut self,
- context: &MTLContext,
- configuration: &PipelineConfiguration,
- ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError> {
- if !self.pipelines.contains_key(configuration) {
- let kernel_name = self.kernel_name(configuration);
-
- let function_constants = MTLFunctionConstantValues::new();
- let activation_val = configuration.activation as u32;
- function_constants.set_value(&activation_val, 52);
-
- let pipeline_state = context.compute_pipeline_state(&kernel_name, Some(&function_constants))?;
- self.pipelines.insert(configuration.clone(), pipeline_state);
- }
- Ok(self.pipelines.get(configuration).unwrap())
- }
-
- pub(crate) fn encode_descriptor(
- &mut self,
- context: &MTLContext,
- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- arguments: &MlpFusedArguments,
- descriptor: &DispatchDescriptor,
- ) -> Result<(), MTLError> {
- let pipeline_state = self.get_or_compile_pipeline(context, &descriptor.pipeline_configuration)?;
- encoder.set_compute_pipeline_state(pipeline_state);
-
- encoder.set_buffer(Some(arguments.input), arguments.input_offset as usize, 0);
- encoder.set_buffer(Some(arguments.weights), 0, 1);
- encoder.set_buffer(Some(arguments.output), 0, 2);
-
- encoder.set_value(&descriptor.params, 3);
- encoder.set_value(&descriptor.hidden_dim, 10);
-
- encoder.dispatch_threadgroups(descriptor.threadgroups, descriptor.threads_per_threadgroup);
- Ok(())
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/mod.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/mod.rs
deleted file mode 100644
index 6b041d0fc..000000000
--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/mod.rs
+++ /dev/null
@@ -1,7 +0,0 @@
-mod dispatch_descriptor;
-mod kernel;
-mod pipeline_configuration;
-mod tile_configuration;
-
-pub(crate) use dispatch_descriptor::DispatchDescriptor;
-pub use kernel::Kernel as GemmKernel;
diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/pipeline_configuration.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/pipeline_configuration.rs
deleted file mode 100644
index fdf5fa36b..000000000
--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/pipeline_configuration.rs
+++ /dev/null
@@ -1,11 +0,0 @@
-use super::tile_configuration::TileConfiguration;
-use crate::backends::metal::kernel::mlp::MlpActivationType;
-
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct PipelineConfiguration {
- pub tile: TileConfiguration,
- pub weights_transposed: bool,
- pub mn_aligned: bool,
- pub k_aligned: bool,
- pub activation: MlpActivationType,
-}
diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/steel_gemm_mlp_fused.h b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/steel_gemm_mlp_fused.h
deleted file mode 100644
index 6f914713f..000000000
--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/steel_gemm_mlp_fused.h
+++ /dev/null
@@ -1,221 +0,0 @@
-#pragma once
-
-#include <metal_stdlib>
-#include <metal_simdgroup>
-#include <metal_simdgroup_matrix>
-
-#include "../../matmul/common/steel/gemm/gemm.h"
-#include "../../matmul/common/steel/gemm/params.h"
-#include "../../common/mlp_epilogue.h"
-
-using namespace metal;
-
-namespace steel {
-
-///////////////////////////////////////////////////////////////////////////////
-// MLP Fused GEMM Kernel
-// Computes paired up/gate projections with fused activation for prefill path.
-// Weight layout: B = [up_weights (hidden_dim cols), gate_weights (hidden_dim
-// cols)] Output = up * activation(gate), written to first hidden_dim columns
-///////////////////////////////////////////////////////////////////////////////
-
-template <
- typename T,
- typename U,
- int BM,
- int BN,
- int BK,
- int WM,
- int WN,
- bool transpose_a,
- bool transpose_b,
- bool MN_aligned,
- bool K_aligned,
- typename AccumType = typename AccumHelper<T>::accum_type>
-struct GEMMMlpFusedKernel {
- STEEL_CONST short tgp_padding_a = 16 / sizeof(T);
- STEEL_CONST short tgp_padding_b = 16 / sizeof(T);
- STEEL_CONST short tgp_mem_size_a =
- transpose_a ? BK * (BM + tgp_padding_a) : BM * (BK + tgp_padding_a);
- // Double B threadgroup memory for up and gate
- STEEL_CONST short tgp_mem_size_b =
- 2 * (transpose_b ? BN * (BK + tgp_padding_b) : BK * (BN + tgp_padding_b));
- STEEL_CONST short tgp_mem_size = tgp_mem_size_a + tgp_mem_size_b;
-
- STEEL_CONST short tgp_size = WM * WN * 32;
-
- using loader_a_t = BlockLoader<
- T,
- transpose_a ? BK : BM,
- transpose_a ? BM : BK,
- transpose_a ? BM + tgp_padding_a : BK + tgp_padding_a,
- !transpose_a,
- tgp_size>;
- using loader_b_t = BlockLoader<
- T,
- transpose_b ? BN : BK,
- transpose_b ? BK : BN,
- transpose_b ? BK + tgp_padding_b : BN + tgp_padding_b,
- transpose_b,
- tgp_size>;
- using mma_t = BlockMMA<
- T,
- U,
- BM,
- BN,
- BK,
- WM,
- WN,
- transpose_a,
- transpose_b,
- transpose_a ? BM + tgp_padding_a : BK + tgp_padding_a,
- transpose_b ? BK + tgp_padding_b : BN + tgp_padding_b,
- AccumType,
- TransformNone<U, AccumType>>;
-
- static METAL_FUNC void run(
- const device T* A [[buffer(0)]],
- const device T* B [[buffer(1)]],
- device U* D [[buffer(2)]],
- const constant GEMMParams* params [[buffer(3)]],
- const constant int& hidden_dim [[buffer(10)]],
- threadgroup T* As [[threadgroup(0)]],
- threadgroup T* Bs_up [[threadgroup(1)]],
- uint simd_lane_id [[thread_index_in_simdgroup]],
- uint simd_group_id [[simdgroup_index_in_threadgroup]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint3 lid [[thread_position_in_threadgroup]]
- ) {
- (void)lid;
-
- const int tid_y = ((tid.y) << params->swizzle_log) +
- ((tid.x) & ((1 << params->swizzle_log) - 1));
- const int tid_x = (tid.x) >> params->swizzle_log;
-
- // Only process tiles within hidden_dim columns (output size)
- int hidden_tiles = (hidden_dim + BN - 1) / BN;
- if (hidden_tiles <= tid_x || params->tiles_m <= tid_y) {
- return;
- }
-
- threadgroup_barrier(mem_flags::mem_none);
-
- const int c_row = tid_y * BM;
- const int c_col = tid_x * BN;
- const size_t c_row_long = size_t(c_row);
- const size_t c_col_long = size_t(c_col);
-
- // Pointer to A (input activations)
- const device T* A_ptr =
- A + (transpose_a ? c_row_long : c_row_long * params->lda);
-
- // Pointers to up and gate weights in B
- // B layout: [up_weights | gate_weights] where each is hidden_dim columns
- const device T* B_up =
- B + (transpose_b ? c_col_long * params->ldb : c_col_long);
- const device T* B_gate =
- B_up + (transpose_b ? hidden_dim * params->ldb : hidden_dim);
-
- device U* D_ptr = D + c_row_long * params->ldd + c_col_long;
-
- // Threadgroup memory layout: [As, Bs_up, Bs_gate]
- threadgroup T* Bs_gate = Bs_up + (tgp_mem_size_b / 2);
-
- // Loaders for A (shared) and B (up and gate)
- thread loader_a_t
- loader_a(A_ptr, params->lda, As, simd_group_id, simd_lane_id);
- thread loader_b_t
- loader_b_up(B_up, params->ldb, Bs_up, simd_group_id, simd_lane_id);
- thread loader_b_t loader_b_gate(
- B_gate,
- params->ldb,
- Bs_gate,
- simd_group_id,
- simd_lane_id
- );
-
- // MMA operations for up and gate
- thread mma_t mma_up(simd_group_id, simd_lane_id);
- thread mma_t mma_gate(simd_group_id, simd_lane_id);
-
- int gemm_k_iterations = params->gemm_k_iterations_aligned;
-
- // Main compute loop
- for (int k = 0; k < gemm_k_iterations; k++) {
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // Load A (shared for both up and gate)
- if (MN_aligned) {
- loader_a.load_unsafe();
- } else {
- short tgp_bm = min(short(BM), short(params->M - c_row));
- short2 tile_dims_A =
- transpose_a ? short2(tgp_bm, BK) : short2(BK, tgp_bm);
- loader_a.load_safe(tile_dims_A);
- }
-
- // Load B_up and B_gate
- if (MN_aligned) {
- loader_b_up.load_unsafe();
- loader_b_gate.load_unsafe();
- } else {
- short tgp_bn = min(short(BN), short(hidden_dim - c_col));
- short2 tile_dims_B =
- transpose_b ? short2(BK, tgp_bn) : short2(tgp_bn, BK);
- loader_b_up.load_safe(tile_dims_B);
- loader_b_gate.load_safe(tile_dims_B);
- }
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // MMA for up and gate
- mma_up.mma(As, Bs_up);
- mma_gate.mma(As, Bs_gate);
-
- loader_a.next();
- loader_b_up.next();
- loader_b_gate.next();
- }
-
- // Handle leftover K
- if (!K_aligned) {
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- int lbk = params->K - params->gemm_k_iterations_aligned * BK;
- short2 tile_dims_A = transpose_a ? short2(BM, lbk) : short2(lbk, BM);
- short2 tile_dims_B = transpose_b ? short2(lbk, BN) : short2(BN, lbk);
-
- loader_a.load_safe(tile_dims_A);
- loader_b_up.load_safe(tile_dims_B);
- loader_b_gate.load_safe(tile_dims_B);
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- mma_up.mma(As, Bs_up);
- mma_gate.mma(As, Bs_gate);
- }
-
- // Apply MLP fused epilogue: out = up * activation(gate)
- // Access the Ctile elements and apply fusion
- constexpr int kElemsPerTile = decltype(mma_up.Ctile)::kElemsPerTile;
-
-#pragma unroll
- for (short i = 0; i < kElemsPerTile; i++) {
- float up_val = static_cast<float>(mma_up.Ctile.elems()[i]);
- float gate_val = static_cast<float>(mma_gate.Ctile.elems()[i]);
- float fused = mlp_fused_epilogue_f32(up_val, gate_val);
- mma_up.Ctile.elems()[i] = static_cast<AccumType>(fused);
- }
-
- // Store result
- if (MN_aligned) {
- mma_up.store_result(D_ptr, params->ldd);
- } else {
- short tgp_bm = min(short(BM), short(params->M - c_row));
- short tgp_bn = min(short(BN), short(hidden_dim - c_col));
- mma_up.store_result_safe(D_ptr, params->ldd, short2(tgp_bn, tgp_bm));
- }
- }
-};
-
-} // namespace steel
diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/steel_gemm_mlp_fused.metal b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/steel_gemm_mlp_fused.metal
deleted file mode 100644
index 850b63b6c..000000000
--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/steel_gemm_mlp_fused.metal
+++ /dev/null
@@ -1,605 +0,0 @@
-#include <metal_stdlib>
-
-#include "../../common/utils.h"
-#include "steel_gemm_mlp_fused.h"
-
-using namespace metal;
-
-///////////////////////////////////////////////////////////////////////////////
-// MLP Fused GEMM Kernel Instantiations
-// For prefill path: computes up * activation(gate) in a single pass
-///////////////////////////////////////////////////////////////////////////////
-
-#define instantiate_mlp_fused_gemm( \
- tname, \
- trans_a, \
- trans_b, \
- itype, \
- otype, \
- bm, \
- bn, \
- bk, \
- wm, \
- wn, \
- aname, \
- mn, \
- k \
-) \
- template [[host_name( \
- "steel_gemm_mlp_fused_" #tname "_" #itype "_" #otype "_bm" #bm "_bn" #bn \
- "_bk" #bk "_wm" #wm "_wn" #wn "_" #aname "_MN_" #mn "_K_" #k \
- )]] [[kernel]] void \
- steel_gemm_mlp_fused< \
- itype, \
- otype, \
- bm, \
- bn, \
- bk, \
- wm, \
- wn, \
- trans_a, \
- trans_b, \
- mn, \
- k>( \
- const device itype* A [[buffer(0)]], \
- const device itype* B [[buffer(1)]], \
- device otype* D [[buffer(2)]], \
- const constant steel::GEMMParams* params [[buffer(3)]], \
- const constant int& hidden_dim [[buffer(10)]], \
- uint simd_lane_id [[thread_index_in_simdgroup]], \
- uint simd_group_id [[simdgroup_index_in_threadgroup]], \
- uint3 tid [[threadgroup_position_in_grid]], \
- uint3 lid [[thread_position_in_threadgroup]] \
- )
-
-template <
- typename T,
- typename U,
- int BM,
- int BN,
- int BK,
- int WM,
- int WN,
- bool transpose_a,
- bool transpose_b,
- bool MN_aligned,
- bool K_aligned>
-[[kernel, max_total_threads_per_threadgroup(WM * WN * 32)]] void
-steel_gemm_mlp_fused(
- const device T* A [[buffer(0)]],
- const device T* B [[buffer(1)]],
- device U* D [[buffer(2)]],
- const constant steel::GEMMParams* params [[buffer(3)]],
- const constant int& hidden_dim [[buffer(10)]],
- uint simd_lane_id [[thread_index_in_simdgroup]],
- uint simd_group_id [[simdgroup_index_in_threadgroup]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint3 lid [[thread_position_in_threadgroup]]
-) {
- using kernel_t = steel::GEMMMlpFusedKernel<
- T,
- U,
- BM,
- BN,
- BK,
- WM,
- WN,
- transpose_a,
- transpose_b,
- MN_aligned,
- K_aligned>;
-
- // Allocate threadgroup memory
- threadgroup T As[kernel_t::tgp_mem_size_a];
- threadgroup T Bs[kernel_t::tgp_mem_size_b];
-
- kernel_t::run(
- A,
- B,
- D,
- params,
- hidden_dim,
- As,
- Bs,
- simd_lane_id,
- simd_group_id,
- tid,
- lid
- );
-}
-
-// Instantiate for common configurations
-// transpose_a = false, transpose_b = true (weights are [K, 2*H], accessed as
-// transposed)
-
-// float16 configurations
-instantiate_mlp_fused_gemm(
- t,
- false,
- true,
- half,
- half,
- 32,
- 32,
- 16,
- 2,
- 2,
- align,
- true,
- true
-);
-instantiate_mlp_fused_gemm(
- t,
- false,
- true,
- half,
- half,
- 32,
- 32,
- 16,
- 2,
- 2,
- align,
- true,
- false
-);
-instantiate_mlp_fused_gemm(
- t,
- false,
- true,
- half,
- half,
- 32,
- 32,
- 16,
- 2,
- 2,
- align,
- false,
- true
-);
-instantiate_mlp_fused_gemm(
- t,
- false,
- true,
- half,
- half,
- 32,
- 32,
- 16,
- 2,
- 2,
- align,
- false,
- false
-);
-
-instantiate_mlp_fused_gemm(
- t,
- false,
- true,
- half,
- half,
- 64,
- 64,
- 16,
- 2,
- 2,
- align,
- true,
- true
-);
-instantiate_mlp_fused_gemm(
- t,
- false,
- true,
- half,
- half,
- 64,
- 64,
- 16,
- 2,
- 2,
- align,
- true,
- false
-);
-instantiate_mlp_fused_gemm(
- t,
- false,
- true,
- half,
- half,
- 64,
- 64,
- 16,
- 2,
- 2,
- align,
- false,
- true
-);
-instantiate_mlp_fused_gemm(
- t,
- false,
- true,
- half,
- half,
- 64,
- 64,
- 16,
- 2,
- 2,
- align,
- false,
- false
-);
-
-// bfloat16 configurations
-instantiate_mlp_fused_gemm(
- t,
- false,
- true,
- bfloat,
- bfloat,
- 32,
- 32,
- 16,
- 2,
- 2,
- align,
- true,
- true
-);
-instantiate_mlp_fused_gemm(
- t,
- false,
- true,
- bfloat,
- bfloat,
- 32,
- 32,
- 16,
- 2,
- 2,
- align,
- true,
- false
-);
-instantiate_mlp_fused_gemm(
- t,
- false,
- true,
- bfloat,
- bfloat,
- 32,
- 32,
- 16,
- 2,
- 2,
- align,
- false,
- true
-);
-instantiate_mlp_fused_gemm(
- t,
- false,
- true,
- bfloat,
- bfloat,
- 32,
- 32,
- 16,
- 2,
- 2,
- align,
- false,
- false
-);
-
-instantiate_mlp_fused_gemm(
- t,
- false,
- true,
- bfloat,
- bfloat,
- 64,
- 64,
- 16,
- 2,
- 2,
- align,
- true,
- true
-);
-instantiate_mlp_fused_gemm(
- t,
- false,
- true,
- bfloat,
- bfloat,
- 64,
- 64,
- 16,
- 2,
- 2,
- align,
- true,
- false
-);
-instantiate_mlp_fused_gemm(
- t,
- false,
- true,
- bfloat,
- bfloat,
- 64,
- 64,
- 16,
- 2,
- 2,
- align,
- false,
- true
-);
-instantiate_mlp_fused_gemm(
- t,
- false,
- true,
- bfloat,
- bfloat,
- 64,
- 64,
- 16,
- 2,
- 2,
- align,
- false,
- false
-);
-
-// float32 configurations
-instantiate_mlp_fused_gemm(
- t,
- false,
- true,
- float,
- float,
- 32,
- 32,
- 16,
- 2,
- 2,
- align,
- true,
- true
-);
-instantiate_mlp_fused_gemm(
- t,
- false,
- true,
- float,
- float,
- 32,
- 32,
- 16,
- 2,
- 2,
- align,
- true,
- false
-);
-instantiate_mlp_fused_gemm(
- t,
- false,
- true,
- float,
- float,
- 32,
- 32,
- 16,
- 2,
- 2,
- align,
- false,
- true
-);
-instantiate_mlp_fused_gemm(
- t,
- false,
- true,
- float,
- float,
- 32,
- 32,
- 16,
- 2,
- 2,
- align,
- false,
- false
-);
-
-// Non-transposed B (weights are [2*H, K])
-instantiate_mlp_fused_gemm(
- n,
- false,
- false,
- half,
- half,
- 32,
- 32,
- 16,
- 2,
- 2,
- align,
- true,
- true
-);
-instantiate_mlp_fused_gemm(
- n,
- false,
- false,
- half,
- half,
- 32,
- 32,
- 16,
- 2,
- 2,
- align,
- true,
- false
-);
-instantiate_mlp_fused_gemm(
- n,
- false,
- false,
- half,
- half,
- 32,
- 32,
- 16,
- 2,
- 2,
- align,
- false,
- true
-);
-instantiate_mlp_fused_gemm(
- n,
- false,
- false,
- half,
- half,
- 32,
- 32,
- 16,
- 2,
- 2,
- align,
- false,
- false
-);
-
-instantiate_mlp_fused_gemm(
- n,
- false,
- false,
- bfloat,
- bfloat,
- 32,
- 32,
- 16,
- 2,
- 2,
- align,
- true,
- true
-);
-instantiate_mlp_fused_gemm(
- n,
- false,
- false,
- bfloat,
- bfloat,
- 32,
- 32,
- 16,
- 2,
- 2,
- align,
- true,
- false
-);
-instantiate_mlp_fused_gemm(
- n,
- false,
- false,
- bfloat,
- bfloat,
- 32,
- 32,
- 16,
- 2,
- 2,
- align,
- false,
- true
-);
-instantiate_mlp_fused_gemm(
- n,
- false,
- false,
- bfloat,
- bfloat,
- 32,
- 32,
- 16,
- 2,
- 2,
- align,
- false,
- false
-);
-
-instantiate_mlp_fused_gemm(
- n,
- false,
- false,
- float,
- float,
- 32,
- 32,
- 16,
- 2,
- 2,
- align,
- true,
- true
-);
-instantiate_mlp_fused_gemm(
- n,
- false,
- false,
- float,
- float,
- 32,
- 32,
- 16,
- 2,
- 2,
- align,
- true,
- false
-);
-instantiate_mlp_fused_gemm(
- n,
- false,
- false,
- float,
- float,
- 32,
- 32,
- 16,
- 2,
- 2,
- align,
- false,
- true
-);
-instantiate_mlp_fused_gemm(
- n,
- false,
- false,
- float,
- float,
- 32,
- 32,
- 16,
- 2,
- 2,
- align,
- false,
- false
-);
diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/tile_configuration.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/tile_configuration.rs
deleted file mode 100644
index 408008478..000000000
--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/tile_configuration.rs
+++ /dev/null
@@ -1,31 +0,0 @@
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub struct TileConfiguration {
- pub block_rows: i32,
- pub block_cols: i32,
- pub block_depth: i32,
-}
-
-impl TileConfiguration {
- pub fn new(
- block_rows: i32,
- block_cols: i32,
- block_depth: i32,
- ) -> Self {
- Self {
- block_rows,
- block_cols,
- block_depth,
- }
- }
-}
-
-pub fn select_tile_configuration(
- batch: i32,
- hidden_dim: i32,
-) -> TileConfiguration {
- if batch >= 64 && hidden_dim >= 64 {
- TileConfiguration::new(64, 64, 16)
- } else {
- TileConfiguration::new(32, 32, 16)
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/dispatch_descriptor.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/dispatch_descriptor.rs
deleted file mode 100644
index b560342ce..000000000
--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/dispatch_descriptor.rs
+++ /dev/null
@@ -1,54 +0,0 @@
-use super::pipeline_configuration::{PipelineConfiguration, select_configuration};
-use crate::{
- DataType,
- backends::metal::{MTLContext, MTLError, MTLSize, kernel::mlp_fused::common::MlpFusedArguments},
-};
-
-#[derive(Debug, Clone)]
-pub struct DispatchDescriptor {
- pub(crate) pipeline_configuration: PipelineConfiguration,
- pub(crate) input_dim: i32,
- pub(crate) hidden_dim: i32,
- pub(crate) weights_ld: i32,
- pub(crate) vector_batch_stride: i64,
- pub(crate) matrix_batch_stride: i64,
- pub(crate) threadgroups: MTLSize,
- pub(crate) threads_per_threadgroup: MTLSize,
-}
-
-impl DispatchDescriptor {
- pub(crate) fn try_new(
- _context: &MTLContext,
- data_type: DataType,
- arguments: &MlpFusedArguments,
- ) -> Result<Option<Self>, MTLError> {
- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
- return Err(MTLError::Generic(format!("Unsupported dtype for MLP fused GEMV: {data_type:?}")));
- }
-
- if arguments.batch != 1 {
- return Ok(None);
- }
-
- let pipeline_configuration = select_configuration(arguments.hidden_dim, arguments.activation);
-
- let output_elements_per_threadgroup = pipeline_configuration.output_elements_per_threadgroup();
- let threadgroup_count_x = ((arguments.hidden_dim as u32 + output_elements_per_threadgroup - 1)
- / output_elements_per_threadgroup) as u64;
- let threadgroup_count_z = arguments.batch_count.max(1) as u64;
-
- let threadgroups = MTLSize::new(threadgroup_count_x as usize, 1, threadgroup_count_z as usize);
- let threads_per_threadgroup = pipeline_configuration.threads_per_threadgroup();
-
- Ok(Some(Self {
- pipeline_configuration,
- input_dim: arguments.input_dim,
- hidden_dim: arguments.hidden_dim,
- weights_ld: arguments.ldb,
- vector_batch_stride: arguments.input_dim as i64,
- matrix_batch_stride: 0,
- threadgroups,
- threads_per_threadgroup,
- }))
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/gemv_mlp_fused.metal b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/gemv_mlp_fused.metal
deleted file mode 100644
index a7a13fee1..000000000
--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/gemv_mlp_fused.metal
+++ /dev/null
@@ -1,284 +0,0 @@
-#include <metal_simdgroup>
-#include <metal_stdlib>
-
-#include "../../common/utils.h"
-#include "../../common/mlp_epilogue.h"
-
-#include "../../matmul/common/steel/utils.h"
-
-using namespace metal;
-
-///////////////////////////////////////////////////////////////////////////////
-/// MLP Fused GEMV - Paired Up/Gate Computation with Activation Fusion
-///////////////////////////////////////////////////////////////////////////////
-
-#define MTL_CONST static constant constexpr const
-
-// MLP fused GEMV kernel for decode path (M=1).
-// Computes paired up and gate projections, then applies: out = up *
-// activation(gate) Output size is hidden_dim (half of the weight matrix
-// columns).
-template <
- typename T,
- const int BM, /* Threadgroup rows (in simdgroups) */
- const int BN, /* Threadgroup cols (in simdgroups) */
- const int SM, /* Simdgroup rows (in threads) */
- const int SN, /* Simdgroup cols (in threads) */
- const int TM, /* Thread rows (in elements) */
- const int TN> /* Thread cols (in elements) */
-struct GEMVMlpFusedKernel {
- using AccT = float;
-
- MTL_CONST int threadsM = BM * SM;
- MTL_CONST int threadsN = BN * SN;
- MTL_CONST int blockM = threadsM * TM;
- MTL_CONST int blockN = threadsN * TN;
-
- static_assert(SM * SN == 32, "simdgroup can only have 32 threads");
-
- MTL_CONST short tgp_mem_size = BN > 1 ? 2 * BN*(blockM + TM) : 0;
- MTL_CONST bool needs_tgp_reduction = BN > 1;
-
- template <typename U = T>
- static METAL_FUNC void load_unsafe(
- const device T* src,
- thread U dst[TN],
- const int src_offset = 0
- ) {
- MTL_PRAGMA_UNROLL
- for (int tn = 0; tn < TN; tn++) {
- dst[tn] = static_cast<U>(src[src_offset + tn]);
- }
- }
-
- template <typename U = T>
- static METAL_FUNC void load_safe(
- const device T* src,
- thread U dst[TN],
- const int src_offset = 0,
- const int src_size = TN
- ) {
- if (src_offset + TN <= src_size) {
- MTL_PRAGMA_UNROLL
- for (int tn = 0; tn < TN; tn++) {
- dst[tn] = static_cast<U>(src[src_offset + tn]);
- }
- } else {
- MTL_PRAGMA_UNROLL
- for (int tn = 0; tn < TN; tn++) {
- dst[tn] = src_offset + tn < src_size
- ? static_cast<U>(src[src_offset + tn])
- : U(0);
- }
- }
- }
-
- static METAL_FUNC void run(
- const device T* mat [[buffer(0)]],
- const device T* in_vec [[buffer(1)]],
- device T* out_vec [[buffer(3)]],
- const int in_vec_size,
- const int hidden_dim, // Output size (half of weight matrix rows)
- const int matrix_ld,
- threadgroup AccT* tgp_memory,
- uint3 tid [[threadgroup_position_in_grid]],
- uint3 lid [[thread_position_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
- ) {
- (void)lid;
-
- // Thread local accumulation results for up and gate
- thread AccT result_up[TM] = {0};
- thread AccT result_gate[TM] = {0};
- thread T inter_up[TN];
- thread T inter_gate[TN];
- thread AccT v_coeff[TN];
-
- const int thrM = SN != 32 ? simd_lid / SN : 0;
- const int thrN = SN != 32 ? simd_lid % SN : int(simd_lid);
-
- const int sgN = BN != 1 ? (simd_gid % BN) : 0;
- const int simdM = BN != 1 ? SM * (simd_gid / BN) : int(SM * simd_gid);
- const int simdN = BN != 1 ? SN * (simd_gid % BN) : 0;
-
- int bm = (simdM + thrM) * TM;
- int bn = (simdN + thrN) * TN;
-
- // Block position - outputs to hidden_dim elements
- int out_row = tid.x * blockM + bm;
-
- if (out_row >= hidden_dim)
- return;
-
- out_row = out_row + TM <= hidden_dim ? out_row : hidden_dim - TM;
-
- // Pointers to up and gate rows in the weight matrix
- // Weight layout: [up_0, up_1, ..., up_H-1, gate_0, gate_1, ..., gate_H-1]
- const device T* mat_up = mat + out_row * matrix_ld;
- const device T* mat_gate = mat + (out_row + hidden_dim) * matrix_ld;
-
- constexpr const uniform<int> loop_stride = make_uniform(blockN);
- const uniform<int> in_size = make_uniform(in_vec_size);
- const uniform<int> n_iter = in_size / loop_stride;
- const uniform<int> last_iter = loop_stride * n_iter;
- const uniform<int> leftover = in_size - last_iter;
-
- // Main loop - compute both up and gate dot products
- for (int i = 0; i < n_iter; ++i) {
- load_unsafe<AccT>(in_vec, v_coeff, bn);
-
- int mat_offset = 0;
- MTL_PRAGMA_UNROLL
- for (int tm = 0; tm < TM; tm++) {
- load_unsafe(mat_up, inter_up, mat_offset + bn);
- load_unsafe(mat_gate, inter_gate, mat_offset + bn);
-
- MTL_PRAGMA_UNROLL
- for (int tn = 0; tn < TN; tn++) {
- result_up[tm] += inter_up[tn] * v_coeff[tn];
- result_gate[tm] += inter_gate[tn] * v_coeff[tn];
- }
-
- mat_offset += matrix_ld;
- }
-
- bn += blockN;
- }
-
- // Handle leftover
- if (leftover > 0) {
- load_safe<AccT>(in_vec, v_coeff, bn, in_vec_size);
-
- MTL_PRAGMA_UNROLL
- for (int tm = 0; tm < TM; tm++) {
- load_safe(&mat_up[tm * matrix_ld], inter_up, bn, in_vec_size);
- load_safe(&mat_gate[tm * matrix_ld], inter_gate, bn, in_vec_size);
-
- MTL_PRAGMA_UNROLL
- for (int tn = 0; tn < TN; tn++) {
- result_up[tm] += inter_up[tn] * v_coeff[tn];
- result_gate[tm] += inter_gate[tn] * v_coeff[tn];
- }
- }
- }
-
- // Simdgroup reduction for both up and gate
- MTL_PRAGMA_UNROLL
- for (int tm = 0; tm < TM; tm++) {
- MTL_PRAGMA_UNROLL
- for (ushort sn = (SN / 2); sn >= 1; sn >>= 1) {
- result_up[tm] += simd_shuffle_down(result_up[tm], sn);
- result_gate[tm] += simd_shuffle_down(result_gate[tm], sn);
- }
- }
-
- // Threadgroup reduction if needed
- if (needs_tgp_reduction) {
- threadgroup AccT* tgp_results_up = tgp_memory + sgN * (blockM + TM) + bm;
- threadgroup AccT* tgp_results_gate =
- tgp_memory + BN * (blockM + TM) + sgN * (blockM + TM) + bm;
-
- if (thrN == 0) {
- MTL_PRAGMA_UNROLL
- for (int tm = 0; tm < TM; tm++) {
- tgp_results_up[tm] = result_up[tm];
- tgp_results_gate[tm] = result_gate[tm];
- }
-
- threadgroup_barrier(mem_flags::mem_none);
-
- if (sgN == 0) {
- MTL_PRAGMA_UNROLL
- for (int sgn = 1; sgn < BN; sgn++) {
- MTL_PRAGMA_UNROLL
- for (int tm = 0; tm < TM; tm++) {
- result_up[tm] += tgp_results_up[sgn * (blockM + TM) + tm];
- result_gate[tm] += tgp_results_gate[sgn * (blockM + TM) + tm];
- }
- }
- }
- }
- }
-
- // Apply MLP fused epilogue and write output
- if (simdN == 0 && thrN == 0) {
- MTL_PRAGMA_UNROLL
- for (int tm = 0; tm < TM; tm++) {
- float fused_result =
- mlp_fused_epilogue_f32(result_up[tm], result_gate[tm]);
- out_vec[out_row + tm] = static_cast<T>(fused_result);
- }
- }
- }
-};
-
-template <
- typename T,
- const int BM,
- const int BN,
- const int SM,
- const int SN,
- const int TM,
- const int TN>
-[[kernel, max_total_threads_per_threadgroup(BM * BN * 32)]] void gemv_mlp_fused(
- const device T* mat [[buffer(0)]],
- const device T* in_vec [[buffer(1)]],
- device T* out_vec [[buffer(3)]],
- const constant int& in_vec_size [[buffer(4)]],
- const constant int& hidden_dim
- [[buffer(5)]], // Output dimension (half of weight rows)
- const constant int& matrix_ld [[buffer(6)]],
- const constant int64_t& vector_batch_stride [[buffer(11)]],
- const constant int64_t& matrix_batch_stride [[buffer(12)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint3 lid [[thread_position_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]
-) {
- using gemv_kernel = GEMVMlpFusedKernel<T, BM, BN, SM, SN, TM, TN>;
- threadgroup typename gemv_kernel::AccT tgp_memory
- [gemv_kernel::tgp_mem_size == 0 ? 1 : gemv_kernel::tgp_mem_size];
-
- // Batch offset
- in_vec += tid.z * vector_batch_stride;
- mat += tid.z * matrix_batch_stride;
- out_vec += tid.z * hidden_dim;
-
- gemv_kernel::run(
- mat,
- in_vec,
- out_vec,
- in_vec_size,
- hidden_dim,
- matrix_ld,
- gemv_kernel::tgp_mem_size == 0 ? nullptr : tgp_memory,
- tid,
- lid,
- simd_gid,
- simd_lid
- );
-}
-
-#define instantiate_gemv_mlp_fused_helper(name, itype, bm, bn, sm, sn, tm, tn) \
- instantiate_kernel( \
- "gemv_mlp_fused_" #name "_bm" #bm "_bn" #bn "_sm" #sm "_sn" #sn \
- "_tm" #tm "_tn" #tn, \
- gemv_mlp_fused, \
- itype, \
- bm, \
- bn, \
- sm, \
- sn, \
- tm, \
- tn \
- )
-
-// clang-format off
-#define instantiate_gemv_mlp_fused_blocks(name, itype) \
- instantiate_gemv_mlp_fused_helper(name, itype, 4, 1, 1, 32, 4, 4) \
- instantiate_gemv_mlp_fused_helper(name, itype, 8, 1, 1, 32, 4, 4) // clang-format on
-
-instantiate_gemv_mlp_fused_blocks(float32, float);
-instantiate_gemv_mlp_fused_blocks(float16, half);
-instantiate_gemv_mlp_fused_blocks(bfloat16, bfloat16_t);
diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/kernel.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/kernel.rs
deleted file mode 100644
index 3cc73b9da..000000000
--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/kernel.rs
+++ /dev/null
@@ -1,95 +0,0 @@
-use std::collections::HashMap;
-
-use super::{DispatchDescriptor, pipeline_configuration::PipelineConfiguration};
-use crate::{
- DataType,
- backends::metal::{
- ComputeEncoderSetValue, FunctionConstantValuesSetValue, MTLComputeCommandEncoder, MTLComputePipelineState,
- MTLContext, MTLError, MTLFunctionConstantValues, ProtocolObject, Retained,
- kernel::mlp_fused::common::MlpFusedArguments,
- },
-};
-
-fn kernel_name(
- data_type: DataType,
- configuration: &PipelineConfiguration,
-) -> Result<String, MTLError> {
- let dtype_name = match data_type {
- DataType::F16 => "float16",
- DataType::BF16 => "bfloat16",
- DataType::F32 => "float32",
- _ => {
- return Err(MTLError::Generic(format!("Unsupported data type for MLP fused GEMV: {data_type:?}")));
- },
- };
-
- Ok(format!(
- "gemv_mlp_fused_{}_bm{}_bn{}_sm{}_sn{}_tm{}_tn{}",
- dtype_name,
- configuration.threadgroup_rows,
- configuration.threadgroup_cols,
- configuration.threads_per_simdgroup_row,
- configuration.threads_per_simdgroup_col,
- configuration.elements_per_thread_row,
- configuration.elements_per_thread_col,
- ))
-}
-
-pub struct Kernel {
- data_type: DataType,
- pipelines: HashMap<PipelineConfiguration, Retained<ProtocolObject<dyn MTLComputePipelineState>>>,
-}
-
-impl Kernel {
- pub fn new(data_type: DataType) -> Result<Self, MTLError> {
- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
- return Err(MTLError::Generic(format!("Unsupported dtype for MLP fused GEMV: {data_type:?}")));
- }
- Ok(Self {
- data_type,
- pipelines: HashMap::new(),
- })
- }
-
- fn get_or_compile_pipeline(
- &mut self,
- context: &MTLContext,
- configuration: &PipelineConfiguration,
- ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError> {
- if !self.pipelines.contains_key(configuration) {
- let kernel_name = kernel_name(self.data_type, configuration)?;
-
- let function_constants = MTLFunctionConstantValues::new();
- let activation_val = configuration.activation as u32;
- function_constants.set_value(&activation_val, 52);
-
- let pipeline_state = context.compute_pipeline_state(&kernel_name, Some(&function_constants))?;
- self.pipelines.insert(*configuration, pipeline_state);
- }
- Ok(self.pipelines.get(configuration).unwrap())
- }
-
- pub(crate) fn encode_descriptor(
- &mut self,
- context: &MTLContext,
- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- arguments: &MlpFusedArguments,
- descriptor: &DispatchDescriptor,
- ) -> Result<(), MTLError> {
- let pipeline_state = self.get_or_compile_pipeline(context, &descriptor.pipeline_configuration)?;
- encoder.set_compute_pipeline_state(pipeline_state);
-
- encoder.set_buffer(Some(arguments.weights), 0, 0);
- encoder.set_buffer(Some(arguments.input), arguments.input_offset as usize, 1);
- encoder.set_buffer(Some(arguments.output), 0, 3);
-
- encoder.set_value(&descriptor.input_dim, 4);
- encoder.set_value(&descriptor.hidden_dim, 5);
- encoder.set_value(&descriptor.weights_ld, 6);
- encoder.set_value(&descriptor.vector_batch_stride, 11);
- encoder.set_value(&descriptor.matrix_batch_stride, 12);
-
- encoder.dispatch_threadgroups(descriptor.threadgroups, descriptor.threads_per_threadgroup);
- Ok(())
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/mod.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/mod.rs
deleted file mode 100644
index 5ee340249..000000000
--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/mod.rs
+++ /dev/null
@@ -1,6 +0,0 @@
-mod dispatch_descriptor;
-mod kernel;
-mod pipeline_configuration;
-
-pub(crate) use dispatch_descriptor::DispatchDescriptor;
-pub use kernel::Kernel as GemvKernel;
diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/pipeline_configuration.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/pipeline_configuration.rs
deleted file mode 100644
index 91130ed54..000000000
--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/pipeline_configuration.rs
+++ /dev/null
@@ -1,43 +0,0 @@
-use crate::backends::metal::{MTLSize, kernel::mlp::MlpActivationType};
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub struct PipelineConfiguration {
- pub threadgroup_rows: u32,
- pub threadgroup_cols: u32,
- pub threads_per_simdgroup_row: u32,
- pub threads_per_simdgroup_col: u32,
- pub elements_per_thread_row: u32,
- pub elements_per_thread_col: u32,
- pub activation: MlpActivationType,
-}
-
-impl PipelineConfiguration {
- pub fn output_elements_per_threadgroup(&self) -> u32 {
- self.threadgroup_rows * self.threads_per_simdgroup_row * self.elements_per_thread_row
- }
-
- pub fn threads_per_threadgroup(&self) -> MTLSize {
- MTLSize::new(32, self.threadgroup_cols as usize, self.threadgroup_rows as usize)
- }
-}
-
-pub fn select_configuration(
- hidden_dim: i32,
- activation: MlpActivationType,
-) -> PipelineConfiguration {
- let threadgroup_rows = if hidden_dim >= 4096 {
- 8
- } else {
- 4
- };
-
- PipelineConfiguration {
- threadgroup_rows,
- threadgroup_cols: 1,
- threads_per_simdgroup_row: 1,
- threads_per_simdgroup_col: 32,
- elements_per_thread_row: 4,
- elements_per_thread_col: 4,
- activation,
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/kernel.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/kernel.rs
deleted file mode 100644
index 1c3eb6ad9..000000000
--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/kernel.rs
+++ /dev/null
@@ -1,93 +0,0 @@
-use super::{
- common::MlpFusedArguments,
- dispatch_descriptor::{MlpFusedDispatchDescriptor, choose_dispatch_descriptor},
- gemm::GemmKernel,
- gemv::GemvKernel,
- split_k::SplitKKernel,
-};
-use crate::{
- DataType,
- backends::metal::{MTLComputeCommandEncoder, MTLContext, MTLError, ProtocolObject},
-};
-
-pub struct MlpFusedKernel {
- data_type: DataType,
- weights_transposed: bool,
- gemm: Option<GemmKernel>,
- gemv: Option<GemvKernel>,
- splitk: Option<SplitKKernel>,
-}
-
-impl MlpFusedKernel {
- pub fn new(
- data_type: DataType,
- weights_transposed: bool,
- ) -> Result<Self, MTLError> {
- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
- return Err(MTLError::Generic(format!("Unsupported dtype for MlpFusedKernel: {data_type:?}")));
- }
-
- Ok(Self {
- data_type,
- weights_transposed,
- gemm: None,
- gemv: None,
- splitk: None,
- })
- }
-
- fn get_or_create_gemm(&mut self) -> Result<&mut GemmKernel, MTLError> {
- if self.gemm.is_none() {
- self.gemm = Some(GemmKernel::new(self.data_type, self.weights_transposed)?);
- }
- Ok(self.gemm.as_mut().unwrap())
- }
-
- fn get_or_create_gemv(&mut self) -> Result<&mut GemvKernel, MTLError> {
- if self.gemv.is_none() {
- self.gemv = Some(GemvKernel::new(self.data_type)?);
- }
- Ok(self.gemv.as_mut().unwrap())
- }
-
- fn get_or_create_splitk(&mut self) -> Result<&mut SplitKKernel, MTLError> {
- if self.splitk.is_none() {
- self.splitk = Some(SplitKKernel::new(self.data_type)?);
- }
- Ok(self.splitk.as_mut().unwrap())
- }
-
- fn encode_dispatch_descriptor(
- &mut self,
- context: &MTLContext,
- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- arguments: &MlpFusedArguments,
- descriptor: &MlpFusedDispatchDescriptor,
- ) -> Result<(), MTLError> {
- match descriptor {
- MlpFusedDispatchDescriptor::Gemv(descriptor) => {
- let gemv = self.get_or_create_gemv()?;
- gemv.encode_descriptor(context, encoder, arguments, descriptor)
- },
- MlpFusedDispatchDescriptor::SplitK(descriptor) => {
- let splitk = self.get_or_create_splitk()?;
- splitk.encode_descriptor(context, encoder, arguments, descriptor)
- },
- MlpFusedDispatchDescriptor::Gemm(descriptor) => {
- let gemm = self.get_or_create_gemm()?;
- gemm.encode_descriptor(context, encoder, arguments, descriptor)
- },
- }
- }
-
- pub fn encode(
- &mut self,
- context: &MTLContext,
- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- arguments: &MlpFusedArguments,
- ) -> Result<(), MTLError> {
- let descriptor = choose_dispatch_descriptor(context, self.data_type, self.weights_transposed, arguments)?;
-
- self.encode_dispatch_descriptor(context, encoder, arguments, &descriptor)
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/mod.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/mod.rs
deleted file mode 100644
index f503655f3..000000000
--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/mod.rs
+++ /dev/null
@@ -1,13 +0,0 @@
-pub mod common;
-mod dispatch_descriptor;
-mod gemm;
-mod gemv;
-mod kernel;
-mod split_k;
-
-pub use common::MlpFusedArguments;
-pub use dispatch_descriptor::{MlpFusedKernelVariant, determine_kernel_variant};
-pub use gemm::GemmKernel;
-pub use gemv::GemvKernel;
-pub use kernel::MlpFusedKernel;
-pub use split_k::SplitKKernel;
diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/dispatch_descriptor.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/dispatch_descriptor.rs
deleted file mode 100644
index 574efb5b9..000000000
--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/dispatch_descriptor.rs
+++ /dev/null
@@ -1,129 +0,0 @@
-use super::{pipeline_configuration::PipelineConfiguration, tile_configuration::select_tile_configuration};
-use crate::{
- DataType,
- backends::metal::{
- MTLContext, MTLError, MTLSize,
- kernel::{matmul::common::GEMMSpiltKMlpFusedParams, mlp_fused::common::MlpFusedArguments},
- },
-};
-
-#[derive(Debug, Clone)]
-pub struct DispatchDescriptor {
- pub(crate) pipeline_configuration: PipelineConfiguration,
- pub(crate) params: GEMMSpiltKMlpFusedParams,
- pub(crate) partition_count: i32,
- pub(crate) output_elements_per_partition: i32,
- pub(crate) accumulator_bytes: usize,
- pub(crate) ldd: i32,
- pub(crate) partial_threadgroups: MTLSize,
- pub(crate) partial_threads_per_threadgroup: MTLSize,
- pub(crate) accum_total_threads: MTLSize,
- pub(crate) accum_threads_per_threadgroup: MTLSize,
-}
-
-impl DispatchDescriptor {
- pub(crate) fn try_new(
- _context: &MTLContext,
- data_type: DataType,
- arguments: &MlpFusedArguments,
- ) -> Result<Option<Self>, MTLError> {
- if !matches!(data_type, DataType::F16 | DataType::BF16) {
- return Ok(None);
- }
-
- if !should_use_splitk(arguments.batch, arguments.hidden_dim, arguments.input_dim) {
- return Ok(None);
- }
-
- let batch = arguments.batch;
- let hidden_dim = arguments.hidden_dim;
- let input_dim = arguments.input_dim;
-
- let tile = select_tile_configuration(batch, hidden_dim);
- let partition_count = compute_partition_count(input_dim);
- let mn_aligned = (batch % tile.tile_rows) == 0 && (hidden_dim % tile.tile_cols) == 0;
- let k_aligned = (input_dim % tile.tile_depth) == 0;
-
- let pipeline_configuration = PipelineConfiguration {
- tile,
- mn_aligned,
- k_aligned,
- activation: arguments.activation,
- };
-
- let tile_count_m = (batch + tile.tile_rows - 1) / tile.tile_rows;
- let tile_count_n = (hidden_dim + tile.tile_cols - 1) / tile.tile_cols;
-
- let gemm_k_iterations = (input_dim / tile.tile_depth) / partition_count;
- let k_elements_per_partition = gemm_k_iterations * tile.tile_depth;
- let output_elements_per_partition = batch * hidden_dim;
-
- let accumulator_element_count = partition_count * output_elements_per_partition;
- let accumulator_bytes = (accumulator_element_count as usize) * std::mem::size_of::<f32>();
-
- let params = GEMMSpiltKMlpFusedParams {
- M: batch,
- N: hidden_dim,
- K: input_dim,
- lda: arguments.lda,
- ldb: arguments.ldb,
- ldc: hidden_dim,
- tiles_n: tile_count_n,
- tiles_m: tile_count_m,
- split_k_partitions: partition_count,
- split_k_partition_stride: output_elements_per_partition,
- split_k_partition_size: k_elements_per_partition,
- gemm_k_iterations_aligned: gemm_k_iterations,
- hidden_dim,
- };
-
- let partial_threads_per_threadgroup =
- MTLSize::new(32, tile.warps_per_col as usize, tile.warps_per_row as usize);
- let partial_threadgroups = MTLSize::new(tile_count_n as usize, tile_count_m as usize, partition_count as usize);
-
- let accum_total_threads = MTLSize::new(hidden_dim as usize, batch as usize, 1);
- let accum_threads_per_threadgroup = MTLSize::new(16.min(hidden_dim as usize), 16.min(batch as usize), 1);
-
- Ok(Some(Self {
- pipeline_configuration,
- params,
- partition_count,
- output_elements_per_partition,
- accumulator_bytes,
- ldd: arguments.ldd,
- partial_threadgroups,
- partial_threads_per_threadgroup,
- accum_total_threads,
- accum_threads_per_threadgroup,
- }))
- }
-}
-
-fn compute_partition_count(k: i32) -> i32 {
- let k_tiles = k / 16;
- if k_tiles < 16 {
- 2
- } else if k_tiles < 32 {
- 4
- } else if k_tiles < 64 {
- 8
- } else {
- 16
- }
-}
-
-pub fn should_use_splitk(
- batch: i32,
- hidden_dim: i32,
- input_dim: i32,
-) -> bool {
- if batch != 1 {
- return false;
- }
- if hidden_dim <= 0 || input_dim <= 0 {
- return false;
- }
- let n_tiles = hidden_dim / 16;
- let k_tiles = input_dim / 16;
- n_tiles <= 32 && k_tiles >= 8
-}
diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/kernel.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/kernel.rs
deleted file mode 100644
index 634bfe559..000000000
--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/kernel.rs
+++ /dev/null
@@ -1,170 +0,0 @@
-use std::collections::HashMap;
-
-use super::{DispatchDescriptor, pipeline_configuration::PipelineConfiguration};
-use crate::{
- DataType,
- backends::{
- common::Context,
- metal::{
- ComputeEncoderSetValue, FunctionConstantValuesSetValue, MTLBuffer, MTLComputeCommandEncoder,
- MTLComputePipelineState, MTLContext, MTLError, MTLFunctionConstantValues, ProtocolObject, Retained,
- kernel::{mlp::MlpActivationType, mlp_fused::common::MlpFusedArguments},
- },
- },
-};
-
-pub struct Kernel {
- data_type: DataType,
- partial_pipelines: HashMap<PipelineConfiguration, Retained<ProtocolObject<dyn MTLComputePipelineState>>>,
- accum_pipelines: HashMap<MlpActivationType, Retained<ProtocolObject<dyn MTLComputePipelineState>>>,
- up_accumulator_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>>,
- gate_accumulator_buffer: Option<Retained<ProtocolObject<dyn MTLBuffer>>>,
- accumulator_buffer_bytes: usize,
-}
-
-impl Kernel {
- pub fn new(data_type: DataType) -> Result<Self, MTLError> {
- if !matches!(data_type, DataType::F16 | DataType::BF16) {
- return Err(MTLError::Generic(format!("Unsupported dtype for MLP fused Split-K: {data_type:?}")));
- }
- Ok(Self {
- data_type,
- partial_pipelines: HashMap::new(),
- accum_pipelines: HashMap::new(),
- up_accumulator_buffer: None,
- gate_accumulator_buffer: None,
- accumulator_buffer_bytes: 0,
- })
- }
-
- fn steel_type_name(&self) -> Result<&'static str, MTLError> {
- match self.data_type {
- DataType::F16 => Ok("float16"),
- DataType::BF16 => Ok("bfloat16"),
- _ => Err(MTLError::Generic(format!("Unsupported dtype for MLP Fused Split-K: {:?}", self.data_type))),
- }
- }
-
- fn partial_kernel_name(
- &self,
- configuration: &PipelineConfiguration,
- ) -> Result<String, MTLError> {
- let in_name = self.steel_type_name()?;
- let mn_tag = if configuration.mn_aligned {
- "taligned"
- } else {
- "naligned"
- };
- let k_tag = if configuration.k_aligned {
- "taligned"
- } else {
- "naligned"
- };
- Ok(format!(
- "steel_gemm_splitk_mlp_fused_nt_{}_float32_bm{}_bn{}_bk{}_wm{}_wn{}_MN_{}_K_{}",
- in_name,
- configuration.tile.tile_rows,
- configuration.tile.tile_cols,
- configuration.tile.tile_depth,
- configuration.tile.warps_per_row,
- configuration.tile.warps_per_col,
- mn_tag,
- k_tag,
- ))
- }
-
- fn accum_kernel_name(&self) -> Result<String, MTLError> {
- let out_name = self.steel_type_name()?;
- Ok(format!("steel_gemm_splitk_mlp_fused_accum_{}_float32", out_name))
- }
-
- fn get_partial_pipeline(
- &mut self,
- context: &MTLContext,
- configuration: &PipelineConfiguration,
- ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError> {
- if !self.partial_pipelines.contains_key(configuration) {
- let kernel_name = self.partial_kernel_name(configuration)?;
- let pipeline_state = context.compute_pipeline_state(&kernel_name, None)?;
- self.partial_pipelines.insert(configuration.clone(), pipeline_state);
- }
- Ok(self.partial_pipelines.get(configuration).unwrap())
- }
-
- fn get_accum_pipeline(
- &mut self,
- context: &MTLContext,
- activation: MlpActivationType,
- ) -> Result<&Retained<ProtocolObject<dyn MTLComputePipelineState>>, MTLError> {
- if !self.accum_pipelines.contains_key(&activation) {
- let kernel_name = self.accum_kernel_name()?;
- let function_constants = MTLFunctionConstantValues::new();
- let activation_val = activation as u32;
- function_constants.set_value(&activation_val, 52);
- let pipeline_state = context.compute_pipeline_state(&kernel_name, Some(&function_constants))?;
- self.accum_pipelines.insert(activation, pipeline_state);
- }
- Ok(self.accum_pipelines.get(&activation).unwrap())
- }
-
- fn ensure_accumulator_buffers(
- &mut self,
- context: &MTLContext,
- required_bytes: usize,
- ) {
- if required_bytes <= self.accumulator_buffer_bytes
- && self.up_accumulator_buffer.is_some()
- && self.gate_accumulator_buffer.is_some()
- {
- return;
- }
- self.up_accumulator_buffer = context.create_buffer(required_bytes).ok();
- self.gate_accumulator_buffer = context.create_buffer(required_bytes).ok();
- self.accumulator_buffer_bytes = required_bytes;
- }
-
- pub(crate) fn encode_descriptor(
- &mut self,
- context: &MTLContext,
- encoder: &ProtocolObject<dyn MTLComputeCommandEncoder>,
- arguments: &MlpFusedArguments,
- descriptor: &DispatchDescriptor,
- ) -> Result<(), MTLError> {
- self.ensure_accumulator_buffers(context, descriptor.accumulator_bytes);
- self.get_partial_pipeline(context, &descriptor.pipeline_configuration)?;
- self.get_accum_pipeline(context, descriptor.pipeline_configuration.activation)?;
-
- let partial_pipeline_state = self
- .partial_pipelines
- .get(&descriptor.pipeline_configuration)
- .expect("Partial pipeline must be initialized");
- let accum_pipeline_state = self
- .accum_pipelines
- .get(&descriptor.pipeline_configuration.activation)
- .expect("Accum pipeline must be initialized");
- let up_accumulator_buffer =
- self.up_accumulator_buffer.as_ref().expect("Up accumulator buffer must be initialized");
- let gate_accumulator_buffer =
- self.gate_accumulator_buffer.as_ref().expect("Gate accumulator buffer must be initialized");
-
- encoder.set_compute_pipeline_state(partial_pipeline_state);
- encoder.set_buffer(Some(arguments.input), arguments.input_offset as usize, 0);
- encoder.set_buffer(Some(arguments.weights), 0, 1);
- encoder.set_buffer(Some(up_accumulator_buffer), 0, 2);
- encoder.set_buffer(Some(gate_accumulator_buffer), 0, 3);
- encoder.set_value(&descriptor.params, 4);
- encoder.dispatch_threadgroups(descriptor.partial_threadgroups, descriptor.partial_threads_per_threadgroup);
-
- encoder.set_compute_pipeline_state(accum_pipeline_state);
- encoder.set_buffer(Some(up_accumulator_buffer), 0, 0);
- encoder.set_buffer(Some(gate_accumulator_buffer), 0, 1);
- encoder.set_buffer(Some(arguments.output), 0, 2);
- encoder.set_value(&descriptor.partition_count, 3);
- encoder.set_value(&descriptor.output_elements_per_partition, 4);
- encoder.set_value(&descriptor.ldd, 5);
-
- encoder.dispatch_threads(descriptor.accum_total_threads, descriptor.accum_threads_per_threadgroup);
-
- Ok(())
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/mod.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/mod.rs
deleted file mode 100644
index efefcd2bd..000000000
--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/mod.rs
+++ /dev/null
@@ -1,7 +0,0 @@
-mod dispatch_descriptor;
-mod kernel;
-mod pipeline_configuration;
-mod tile_configuration;
-
-pub(crate) use dispatch_descriptor::DispatchDescriptor;
-pub use kernel::Kernel as SplitKKernel;
diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/pipeline_configuration.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/pipeline_configuration.rs
deleted file mode 100644
index 4f21c8f5e..000000000
--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/pipeline_configuration.rs
+++ /dev/null
@@ -1,10 +0,0 @@
-use super::tile_configuration::TileConfiguration;
-use crate::backends::metal::kernel::mlp::MlpActivationType;
-
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct PipelineConfiguration {
- pub tile: TileConfiguration,
- pub mn_aligned: bool,
- pub k_aligned: bool,
- pub activation: MlpActivationType,
-}
diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/steel_gemm_splitk_mlp_fused.h b/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/steel_gemm_splitk_mlp_fused.h
deleted file mode 100644
index 8c0522fc9..000000000
--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/steel_gemm_splitk_mlp_fused.h
+++ /dev/null
@@ -1,235 +0,0 @@
-#pragma once
-
-#include <metal_stdlib>
-#include <metal_simdgroup>
-#include <metal_simdgroup_matrix>
-
-#include "../../matmul/common/steel/gemm/gemm.h"
-#include "matmul.h"
-#include "../../common/mlp_epilogue.h"
-
-using namespace metal;
-using namespace steel;
-
-///////////////////////////////////////////////////////////////////////////////
-// MLP Fused Split-K GEMM Kernel
-// Computes paired up/gate projections with fused activation using split-K.
-///////////////////////////////////////////////////////////////////////////////
-
-template <
- typename T,
- typename U,
- int BM,
- int BN,
- int BK,
- int WM,
- int WN,
- bool transpose_a,
- bool transpose_b,
- bool MN_aligned,
- bool K_aligned>
-[[kernel, max_total_threads_per_threadgroup(WM * WN * 32)]] void
-gemm_splitk_mlp_fused(
- const device T* A [[buffer(0)]],
- const device T* B [[buffer(1)]],
- device U* C [[buffer(2)]],
- const constant uzu::matmul::GEMMSpiltKMlpFusedParams* params [[buffer(3)]],
- uint simd_lane_id [[thread_index_in_simdgroup]],
- uint simd_group_id [[simdgroup_index_in_threadgroup]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint3 lid [[thread_position_in_threadgroup]]
-) {
- (void)lid;
-
- using gemm_kernel = GEMMKernel<
- T,
- U,
- BM,
- BN,
- BK,
- WM,
- WN,
- transpose_a,
- transpose_b,
- MN_aligned,
- K_aligned>;
- using loader_a_t = typename gemm_kernel::loader_a_t;
- using loader_b_t = typename gemm_kernel::loader_b_t;
- using mma_t = typename gemm_kernel::mma_t;
-
- threadgroup T As[gemm_kernel::tgp_mem_size_a];
- threadgroup T Bs_up[gemm_kernel::tgp_mem_size_b];
- threadgroup T Bs_gate[gemm_kernel::tgp_mem_size_b];
-
- const int tid_x = tid.x;
- const int tid_y = tid.y;
- const int tid_z = tid.z;
-
- // Only process tiles within hidden_dim columns
- int hidden_tiles = (params->hidden_dim + BN - 1) / BN;
- if (hidden_tiles <= tid_x || params->tiles_m <= tid_y) {
- return;
- }
-
- // Find block in A, B, C
- const int c_row = tid_y * BM;
- const int c_col = tid_x * BN;
- const int k_start = params->split_k_partition_size * tid_z;
-
- const size_t c_row_long = size_t(c_row);
- const size_t c_col_long = size_t(c_col);
- const size_t k_start_long = size_t(k_start);
-
- // Pointers to A (input)
- const device T* A_ptr =
- A + (transpose_a ? (c_row_long + k_start_long * params->lda)
- : (k_start_long + c_row_long * params->lda));
-
- // Pointers to B (up and gate weights)
- // B layout: [up_weights | gate_weights] where each is hidden_dim columns
- const device T* B_up =
- B + (transpose_b ? (k_start_long + c_col_long * params->ldb)
- : (c_col_long + k_start_long * params->ldb));
- const device T* B_gate =
- B_up +
- (transpose_b ? params->hidden_dim * params->ldb : params->hidden_dim);
-
- // C output for partial sums (up and gate interleaved in split_k dimension)
- // Layout: [up_partitions, gate_partitions] in the partition stride
- device U* C_up = C + (size_t(params->split_k_partition_stride) * tid_z) +
- (c_row_long * params->ldc + c_col_long);
- device U* C_gate = C_up + (size_t(params->split_k_partition_stride) *
- params->split_k_partitions);
-
- // Loaders
- thread loader_a_t
- loader_a(A_ptr, params->lda, As, simd_group_id, simd_lane_id);
- thread loader_b_t
- loader_b_up(B_up, params->ldb, Bs_up, simd_group_id, simd_lane_id);
- thread loader_b_t
- loader_b_gate(B_gate, params->ldb, Bs_gate, simd_group_id, simd_lane_id);
-
- // MMA operations
- thread mma_t mma_up(simd_group_id, simd_lane_id);
- thread mma_t mma_gate(simd_group_id, simd_lane_id);
-
- int gemm_k_iterations = params->gemm_k_iterations_aligned;
-
- short tgp_bm = min(BM, params->M - c_row);
- short tgp_bn = min(BN, params->hidden_dim - c_col);
- short leftover_bk = params->K % BK;
-
- // Main loop
- for (int k = 0; k < gemm_k_iterations; k++) {
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- if (MN_aligned) {
- loader_a.load_unsafe();
- loader_b_up.load_unsafe();
- loader_b_gate.load_unsafe();
- } else if (tgp_bm == BM && tgp_bn == BN) {
- loader_a.load_unsafe();
- loader_b_up.load_unsafe();
- loader_b_gate.load_unsafe();
- } else {
- short2 tile_dims_A =
- transpose_a ? short2(tgp_bm, BK) : short2(BK, tgp_bm);
- short2 tile_dims_B =
- transpose_b ? short2(BK, tgp_bn) : short2(tgp_bn, BK);
- loader_a.load_safe(tile_dims_A);
- loader_b_up.load_safe(tile_dims_B);
- loader_b_gate.load_safe(tile_dims_B);
- }
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- mma_up.mma(As, Bs_up);
- mma_gate.mma(As, Bs_gate);
-
- loader_a.next();
- loader_b_up.next();
- loader_b_gate.next();
- }
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // Handle remaining K for last partition
- if ((tid_z + 1) == (params->split_k_partitions)) {
- int gemm_k_iter_remaining =
- (params->K - (k_start + params->split_k_partition_size)) / BK;
- for (int k = 0; k < gemm_k_iter_remaining; k++) {
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- if (!K_aligned) {
- short2 tile_dims_A = transpose_a ? short2(tgp_bm, leftover_bk)
- : short2(leftover_bk, tgp_bm);
- short2 tile_dims_B = transpose_b ? short2(leftover_bk, tgp_bn)
- : short2(tgp_bn, leftover_bk);
- loader_a.load_safe(tile_dims_A);
- loader_b_up.load_safe(tile_dims_B);
- loader_b_gate.load_safe(tile_dims_B);
- } else {
- loader_a.load_unsafe();
- loader_b_up.load_unsafe();
- loader_b_gate.load_unsafe();
- }
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- mma_up.mma(As, Bs_up);
- mma_gate.mma(As, Bs_gate);
-
- loader_a.next();
- loader_b_up.next();
- loader_b_gate.next();
- }
- }
-
- // Store partial results
- if (MN_aligned || (tgp_bm == BM && tgp_bn == BN)) {
- mma_up.store_result(C_up, params->ldc);
- mma_gate.store_result(C_gate, params->ldc);
- } else {
- mma_up.store_result_safe(C_up, params->ldc, short2(tgp_bn, tgp_bm));
- mma_gate.store_result_safe(C_gate, params->ldc, short2(tgp_bn, tgp_bm));
- }
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// MLP Fused Split-K accumulation kernel
-// Accumulates partial sums and applies MLP fused epilogue
-///////////////////////////////////////////////////////////////////////////////
-
-template <typename AccT, typename OutT>
-[[kernel, max_total_threads_per_threadgroup(256)]] void
-gemm_splitk_mlp_fused_accum(
- const device AccT* C_split [[buffer(0)]],
- device OutT* D [[buffer(1)]],
- const constant int& k_partitions [[buffer(2)]],
- const constant int& partition_stride [[buffer(3)]],
- const constant int& ldd [[buffer(4)]],
- uint2 gid [[thread_position_in_grid]]
-) {
- // Adjust pointers
- D += gid.x + gid.y * size_t(ldd);
- const device AccT* C_up = C_split + gid.x + gid.y * size_t(ldd);
- const device AccT* C_gate = C_up + (size_t(partition_stride) * k_partitions);
-
- // Accumulate up and gate partial sums
- AccT out_up = 0;
- AccT out_gate = 0;
-
- size_t offset = 0;
- for (int i = 0; i < k_partitions; i++) {
- out_up += C_up[offset];
- out_gate += C_gate[offset];
- offset += partition_stride;
- }
-
- // Apply MLP fused epilogue: out = up * activation(gate)
- float fused = mlp_fused_epilogue_f32(
- static_cast<float>(out_up),
- static_cast<float>(out_gate)
- );
- D[0] = static_cast<OutT>(fused);
-}
diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/steel_gemm_splitk_mlp_fused.metal b/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/steel_gemm_splitk_mlp_fused.metal
deleted file mode 100644
index e58783794..000000000
--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/steel_gemm_splitk_mlp_fused.metal
+++ /dev/null
@@ -1,67 +0,0 @@
-// clang-format off
-#include "../../common/utils.h"
-#include "steel_gemm_splitk_mlp_fused.h"
-
-#define instantiate_gemm( \
- tname, \
- trans_a, \
- trans_b, \
- iname, \
- itype, \
- oname, \
- otype, \
- bm, \
- bn, \
- bk, \
- wm, \
- wn, \
- aname, \
- mn_aligned, \
- kname, \
- k_aligned) \
- instantiate_kernel( \
- "steel_gemm_splitk_mlp_fused_" #tname "_" #iname "_" #oname \
- "_bm" #bm "_bn" #bn "_bk" #bk "_wm" #wm "_wn" #wn \
- "_MN_" #aname "_K_" #kname, \
- gemm_splitk_mlp_fused, \
- itype, \
- otype, \
- bm, \
- bn, \
- bk, \
- wm, \
- wn, \
- trans_a, \
- trans_b, \
- mn_aligned, \
- k_aligned)
-
-#define instantiate_gemm_aligned_helper(tname, trans_a, trans_b, iname, itype, oname, otype, bm, bn, bk, wm, wn) \
- instantiate_gemm(tname, trans_a, trans_b, iname, itype, oname, otype, bm, bn, bk, wm, wn, taligned, true, taligned, true) \
- instantiate_gemm(tname, trans_a, trans_b, iname, itype, oname, otype, bm, bn, bk, wm, wn, taligned, true, naligned, false) \
- instantiate_gemm(tname, trans_a, trans_b, iname, itype, oname, otype, bm, bn, bk, wm, wn, naligned, false, taligned, true) \
- instantiate_gemm(tname, trans_a, trans_b, iname, itype, oname, otype, bm, bn, bk, wm, wn, naligned, false, naligned, false)
-
-// Only NT transpose (common case for MLP)
-#define instantiate_gemm_transpose_helper(iname, itype, oname, otype, bm, bn, bk, wm, wn) \
- instantiate_gemm_aligned_helper(nt, false, true, iname, itype, oname, otype, bm, bn, bk, wm, wn)
-
-#define instantiate_gemm_shapes_helper(iname, itype, oname, otype) \
- instantiate_gemm_transpose_helper(iname, itype, oname, otype, 16, 16, 16, 2, 2) \
- instantiate_gemm_transpose_helper(iname, itype, oname, otype, 16, 32, 16, 2, 2) \
- instantiate_gemm_transpose_helper(iname, itype, oname, otype, 32, 16, 16, 2, 2) \
- instantiate_gemm_transpose_helper(iname, itype, oname, otype, 32, 32, 16, 2, 2)
-
-instantiate_gemm_shapes_helper(float16, half, float32, float);
-instantiate_gemm_shapes_helper(bfloat16, bfloat16_t, float32, float);
-instantiate_gemm_shapes_helper(float32, float, float32, float);
-
-#define instantiate_accum(oname, otype, aname, atype) \
- instantiate_kernel( \
- "steel_gemm_splitk_mlp_fused_accum_" #oname "_" #aname, \
- gemm_splitk_mlp_fused_accum, atype, otype)
-
-instantiate_accum(bfloat16, bfloat16_t, float32, float);
-instantiate_accum(float16, half, float32, float);
-instantiate_accum(float32, float, float32, float);
-// clang-format on
diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/tile_configuration.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/tile_configuration.rs
deleted file mode 100644
index c1de7141a..000000000
--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/tile_configuration.rs
+++ /dev/null
@@ -1,31 +0,0 @@
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub struct TileConfiguration {
- pub tile_rows: i32,
- pub tile_cols: i32,
- pub tile_depth: i32,
- pub warps_per_row: i32,
- pub warps_per_col: i32,
-}
-
-pub fn select_tile_configuration(
- batch: i32,
- hidden_dim: i32,
-) -> TileConfiguration {
- let tile_rows = if batch < 40 {
- 16
- } else {
- 32
- };
- let tile_cols = if hidden_dim < 40 {
- 16
- } else {
- 32
- };
- TileConfiguration {
- tile_rows,
- tile_cols,
- tile_depth: 16,
- warps_per_row: 2,
- warps_per_col: 2,
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rs
index edf2b6d24..828bf05c5 100644
--- a/crates/uzu/src/backends/metal/kernel/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/mod.rs
@@ -11,7 +11,6 @@ pub use mlp::{
MLP_ACTIVATION_FC_INDEX, MLP_FUSED_FC_INDEX, MLP_HIDDEN_DIM_FC_INDEX, MlpActivationType, MlpFusedConfig,
make_non_fused_function_constants,
};
-pub mod mlp_fused;
pub mod moe;
pub mod quant_matmul;
pub mod ssm;
diff --git a/crates/uzu/tests/mlp_fused_correctness_test.rs b/crates/uzu/tests/mlp_fused_correctness_test.rs
deleted file mode 100644
index 7d407a348..000000000
--- a/crates/uzu/tests/mlp_fused_correctness_test.rs
+++ /dev/null
@@ -1,358 +0,0 @@
-//! Correctness tests for MLP fused kernels
-//! Compares fused (matmul + activation) output against separate operations
-
-use bytemuck;
-use half::f16;
-use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue, MTLDeviceExt, MTLResourceOptions};
-use objc2::rc::autoreleasepool;
-use uzu::{
- DataType,
- backends::{
- common::Context,
- metal::{
- MTLContext,
- kernel::{
- mlp::MlpActivationType,
- mlp_fused::{MlpFusedArguments, MlpFusedKernel},
- },
- },
- },
-};
-
-/// CPU reference implementation for SiLU activation
-fn silu(x: f32) -> f32 {
- x / (1.0 + (-x).exp())
-}
-
-/// CPU reference implementation for GELU activation
-fn gelu(x: f32) -> f32 {
- let k0 = 0.044715f32;
- let k1 = 0.7978845608f32;
- let t = k1 * (x + k0 * x * x * x);
- 0.5 * x * (1.0 + t.tanh())
-}
-
-/// CPU reference for MLP fused: up * activation(gate)
-fn mlp_fused_reference(
- input: &[f32],
- weights_up: &[f32],
- weights_gate: &[f32],
- m: usize,
- k: usize,
- hidden_dim: usize,
- activation: MlpActivationType,
-) -> Vec<f32> {
- let mut result = vec![0.0f32; m * hidden_dim];
-
- for row in 0..m {
- for col in 0..hidden_dim {
- // Compute up and gate dot products
- let mut up_sum = 0.0f32;
- let mut gate_sum = 0.0f32;
- for i in 0..k {
- let input_val = input[row * k + i];
- up_sum += input_val * weights_up[col * k + i];
- gate_sum += input_val * weights_gate[col * k + i];
- }
-
- // Apply fused activation
- let activated_gate = match activation {
- MlpActivationType::SiLU => silu(gate_sum),
- MlpActivationType::Gelu => gelu(gate_sum),
- };
- result[row * hidden_dim + col] = up_sum * activated_gate;
- }
- }
-
- result
-}
-
-/// Run MLP fused kernel (decode path, M=1)
-fn run_fused_gemv(
- ctx: &MTLContext,
- input: &[f16],
- weights: &[f16], // [2*hidden_dim, k] - up then gate weights concatenated
- k: usize,
- hidden_dim: usize,
- activation: MlpActivationType,
-) -> Vec<f16> {
- let input_buf = ctx
- .device
- .new_buffer_with_data(bytemuck::cast_slice(input), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
- let weights_buf = ctx
- .device
- .new_buffer_with_data(bytemuck::cast_slice(weights), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
- let output_buf = ctx
- .device
- .new_buffer(hidden_dim * core::mem::size_of::<f16>(), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
-
- let mut kernel = MlpFusedKernel::new(DataType::F16, true).expect("kernel");
-
- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();
- let enc = cb.new_compute_command_encoder().expect("Failed to create compute encoder");
-
- let args = MlpFusedArguments {
- input: &input_buf,
- input_offset: 0,
- weights: &weights_buf,
- output: &output_buf,
- batch: 1,
- input_dim: k as i32,
- hidden_dim: hidden_dim as i32,
- lda: k as i32,
- ldb: k as i32,
- ldd: hidden_dim as i32,
- batch_count: 1,
- activation,
- };
-
- kernel.encode(ctx, &enc, &args).expect("encode");
- enc.end_encoding();
- cb.commit();
- cb.wait_until_completed();
-
- unsafe {
- let ptr = output_buf.contents().as_ptr() as *const f16;
- std::slice::from_raw_parts(ptr, hidden_dim).to_vec()
- }
-}
-
-/// Run MLP fused kernel (prefill path, M>1)
-fn run_fused_gemm(
- ctx: &MTLContext,
- input: &[f16],
- weights: &[f16], // [2*hidden_dim, k] - up then gate weights concatenated
- m: usize,
- k: usize,
- hidden_dim: usize,
- activation: MlpActivationType,
-) -> Vec<f16> {
- let input_buf = ctx
- .device
- .new_buffer_with_data(bytemuck::cast_slice(input), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
- let weights_buf = ctx
- .device
- .new_buffer_with_data(bytemuck::cast_slice(weights), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
- let output_buf = ctx
- .device
- .new_buffer(m * hidden_dim * core::mem::size_of::<f16>(), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
-
- let mut kernel = MlpFusedKernel::new(DataType::F16, true).expect("kernel");
-
- let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();
- let enc = cb.new_compute_command_encoder().expect("Failed to create compute encoder");
-
- let args = MlpFusedArguments {
- input: &input_buf,
- input_offset: 0,
- weights: &weights_buf,
- output: &output_buf,
- batch: m as i32,
- input_dim: k as i32,
- hidden_dim: hidden_dim as i32,
- lda: k as i32,
- ldb: k as i32,
- ldd: hidden_dim as i32,
- batch_count: 1,
- activation,
- };
-
- kernel.encode(ctx, &enc, &args).expect("encode");
- enc.end_encoding();
- cb.commit();
- cb.wait_until_completed();
-
- unsafe {
- let ptr = output_buf.contents().as_ptr() as *const f16;
- std::slice::from_raw_parts(ptr, m * hidden_dim).to_vec()
- }
-}
-
-fn compare_outputs(
- expected: &[f32],
- actual: &[f16],
- rtol: f32,
- atol: f32,
-) -> (bool, f32) {
- let mut max_diff = 0.0f32;
- let mut all_close = true;
-
- for (i, (&exp, &act)) in expected.iter().zip(actual.iter()).enumerate() {
- let act_f32 = f32::from(act);
- let diff = (exp - act_f32).abs();
- let rel_diff = diff / (exp.abs() + 1e-8);
-
- if diff > atol && rel_diff > rtol {
- if all_close {
- eprintln!(
- "First mismatch at index {}: expected {}, got {} (diff={}, rel={})",
- i, exp, act_f32, diff, rel_diff
- );
- }
- all_close = false;
- }
- max_diff = max_diff.max(diff);
- }
-
- (all_close, max_diff)
-}
-
-#[test]
-fn test_mlp_fused_gemv_silu_small() {
- autoreleasepool(|_| {
- let ctx = match MTLContext::new().ok() {
- Some(c) => c,
- None => {
- eprintln!("No Metal device available, skipping test");
- return;
- },
- };
-
- let k = 64;
- let hidden_dim = 32;
- let m = 1;
-
- // Generate random-ish test data
- let input: Vec<f32> = (0..k).map(|i| (i as f32 * 0.1).sin() * 0.5).collect();
- let weights_up: Vec<f32> = (0..hidden_dim * k).map(|i| (i as f32 * 0.017).cos() * 0.3).collect();
- let weights_gate: Vec<f32> = (0..hidden_dim * k).map(|i| (i as f32 * 0.023).sin() * 0.3).collect();
-
- // Create concatenated weights [up; gate] row-major
- let mut weights_concat: Vec<f32> = Vec::with_capacity(2 * hidden_dim * k);
- weights_concat.extend(&weights_up);
- weights_concat.extend(&weights_gate);
-
- // CPU reference
- let expected =
- mlp_fused_reference(&input, &weights_up, &weights_gate, m, k, hidden_dim, MlpActivationType::SiLU);
-
- // Convert to f16
- let input_f16: Vec<f16> = input.iter().map(|&x| f16::from_f32(x)).collect();
- let weights_f16: Vec<f16> = weights_concat.iter().map(|&x| f16::from_f32(x)).collect();
-
- // Run fused kernel
- let actual = run_fused_gemv(&ctx, &input_f16, &weights_f16, k, hidden_dim, MlpActivationType::SiLU);
-
- let (close, max_diff) = compare_outputs(&expected, &actual, 0.01, 0.01);
- println!("GEMV SiLU small: max_diff = {}", max_diff);
- assert!(close, "GEMV SiLU small test failed, max_diff = {}", max_diff);
- });
-}
-
-#[test]
-fn test_mlp_fused_gemv_gelu_small() {
- autoreleasepool(|_| {
- let ctx = match MTLContext::new().ok() {
- Some(c) => c,
- None => {
- eprintln!("No Metal device available, skipping test");
- return;
- },
- };
-
- let k = 64;
- let hidden_dim = 32;
- let m = 1;
-
- let input: Vec<f32> = (0..k).map(|i| (i as f32 * 0.1).sin() * 0.5).collect();
- let weights_up: Vec<f32> = (0..hidden_dim * k).map(|i| (i as f32 * 0.017).cos() * 0.3).collect();
- let weights_gate: Vec<f32> = (0..hidden_dim * k).map(|i| (i as f32 * 0.023).sin() * 0.3).collect();
-
- let mut weights_concat: Vec<f32> = Vec::with_capacity(2 * hidden_dim * k);
- weights_concat.extend(&weights_up);
- weights_concat.extend(&weights_gate);
-
- let expected =
- mlp_fused_reference(&input, &weights_up, &weights_gate, m, k, hidden_dim, MlpActivationType::Gelu);
-
- let input_f16: Vec<f16> = input.iter().map(|&x| f16::from_f32(x)).collect();
- let weights_f16: Vec<f16> = weights_concat.iter().map(|&x| f16::from_f32(x)).collect();
-
- let actual = run_fused_gemv(&ctx, &input_f16, &weights_f16, k, hidden_dim, MlpActivationType::Gelu);
-
- let (close, max_diff) = compare_outputs(&expected, &actual, 0.01, 0.01);
- println!("GEMV GELU small: max_diff = {}", max_diff);
- assert!(close, "GEMV GELU small test failed, max_diff = {}", max_diff);
- });
-}
-
-#[test]
-fn test_mlp_fused_gemm_silu_medium() {
- autoreleasepool(|_| {
- let ctx = match MTLContext::new().ok() {
- Some(c) => c,
- None => {
- eprintln!("No Metal device available, skipping test");
- return;
- },
- };
-
- let m = 32;
- let k = 64;
- let hidden_dim = 64;
-
- let input: Vec<f32> = (0..m * k).map(|i| (i as f32 * 0.1).sin() * 0.5).collect();
- let weights_up: Vec<f32> = (0..hidden_dim * k).map(|i| (i as f32 * 0.017).cos() * 0.3).collect();
- let weights_gate: Vec<f32> = (0..hidden_dim * k).map(|i| (i as f32 * 0.023).sin() * 0.3).collect();
-
- let mut weights_concat: Vec<f32> = Vec::with_capacity(2 * hidden_dim * k);
- weights_concat.extend(&weights_up);
- weights_concat.extend(&weights_gate);
-
- let expected =
- mlp_fused_reference(&input, &weights_up, &weights_gate, m, k, hidden_dim, MlpActivationType::SiLU);
-
- let input_f16: Vec<f16> = input.iter().map(|&x| f16::from_f32(x)).collect();
- let weights_f16: Vec<f16> = weights_concat.iter().map(|&x| f16::from_f32(x)).collect();
-
- let actual = run_fused_gemm(&ctx, &input_f16, &weights_f16, m, k, hidden_dim, MlpActivationType::SiLU);
-
- let (close, max_diff) = compare_outputs(&expected, &actual, 0.05, 0.05);
- println!("GEMM SiLU medium: max_diff = {}", max_diff);
- assert!(close, "GEMM SiLU medium test failed, max_diff = {}", max_diff);
- });
-}
-
-#[test]
-fn test_mlp_fused_gemv_realistic_dimensions() {
- autoreleasepool(|_| {
- let ctx = match MTLContext::new().ok() {
- Some(c) => c,
- None => {
- eprintln!("No Metal device available, skipping test");
- return;
- },
- };
-
- // Realistic LLM dimensions (scaled down for test speed)
- let k = 512; // input dim
- let hidden_dim = 256; // hidden dim (half of up+gate)
- let m = 1;
-
- let input: Vec<f32> = (0..k).map(|i| (i as f32 * 0.01).sin() * 0.5).collect();
- let weights_up: Vec<f32> = (0..hidden_dim * k).map(|i| (i as f32 * 0.001).cos() * 0.1).collect();
- let weights_gate: Vec<f32> = (0..hidden_dim * k).map(|i| (i as f32 * 0.002).sin() * 0.1).collect();
-
- let mut weights_concat: Vec<f32> = Vec::with_capacity(2 * hidden_dim * k);
- weights_concat.extend(&weights_up);
- weights_concat.extend(&weights_gate);
-
- let expected =
- mlp_fused_reference(&input, &weights_up, &weights_gate, m, k, hidden_dim, MlpActivationType::SiLU);
-
- let input_f16: Vec<f16> = input.iter().map(|&x| f16::from_f32(x)).collect();
- let weights_f16: Vec<f16> = weights_concat.iter().map(|&x| f16::from_f32(x)).collect();
-
- let actual = run_fused_gemv(&ctx, &input_f16, &weights_f16, k, hidden_dim, MlpActivationType::SiLU);
-
- let (close, max_diff) = compare_outputs(&expected, &actual, 0.05, 0.02);
- println!("GEMV realistic: max_diff = {}", max_diff);
- assert!(close, "GEMV realistic test failed, max_diff = {}", max_diff);
- });
-}
diff --git a/crates/uzu/tests/mlp_fused_performance_test.rs b/crates/uzu/tests/mlp_fused_performance_test.rs
deleted file mode 100644
index 2333f1747..000000000
--- a/crates/uzu/tests/mlp_fused_performance_test.rs
+++ /dev/null
@@ -1,379 +0,0 @@
-//! MLP Fused Kernel Performance Benchmark
-//! Compares fused vs unfused MLP paths to measure speedup
-
-use std::time::{Duration, Instant};
-
-use half::f16;
-use metal::{MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue, MTLDeviceExt, MTLResourceOptions};
-use objc2::rc::autoreleasepool;
-use uzu::{
- DataType,
- backends::{
- common::Context,
- metal::{
- MTLContext,
- kernel::{
- matmul::{MatmulArguments, MatmulKernel, determine_kernel_variant as determine_matmul_variant},
- mlp::MlpActivationType,
- mlp_fused::{
- MlpFusedArguments, MlpFusedKernel, determine_kernel_variant as determine_mlp_fused_variant,
- },
- },
- },
- },
-};
-
-struct BenchmarkResult {
- fused_time_microseconds: f64,
- unfused_time_microseconds: f64,
- speedup: f64,
- fused_kernel_name: &'static str,
- unfused_kernel_name: &'static str,
-}
-
-impl BenchmarkResult {
- fn new(
- fused_duration: Duration,
- unfused_duration: Duration,
- fused_kernel_name: &'static str,
- unfused_kernel_name: &'static str,
- ) -> Self {
- let fused_microseconds = fused_duration.as_secs_f64() * 1_000_000.0;
- let unfused_microseconds = unfused_duration.as_secs_f64() * 1_000_000.0;
- Self {
- fused_time_microseconds: fused_microseconds,
- unfused_time_microseconds: unfused_microseconds,
- speedup: unfused_microseconds / fused_microseconds,
- fused_kernel_name,
- unfused_kernel_name,
- }
- }
-}
-
-fn benchmark_gemv_fused(
- context: &MTLContext,
- input_dimension: usize,
- hidden_dimension: usize,
- iteration_count: usize,
- warmup_count: usize,
-) -> BenchmarkResult {
- let input_buffer = context
- .device
- .new_buffer(input_dimension * std::mem::size_of::<f16>(), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
- let weights_buffer = context
- .device
- .new_buffer(
- 2 * hidden_dimension * input_dimension * std::mem::size_of::<f16>(),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
- .expect("Failed to create buffer");
- let output_buffer = context
- .device
- .new_buffer(hidden_dimension * std::mem::size_of::<f16>(), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
- let fused_up_buffer = context
- .device
- .new_buffer(2 * hidden_dimension * std::mem::size_of::<f16>(), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
-
- let mut fused_kernel = MlpFusedKernel::new(DataType::F16, true).expect("fused kernel");
- let mut matmul_kernel = MatmulKernel::new(DataType::F16).expect("matmul");
-
- let fused_args = MlpFusedArguments {
- input: &input_buffer,
- input_offset: 0,
- weights: &weights_buffer,
- output: &output_buffer,
- batch: 1,
- input_dim: input_dimension as i32,
- hidden_dim: hidden_dimension as i32,
- lda: input_dimension as i32,
- ldb: input_dimension as i32,
- ldd: hidden_dimension as i32,
- batch_count: 1,
- activation: MlpActivationType::SiLU,
- };
-
- // Warmup fused
- for _ in 0..warmup_count {
- let command_buffer =
- context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();
- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
- fused_kernel.encode(context, &encoder, &fused_args).unwrap();
- encoder.end_encoding();
- command_buffer.commit();
- command_buffer.wait_until_completed();
- }
-
- // Benchmark fused
- let fused_start = Instant::now();
- for _ in 0..iteration_count {
- let command_buffer =
- context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();
- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
- fused_kernel.encode(context, &encoder, &fused_args).unwrap();
- encoder.end_encoding();
- command_buffer.commit();
- command_buffer.wait_until_completed();
- }
- let fused_duration = fused_start.elapsed() / iteration_count as u32;
-
- let matmul_args = MatmulArguments {
- a: &input_buffer,
- a_offset: 0,
- b: &weights_buffer,
- c: None,
- d: &fused_up_buffer,
- bias: None,
- batch: 1,
- input_dim: input_dimension as i32,
- output_dim: (2 * hidden_dimension) as i32,
- lda: input_dimension as i32,
- ldb: input_dimension as i32,
- ldd: (2 * hidden_dimension) as i32,
- batch_count: 1,
- alpha: 1.0,
- beta: 0.0,
- transpose_a: false,
- transpose_b: true,
- };
-
- let fused_kernel_name =
- determine_mlp_fused_variant(context, DataType::F16, true, &fused_args).map(|v| v.as_str()).unwrap_or("?");
-
- let unfused_kernel_name =
- determine_matmul_variant(context, DataType::F16, &matmul_args).map(|v| v.as_str()).unwrap_or("?");
-
- // Warmup unfused
- for _ in 0..warmup_count {
- let command_buffer =
- context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();
- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
- matmul_kernel.encode(context, &encoder, matmul_args.clone()).unwrap();
- encoder.end_encoding();
- command_buffer.commit();
- command_buffer.wait_until_completed();
- }
-
- // Benchmark unfused
- let unfused_start = Instant::now();
- for _ in 0..iteration_count {
- let command_buffer =
- context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();
- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
- matmul_kernel.encode(context, &encoder, matmul_args.clone()).unwrap();
- encoder.end_encoding();
- command_buffer.commit();
- command_buffer.wait_until_completed();
- }
- let unfused_duration = unfused_start.elapsed() / iteration_count as u32;
-
- BenchmarkResult::new(fused_duration, unfused_duration, fused_kernel_name, unfused_kernel_name)
-}
-
-fn benchmark_gemm_fused(
- context: &MTLContext,
- batch_size: usize,
- input_dimension: usize,
- hidden_dimension: usize,
- iteration_count: usize,
- warmup_count: usize,
-) -> BenchmarkResult {
- let input_buffer = context
- .device
- .new_buffer(batch_size * input_dimension * std::mem::size_of::<f16>(), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
- let weights_buffer = context
- .device
- .new_buffer(
- 2 * hidden_dimension * input_dimension * std::mem::size_of::<f16>(),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
- .expect("Failed to create buffer");
- let output_buffer = context
- .device
- .new_buffer(batch_size * hidden_dimension * std::mem::size_of::<f16>(), MTLResourceOptions::STORAGE_MODE_SHARED)
- .expect("Failed to create buffer");
- let fused_up_buffer = context
- .device
- .new_buffer(
- batch_size * 2 * hidden_dimension * std::mem::size_of::<f16>(),
- MTLResourceOptions::STORAGE_MODE_SHARED,
- )
- .expect("Failed to create buffer");
-
- let mut fused_kernel = MlpFusedKernel::new(DataType::F16, true).expect("fused kernel");
- let mut matmul_kernel = MatmulKernel::new(DataType::F16).expect("matmul");
-
- let fused_args = MlpFusedArguments {
- input: &input_buffer,
- input_offset: 0,
- weights: &weights_buffer,
- output: &output_buffer,
- batch: batch_size as i32,
- input_dim: input_dimension as i32,
- hidden_dim: hidden_dimension as i32,
- lda: input_dimension as i32,
- ldb: input_dimension as i32,
- ldd: hidden_dimension as i32,
- batch_count: 1,
- activation: MlpActivationType::SiLU,
- };
-
- // Warmup fused
- for _ in 0..warmup_count {
- let command_buffer =
- context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();
- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
- fused_kernel.encode(context, &encoder, &fused_args).unwrap();
- encoder.end_encoding();
- command_buffer.commit();
- command_buffer.wait_until_completed();
- }
-
- let fused_start = Instant::now();
- for _ in 0..iteration_count {
- let command_buffer =
- context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();
- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
- fused_kernel.encode(context, &encoder, &fused_args).unwrap();
- encoder.end_encoding();
- command_buffer.commit();
- command_buffer.wait_until_completed();
- }
- let fused_duration = fused_start.elapsed() / iteration_count as u32;
-
- let matmul_args = MatmulArguments {
- a: &input_buffer,
- a_offset: 0,
- b: &weights_buffer,
- c: None,
- d: &fused_up_buffer,
- bias: None,
- batch: batch_size as i32,
- input_dim: input_dimension as i32,
- output_dim: (2 * hidden_dimension) as i32,
- lda: input_dimension as i32,
- ldb: input_dimension as i32,
- ldd: (2 * hidden_dimension) as i32,
- batch_count: 1,
- alpha: 1.0,
- beta: 0.0,
- transpose_a: false,
- transpose_b: true,
- };
-
- let fused_kernel_name =
- determine_mlp_fused_variant(context, DataType::F16, true, &fused_args).map(|v| v.as_str()).unwrap_or("?");
-
- let unfused_kernel_name =
- determine_matmul_variant(context, DataType::F16, &matmul_args).map(|v| v.as_str()).unwrap_or("?");
-
- // Warmup unfused
- for _ in 0..warmup_count {
- let command_buffer =
- context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();
- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
- matmul_kernel.encode(context, &encoder, matmul_args.clone()).unwrap();
- encoder.end_encoding();
- command_buffer.commit();
- command_buffer.wait_until_completed();
- }
-
- let unfused_start = Instant::now();
- for _ in 0..iteration_count {
- let command_buffer =
- context.command_queue.command_buffer().expect("Failed to create command buffer").to_owned();
- let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute encoder");
- matmul_kernel.encode(context, &encoder, matmul_args.clone()).unwrap();
- encoder.end_encoding();
- command_buffer.commit();
- command_buffer.wait_until_completed();
- }
- let unfused_duration = unfused_start.elapsed() / iteration_count as u32;
-
- BenchmarkResult::new(fused_duration, unfused_duration, fused_kernel_name, unfused_kernel_name)
-}
-
-/// MLP shapes from real model configs
-/// Format: (input_dimension, hidden_dimension)
-const MLP_SHAPES: &[(usize, usize)] = &[
- (896, 4864), // Qwen2.5-Coder-0.5B
- (1024, 3072), // Qwen3-0.6B
- (1024, 4608), // LFM2-350M
- (1152, 6912), // gemma-3-1b
- (1536, 6912), // LFM2-700M
- (1536, 8960), // Qwen2.5-Coder-1.5B, DeepSeek-R1-Distill-Qwen-1.5B
- (2048, 6144), // Qwen3-1.7B
- (2048, 8192), // Llamba-1B, SmolLM2-1.7B, Llama-3.2-1B, LFM2-1.2B
- (2048, 10752), // LFM2-2.6B
- (2048, 11008), // Qwen2.5-Coder-3B
- (2560, 9728), // Qwen3-4B
- (2560, 10240), // gemma-3-4b
- (3072, 8192), // Llama-3.2-3B, Llamba-3B
- (3584, 18944), // Qwen2.5-Coder-7B
- (4096, 12288), // Qwen3-8B
- (4096, 14336), // Llamba-8B, Llama-3.1-8B
- (4096, 16384), // rnj-1-instruct
- (5120, 13824), // Qwen2.5-Coder-14B
- (5120, 17408), // Qwen3-14B
- (5120, 25600), // Qwen3-32B
- (5376, 21504), // gemma-3-27b
- (6144, 16384), // Codestral-22B
-];
-
-/// Batch sizes to test: decode (1) and prefill sizes
-const BATCH_SIZES: &[usize] = &[1, 8, 16, 32, 64, 128, 256, 512];
-
-#[test]
-#[ignore] // Run with: cargo test --package uzu --test mlp_fused_performance_test -- --ignored --nocapture
-fn mlp_fused_performance_benchmark() {
- autoreleasepool(|_| {
- let context = match MTLContext::new().ok() {
- Some(context) => context,
- None => {
- eprintln!("No Metal device available");
- return;
- },
- };
-
- let iteration_count = 100;
- let warmup_count = 10;
-
- println!(
- "{:>10} {:>10} {:>10} {:>10} {:>12} {:>10} {:>12} {:>10}",
- "batch", "input", "hidden", "Fused", "Fused (µs)", "Unfused", "Unfused (µs)", "Speedup"
- );
- println!("{}", "-".repeat(98));
-
- for &batch_size in BATCH_SIZES {
- for &(input_dimension, hidden_dimension) in MLP_SHAPES {
- let result = if batch_size == 1 {
- benchmark_gemv_fused(&context, input_dimension, hidden_dimension, iteration_count, warmup_count)
- } else {
- benchmark_gemm_fused(
- &context,
- batch_size,
- input_dimension,
- hidden_dimension,
- iteration_count,
- warmup_count,
- )
- };
- let speedup_string = format!("{:.2}x", result.speedup);
- println!(
- "{:>10} {:>10} {:>10} {:>10} {:>12.1} {:>10} {:>12.1} {:>10}",
- batch_size,
- input_dimension,
- hidden_dimension,
- result.fused_kernel_name,
- result.fused_time_microseconds,
- result.unfused_kernel_name,
- result.unfused_time_microseconds,
- speedup_string
- );
- }
- }
- });
-}