A high-performance inference engine for AI models
dispatch descriptor
Eugene Bokhan230c85889b43
diff --git a/crates/matmul-benchmark/src/main.rs b/crates/matmul-benchmark/src/main.rsindex 41c86c578..716629e68 100644--- a/crates/matmul-benchmark/src/main.rs+++ b/crates/matmul-benchmark/src/main.rs@@ -533,6 +533,7 @@ fn benchmark_uzu(alpha: 1.0,beta: 0.0,},+ None,).expect("encode");enc.end_encoding();@@ -564,6 +565,7 @@ fn benchmark_uzu(alpha: 1.0,beta: 0.0,},+ None,).expect("encode");enc.end_encoding();@@ -804,6 +806,7 @@ fn run_accuracy_test_f16(alpha: 1.0,beta: 0.0,},+ None,).unwrap();enc.end_encoding();@@ -869,6 +872,7 @@ fn run_accuracy_test_f32(alpha: 1.0,beta: 0.0,},+ None,).unwrap();enc.end_encoding();@@ -945,6 +949,7 @@ fn run_accuracy_test_bf16(alpha: 1.0,beta: 0.0,},+ None,).unwrap();enc.end_encoding();diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rsindex 12833c077..835a07c4d 100644--- a/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs@@ -25,7 +25,7 @@ pub struct FullPrecisionEmbeddingReadout {impl FullPrecisionEmbeddingReadout {pub fn new(- mtl_context: &MTLContext,+ _mtl_context: &MTLContext,data_type: DataType,vocab_size: usize,model_dim: usize,@@ -69,10 +69,8 @@ impl FullPrecisionEmbeddingReadout {let weights_buffer = unsafe { weights.mtl_buffer().to_owned() };- // Weights are [vocab_size, model_dim], we compute input @ weights^T- // MatmulKernel with transpose_b=true handles this- let kernel = MatmulKernel::new(mtl_context, data_type, false, true)- .map_err(EmbeddingError::MetalError)?;+ let kernel =+ MatmulKernel::new(data_type).map_err(EmbeddingError::MetalError)?;Ok(Self {kernel: RefCell::new(kernel),@@ -130,6 +128,7 @@ impl EncodableBlock for FullPrecisionEmbeddingReadout {b: &self.weights_buffer,c: None,d: output_buffer,+ bias: None,batch: batch_size as i32,input_dim: self.model_dim as i32,output_dim: self.vocab_size as i32,@@ -139,6 +138,8 @@ impl EncodableBlock for FullPrecisionEmbeddingReadout {batch_count: 1,alpha: 1.0,beta: 0.0,+ transpose_a: false,+ transpose_b: true,};self.kerneldiff --git a/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs b/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rsindex 5d77b2ba3..857b2b592 100644--- a/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs+++ b/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs@@ -16,7 +16,7 @@ use crate::{pub struct FullPrecisionLinear {kernel: RefCell<MatmulKernel>,- biases_buffer: Option<MTLBuffer>,+ bias_buffer: Option<MTLBuffer>,weights_buffer: MTLBuffer,input_dim: usize,output_dim: usize,@@ -26,7 +26,7 @@ pub struct FullPrecisionLinear {impl FullPrecisionLinear {pub fn new(- mtl_context: &MTLContext,+ _mtl_context: &MTLContext,precision: DataType,input_dim: usize,output_dim: usize,@@ -65,7 +65,7 @@ impl FullPrecisionLinear {let weights_buffer: MTLBuffer =unsafe { weights.mtl_buffer() }.to_owned();- let biases_buffer = match parameter_tree.leaf("biases") {+ let bias_buffer = match parameter_tree.leaf("biases") {Ok(mut biases) => {if biases.shape() != [output_dim] {return Err(MTLError::Generic(format!(@@ -81,18 +81,18 @@ impl FullPrecisionLinear {precision)));}- let biases_buffer: MTLBuffer =+ let bias_buffer: MTLBuffer =unsafe { biases.mtl_buffer() }.to_owned();- Some(biases_buffer)+ Some(bias_buffer)},Err(_) => None,};- let kernel = MatmulKernel::new(mtl_context, precision, false, true)?;+ let kernel = MatmulKernel::new(precision)?;Ok(Self {kernel: RefCell::new(kernel),- biases_buffer,+ bias_buffer,weights_buffer,input_dim,output_dim,@@ -126,6 +126,7 @@ impl EncodableBlock for FullPrecisionLinear {b: &self.weights_buffer,c: None,d: output_buffer,+ bias: self.bias_buffer.as_ref(),batch: batch_size as i32,input_dim: self.input_dim as i32,output_dim: self.output_dim as i32,@@ -135,18 +136,14 @@ impl EncodableBlock for FullPrecisionLinear {batch_count: 1,alpha: 1.0,beta: 0.0,+ transpose_a: false,+ transpose_b: true,};let mut kernel = self.kernel.borrow_mut();- if let Some(bias_buf) = &self.biases_buffer {- kernel- .encode_with_bias(state.mtl_context(), encoder, args, bias_buf)- .expect("Failed to encode matmul kernel");- } else {- kernel- .encode(state.mtl_context(), encoder, args)- .expect("Failed to encode matmul kernel");- }+ kernel+ .encode(state.mtl_context(), encoder, args)+ .expect("Failed to encode matmul kernel");encoder.end_encoding();@@ -181,6 +178,7 @@ impl EncodableBlock for FullPrecisionLinear {b: &self.weights_buffer,c: None,d: output_buffer,+ bias: self.bias_buffer.as_ref(),batch: batch_size as i32,input_dim: self.input_dim as i32,output_dim: self.output_dim as i32,@@ -190,17 +188,13 @@ impl EncodableBlock for FullPrecisionLinear {batch_count: 1,alpha: 1.0,beta: 0.0,+ transpose_a: false,+ transpose_b: true,};let mut kernel = self.kernel.borrow_mut();- if let Some(bias_buf) = &self.biases_buffer {- kernel- .encode_with_bias(state.mtl_context(), encoder, args, bias_buf)- .expect("Failed to encode matmul kernel");- } else {- kernel- .encode(state.mtl_context(), encoder, args)- .expect("Failed to encode matmul kernel");- }+ kernel+ .encode(state.mtl_context(), encoder, args)+ .expect("Failed to encode matmul kernel");}}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/matmul_arguments.rs b/crates/uzu/src/backends/metal/kernel/matmul/common/matmul_arguments.rsindex 56dc4e2f6..dfb3cf769 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/common/matmul_arguments.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/matmul_arguments.rs@@ -8,6 +8,7 @@ pub struct MatmulArguments<'a> {pub b: &'a MTLBuffer,pub c: Option<&'a MTLBuffer>,pub d: &'a MTLBuffer,+ pub bias: Option<&'a MTLBuffer>,/// M dimension - batch/number of tokens (rows of A, rows of D)pub batch: i32,/// K dimension - input_dim/reduction dimension (cols of A, rows of B)@@ -22,4 +23,6 @@ pub struct MatmulArguments<'a> {/// Scaling factors for fused addmm (alpha * A @ B + beta * C)pub alpha: f32,pub beta: f32,+ pub transpose_a: bool,+ pub transpose_b: bool,}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/dispatch_descriptor.rs b/crates/uzu/src/backends/metal/kernel/matmul/dispatch_descriptor.rsnew file mode 100644index 000000000..c4934b8ee--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/dispatch_descriptor.rs@@ -0,0 +1,34 @@+use super::{common::MatmulArguments, gemm, gemv, split_k};+use crate::{+ DataType,+ backends::metal::{MTLContext, MTLError},+};++#[derive(Debug, Clone)]+pub(crate) enum MatmulDispatchDescriptor {+ Gemv(gemv::DispatchDescriptor),+ SplitK(split_k::DispatchDescriptor),+ Gemm(gemm::DispatchDescriptor),+}++pub(crate) fn choose_dispatch_descriptor(+ context: &MTLContext,+ data_type: DataType,+ arguments: &MatmulArguments,+) -> Result<MatmulDispatchDescriptor, MTLError> {+ if let Some(descriptor) =+ gemv::DispatchDescriptor::try_new(context, data_type, arguments)?+ {+ return Ok(MatmulDispatchDescriptor::Gemv(descriptor));+ }++ if let Some(descriptor) =+ split_k::DispatchDescriptor::try_new(context, data_type, arguments)?+ {+ return Ok(MatmulDispatchDescriptor::SplitK(descriptor));+ }++ Ok(MatmulDispatchDescriptor::Gemm(+ gemm::DispatchDescriptor::new(context, data_type, arguments)?,+ ))+}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/dispatch_descriptor.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm/dispatch_descriptor.rsnew file mode 100644index 000000000..2861f4a3d--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm/dispatch_descriptor.rs@@ -0,0 +1,200 @@+use metal::MTLSize;++use super::{+ pipeline_configuration::PipelineConfiguration,+ tile_configuration::TileConfiguration,+};+use crate::{+ DataType,+ backends::metal::{+ DeviceClass, MTLContext, MTLError,+ kernel::matmul::common::{+ GEMMAddMMParams, GEMMParams, MatmulArguments,+ },+ },+};++#[derive(Debug, Clone)]+pub(crate) struct DispatchDescriptor {+ pub(crate) pipeline_configuration: PipelineConfiguration,+ pub(crate) params: GEMMParams,+ pub(crate) addmm_params: Option<GEMMAddMMParams>,+ pub(crate) threadgroups: MTLSize,+ pub(crate) threads_per_threadgroup: MTLSize,+}++impl DispatchDescriptor {+ pub(crate) fn new(+ context: &MTLContext,+ data_type: DataType,+ arguments: &MatmulArguments,+ ) -> Result<Self, MTLError> {+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)+ {+ return Err(MTLError::Generic(format!(+ "Unsupported dtype for GEMM: {data_type:?}"+ )));+ }++ let tile = select_tile(context, data_type, arguments);++ let m = arguments.batch;+ let n = arguments.output_dim;+ let k = arguments.input_dim;++ let pipeline_configuration = PipelineConfiguration {+ tile,+ transpose_a: arguments.transpose_a,+ transpose_b: arguments.transpose_b,+ align_m: (m % tile.block_rows) == 0,+ align_n: (n % tile.block_cols) == 0,+ align_k: (k % tile.block_depth) == 0,+ has_batch: arguments.batch_count > 1,+ use_out_source: arguments.c.is_some(),+ do_axpby: arguments.c.is_some()+ && (arguments.alpha != 1.0 || arguments.beta != 0.0),+ };++ let tiles_n = (n + tile.block_cols - 1) / tile.block_cols;+ let tiles_m = (m + tile.block_rows - 1) / tile.block_rows;+ let swizzle_log = tile.swizzle_log2;++ let tile_swizzle = 1 << swizzle_log;+ let tm_swizzled = (tiles_m + tile_swizzle - 1) / tile_swizzle;+ let tn_swizzled = tiles_n * tile_swizzle;++ let elements_per_matrix_a =+ (arguments.batch as i64) * (arguments.lda as i64);+ let elements_per_matrix_b = if arguments.transpose_b {+ (arguments.output_dim as i64) * (arguments.ldb as i64)+ } else {+ (arguments.input_dim as i64) * (arguments.ldb as i64)+ };+ let elements_per_matrix_d =+ (arguments.batch as i64) * (arguments.ldd as i64);++ let params = GEMMParams {+ M: m,+ N: n,+ K: k,+ lda: arguments.lda,+ ldb: arguments.ldb,+ ldd: arguments.ldd,+ tiles_n,+ tiles_m,+ batch_stride_a: elements_per_matrix_a,+ batch_stride_b: elements_per_matrix_b,+ batch_stride_d: elements_per_matrix_d,+ swizzle_log,+ gemm_k_iterations_aligned: k / tile.block_depth,+ batch_ndim: 1,+ };++ let addmm_params = if pipeline_configuration.use_out_source {+ let batch_stride_c = if arguments.batch_count > 1 {+ (arguments.ldd as i64) * (arguments.output_dim as i64)+ } else {+ 0+ };+ Some(GEMMAddMMParams {+ ldc: arguments.ldd,+ fdc: 1,+ batch_stride_c,+ alpha: arguments.alpha,+ beta: arguments.beta,+ })+ } else {+ None+ };++ let threads_per_threadgroup =+ MTLSize::new(32, tile.warps_per_col, tile.warps_per_row);+ let threadgroups = MTLSize::new(+ tn_swizzled as u64,+ tm_swizzled as u64,+ arguments.batch_count as u64,+ );++ Ok(Self {+ pipeline_configuration,+ params,+ addmm_params,+ threadgroups,+ threads_per_threadgroup,+ })+ }+}++fn select_tile(+ context: &MTLContext,+ data_type: DataType,+ arguments: &MatmulArguments,+) -> TileConfiguration {+ let overall_work_elements = (arguments.batch_count as i64)+ * (arguments.batch as i64)+ * (arguments.output_dim as i64);+ let is_float32 = matches!(data_type, DataType::F32);+ let prefer_half_or_tf32 = !is_float32 || context.tf32_enabled();++ if context.is_nax_available() && prefer_half_or_tf32 {+ let base_tile = TileConfiguration::new(128, 128, 512, 4, 4, 0);+ let tile_rows =+ (arguments.batch + base_tile.block_rows - 1) / base_tile.block_rows;+ let swizzle_log2 = if tile_rows <= 3 {+ 0+ } else {+ 1+ };+ return TileConfiguration {+ swizzle_log2,+ ..base_tile+ };+ }++ match context.architecture.device_class {+ DeviceClass::Integrated+ | DeviceClass::Phone+ | DeviceClass::Unknown(_) => {+ if prefer_half_or_tf32 {+ if !arguments.transpose_a && arguments.transpose_b {+ TileConfiguration::new(64, 32, 32, 2, 2, 0)+ } else {+ TileConfiguration::new(64, 64, 16, 1, 2, 0)+ }+ } else if !arguments.transpose_a && arguments.transpose_b {+ TileConfiguration::new(32, 64, 16, 1, 2, 0)+ } else {+ TileConfiguration::new(64, 32, 32, 2, 2, 0)+ }+ },+ DeviceClass::Desktop => {+ if overall_work_elements >= (1_i64 << 20) {+ if prefer_half_or_tf32 {+ if 2 * std::cmp::max(arguments.batch, arguments.output_dim)+ > arguments.input_dim+ {+ TileConfiguration::new(64, 64, 16, 2, 2, 0)+ } else if !arguments.transpose_a && arguments.transpose_b {+ TileConfiguration::new(64, 32, 32, 2, 2, 0)+ } else {+ TileConfiguration::new(32, 64, 16, 1, 2, 0)+ }+ } else if !arguments.transpose_a && arguments.transpose_b {+ TileConfiguration::new(32, 64, 16, 1, 2, 0)+ } else {+ TileConfiguration::new(64, 32, 32, 2, 2, 0)+ }+ } else if prefer_half_or_tf32 {+ if !arguments.transpose_a && arguments.transpose_b {+ TileConfiguration::new(64, 32, 32, 2, 2, 0)+ } else {+ TileConfiguration::new(64, 64, 16, 1, 2, 0)+ }+ } else if !arguments.transpose_a && arguments.transpose_b {+ TileConfiguration::new(32, 64, 16, 1, 2, 0)+ } else {+ TileConfiguration::new(64, 32, 32, 2, 2, 0)+ }+ },+ }+}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm/kernel.rsindex 9742f81ae..cef66acbd 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs@@ -2,17 +2,15 @@ use std::collections::HashMap;use metal::{ComputeCommandEncoderRef, ComputePipelineState as MTLComputePipelineState,- MTLSize,};use super::{- pipeline_configuration::PipelineConfiguration,- tile_configuration::TileConfiguration,+ DispatchDescriptor, pipeline_configuration::PipelineConfiguration,};use crate::{DataType,backends::metal::{- DeviceClass, MTLContext, MTLError,+ MTLContext, MTLError,kernel::matmul::common::{GEMMAddMMParams, GEMMParams, MatmulArguments,transpose_configuration,@@ -22,17 +20,11 @@ use crate::{pub struct Kernel {data_type: DataType,- transpose_a: bool,- transpose_b: bool,pipelines: HashMap<PipelineConfiguration, MTLComputePipelineState>,}impl Kernel {- pub fn new(- data_type: DataType,- transpose_a: bool,- transpose_b: bool,- ) -> Result<Self, MTLError> {+ pub fn new(data_type: DataType) -> Result<Self, MTLError> {if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32){return Err(MTLError::Generic(format!(@@ -41,8 +33,6 @@ impl Kernel {}Ok(Self {data_type,- transpose_a,- transpose_b,pipelines: HashMap::new(),})}@@ -58,12 +48,14 @@ impl Kernel {fn kernel_name(&self,- tile: &TileConfiguration,+ configuration: &PipelineConfiguration,) -> String {let type_name = self.type_name();- let transpose_suffix =- transpose_configuration(self.transpose_a, self.transpose_b);- let prefix = if tile.is_nax() {+ let transpose_suffix = transpose_configuration(+ configuration.transpose_a,+ configuration.transpose_b,+ );+ let prefix = if configuration.tile.is_nax() {"steel_gemm_nax"} else {"steel_gemm"@@ -74,263 +66,114 @@ impl Kernel {transpose_suffix.as_str(),type_name,type_name,- tile.block_rows,- tile.block_cols,- tile.block_depth,- tile.warps_per_row,- tile.warps_per_col+ configuration.tile.block_rows,+ configuration.tile.block_cols,+ configuration.tile.block_depth,+ configuration.tile.warps_per_row,+ configuration.tile.warps_per_col)}- pub fn select_tile(- &self,- mtl: &MTLContext,- args: &MatmulArguments,- ) -> TileConfiguration {- let overall_work_elements = (args.batch_count as i64)- * (args.batch as i64)- * (args.output_dim as i64);- let is_float32 = matches!(self.data_type, DataType::F32);- let prefer_half_or_tf32 = !is_float32 || mtl.tf32_enabled();-- if mtl.is_nax_available() && prefer_half_or_tf32 {- let base_tile = TileConfiguration::new(128, 128, 512, 4, 4, 0);- let tile_rows =- (args.batch + base_tile.block_rows - 1) / base_tile.block_rows;- let swizzle_log2 = if tile_rows <= 3 {- 0- } else {- 1- };- return TileConfiguration {- swizzle_log2,- ..base_tile- };- }-- let device_class_code = match mtl.architecture.device_class {- DeviceClass::Desktop => 'd',- DeviceClass::Integrated => 'g',- DeviceClass::Phone => 'p',- DeviceClass::Unknown(_) => 'g',- };-- match device_class_code {- 'g' | 'p' => {- if prefer_half_or_tf32 {- if !self.transpose_a && self.transpose_b {- TileConfiguration::new(64, 32, 32, 2, 2, 0)- } else {- TileConfiguration::new(64, 64, 16, 1, 2, 0)- }- } else if !self.transpose_a && self.transpose_b {- TileConfiguration::new(32, 64, 16, 1, 2, 0)- } else {- TileConfiguration::new(64, 32, 32, 2, 2, 0)- }- },- 'd' => {- if overall_work_elements >= (1_i64 << 20) {- if prefer_half_or_tf32 {- if 2 * std::cmp::max(args.batch, args.output_dim)- > args.input_dim- {- TileConfiguration::new(64, 64, 16, 2, 2, 0)- } else if !self.transpose_a && self.transpose_b {- TileConfiguration::new(64, 32, 32, 2, 2, 0)- } else {- TileConfiguration::new(32, 64, 16, 1, 2, 0)- }- } else if !self.transpose_a && self.transpose_b {- TileConfiguration::new(32, 64, 16, 1, 2, 0)- } else {- TileConfiguration::new(64, 32, 32, 2, 2, 0)- }- } else if prefer_half_or_tf32 {- if !self.transpose_a && self.transpose_b {- TileConfiguration::new(64, 32, 32, 2, 2, 0)- } else {- TileConfiguration::new(64, 64, 16, 1, 2, 0)- }- } else if !self.transpose_a && self.transpose_b {- TileConfiguration::new(32, 64, 16, 1, 2, 0)- } else {- TileConfiguration::new(64, 32, 32, 2, 2, 0)- }- },- _ => TileConfiguration::new(64, 64, 16, 2, 2, 0),- }- }-fn get_or_compile_pipeline(&mut self,- mtl: &MTLContext,- config: &PipelineConfiguration,+ context: &MTLContext,+ configuration: &PipelineConfiguration,) -> Result<&MTLComputePipelineState, MTLError> {- if !self.pipelines.contains_key(config) {- let name = self.kernel_name(&config.tile);- let fcv = metal::FunctionConstantValues::new();- fcv.set_constant_value_at_index(- &config.has_batch as *const bool as *const _,+ if !self.pipelines.contains_key(configuration) {+ let kernel_name = self.kernel_name(configuration);+ let function_constants = metal::FunctionConstantValues::new();+ function_constants.set_constant_value_at_index(+ &configuration.has_batch as *const bool as *const _,metal::MTLDataType::Bool,10,);- fcv.set_constant_value_at_index(- &config.use_out_source as *const bool as *const _,+ function_constants.set_constant_value_at_index(+ &configuration.use_out_source as *const bool as *const _,metal::MTLDataType::Bool,100,);- fcv.set_constant_value_at_index(- &config.do_axpby as *const bool as *const _,+ function_constants.set_constant_value_at_index(+ &configuration.do_axpby as *const bool as *const _,metal::MTLDataType::Bool,110,);- fcv.set_constant_value_at_index(- &config.align_m as *const bool as *const _,+ function_constants.set_constant_value_at_index(+ &configuration.align_m as *const bool as *const _,metal::MTLDataType::Bool,200,);- fcv.set_constant_value_at_index(- &config.align_n as *const bool as *const _,+ function_constants.set_constant_value_at_index(+ &configuration.align_n as *const bool as *const _,metal::MTLDataType::Bool,201,);- fcv.set_constant_value_at_index(- &config.align_k as *const bool as *const _,+ function_constants.set_constant_value_at_index(+ &configuration.align_k as *const bool as *const _,metal::MTLDataType::Bool,202,);let cache_key = format!("{}_am{}_an{}_ak{}_hb{}_uo{}_ax{}",- name,- config.align_m as u8,- config.align_n as u8,- config.align_k as u8,- config.has_batch as u8,- config.use_out_source as u8,- config.do_axpby as u8+ kernel_name,+ configuration.align_m as u8,+ configuration.align_n as u8,+ configuration.align_k as u8,+ configuration.has_batch as u8,+ configuration.use_out_source as u8,+ configuration.do_axpby as u8);- let (ps, _) = mtl.compute_pipeline_state_with_reflection_cached(- &cache_key,- &name,- Some(&fcv),- )?;- self.pipelines.insert(config.clone(), ps);+ let (pipeline_state, _) = context+ .compute_pipeline_state_with_reflection_cached(+ &cache_key,+ &kernel_name,+ Some(&function_constants),+ )?;+ self.pipelines.insert(configuration.clone(), pipeline_state);}- Ok(self.pipelines.get(config).unwrap())+ Ok(self.pipelines.get(configuration).unwrap())}- pub fn encode(+ pub(crate) fn encode_descriptor(&mut self,- mtl: &MTLContext,- enc: &ComputeCommandEncoderRef,- args: &MatmulArguments,- ) -> Result<(), MTLError> {- let tile = self.select_tile(mtl, args);- self.encode_with_tile(mtl, enc, args, &tile)- }-- pub fn encode_with_tile(- &mut self,- mtl: &MTLContext,- enc: &ComputeCommandEncoderRef,- args: &MatmulArguments,- tile: &TileConfiguration,- ) -> Result<(), MTLError> {- let m = args.batch;- let n = args.output_dim;- let k = args.input_dim;-- let config = PipelineConfiguration {- tile: *tile,- align_m: (m % tile.block_rows) == 0,- align_n: (n % tile.block_cols) == 0,- align_k: (k % tile.block_depth) == 0,- has_batch: args.batch_count > 1,- use_out_source: args.c.is_some(),- do_axpby: args.c.is_some()- && (args.alpha != 1.0 || args.beta != 0.0),- };-- let ps = self.get_or_compile_pipeline(mtl, &config)?;- enc.set_compute_pipeline_state(ps);-- enc.set_buffer(0, Some(args.a), args.a_offset);- enc.set_buffer(1, Some(args.b), 0);- if config.use_out_source {- if let Some(c_buf) = args.c {- enc.set_buffer(2, Some(c_buf), 0);+ context: &MTLContext,+ encoder: &ComputeCommandEncoderRef,+ arguments: &MatmulArguments,+ descriptor: &DispatchDescriptor,+ ) -> Result<bool, MTLError> {+ let pipeline_state = self.get_or_compile_pipeline(+ context,+ &descriptor.pipeline_configuration,+ )?;+ encoder.set_compute_pipeline_state(pipeline_state);++ encoder.set_buffer(0, Some(arguments.a), arguments.a_offset);+ encoder.set_buffer(1, Some(arguments.b), 0);+ if descriptor.pipeline_configuration.use_out_source {+ if let Some(c_buffer) = arguments.c {+ encoder.set_buffer(2, Some(c_buffer), 0);}}- enc.set_buffer(3, Some(args.d), 0);-- let tiles_n = (n + tile.block_cols - 1) / tile.block_cols;- let tiles_m = (m + tile.block_rows - 1) / tile.block_rows;- let swizzle_log = tile.swizzle_log2;+ encoder.set_buffer(3, Some(arguments.d), 0);- let tile_swizzle = 1 << swizzle_log;- let tm_swizzled = (tiles_m + tile_swizzle - 1) / tile_swizzle;- let tn_swizzled = tiles_n * tile_swizzle;-- let elements_per_matrix_a = (args.batch as i64) * (args.lda as i64);- let elements_per_matrix_b = if self.transpose_b {- (args.output_dim as i64) * (args.ldb as i64)- } else {- (args.input_dim as i64) * (args.ldb as i64)- };- let elements_per_matrix_d = (args.batch as i64) * (args.ldd as i64);-- let params = GEMMParams {- M: m,- N: n,- K: k,- lda: args.lda,- ldb: args.ldb,- ldd: args.ldd,- tiles_n,- tiles_m,- batch_stride_a: elements_per_matrix_a,- batch_stride_b: elements_per_matrix_b,- batch_stride_d: elements_per_matrix_d,- swizzle_log,- gemm_k_iterations_aligned: k / tile.block_depth,- batch_ndim: 1,- };- enc.set_bytes(+ encoder.set_bytes(4,std::mem::size_of::<GEMMParams>() as u64,- ¶ms as *const GEMMParams as *const _,+ &descriptor.params as *const _ as *const _,);- if config.use_out_source {- let batch_stride_c = if args.batch_count > 1 {- (args.ldd as i64) * (args.output_dim as i64)- } else {- 0- };- let addmm_params = GEMMAddMMParams {- ldc: args.ldd,- fdc: 1,- batch_stride_c,- alpha: args.alpha,- beta: args.beta,- };- enc.set_bytes(+ if let Some(addmm_params) = &descriptor.addmm_params {+ encoder.set_bytes(5,std::mem::size_of::<GEMMAddMMParams>() as u64,- &addmm_params as *const GEMMAddMMParams as *const _,+ addmm_params as *const _ as *const _,);}- let threads_per_tg =- MTLSize::new(32, tile.warps_per_col, tile.warps_per_row);- let tgs = MTLSize::new(- tn_swizzled as u64,- tm_swizzled as u64,- args.batch_count as u64,+ encoder.dispatch_thread_groups(+ descriptor.threadgroups,+ descriptor.threads_per_threadgroup,);- enc.dispatch_thread_groups(tgs, threads_per_tg);- Ok(())+ Ok(false)}}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm/mod.rsindex f78544f8c..b8eda5d5d 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm/mod.rs@@ -1,5 +1,7 @@+mod dispatch_descriptor;mod kernel;mod pipeline_configuration;mod tile_configuration;pub use kernel::Kernel as GemmKernel;+pub(crate) use dispatch_descriptor::DispatchDescriptor;diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/pipeline_configuration.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm/pipeline_configuration.rsindex e92065b03..252792c91 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/pipeline_configuration.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm/pipeline_configuration.rs@@ -3,6 +3,8 @@ use super::tile_configuration::TileConfiguration;#[derive(Debug, Clone, PartialEq, Eq, Hash)]pub struct PipelineConfiguration {pub tile: TileConfiguration,+ pub transpose_a: bool,+ pub transpose_b: bool,pub align_m: bool,pub align_n: bool,pub align_k: bool,diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/dispatch_descriptor.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv/dispatch_descriptor.rsnew file mode 100644index 000000000..94344f824--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv/dispatch_descriptor.rs@@ -0,0 +1,175 @@+use metal::MTLSize;++use super::pipeline_configuration::{+ PipelineConfiguration, select_configuration,+};+use crate::{+ DataType,+ backends::metal::{+ MTLContext, MTLError, kernel::matmul::common::MatmulArguments,+ },+};++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub(crate) enum AxpbySource {+ None,+ Bias,+ C,+}++#[derive(Debug, Clone)]+pub(crate) struct DispatchDescriptor {+ pub(crate) pipeline_configuration: PipelineConfiguration,+ pub(crate) matrix_is_rhs: bool,+ pub(crate) axpby_source: AxpbySource,+ pub(crate) input_dimension: i32,+ pub(crate) output_dimension: i32,+ pub(crate) matrix_leading_dim: i32,+ pub(crate) alpha: f32,+ pub(crate) beta: f32,+ pub(crate) batch_ndim: i32,+ pub(crate) batch_shape: [i32; 1],+ pub(crate) vector_batch_stride: [i64; 1],+ pub(crate) matrix_batch_stride: [i64; 1],+ pub(crate) bias_batch_stride: [i64; 1],+ pub(crate) bias_stride: i32,+ pub(crate) threadgroups: MTLSize,+ pub(crate) threads_per_threadgroup: MTLSize,+}++impl DispatchDescriptor {+ pub(crate) fn try_new(+ _context: &MTLContext,+ data_type: DataType,+ arguments: &MatmulArguments,+ ) -> Result<Option<Self>, MTLError> {+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)+ {+ return Err(MTLError::Generic(format!(+ "Unsupported data type for GEMV: {data_type:?}"+ )));+ }++ if arguments.transpose_a || !arguments.transpose_b {+ return Ok(None);+ }++ let m = arguments.batch;+ let n = arguments.output_dim;+ if m != 1 && n != 1 {+ return Ok(None);+ }++ let matrix_is_rhs = n != 1;+ let transpose_matrix = if matrix_is_rhs {+ !arguments.transpose_b+ } else {+ arguments.transpose_a+ };++ let axpby_source = if arguments.c.is_some() {+ AxpbySource::C+ } else if arguments.bias.is_some() {+ AxpbySource::Bias+ } else {+ AxpbySource::None+ };++ let (do_axpby, alpha, beta, bias_stride) = match axpby_source {+ AxpbySource::None => (false, 1.0f32, 0.0f32, 0),+ AxpbySource::Bias => (true, 1.0f32, 1.0f32, 1),+ AxpbySource::C => (true, arguments.alpha, arguments.beta, arguments.ldd),+ };++ let output_dimension = if matrix_is_rhs {+ arguments.output_dim+ } else {+ arguments.batch+ };+ let pipeline_configuration = select_configuration(+ arguments.transpose_a,+ arguments.transpose_b,+ transpose_matrix,+ arguments.input_dim,+ output_dimension,+ false,+ do_axpby,+ );++ let input_dimension = arguments.input_dim;+ let matrix_leading_dim = if matrix_is_rhs {+ arguments.ldb+ } else {+ arguments.lda+ };++ let batch_ndim = 1i32;+ let batch_groups = arguments.batch_count.max(1);+ let batch_shape = [if arguments.batch_count > 1 {+ arguments.batch_count+ } else {+ 1+ }];++ let elements_per_matrix_a =+ (arguments.batch as i64) * (arguments.lda as i64);+ let elements_per_matrix_b = if arguments.transpose_b {+ (arguments.output_dim as i64) * (arguments.ldb as i64)+ } else {+ (arguments.input_dim as i64) * (arguments.ldb as i64)+ };++ let vector_batch_stride = [if matrix_is_rhs {+ elements_per_matrix_a+ } else {+ elements_per_matrix_b+ }];++ let matrix_batch_stride = [if matrix_is_rhs {+ elements_per_matrix_b+ } else {+ elements_per_matrix_a+ }];++ let bias_batch_stride = [if arguments.batch_count > 1 {+ (output_dimension as i64) * (arguments.ldd as i64)+ } else {+ 0+ }];++ let output_elements_per_threadgroup =+ pipeline_configuration.output_elements_per_threadgroup();+ let threadgroup_count_x =+ ((output_dimension as u32 + output_elements_per_threadgroup - 1)+ / output_elements_per_threadgroup) as u64;+ let threadgroup_count_z = batch_groups.max(1) as u64;++ let threadgroups =+ MTLSize::new(threadgroup_count_x, 1, threadgroup_count_z);+ let threads_per_threadgroup =+ pipeline_configuration.threads_per_threadgroup();++ Ok(Some(Self {+ pipeline_configuration,+ matrix_is_rhs,+ axpby_source,+ input_dimension,+ output_dimension,+ matrix_leading_dim,+ alpha,+ beta,+ batch_ndim,+ batch_shape,+ vector_batch_stride,+ matrix_batch_stride,+ bias_batch_stride,+ bias_stride,+ threadgroups,+ threads_per_threadgroup,+ }))+ }++ pub(crate) fn bias_is_fused(&self) -> bool {+ matches!(self.axpby_source, AxpbySource::Bias)+ }+}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv/kernel.rsindex 53b50be63..bcf872da8 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs@@ -1,11 +1,11 @@use std::collections::HashMap;-use metal::{- Buffer as MTLBuffer, ComputeCommandEncoderRef, ComputePipelineState,- MTLSize,-};+use metal::{ComputeCommandEncoderRef, ComputePipelineState};-use super::configuration::{Configuration, select_configuration};+use super::{+ dispatch_descriptor::{AxpbySource, DispatchDescriptor},+ pipeline_configuration::PipelineConfiguration,+};use crate::{DataType,backends::metal::{@@ -15,7 +15,7 @@ use crate::{fn gemv_kernel_name(data_type: DataType,- configuration: &Configuration,+ config: &PipelineConfiguration,) -> Result<String, MTLError> {let dtype_name = match data_type {DataType::F16 => "float16",@@ -29,7 +29,7 @@ fn gemv_kernel_name(},};- let prefix = if configuration.transpose_matrix {+ let prefix = if config.transpose_matrix {"gemv_t"} else {"gemv"@@ -37,303 +37,174 @@ fn gemv_kernel_name(Ok(format!("{prefix}_{dtype_name}_bm{}_bn{}_sm{}_sn{}_tm{}_tn{}_nc{}_axpby{}",- configuration.threadgroup_rows,- configuration.threadgroup_cols,- configuration.threads_per_simdgroup_row,- configuration.threads_per_simdgroup_col,- configuration.elements_per_thread_row,- configuration.elements_per_thread_col,- configuration.non_contiguous_batch as u8,- configuration.do_axpby as u8,+ config.threadgroup_rows,+ config.threadgroup_cols,+ config.threads_per_simdgroup_row,+ config.threads_per_simdgroup_col,+ config.elements_per_thread_row,+ config.elements_per_thread_col,+ config.non_contiguous_batch as u8,+ config.do_axpby as u8,))}pub struct Kernel {data_type: DataType,- lhs_is_transposed: bool,- rhs_is_transposed: bool,- pipelines: HashMap<Configuration, ComputePipelineState>,+ pipelines: HashMap<PipelineConfiguration, ComputePipelineState>,}impl Kernel {- pub fn new(- data_type: DataType,- lhs_is_transposed: bool,- rhs_is_transposed: bool,- ) -> Self {- Self {+ pub fn new(data_type: DataType) -> Result<Self, MTLError> {+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)+ {+ return Err(MTLError::Generic(format!(+ "Unsupported data type for GEMV: {:?}",+ data_type+ )));+ }+ Ok(Self {data_type,- lhs_is_transposed,- rhs_is_transposed,pipelines: HashMap::new(),- }+ })}fn get_pipeline(&mut self,context: &MTLContext,- configuration: Configuration,+ config: &PipelineConfiguration,) -> Result<&ComputePipelineState, MTLError> {- if !self.pipelines.contains_key(&configuration) {- let kernel_name = gemv_kernel_name(self.data_type, &configuration)?;+ if !self.pipelines.contains_key(config) {+ let kernel_name = gemv_kernel_name(self.data_type, config)?;let pipeline =context.compute_pipeline_state(&kernel_name, None)?;- self.pipelines.insert(configuration, pipeline);+ self.pipelines.insert(*config, pipeline);}- Ok(self.pipelines.get(&configuration).unwrap())+ Ok(self.pipelines.get(config).unwrap())}- #[allow(clippy::too_many_arguments)]- fn encode_with_configuration(+ pub(crate) fn encode_descriptor(&mut self,context: &MTLContext,encoder: &ComputeCommandEncoderRef,arguments: &MatmulArguments,- configuration: Configuration,- matrix_is_rhs: bool,- bias_buffer: Option<&MTLBuffer>,- alpha: f32,- beta: f32,- bias_stride: i32,- ) -> Result<(), MTLError> {- let pipeline = self.get_pipeline(context, configuration)?;+ descriptor: &DispatchDescriptor,+ ) -> Result<bool, MTLError> {+ let pipeline =+ self.get_pipeline(context, &descriptor.pipeline_configuration)?;encoder.set_compute_pipeline_state(pipeline);- let (buf0, off0) = if matrix_is_rhs {+ let (buf0, off0) = if descriptor.matrix_is_rhs {(arguments.b, 0)} else {(arguments.a, arguments.a_offset)};encoder.set_buffer(0, Some(buf0), off0);- let (buf1, off1) = if matrix_is_rhs {+ let (buf1, off1) = if descriptor.matrix_is_rhs {(arguments.a, arguments.a_offset)} else {(arguments.b, 0)};encoder.set_buffer(1, Some(buf1), off1);- if configuration.do_axpby {- if let Some(bias) = bias_buffer {- encoder.set_buffer(2, Some(bias), 0);+ if descriptor.pipeline_configuration.do_axpby {+ match descriptor.axpby_source {+ AxpbySource::None => {+ return Err(MTLError::Generic(+ "GEMV descriptor mismatch: do_axpby=true but axpby_source=None"+ .to_owned(),+ ));+ },+ AxpbySource::Bias => {+ let bias = arguments.bias.ok_or_else(|| {+ MTLError::Generic(+ "GEMV descriptor requires bias buffer".to_owned(),+ )+ })?;+ encoder.set_buffer(2, Some(bias), 0);+ },+ AxpbySource::C => {+ let c_buffer = arguments.c.ok_or_else(|| {+ MTLError::Generic(+ "GEMV descriptor requires C buffer".to_owned(),+ )+ })?;+ encoder.set_buffer(2, Some(c_buffer), 0);+ },}}encoder.set_buffer(3, Some(arguments.d), 0);- let input_dimension = arguments.input_dim;- let output_dimension = if matrix_is_rhs {- arguments.output_dim- } else {- arguments.batch- };- let matrix_leading_dim = if matrix_is_rhs {- arguments.ldb- } else {- arguments.lda- };-encoder.set_bytes(4,std::mem::size_of::<i32>() as u64,- &input_dimension as *const i32 as *const std::ffi::c_void,+ &descriptor.input_dimension as *const i32+ as *const std::ffi::c_void,);encoder.set_bytes(5,std::mem::size_of::<i32>() as u64,- &output_dimension as *const i32 as *const std::ffi::c_void,+ &descriptor.output_dimension as *const i32+ as *const std::ffi::c_void,);encoder.set_bytes(6,std::mem::size_of::<i32>() as u64,- &matrix_leading_dim as *const i32 as *const std::ffi::c_void,+ &descriptor.matrix_leading_dim as *const i32+ as *const std::ffi::c_void,);encoder.set_bytes(7,std::mem::size_of::<f32>() as u64,- &alpha as *const f32 as *const std::ffi::c_void,+ &descriptor.alpha as *const f32 as *const std::ffi::c_void,);encoder.set_bytes(8,std::mem::size_of::<f32>() as u64,- &beta as *const f32 as *const std::ffi::c_void,+ &descriptor.beta as *const f32 as *const std::ffi::c_void,);- let batch_shape = if arguments.batch_count > 1 {- vec![arguments.batch_count]- } else {- vec![1]- };- let batch_ndim = 1i32;- let batch_groups = arguments.batch_count.max(1);-- let elements_per_matrix_a =- (arguments.batch as i64) * (arguments.lda as i64);- let elements_per_matrix_b = if self.rhs_is_transposed {- (arguments.output_dim as i64) * (arguments.ldb as i64)- } else {- (arguments.input_dim as i64) * (arguments.ldb as i64)- };-- let vector_batch_stride = if matrix_is_rhs {- vec![elements_per_matrix_a]- } else {- vec![elements_per_matrix_b]- };-- let matrix_batch_stride = if matrix_is_rhs {- vec![elements_per_matrix_b]- } else {- vec![elements_per_matrix_a]- };-- let bias_batch_stride = if arguments.batch_count > 1 {- vec![(output_dimension as i64) * (arguments.ldd as i64)]- } else {- vec![0]- };-encoder.set_bytes(9,std::mem::size_of::<i32>() as u64,- &batch_ndim as *const i32 as *const std::ffi::c_void,+ &descriptor.batch_ndim as *const i32 as *const std::ffi::c_void,);encoder.set_bytes(10,- (std::mem::size_of::<i32>() * batch_shape.len()) as u64,- batch_shape.as_ptr() as *const std::ffi::c_void,+ (std::mem::size_of::<i32>() * descriptor.batch_shape.len()) as u64,+ descriptor.batch_shape.as_ptr() as *const std::ffi::c_void,);encoder.set_bytes(11,- (std::mem::size_of::<i64>() * vector_batch_stride.len()) as u64,- vector_batch_stride.as_ptr() as *const std::ffi::c_void,+ (std::mem::size_of::<i64>() * descriptor.vector_batch_stride.len())+ as u64,+ descriptor.vector_batch_stride.as_ptr() as *const std::ffi::c_void,);encoder.set_bytes(12,- (std::mem::size_of::<i64>() * matrix_batch_stride.len()) as u64,- matrix_batch_stride.as_ptr() as *const std::ffi::c_void,+ (std::mem::size_of::<i64>() * descriptor.matrix_batch_stride.len())+ as u64,+ descriptor.matrix_batch_stride.as_ptr() as *const std::ffi::c_void,);encoder.set_bytes(13,- (std::mem::size_of::<i64>() * bias_batch_stride.len()) as u64,- bias_batch_stride.as_ptr() as *const std::ffi::c_void,+ (std::mem::size_of::<i64>() * descriptor.bias_batch_stride.len())+ as u64,+ descriptor.bias_batch_stride.as_ptr() as *const std::ffi::c_void,);encoder.set_bytes(14,std::mem::size_of::<i32>() as u64,- &bias_stride as *const i32 as *const std::ffi::c_void,+ &descriptor.bias_stride as *const i32 as *const std::ffi::c_void,);- let output_elements_per_threadgroup =- configuration.output_elements_per_threadgroup();- let threadgroup_count_x =- ((output_dimension as u32 + output_elements_per_threadgroup - 1)- / output_elements_per_threadgroup) as u64;- let threadgroup_count_z = batch_groups.max(1) as u64;-- let threadgroup_count =- MTLSize::new(threadgroup_count_x, 1, threadgroup_count_z);- let threads_per_threadgroup = configuration.threads_per_threadgroup();-- encoder- .dispatch_thread_groups(threadgroup_count, threads_per_threadgroup);- Ok(())- }-- pub fn encode(- &mut self,- context: &MTLContext,- encoder: &ComputeCommandEncoderRef,- arguments: &MatmulArguments,- ) -> Result<(), MTLError> {- self.encode_internal(context, encoder, arguments, None)- }-- pub fn encode_with_bias(- &mut self,- context: &MTLContext,- encoder: &ComputeCommandEncoderRef,- arguments: &MatmulArguments,- bias: &MTLBuffer,- ) -> Result<(), MTLError> {- self.encode_internal(context, encoder, arguments, Some(bias))- }-- fn encode_internal(- &mut self,- context: &MTLContext,- encoder: &ComputeCommandEncoderRef,- arguments: &MatmulArguments,- bias: Option<&MTLBuffer>,- ) -> Result<(), MTLError> {- let m = arguments.batch;- let n = arguments.output_dim;- if m != 1 && n != 1 {- return Ok(());- }-- let matrix_is_rhs = n != 1;- let transpose_matrix = if matrix_is_rhs {- !self.rhs_is_transposed- } else {- self.lhs_is_transposed- };-- let has_non_contiguous_batch = false;-- let (do_axpby, bias_buffer, alpha, beta, bias_stride) =- if let Some(bias_buf) = bias {- (true, Some(bias_buf), 1.0f32, 1.0f32, 1)- } else if let Some(c_buf) = arguments.c {- (- true,- Some(c_buf),- arguments.alpha,- arguments.beta,- arguments.ldd,- )- } else {- (false, None, 1.0f32, 0.0f32, 0)- };-- let configuration = select_configuration(- transpose_matrix,- arguments.input_dim,- if matrix_is_rhs {- arguments.output_dim- } else {- arguments.batch- },- has_non_contiguous_batch,- do_axpby,+ encoder.dispatch_thread_groups(+ descriptor.threadgroups,+ descriptor.threads_per_threadgroup,);- if crate::utils::env_utils::debug_matmul_enabled() {- let kernel_name = gemv_kernel_name(self.data_type, &configuration)- .unwrap_or_default();- eprintln!(- "[matmul] GEMV m={} k={} n={} batch={} dtype={:?} transpose={} kernel={}",- arguments.batch,- arguments.input_dim,- arguments.output_dim,- arguments.batch_count,- self.data_type,- configuration.transpose_matrix,- kernel_name- );- }-- self.encode_with_configuration(- context,- encoder,- arguments,- configuration,- matrix_is_rhs,- bias_buffer,- alpha,- beta,- bias_stride,- )+ Ok(descriptor.bias_is_fused())}}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv/mod.rsindex bbd057f95..ca3ea07c9 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv/mod.rs@@ -1,4 +1,6 @@-mod configuration;+mod dispatch_descriptor;mod kernel;+mod pipeline_configuration;pub use kernel::Kernel as GemvKernel;+pub(crate) use dispatch_descriptor::DispatchDescriptor;diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/configuration.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv/pipeline_configuration.rssimilarity index 91%rename from crates/uzu/src/backends/metal/kernel/matmul/gemv/configuration.rsrename to crates/uzu/src/backends/metal/kernel/matmul/gemv/pipeline_configuration.rsindex 0b11354f0..b89852501 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv/configuration.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv/pipeline_configuration.rs@@ -1,7 +1,9 @@use metal::MTLSize;#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]-pub struct Configuration {+pub struct PipelineConfiguration {+ pub transpose_a: bool,+ pub transpose_b: bool,pub transpose_matrix: bool,pub threadgroup_rows: u32,pub threadgroup_cols: u32,@@ -13,7 +15,7 @@ pub struct Configuration {pub do_axpby: bool,}-impl Configuration {+impl PipelineConfiguration {pub fn output_elements_per_threadgroup(&self) -> u32 {if self.transpose_matrix {self.threadgroup_cols@@ -36,12 +38,14 @@ impl Configuration {}pub fn select_configuration(+ transpose_a: bool,+ transpose_b: bool,transpose_matrix: bool,input_dimension: i32,output_dimension: i32,non_contiguous_batch: bool,do_axpby: bool,-) -> Configuration {+) -> PipelineConfiguration {let (threadgroup_rows, threadgroup_cols);let (threads_per_simdgroup_row, threads_per_simdgroup_col);let (elements_per_thread_row, elements_per_thread_col);@@ -107,7 +111,9 @@ pub fn select_configuration(elements_per_thread_col = 4;}- Configuration {+ PipelineConfiguration {+ transpose_a,+ transpose_b,transpose_matrix,threadgroup_rows,threadgroup_cols,diff --git a/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/kernel.rsindex fedf3cfe2..4d84aedd4 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs@@ -1,8 +1,11 @@-use metal::{Buffer as MTLBuffer, ComputeCommandEncoderRef};+use metal::ComputeCommandEncoderRef;use super::{super::{KernelDataType, TensorAddBias},common::MatmulArguments,+ dispatch_descriptor::{+ MatmulDispatchDescriptor, choose_dispatch_descriptor,+ },gemm,gemv::GemvKernel,split_k::SplitKGemm,@@ -10,13 +13,10 @@ use super::{use crate::{DataType,backends::metal::{MTLContext, MTLError},- utils::env_utils::debug_matmul_enabled,};pub struct MatmulKernel {data_type: DataType,- lhs_is_transposed: bool,- rhs_is_transposed: bool,gemm: Option<gemm::GemmKernel>,gemv: Option<GemvKernel>,splitk: Option<SplitKGemm>,@@ -24,12 +24,7 @@ pub struct MatmulKernel {}impl MatmulKernel {- pub fn new(- _mtl: &MTLContext,- data_type: DataType,- lhs_is_transposed: bool,- rhs_is_transposed: bool,- ) -> Result<Self, MTLError> {+ pub fn new(data_type: DataType) -> Result<Self, MTLError> {if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32){return Err(MTLError::Generic(format!(@@ -39,8 +34,6 @@ impl MatmulKernel {Ok(Self {data_type,- lhs_is_transposed,- rhs_is_transposed,gemm: None,gemv: None,splitk: None,@@ -48,177 +41,90 @@ impl MatmulKernel {})}- fn maybe_use_gemv_impl(- &mut self,- mtl: &MTLContext,- enc: &ComputeCommandEncoderRef,- args: &MatmulArguments,- bias: Option<&MTLBuffer>,- ) -> Result<bool, MTLError> {- let m = args.batch;- let n = args.output_dim;- if self.lhs_is_transposed || !self.rhs_is_transposed {- return Ok(false);- }-- if m != 1 && n != 1 {- return Ok(false);- }- let gemv = self.gemv.get_or_insert_with(|| {- GemvKernel::new(- self.data_type,- self.lhs_is_transposed,- self.rhs_is_transposed,- )- });- if let Some(bias) = bias {- gemv.encode_with_bias(mtl, enc, args, bias)?;- } else {- gemv.encode(mtl, enc, args)?;- }- Ok(true)- }-- fn maybe_use_gemv(- &mut self,- mtl: &MTLContext,- enc: &ComputeCommandEncoderRef,- args: &MatmulArguments,- ) -> Result<bool, MTLError> {- self.maybe_use_gemv_impl(mtl, enc, args, None)- }-- fn maybe_use_gemv_with_bias(- &mut self,- mtl: &MTLContext,- enc: &ComputeCommandEncoderRef,- args: &MatmulArguments,- bias: &MTLBuffer,- ) -> Result<bool, MTLError> {- self.maybe_use_gemv_impl(mtl, enc, args, Some(bias))- }-- fn maybe_use_splitk(- &mut self,- mtl: &MTLContext,- enc: &ComputeCommandEncoderRef,- args: &MatmulArguments,- ) -> Result<bool, MTLError> {- let m = args.batch;- let n = args.output_dim;- let k = args.input_dim;- let batch_count = args.batch_count;-- if !SplitKGemm::should_use_splitk(m, n, k, batch_count) {- return Ok(false);- }-- let splitk = self.splitk.get_or_insert_with(|| {- SplitKGemm::new(- self.data_type,- self.lhs_is_transposed,- self.rhs_is_transposed,- )- });- splitk.encode(mtl, enc, args)?;- Ok(true)- }-fn get_or_create_gemm(&mut self) -> Result<&mut gemm::GemmKernel, MTLError> {if self.gemm.is_none() {- self.gemm = Some(gemm::GemmKernel::new(- self.data_type,- self.lhs_is_transposed,- self.rhs_is_transposed,- )?);+ self.gemm = Some(gemm::GemmKernel::new(self.data_type)?);}Ok(self.gemm.as_mut().unwrap())}- fn encode_gemm(- &mut self,- mtl: &MTLContext,- enc: &ComputeCommandEncoderRef,- args: &MatmulArguments,- ) -> Result<(), MTLError> {- let gemm = self.get_or_create_gemm()?;- gemm.encode(mtl, enc, args)- }-- pub fn encode(- &mut self,- mtl: &MTLContext,- enc: &ComputeCommandEncoderRef,- mut args: MatmulArguments,- ) -> Result<(), MTLError> {- self.apply_batch_collapse(&mut args);-- if self.maybe_use_gemv(mtl, enc, &args)? {- if debug_matmul_enabled() {- self.log_gemv(&args);- }- return Ok(());+ fn get_or_create_gemv(&mut self) -> Result<&mut GemvKernel, MTLError> {+ if self.gemv.is_none() {+ self.gemv = Some(GemvKernel::new(self.data_type)?);}+ Ok(self.gemv.as_mut().unwrap())+ }- if self.maybe_use_splitk(mtl, enc, &args)? {- if debug_matmul_enabled() {- self.log_splitk(&args);- }- return Ok(());+ fn get_or_create_splitk(&mut self) -> Result<&mut SplitKGemm, MTLError> {+ if self.splitk.is_none() {+ self.splitk = Some(SplitKGemm::new(self.data_type)?);}+ Ok(self.splitk.as_mut().unwrap())+ }- if debug_matmul_enabled() {- self.log_gemm(&args, mtl);+ fn encode_dispatch_descriptor(+ &mut self,+ context: &MTLContext,+ encoder: &ComputeCommandEncoderRef,+ arguments: &MatmulArguments,+ descriptor: &MatmulDispatchDescriptor,+ ) -> Result<bool, MTLError> {+ match descriptor {+ MatmulDispatchDescriptor::Gemv(descriptor) => {+ let gemv = self.get_or_create_gemv()?;+ gemv.encode_descriptor(context, encoder, arguments, descriptor)+ },+ MatmulDispatchDescriptor::SplitK(descriptor) => {+ let splitk = self.get_or_create_splitk()?;+ splitk+ .encode_descriptor(context, encoder, arguments, descriptor)+ },+ MatmulDispatchDescriptor::Gemm(descriptor) => {+ let gemm = self.get_or_create_gemm()?;+ gemm.encode_descriptor(context, encoder, arguments, descriptor)+ },}-- self.encode_gemm(mtl, enc, &args)}- pub fn encode_with_bias(+ pub fn encode(&mut self,- mtl: &MTLContext,- enc: &ComputeCommandEncoderRef,- mut args: MatmulArguments,- bias: &MTLBuffer,+ context: &MTLContext,+ encoder: &ComputeCommandEncoderRef,+ mut arguments: MatmulArguments,) -> Result<(), MTLError> {- self.apply_batch_collapse(&mut args);+ Self::apply_batch_collapse(&mut arguments);- if self.maybe_use_gemv_with_bias(mtl, enc, &args, bias)? {- if debug_matmul_enabled() {- self.log_gemv(&args);- }- return Ok(());- }+ let descriptor =+ choose_dispatch_descriptor(context, self.data_type, &arguments)?;- if self.maybe_use_splitk(mtl, enc, &args)? {- if debug_matmul_enabled() {- self.log_splitk(&args);- }- self.apply_bias_add(mtl, enc, &args, bias)?;- return Ok(());- }+ let bias_fused = self.encode_dispatch_descriptor(+ context,+ encoder,+ &arguments,+ &descriptor,+ )?;- if debug_matmul_enabled() {- self.log_gemm(&args, mtl);+ if let Some(bias) = arguments.bias {+ if !bias_fused {+ self.apply_bias_add(context, encoder, &arguments, bias)?;+ }}- self.encode_gemm(mtl, enc, &args)?;- self.apply_bias_add(mtl, enc, &args, bias)?;Ok(())}fn apply_bias_add(&mut self,- mtl: &MTLContext,- enc: &ComputeCommandEncoderRef,- args: &MatmulArguments,- bias: &MTLBuffer,+ context: &MTLContext,+ encoder: &ComputeCommandEncoderRef,+ arguments: &MatmulArguments,+ bias: &metal::Buffer,) -> Result<(), MTLError> {- let m = args.batch as usize;- let n = args.output_dim as usize;- let batch_count = args.batch_count as usize;+ let m = arguments.batch as usize;+ let n = arguments.output_dim as usize;+ let batch_count = arguments.batch_count as usize;let total_len = m * n * batch_count;if total_len == 0 {return Ok(());@@ -226,80 +132,33 @@ impl MatmulKernel {if self.bias_add.is_none() {self.bias_add = Some(TensorAddBias::new(- mtl,+ context,KernelDataType::from(self.data_type),)?);}let bias_add = self.bias_add.as_ref().unwrap();- bias_add- .encode_with_encoder(args.d, bias, args.d, n, total_len, enc, None);+ bias_add.encode_with_encoder(+ arguments.d,+ bias,+ arguments.d,+ n,+ total_len,+ encoder,+ None,+ );Ok(())}- fn apply_batch_collapse(- &self,- args: &mut MatmulArguments,- ) {- if self.lhs_is_transposed {+ fn apply_batch_collapse(arguments: &mut MatmulArguments) {+ if arguments.transpose_a {return;}- if args.batch_count <= 1 {+ if arguments.batch_count <= 1 {return;}- if args.lda == args.input_dim && self.rhs_is_transposed {- args.batch *= args.batch_count;- args.batch_count = 1;- }- }-- fn log_gemv(- &self,- args: &MatmulArguments,- ) {- eprintln!(- "[matmul] GEMV m={} k={} n={} batch={} dtype={:?}",- args.batch,- args.input_dim,- args.output_dim,- args.batch_count,- self.data_type- );- }-- fn log_splitk(- &self,- args: &MatmulArguments,- ) {- eprintln!(- "[matmul] SplitK m={} k={} n={} batch={} dtype={:?}",- args.batch,- args.input_dim,- args.output_dim,- args.batch_count,- self.data_type- );- }-- fn log_gemm(- &self,- args: &MatmulArguments,- mtl: &MTLContext,- ) {- let gemm_ref = self.gemm.as_ref();- if let Some(gemm) = gemm_ref {- let tile = gemm.select_tile(mtl, args);- eprintln!(- "[matmul] GEMM m={} k={} n={} batch={} dtype={:?} tile={}x{}x{} nax={}",- args.batch,- args.input_dim,- args.output_dim,- args.batch_count,- self.data_type,- tile.block_rows,- tile.block_cols,- tile.block_depth,- tile.is_nax()- );+ if arguments.lda == arguments.input_dim && arguments.transpose_b {+ arguments.batch *= arguments.batch_count;+ arguments.batch_count = 1;}}}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/mod.rsindex 2ef5b3948..581cc089d 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/mod.rs@@ -1,4 +1,5 @@pub mod common;+mod dispatch_descriptor;mod gemm;mod gemv;mod kernel;diff --git a/crates/uzu/src/backends/metal/kernel/matmul/split_k/dispatch_descriptor.rs b/crates/uzu/src/backends/metal/kernel/matmul/split_k/dispatch_descriptor.rsnew file mode 100644index 000000000..a052e8d66--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/split_k/dispatch_descriptor.rs@@ -0,0 +1,152 @@+use metal::MTLSize;++use super::{+ pipeline_configuration::PipelineConfiguration,+ tile_configuration::select_tile_configuration,+};+use crate::{+ DataType,+ backends::metal::{+ MTLContext, MTLError,+ kernel::matmul::common::{+ GEMMSpiltKParams as SplitKGEMMParams, MatmulArguments,+ },+ },+};++#[derive(Debug, Clone)]+pub(crate) struct DispatchDescriptor {+ pub(crate) pipeline_configuration: PipelineConfiguration,+ pub(crate) params: SplitKGEMMParams,+ pub(crate) partition_count: i32,+ pub(crate) output_elements_per_partition: i32,+ pub(crate) accumulator_bytes: usize,+ pub(crate) partial_threadgroups: MTLSize,+ pub(crate) partial_threads_per_threadgroup: MTLSize,+ pub(crate) accum_total_threads: MTLSize,+ pub(crate) accum_threads_per_threadgroup: MTLSize,+}++impl DispatchDescriptor {+ pub(crate) fn try_new(+ _context: &MTLContext,+ data_type: DataType,+ arguments: &MatmulArguments,+ ) -> Result<Option<Self>, MTLError> {+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)+ {+ return Err(MTLError::Generic(format!(+ "Unsupported dtype for Split-K: {:?}",+ data_type+ )));+ }++ if arguments.c.is_some() {+ return Ok(None);+ }++ let m = arguments.batch;+ let n = arguments.output_dim;+ let k = arguments.input_dim;+ let batch_count = arguments.batch_count;++ if !should_use_splitk(m, n, k, batch_count) {+ return Ok(None);+ }++ let tile = select_tile_configuration(m, n);+ let partition_count = compute_partition_count(k);+ let mn_aligned =+ (m % tile.tile_rows) == 0 && (n % tile.tile_cols) == 0;+ let k_aligned = (k % tile.tile_depth) == 0;++ let pipeline_configuration = PipelineConfiguration {+ tile,+ transpose_a: arguments.transpose_a,+ transpose_b: arguments.transpose_b,+ mn_aligned,+ k_aligned,+ };++ let tile_count_m = (m + tile.tile_rows - 1) / tile.tile_rows;+ let tile_count_n = (n + tile.tile_cols - 1) / tile.tile_cols;++ let gemm_k_iterations = (k / tile.tile_depth) / partition_count;+ let k_elements_per_partition = gemm_k_iterations * tile.tile_depth;+ let output_elements_per_partition = m * n;++ let accumulator_element_count = partition_count+ * output_elements_per_partition+ * arguments.batch_count;+ let accumulator_bytes =+ (accumulator_element_count as usize) * std::mem::size_of::<f32>();++ let params = SplitKGEMMParams {+ M: m,+ N: n,+ K: k,+ lda: arguments.lda,+ ldb: arguments.ldb,+ ldc: n,+ tiles_n: tile_count_n,+ tiles_m: tile_count_m,+ split_k_partitions: partition_count,+ split_k_partition_stride: output_elements_per_partition,+ split_k_partition_size: k_elements_per_partition,+ gemm_k_iterations_aligned: gemm_k_iterations,+ };++ let partial_threads_per_threadgroup = MTLSize::new(+ 32,+ tile.warps_per_col as u64,+ tile.warps_per_row as u64,+ );+ let partial_threadgroups = MTLSize::new(+ tile_count_n as u64,+ tile_count_m as u64,+ partition_count as u64,+ );++ let accum_total_threads = MTLSize::new(n as u64, m as u64, 1);+ let accum_threads_per_threadgroup =+ MTLSize::new(16.min(n as u64), 16.min(m as u64), 1);++ Ok(Some(Self {+ pipeline_configuration,+ params,+ partition_count,+ output_elements_per_partition,+ accumulator_bytes,+ partial_threadgroups,+ partial_threads_per_threadgroup,+ accum_total_threads,+ accum_threads_per_threadgroup,+ }))+ }+}++fn compute_partition_count(k: i32) -> i32 {+ let k_tiles = k / 16;+ if k_tiles < 16 {+ 2+ } else if k_tiles < 32 {+ 4+ } else if k_tiles < 64 {+ 8+ } else {+ 16+ }+}++fn should_use_splitk(m: i32, n: i32, k: i32, batch_count: i32) -> bool {+ if batch_count != 1 {+ return false;+ }+ if m <= 0 || n <= 0 || k <= 0 {+ return false;+ }+ let m_tiles = m / 16;+ let n_tiles = n / 16;+ let k_tiles = k / 16;+ (m_tiles * n_tiles) <= 32 && k_tiles >= 8+}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/split_k/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/split_k/kernel.rsindex 531755fa0..5809da8b6 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/split_k/kernel.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/split_k/kernel.rs@@ -1,43 +1,48 @@use std::collections::HashMap;use metal::{- ComputeCommandEncoderRef, ComputePipelineState as MTLComputePipelineState,- MTLSize,+ Buffer as MTLBuffer, ComputeCommandEncoderRef,+ ComputePipelineState as MTLComputePipelineState,};use super::{- pipeline_configuration::PipelineConfiguration,- tile_configuration::{TileConfiguration, select_tile_configuration},+ DispatchDescriptor, pipeline_configuration::PipelineConfiguration,};use crate::{DataType,backends::metal::{MTLContext, MTLError,- kernel::matmul::common::{GEMMSpiltKParams as SplitKGEMMParams, MatmulArguments},+ kernel::matmul::common::{+ GEMMSpiltKParams as SplitKGEMMParams, MatmulArguments,+ transpose_configuration,+ },},};pub struct Kernel {data_type: DataType,- transpose_a: bool,- transpose_b: bool,partial_pipelines: HashMap<PipelineConfiguration, MTLComputePipelineState>,accum_pipeline: Option<MTLComputePipelineState>,+ accumulator_buffer: Option<MTLBuffer>,+ accumulator_buffer_bytes: usize,}impl Kernel {- pub fn new(- data_type: DataType,- transpose_a: bool,- transpose_b: bool,- ) -> Self {- Self {+ pub fn new(data_type: DataType) -> Result<Self, MTLError> {+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)+ {+ return Err(MTLError::Generic(format!(+ "Unsupported dtype for Split-K: {:?}",+ data_type+ )));+ }+ Ok(Self {data_type,- transpose_a,- transpose_b,partial_pipelines: HashMap::new(),accum_pipeline: None,- }+ accumulator_buffer: None,+ accumulator_buffer_bytes: 0,+ })}fn steel_type_name(&self) -> Result<&'static str, MTLError> {@@ -56,36 +61,35 @@ impl Kernel {"float32"}- fn transpose_suffix(&self) -> &'static str {- match (self.transpose_a, self.transpose_b) {- (false, false) => "nn",- (false, true) => "nt",- (true, false) => "tn",- (true, true) => "tt",- }- }-fn partial_kernel_name(&self,- tile: &TileConfiguration,- mn_aligned: bool,- k_aligned: bool,+ config: &PipelineConfiguration,) -> Result<String, MTLError> {let in_name = self.steel_type_name()?;let out_name = self.splitk_partial_out_name();- let transpose_suffix = self.transpose_suffix();- let mn_tag = if mn_aligned { "taligned" } else { "naligned" };- let k_tag = if k_aligned { "taligned" } else { "naligned" };+ let transpose_suffix =+ transpose_configuration(config.transpose_a, config.transpose_b)+ .as_str();+ let mn_tag = if config.mn_aligned {+ "taligned"+ } else {+ "naligned"+ };+ let k_tag = if config.k_aligned {+ "taligned"+ } else {+ "naligned"+ };Ok(format!("steel_gemm_splitk_{}_{}_{}_bm{}_bn{}_bk{}_wm{}_wn{}_MN_{}_K_{}",transpose_suffix,in_name,out_name,- tile.tile_rows,- tile.tile_cols,- tile.tile_depth,- tile.warps_per_row,- tile.warps_per_col,+ config.tile.tile_rows,+ config.tile.tile_cols,+ config.tile.tile_depth,+ config.tile.warps_per_row,+ config.tile.warps_per_col,mn_tag,k_tag,))@@ -106,8 +110,7 @@ impl Kernel {config: &PipelineConfiguration,) -> Result<&MTLComputePipelineState, MTLError> {if !self.partial_pipelines.contains_key(config) {- let name =- self.partial_kernel_name(&config.tile, config.mn_aligned, config.k_aligned)?;+ let name = self.partial_kernel_name(config)?;let ps = mtl.compute_pipeline_state(&name, None)?;self.partial_pipelines.insert(config.clone(), ps);}@@ -126,19 +129,6 @@ impl Kernel {Ok(self.accum_pipeline.as_ref().unwrap())}- fn compute_partition_count(k: i32) -> i32 {- let k_tiles = k / 16;- if k_tiles < 16 {- 2- } else if k_tiles < 32 {- 4- } else if k_tiles < 64 {- 8- } else {- 16- }- }-pub fn should_use_splitk(m: i32,n: i32,@@ -157,108 +147,86 @@ impl Kernel {(m_tiles * n_tiles) <= 32 && k_tiles >= 8}- pub fn encode(+ pub(crate) fn encode_descriptor(&mut self,- mtl: &MTLContext,- enc: &ComputeCommandEncoderRef,- args: &MatmulArguments,- ) -> Result<(), MTLError> {- let m = args.batch;- let n = args.output_dim;- let k = args.input_dim;-- let tile = select_tile_configuration(m, n);- let partition_count = Self::compute_partition_count(k);- let mn_aligned =- (m % tile.tile_rows) == 0 && (n % tile.tile_cols) == 0;- let k_aligned = (k % tile.tile_depth) == 0;-- let config = PipelineConfiguration {- tile,- mn_aligned,- k_aligned,- };-- let tile_count_m = (m + tile.tile_rows - 1) / tile.tile_rows;- let tile_count_n = (n + tile.tile_cols - 1) / tile.tile_cols;-- let gemm_k_iterations = (k / tile.tile_depth) / partition_count;- let k_elements_per_partition = gemm_k_iterations * tile.tile_depth;- let output_elements_per_partition = m * n;-- let accumulator_element_count =- partition_count * output_elements_per_partition * args.batch_count;- let accumulator_bytes =- (accumulator_element_count as usize) * std::mem::size_of::<f32>();- let accumulator_buffer = mtl.device.new_buffer(- accumulator_bytes as u64,- metal::MTLResourceOptions::StorageModePrivate,- );-- let params = SplitKGEMMParams {- M: m,- N: n,- K: k,- lda: args.lda,- ldb: args.ldb,- ldc: n,- tiles_n: tile_count_n,- tiles_m: tile_count_m,- split_k_partitions: partition_count,- split_k_partition_stride: output_elements_per_partition,- split_k_partition_size: k_elements_per_partition,- gemm_k_iterations_aligned: gemm_k_iterations,- };-- let partial_ps = self.get_partial_pipeline(mtl, &config)?;-- enc.set_compute_pipeline_state(partial_ps);- enc.set_buffer(0, Some(args.a), args.a_offset);- enc.set_buffer(1, Some(args.b), 0);- enc.set_buffer(2, Some(&accumulator_buffer), 0);- enc.set_bytes(+ context: &MTLContext,+ encoder: &ComputeCommandEncoderRef,+ arguments: &MatmulArguments,+ descriptor: &DispatchDescriptor,+ ) -> Result<bool, MTLError> {+ self.ensure_accumulator_buffer(context, descriptor.accumulator_bytes);+ let accumulator_buffer = self+ .accumulator_buffer+ .as_ref()+ .cloned()+ .expect("Accumulator buffer must be initialized");++ let partial_pipeline_state = self.get_partial_pipeline(+ context,+ &descriptor.pipeline_configuration,+ )?;++ encoder.set_compute_pipeline_state(partial_pipeline_state);+ encoder.set_buffer(0, Some(arguments.a), arguments.a_offset);+ encoder.set_buffer(1, Some(arguments.b), 0);+ encoder.set_buffer(2, Some(&accumulator_buffer), 0);+ encoder.set_bytes(3,std::mem::size_of::<SplitKGEMMParams>() as u64,- ¶ms as *const SplitKGEMMParams as *const _,+ &descriptor.params as *const _ as *const _,);-- let threads_per_threadgroup = MTLSize::new(- 32,- tile.warps_per_col as u64,- tile.warps_per_row as u64,+ encoder.dispatch_thread_groups(+ descriptor.partial_threadgroups,+ descriptor.partial_threads_per_threadgroup,);- let threadgroups = MTLSize::new(- tile_count_n as u64,- tile_count_m as u64,- partition_count as u64,- );- enc.dispatch_thread_groups(threadgroups, threads_per_threadgroup);- let accum_ps = self.get_accum_pipeline(mtl)?;- enc.set_compute_pipeline_state(accum_ps);- enc.set_buffer(0, Some(&accumulator_buffer), 0);- enc.set_buffer(1, Some(args.d), 0);- enc.set_bytes(+ let accum_pipeline_state = self.get_accum_pipeline(context)?;+ encoder.set_compute_pipeline_state(accum_pipeline_state);+ encoder.set_buffer(0, Some(&accumulator_buffer), 0);+ encoder.set_buffer(1, Some(arguments.d), 0);++ let partition_count = descriptor.partition_count;+ let output_elements_per_partition =+ descriptor.output_elements_per_partition;+ encoder.set_bytes(2,- 4,+ std::mem::size_of::<i32>() as u64,&partition_count as *const i32 as *const std::ffi::c_void,);- enc.set_bytes(+ encoder.set_bytes(3,- 4,- &output_elements_per_partition as *const i32 as *const std::ffi::c_void,+ std::mem::size_of::<i32>() as u64,+ &output_elements_per_partition as *const i32+ as *const std::ffi::c_void,);- enc.set_bytes(- 4,+ encoder.set_bytes(4,- &(args.ldd) as *const i32 as *const std::ffi::c_void,+ std::mem::size_of::<i32>() as u64,+ &(arguments.ldd) as *const i32 as *const std::ffi::c_void,+ );++ encoder.dispatch_threads(+ descriptor.accum_total_threads,+ descriptor.accum_threads_per_threadgroup,);- let accum_total_threads = MTLSize::new(n as u64, m as u64, 1);- let accum_threads_per_threadgroup =- MTLSize::new(16.min(n as u64), 16.min(m as u64), 1);- enc.dispatch_threads(accum_total_threads, accum_threads_per_threadgroup);+ Ok(false)+ }- Ok(())+ fn ensure_accumulator_buffer(+ &mut self,+ mtl: &MTLContext,+ required_bytes: usize,+ ) {+ if required_bytes <= self.accumulator_buffer_bytes+ && self.accumulator_buffer.is_some()+ {+ return;+ }+ self.accumulator_buffer = Some(mtl.device.new_buffer(+ required_bytes as u64,+ metal::MTLResourceOptions::StorageModePrivate,+ ));+ self.accumulator_buffer_bytes = required_bytes;}}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/split_k/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/split_k/mod.rsindex 54a1730c2..22383d1f8 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/split_k/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/split_k/mod.rs@@ -1,5 +1,7 @@+mod dispatch_descriptor;mod kernel;mod pipeline_configuration;mod tile_configuration;pub use kernel::Kernel as SplitKGemm;+pub(crate) use dispatch_descriptor::DispatchDescriptor;diff --git a/crates/uzu/src/backends/metal/kernel/matmul/split_k/pipeline_configuration.rs b/crates/uzu/src/backends/metal/kernel/matmul/split_k/pipeline_configuration.rsindex b745b347a..1eab77254 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/split_k/pipeline_configuration.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/split_k/pipeline_configuration.rs@@ -3,6 +3,8 @@ use super::tile_configuration::TileConfiguration;#[derive(Debug, Clone, PartialEq, Eq, Hash)]pub struct PipelineConfiguration {pub tile: TileConfiguration,+ pub transpose_a: bool,+ pub transpose_b: bool,pub mn_aligned: bool,pub k_aligned: bool,}diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rsindex 1698da06d..75d71378d 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs@@ -199,20 +199,6 @@ impl QuantizedMatmulKernel {}})?;- if crate::utils::env_utils::debug_matmul_enabled() {- eprintln!(- "[qmm] {:?} m={} k={} n={} quant={:?} block={}x{} transposed={}",- variant,- args.batch,- args.input_dim,- args.output_dim,- args.quantization_type,- bm,- bn,- self.weights_transposed- );- }-encoder.set_compute_pipeline_state(pipeline);// Set buffersdiff --git a/crates/uzu/src/utils/env_utils.rs b/crates/uzu/src/utils/env_utils.rsindex f87bf1211..27593aef8 100644--- a/crates/uzu/src/utils/env_utils.rs+++ b/crates/uzu/src/utils/env_utils.rs@@ -3,7 +3,6 @@ pub enum MetalEnvVar {DeviceWrapperType,CaptureFirstDecode,CaptureFirstPrefill,- DebugMatmul,}impl MetalEnvVar {@@ -12,7 +11,6 @@ impl MetalEnvVar {MetalEnvVar::DeviceWrapperType => "METAL_DEVICE_WRAPPER_TYPE",MetalEnvVar::CaptureFirstDecode => "UZU_CAPTURE_FIRST_DECODE",MetalEnvVar::CaptureFirstPrefill => "UZU_CAPTURE_FIRST_PREFILL",- MetalEnvVar::DebugMatmul => "UZU_DEBUG_MATMUL",}}@@ -25,11 +23,3 @@ impl MetalEnvVar {matches!(upper.as_str(), "1" | "YES" | "TRUE")}}--use std::sync::OnceLock;--static DEBUG_MATMUL_ENABLED: OnceLock<bool> = OnceLock::new();--pub fn debug_matmul_enabled() -> bool {- *DEBUG_MATMUL_ENABLED.get_or_init(|| MetalEnvVar::DebugMatmul.is_enabled())-}diff --git a/crates/uzu/tests/matmul_correctness_test.rs b/crates/uzu/tests/matmul_correctness_test.rsindex 678504e49..98ff7029d 100644--- a/crates/uzu/tests/matmul_correctness_test.rs+++ b/crates/uzu/tests/matmul_correctness_test.rs@@ -72,6 +72,7 @@ fn run_metal_matmul(alpha: 1.0,beta: 0.0,},+ None,).expect("encode");enc.end_encoding();diff --git a/crates/uzu/tests/mlp_fused_performance_test.rs b/crates/uzu/tests/mlp_fused_performance_test.rsindex 99cc81695..317fa812e 100644--- a/crates/uzu/tests/mlp_fused_performance_test.rs+++ b/crates/uzu/tests/mlp_fused_performance_test.rs@@ -151,6 +151,7 @@ fn benchmark_gemv_fused(alpha: 1.0,beta: 0.0,},+ None,).unwrap();encoder.end_encoding();@@ -184,6 +185,7 @@ fn benchmark_gemv_fused(alpha: 1.0,beta: 0.0,},+ None,).unwrap();encoder.end_encoding();@@ -300,6 +302,7 @@ fn benchmark_gemm_fused(alpha: 1.0,beta: 0.0,},+ None,).unwrap();encoder.end_encoding();@@ -332,6 +335,7 @@ fn benchmark_gemm_fused(alpha: 1.0,beta: 0.0,},+ None,).unwrap();encoder.end_encoding();