A high-performance inference engine for AI models
reorganize code
Eugene Bokhan53935ffe6701
diff --git a/crates/uzu/build.rs b/crates/uzu/build.rsindex 564b34121..aa2258fba 100644--- a/crates/uzu/build.rs+++ b/crates/uzu/build.rs@@ -23,8 +23,9 @@ fn main() {/// These headers are also used by Metal shaders, ensuring type consistency.fn generate_metal_bindings() {let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());- let matmul_dir = manifest_dir.join("src/backends/metal/kernel/matmul");- let types_header = matmul_dir.join("shared_types.h");+ let matmul_common_dir =+ manifest_dir.join("src/backends/metal/kernel/matmul/common");+ let types_header = matmul_common_dir.join("shared_types.h");// Only regenerate if header changedprintln!("cargo:rerun-if-changed={}", types_header.display());@@ -43,13 +44,14 @@ fn generate_metal_bindings() {.allowlist_type("GEMMParams").allowlist_type("GEMMAddMMParams").allowlist_type("GEMMSpiltKParams")+ .allowlist_type("GEMMSpiltKMlpFusedParams")// Use core types.use_core().generate().expect("Failed to generate bindings from shared_types.h");// Write bindings to the source directory so it can be checked in- let bindings_path = matmul_dir.join("shared_types.rs");+ let bindings_path = matmul_common_dir.join("shared_types.rs");bindings.write_to_file(&bindings_path).expect("Failed to write shared_types.rs");@@ -260,7 +262,10 @@ fn macos_sdk_version(sdk: &str) -> Option<(u32, u32)> {Some((major, minor))}-fn should_enable_nax(target_os: &str, sdk: &str) -> bool {+fn should_enable_nax(+ target_os: &str,+ sdk: &str,+) -> bool {if target_os != "macos" && target_os != "ios" {return false;}@@ -269,8 +274,8 @@ fn should_enable_nax(target_os: &str, sdk: &str) -> bool {return false;}// Check host OS version- let os_ok =- host_macos_major_version().is_some_and(|major| major >= NAX_MIN_OS_MAJOR);+ let os_ok = host_macos_major_version()+ .is_some_and(|major| major >= NAX_MIN_OS_MAJOR);if !os_ok {return false;}@@ -380,6 +385,11 @@ fn compile_metal_files(if matmul_dir.exists() {include_dirs.insert(matmul_dir.clone());+ let matmul_common_dir = matmul_dir.join("common");+ if matmul_common_dir.exists() {+ include_dirs.insert(matmul_common_dir);+ }+let gemv_dir = matmul_dir.join("gemv");if gemv_dir.exists() {include_dirs.insert(gemv_dir);diff --git a/crates/uzu/src/backends/metal/encodable_block/mlp.rs b/crates/uzu/src/backends/metal/encodable_block/mlp.rsindex 76ed0bb31..3393dbde5 100644--- a/crates/uzu/src/backends/metal/encodable_block/mlp.rs+++ b/crates/uzu/src/backends/metal/encodable_block/mlp.rs@@ -11,11 +11,11 @@ use crate::{MTLContext,forward_pass::{ArrayId, ForwardPassState},kernel::{- matmul::{- MlpFusedGemmArguments, MlpFusedGemmKernel,- MlpFusedGemvArguments, MlpFusedGemvKernel,- },mlp::{MlpActivationType, MlpGateActMulEncodable},+ mlp_fused::{+ GemmArguments as MlpFusedGemmArguments, GemmKernel as MlpFusedGemmKernel,+ GemvArguments as MlpFusedGemvArguments, GemvKernel as MlpFusedGemvKernel,+ },quant_matmul::{MlpFusedQmmArguments, MlpFusedQmmKernel, MlpFusedQmvArguments,MlpFusedQmvKernel,diff --git a/crates/uzu/src/backends/metal/kernel/attention/gemm_attention.metal b/crates/uzu/src/backends/metal/kernel/attention/gemm_attention.metalindex c348112a6..bcff92b15 100644--- a/crates/uzu/src/backends/metal/kernel/attention/gemm_attention.metal+++ b/crates/uzu/src/backends/metal/kernel/attention/gemm_attention.metal@@ -1,7 +1,7 @@#include <metal_stdlib>-#include "../matmul/loader.h"-#include "../matmul/mma.h"+#include "../matmul/common/loader.h"+#include "../matmul/common/mma.h"#include "gemm_types.h"diff --git a/crates/uzu/src/backends/metal/kernel/matmul/defines.h b/crates/uzu/src/backends/metal/kernel/matmul/common/defines.hsimilarity index 100%rename from crates/uzu/src/backends/metal/kernel/matmul/defines.hrename to crates/uzu/src/backends/metal/kernel/matmul/common/defines.hdiff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm.h b/crates/uzu/src/backends/metal/kernel/matmul/common/gemm.hsimilarity index 100%rename from crates/uzu/src/backends/metal/kernel/matmul/gemm.hrename to crates/uzu/src/backends/metal/kernel/matmul/common/gemm.hdiff --git a/crates/uzu/src/backends/metal/kernel/matmul/loader.h b/crates/uzu/src/backends/metal/kernel/matmul/common/loader.hsimilarity index 100%rename from crates/uzu/src/backends/metal/kernel/matmul/loader.hrename to crates/uzu/src/backends/metal/kernel/matmul/common/loader.hdiff --git a/crates/uzu/src/backends/metal/kernel/matmul/arguments.rs b/crates/uzu/src/backends/metal/kernel/matmul/common/matmul_arguments.rssimilarity index 100%rename from crates/uzu/src/backends/metal/kernel/matmul/arguments.rsrename to crates/uzu/src/backends/metal/kernel/matmul/common/matmul_arguments.rsdiff --git a/crates/uzu/src/backends/metal/kernel/matmul/mma.h b/crates/uzu/src/backends/metal/kernel/matmul/common/mma.hsimilarity index 100%rename from crates/uzu/src/backends/metal/kernel/matmul/mma.hrename to crates/uzu/src/backends/metal/kernel/matmul/common/mma.hdiff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/common/mod.rsnew file mode 100644index 000000000..0ca76a5d9--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/mod.rs@@ -0,0 +1,10 @@+mod matmul_arguments;+#[allow(dead_code, clippy::unnecessary_operation, clippy::identity_op)]+mod shared_types;+mod transpose_configuration;++pub use matmul_arguments::MatmulArguments;+pub use shared_types::{+ GEMMAddMMParams, GEMMParams, GEMMSpiltKMlpFusedParams, GEMMSpiltKParams,+};+pub(crate) use transpose_configuration::transpose_configuration;diff --git a/crates/uzu/src/backends/metal/kernel/matmul/shared_types.h b/crates/uzu/src/backends/metal/kernel/matmul/common/shared_types.hsimilarity index 84%rename from crates/uzu/src/backends/metal/kernel/matmul/shared_types.hrename to crates/uzu/src/backends/metal/kernel/matmul/common/shared_types.hindex 9a957631f..ad116a541 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/shared_types.h+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/shared_types.h@@ -67,6 +67,27 @@ struct GEMMSpiltKParams {int gemm_k_iterations_aligned;};+// Split-K MLP Fused GEMM parameters+struct GEMMSpiltKMlpFusedParams {+ int M;+ int N;+ int K;++ int lda;+ int ldb;+ int ldc;++ int tiles_n;+ int tiles_m;++ int split_k_partitions;+ int split_k_partition_stride;+ int split_k_partition_size;++ int gemm_k_iterations_aligned;+ int hidden_dim;+};+// AddMM parameters (alpha * A @ B + beta * C)struct GEMMAddMMParams {int ldc;diff --git a/crates/uzu/src/backends/metal/kernel/matmul/shared_types.rs b/crates/uzu/src/backends/metal/kernel/matmul/common/shared_types.rssimilarity index 67%rename from crates/uzu/src/backends/metal/kernel/matmul/shared_types.rsrename to crates/uzu/src/backends/metal/kernel/matmul/common/shared_types.rsindex 97e68a5bf..3e4c612d2 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/shared_types.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/shared_types.rs@@ -114,6 +114,70 @@ const _: () = {};#[repr(C)]#[derive(Debug, Default, Copy, Clone)]+pub struct GEMMSpiltKMlpFusedParams {+ pub M: ::core::ffi::c_int,+ pub N: ::core::ffi::c_int,+ pub K: ::core::ffi::c_int,+ pub lda: ::core::ffi::c_int,+ pub ldb: ::core::ffi::c_int,+ pub ldc: ::core::ffi::c_int,+ pub tiles_n: ::core::ffi::c_int,+ pub tiles_m: ::core::ffi::c_int,+ pub split_k_partitions: ::core::ffi::c_int,+ pub split_k_partition_stride: ::core::ffi::c_int,+ pub split_k_partition_size: ::core::ffi::c_int,+ pub gemm_k_iterations_aligned: ::core::ffi::c_int,+ pub hidden_dim: ::core::ffi::c_int,+}+#[allow(clippy::unnecessary_operation, clippy::identity_op)]+const _: () = {+ ["Size of GEMMSpiltKMlpFusedParams"]+ [::core::mem::size_of::<GEMMSpiltKMlpFusedParams>() - 52usize];+ ["Alignment of GEMMSpiltKMlpFusedParams"]+ [::core::mem::align_of::<GEMMSpiltKMlpFusedParams>() - 4usize];+ ["Offset of field: GEMMSpiltKMlpFusedParams::M"]+ [::core::mem::offset_of!(GEMMSpiltKMlpFusedParams, M) - 0usize];+ ["Offset of field: GEMMSpiltKMlpFusedParams::N"]+ [::core::mem::offset_of!(GEMMSpiltKMlpFusedParams, N) - 4usize];+ ["Offset of field: GEMMSpiltKMlpFusedParams::K"]+ [::core::mem::offset_of!(GEMMSpiltKMlpFusedParams, K) - 8usize];+ ["Offset of field: GEMMSpiltKMlpFusedParams::lda"]+ [::core::mem::offset_of!(GEMMSpiltKMlpFusedParams, lda) - 12usize];+ ["Offset of field: GEMMSpiltKMlpFusedParams::ldb"]+ [::core::mem::offset_of!(GEMMSpiltKMlpFusedParams, ldb) - 16usize];+ ["Offset of field: GEMMSpiltKMlpFusedParams::ldc"]+ [::core::mem::offset_of!(GEMMSpiltKMlpFusedParams, ldc) - 20usize];+ ["Offset of field: GEMMSpiltKMlpFusedParams::tiles_n"]+ [::core::mem::offset_of!(GEMMSpiltKMlpFusedParams, tiles_n) - 24usize];+ ["Offset of field: GEMMSpiltKMlpFusedParams::tiles_m"]+ [::core::mem::offset_of!(GEMMSpiltKMlpFusedParams, tiles_m) - 28usize];+ ["Offset of field: GEMMSpiltKMlpFusedParams::split_k_partitions"][::core::mem::offset_of!(+ GEMMSpiltKMlpFusedParams,+ split_k_partitions+ )+ - 32usize];+ ["Offset of field: GEMMSpiltKMlpFusedParams::split_k_partition_stride"][::core::mem::offset_of!(+ GEMMSpiltKMlpFusedParams,+ split_k_partition_stride+ )+ - 36usize];+ ["Offset of field: GEMMSpiltKMlpFusedParams::split_k_partition_size"][::core::mem::offset_of!(+ GEMMSpiltKMlpFusedParams,+ split_k_partition_size+ )+ - 40usize];+ ["Offset of field: GEMMSpiltKMlpFusedParams::gemm_k_iterations_aligned"][::core::mem::offset_of!(+ GEMMSpiltKMlpFusedParams,+ gemm_k_iterations_aligned+ )+ - 44usize];+ ["Offset of field: GEMMSpiltKMlpFusedParams::hidden_dim"][::core::mem::offset_of!(+ GEMMSpiltKMlpFusedParams,+ hidden_dim+ ) - 48usize];+};+#[repr(C)]+#[derive(Debug, Default, Copy, Clone)]pub struct GEMMAddMMParams {pub ldc: ::core::ffi::c_int,pub fdc: ::core::ffi::c_int,diff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/defines.h b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/defines.hsimilarity index 100%rename from crates/uzu/src/backends/metal/kernel/matmul/steel/defines.hrename to crates/uzu/src/backends/metal/kernel/matmul/common/steel/defines.hdiff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/gemm.h b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/gemm.hsimilarity index 100%rename from crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/gemm.hrename to crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/gemm.hdiff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/gemm_nax.h b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/gemm_nax.hsimilarity index 100%rename from crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/gemm_nax.hrename to crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/gemm_nax.hdiff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/loader.h b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/loader.hsimilarity index 100%rename from crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/loader.hrename to crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/loader.hdiff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/mma.h b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/mma.hsimilarity index 100%rename from crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/mma.hrename to crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/mma.hdiff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/nax.h b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/nax.hsimilarity index 100%rename from crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/nax.hrename to crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/nax.hdiff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/params.h b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/params.hsimilarity index 100%rename from crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/params.hrename to crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/params.hdiff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/transforms.h b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/transforms.hsimilarity index 98%rename from crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/transforms.hrename to crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/transforms.hindex 76a87e808..8dd550212 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/transforms.h+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/transforms.h@@ -67,4 +67,4 @@ struct BlockSwizzle {}};-} // namespace steel\ No newline at end of file+} // namespace steeldiff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/utils.h b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/utils.hsimilarity index 100%rename from crates/uzu/src/backends/metal/kernel/matmul/steel/utils.hrename to crates/uzu/src/backends/metal/kernel/matmul/common/steel/utils.hdiff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/utils/integral_constant.h b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/utils/integral_constant.hsimilarity index 99%rename from crates/uzu/src/backends/metal/kernel/matmul/steel/utils/integral_constant.hrename to crates/uzu/src/backends/metal/kernel/matmul/common/steel/utils/integral_constant.hindex 39007d45a..7fc91a29a 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/steel/utils/integral_constant.h+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/utils/integral_constant.h@@ -129,4 +129,4 @@ METAL_FUNC constexpr auto sum(T x, Us... us) {} // namespace steel-#pragma METAL internals : disable\ No newline at end of file+#pragma METAL internals : disablediff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/utils/type_traits.h b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/utils/type_traits.hsimilarity index 96%rename from crates/uzu/src/backends/metal/kernel/matmul/steel/utils/type_traits.hrename to crates/uzu/src/backends/metal/kernel/matmul/common/steel/utils/type_traits.hindex 3c8aca4b9..6423de3b3 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/steel/utils/type_traits.h+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/utils/type_traits.h@@ -52,4 +52,4 @@ using pointer_element_t = typename pointer_element<remove_cv_t<T>>::type;} // namespace metal-#pragma METAL internals : disable\ No newline at end of file+#pragma METAL internals : disablediff --git a/crates/uzu/src/backends/metal/kernel/matmul/transpose.rs b/crates/uzu/src/backends/metal/kernel/matmul/common/transpose_configuration.rssimilarity index 89%rename from crates/uzu/src/backends/metal/kernel/matmul/transpose.rsrename to crates/uzu/src/backends/metal/kernel/matmul/common/transpose_configuration.rsindex d75b4d2c8..b2785dd25 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/transpose.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/transpose_configuration.rs@@ -1,5 +1,5 @@#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]-pub(super) enum TransposeConfiguration {+pub(crate) enum TransposeConfiguration {NN,NT,TN,@@ -17,7 +17,7 @@ impl TransposeConfiguration {}}-pub(super) fn transpose_configuration(+pub(crate) fn transpose_configuration(a_t: bool,b_t: bool,) -> TransposeConfiguration {diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gather.rs b/crates/uzu/src/backends/metal/kernel/matmul/gather.rsdeleted file mode 100644index 3f94225bd..000000000--- a/crates/uzu/src/backends/metal/kernel/matmul/gather.rs+++ /dev/null@@ -1,586 +0,0 @@-use std::collections::HashMap;--use metal::{- Buffer as MTLBuffer, ComputeCommandEncoderRef,- ComputePipelineState as MTLComputePipelineState, MTLSize,-};--use super::shared_types::GEMMParams;-use crate::{- DataType,- backends::metal::{DeviceClass, MTLContext, MTLError},-};--#[derive(Debug, Clone, PartialEq, Eq, Hash)]-struct RhsOnlyPipelineKey {- name: String,- align_m: bool,- align_n: bool,- align_k: bool,-}--#[derive(Debug, Clone, PartialEq, Eq, Hash)]-struct GatherPipelineKey {- name: String,- has_batch: bool,- align_m: bool,- align_n: bool,- align_k: bool,-}--#[derive(Debug, Clone)]-pub struct GatherMmRhsArguments<'a> {- pub a: &'a MTLBuffer,- pub b: &'a MTLBuffer,- pub rhs_indices: &'a MTLBuffer,- pub out: &'a MTLBuffer,- pub m: i32,- pub n: i32,- pub k: i32,- pub lda: i32,- pub ldb: i32,- pub batch_stride_b: i64,- pub transpose_b: bool,- /// Number of matrices available on the RHS (E dimension in the reference backend).- pub rhs_matrix_count: i32,-}--#[derive(Debug, Clone)]-pub struct GatherMmArguments<'a> {- pub a: &'a MTLBuffer,- pub b: &'a MTLBuffer,- pub lhs_indices: &'a MTLBuffer,- pub rhs_indices: &'a MTLBuffer,- pub out: &'a MTLBuffer,- pub m: i32,- pub n: i32,- pub k: i32,- pub lda: i32,- pub ldb: i32,- /// Grid Z dimension for the gather kernel (out.size / (M*N) in the reference backend).- pub batch_size_out: i32,- /// Number of batch dimensions for the output (out.ndim - 2 in the reference backend).- pub batch_ndim: i32,-- // LHS/RHS indices layout (used when has_batch=true).- pub lhs_indices_shape: &'a [i32],- pub lhs_indices_strides: &'a [i64],- pub rhs_indices_strides: &'a [i64],-- // A tensor batch layout (for elem_to_loc on selected matrix index).- pub batch_ndim_a: i32,- pub a_shape: &'a [i32],- pub a_strides: &'a [i64],-- // B tensor batch layout (for elem_to_loc on selected matrix index).- pub batch_ndim_b: i32,- pub b_shape: &'a [i32],- pub b_strides: &'a [i64],-- pub transpose_a: bool,- pub transpose_b: bool,-}--pub struct GatherGemm {- data_type: DataType,- rhs_only_pipelines: HashMap<RhsOnlyPipelineKey, MTLComputePipelineState>,- gather_pipelines: HashMap<GatherPipelineKey, MTLComputePipelineState>,-}--impl GatherGemm {- pub fn new(data_type: DataType) -> Self {- Self {- data_type,- rhs_only_pipelines: HashMap::new(),- gather_pipelines: HashMap::new(),- }- }-- fn steel_type_name(&self) -> Result<&'static str, MTLError> {- match self.data_type {- DataType::F16 => Ok("float16"),- DataType::BF16 => Ok("bfloat16"),- DataType::F32 => Ok("float32"),- other => Err(MTLError::Generic(format!(- "Unsupported data type for gather GEMM: {other:?}"- ))),- }- }-- fn device_class_code(mtl: &MTLContext) -> char {- match mtl.architecture.device_class {- DeviceClass::Desktop => 'd',- DeviceClass::Integrated => 'g',- DeviceClass::Phone => 'p',- DeviceClass::Unknown(_) => 'g',- }- }-- fn select_gemm_tparams(- device_class: char,- data_type: DataType,- transpose_a: bool,- transpose_b: bool,- m: i32,- n: i32,- k: i32,- batch_size_out: i32,- ) -> (i32, i32, i32, i32, i32) {- // Matches the initialization and tparam selection logic used by the- // reference backend for gather/segmented GEMM kernels.- let mut bm = 64;- let mut bn = 64;- let mut bk = 16;- let mut wm = 2;- let mut wn = 2;-- let is_float32 = matches!(data_type, DataType::F32);- let work_elements = (batch_size_out as i64) * (m as i64) * (n as i64);-- match device_class {- 'g' | 'p' => {- if !transpose_a && transpose_b {- // nt- bm = 64;- bn = 32;- bk = 32;- wm = 2;- wn = 2;- } else if !is_float32 {- // half and bfloat- bm = 64;- bn = 64;- bk = 16;- wm = 1;- wn = 2;- }- },- 'd' => {- if work_elements >= (1_i64 << 20) {- // large matmul- if !is_float32 {- if 2 * std::cmp::max(m, n) > k {- // Reasonable K- bm = 64;- bn = 64;- bk = 16;- wm = 1;- wn = 2;- } else if !transpose_a && transpose_b {- // nt with large k- bm = 64;- bn = 32;- bk = 32;- wm = 2;- wn = 2;- } else {- // nn with large K- bm = 32;- bn = 64;- bk = 16;- wm = 1;- wn = 2;- }- }- } else if !is_float32 {- // smaller matmul, half and bfloat- if !transpose_a && transpose_b {- // nt- bm = 64;- bn = 32;- bk = 32;- wm = 2;- wn = 2;- } else {- // nn- bm = 64;- bn = 64;- bk = 16;- wm = 1;- wn = 2;- }- }- },- _ => {},- }-- (bm, bn, bk, wm, wn)- }-- fn gather_kernel_name(- &self,- prefix: &str,- transpose_a: bool,- transpose_b: bool,- bm: i32,- bn: i32,- bk: i32,- wm: i32,- wn: i32,- ) -> Result<String, MTLError> {- let tname = format!(- "{}{}",- if transpose_a {- 't'- } else {- 'n'- },- if transpose_b {- 't'- } else {- 'n'- }- );- let dtype = self.steel_type_name()?;- Ok(format!(- "{prefix}_{tname}_{dtype}_{dtype}_bm{bm}_bn{bn}_bk{bk}_wm{wm}_wn{wn}",- ))- }-- fn set_i32_slice_bytes(- enc: &ComputeCommandEncoderRef,- index: u64,- data: &[i32],- ) {- if data.is_empty() {- let zero = 0_i32;- enc.set_bytes(- index,- std::mem::size_of::<i32>() as u64,- &zero as *const i32 as *const _,- );- } else {- enc.set_bytes(- index,- (data.len() * std::mem::size_of::<i32>()) as u64,- data.as_ptr() as *const _,- );- }- }-- fn set_i64_slice_bytes(- enc: &ComputeCommandEncoderRef,- index: u64,- data: &[i64],- ) {- if data.is_empty() {- let zero = 0_i64;- enc.set_bytes(- index,- std::mem::size_of::<i64>() as u64,- &zero as *const i64 as *const _,- );- } else {- enc.set_bytes(- index,- (data.len() * std::mem::size_of::<i64>()) as u64,- data.as_ptr() as *const _,- );- }- }-- fn get_or_compile_rhs_only_pipeline(- &mut self,- mtl: &MTLContext,- name: &str,- align_m: bool,- align_n: bool,- align_k: bool,- ) -> Result<&MTLComputePipelineState, MTLError> {- let key = RhsOnlyPipelineKey {- name: name.to_string(),- align_m,- align_n,- align_k,- };-- if !self.rhs_only_pipelines.contains_key(&key) {- let fcv = metal::FunctionConstantValues::new();- fcv.set_constant_value_at_index(- &align_m as *const bool as *const _,- metal::MTLDataType::Bool,- 200,- );- fcv.set_constant_value_at_index(- &align_n as *const bool as *const _,- metal::MTLDataType::Bool,- 201,- );- fcv.set_constant_value_at_index(- &align_k as *const bool as *const _,- metal::MTLDataType::Bool,- 202,- );-- let cache_key = format!(- "{name}_am{}_an{}_ak{}",- align_m as u8, align_n as u8, align_k as u8- );- let (ps, _) = mtl.compute_pipeline_state_with_reflection_cached(- &cache_key,- name,- Some(&fcv),- )?;- self.rhs_only_pipelines.insert(key.clone(), ps);- }-- Ok(self.rhs_only_pipelines.get(&key).unwrap())- }-- fn get_or_compile_gather_pipeline(- &mut self,- mtl: &MTLContext,- name: &str,- has_batch: bool,- align_m: bool,- align_n: bool,- align_k: bool,- ) -> Result<&MTLComputePipelineState, MTLError> {- let key = GatherPipelineKey {- name: name.to_string(),- has_batch,- align_m,- align_n,- align_k,- };-- if !self.gather_pipelines.contains_key(&key) {- let fcv = metal::FunctionConstantValues::new();- fcv.set_constant_value_at_index(- &has_batch as *const bool as *const _,- metal::MTLDataType::Bool,- 10,- );- fcv.set_constant_value_at_index(- &align_m as *const bool as *const _,- metal::MTLDataType::Bool,- 200,- );- fcv.set_constant_value_at_index(- &align_n as *const bool as *const _,- metal::MTLDataType::Bool,- 201,- );- fcv.set_constant_value_at_index(- &align_k as *const bool as *const _,- metal::MTLDataType::Bool,- 202,- );-- let cache_key = format!(- "{name}_hb{}_am{}_an{}_ak{}",- has_batch as u8, align_m as u8, align_n as u8, align_k as u8- );- let (ps, _) = mtl.compute_pipeline_state_with_reflection_cached(- &cache_key,- name,- Some(&fcv),- )?;- self.gather_pipelines.insert(key.clone(), ps);- }-- Ok(self.gather_pipelines.get(&key).unwrap())- }-- pub fn encode_mm_rhs(- &mut self,- mtl: &MTLContext,- enc: &ComputeCommandEncoderRef,- args: &GatherMmRhsArguments,- ) -> Result<(), MTLError> {- // NAX gather kernels are only instantiated for float16/bfloat16.- let prefer_nax =- mtl.is_nax_available() && !matches!(self.data_type, DataType::F32);-- let (bm, bn, bk, wm, wn, use_nax) = if prefer_nax {- // Mirrors rhs-only NAX gather tile selection.- let e = std::cmp::max(1, args.rhs_matrix_count);- let ratio = args.m / e;- let (bm, wm) = if ratio > 48 {- (64, 2)- } else if ratio > 24 {- (32, 1)- } else {- (16, 1)- };- (bm, 128, 128, wm, 4, true)- } else {- (16, 64, 16, 1, 2, false)- };-- let align_m = (args.m % bm) == 0;- let align_n = (args.n % bn) == 0;- let align_k = (args.k % bk) == 0;-- let prefix = if use_nax {- "steel_gather_mm_rhs_nax"- } else {- "steel_gather_mm_rhs"- };-- // gather_mm_rhs only supports transpose_a=false (nn/nt instantiations).- let kname = self.gather_kernel_name(- prefix,- /*transpose_a=*/ false,- args.transpose_b,- bm,- bn,- bk,- wm,- wn,- )?;-- let ps = self.get_or_compile_rhs_only_pipeline(- mtl, &kname, align_m, align_n, align_k,- )?;-- enc.set_compute_pipeline_state(ps);- enc.set_buffer(0, Some(args.a), 0);- enc.set_buffer(1, Some(args.b), 0);- enc.set_buffer(2, Some(args.rhs_indices), 0);- enc.set_buffer(3, Some(args.out), 0);-- let tiles_n = (args.n + bn - 1) / bn;- let tiles_m = (args.m + bm - 1) / bm;- let params = GEMMParams {- M: args.m,- N: args.n,- K: args.k,- lda: args.lda,- ldb: args.ldb,- ldd: args.n,- tiles_n,- tiles_m,- batch_stride_a: 0,- batch_stride_b: args.batch_stride_b,- batch_stride_d: 0,- swizzle_log: 0,- gemm_k_iterations_aligned: args.k / bk,- batch_ndim: 0,- };- enc.set_bytes(- 4,- std::mem::size_of::<GEMMParams>() as u64,- ¶ms as *const GEMMParams as *const _,- );-- let threads_per_tg = MTLSize::new(32, wn as u64, wm as u64);- let tgs = MTLSize::new(tiles_n as u64, tiles_m as u64, 1);- enc.dispatch_thread_groups(tgs, threads_per_tg);- Ok(())- }-- pub fn encode_mm(- &mut self,- mtl: &MTLContext,- enc: &ComputeCommandEncoderRef,- args: &GatherMmArguments,- ) -> Result<(), MTLError> {- let device_class = Self::device_class_code(mtl);- let (bm, bn, bk, wm, wn) = Self::select_gemm_tparams(- device_class,- self.data_type,- args.transpose_a,- args.transpose_b,- args.m,- args.n,- args.k,- args.batch_size_out,- );-- let has_batch = args.batch_ndim > 1;- let align_m = (args.m % bm) == 0;- let align_n = (args.n % bn) == 0;- let align_k = (args.k % bk) == 0;-- let kname = self.gather_kernel_name(- "steel_gather_mm",- args.transpose_a,- args.transpose_b,- bm,- bn,- bk,- wm,- wn,- )?;-- let ps = self.get_or_compile_gather_pipeline(- mtl, &kname, has_batch, align_m, align_n, align_k,- )?;-- enc.set_compute_pipeline_state(ps);-- enc.set_buffer(0, Some(args.a), 0);- enc.set_buffer(1, Some(args.b), 0);- enc.set_buffer(2, Some(args.lhs_indices), 0);- enc.set_buffer(3, Some(args.rhs_indices), 0);- enc.set_buffer(4, Some(args.out), 0);-- let tiles_n = (args.n + bn - 1) / bn;- let tiles_m = (args.m + bm - 1) / bm;- let batch_stride_a = if args.batch_ndim > 0 {- *args.lhs_indices_strides.first().unwrap_or(&0)- } else {- 0- };- let batch_stride_b = if args.batch_ndim > 0 {- *args.rhs_indices_strides.first().unwrap_or(&0)- } else {- 0- };-- let params = GEMMParams {- M: args.m,- N: args.n,- K: args.k,- lda: args.lda,- ldb: args.ldb,- ldd: args.n,- tiles_n,- tiles_m,- batch_stride_a,- batch_stride_b,- batch_stride_d: (args.m as i64) * (args.n as i64),- swizzle_log: 0,- gemm_k_iterations_aligned: args.k / bk,- batch_ndim: args.batch_ndim,- };-- enc.set_bytes(- 5,- std::mem::size_of::<GEMMParams>() as u64,- ¶ms as *const GEMMParams as *const _,- );-- // Note: many of these are only used when has_batch=true, but they are- // present in the kernel signature unconditionally.- Self::set_i32_slice_bytes(enc, 6, args.lhs_indices_shape);- Self::set_i64_slice_bytes(enc, 7, args.lhs_indices_strides);- Self::set_i64_slice_bytes(enc, 8, args.rhs_indices_strides);-- enc.set_bytes(- 9,- std::mem::size_of::<i32>() as u64,- &args.batch_ndim_a as *const i32 as *const _,- );- Self::set_i32_slice_bytes(enc, 10, args.a_shape);- Self::set_i64_slice_bytes(enc, 11, args.a_strides);-- enc.set_bytes(- 12,- std::mem::size_of::<i32>() as u64,- &args.batch_ndim_b as *const i32 as *const _,- );- Self::set_i32_slice_bytes(enc, 13, args.b_shape);- Self::set_i64_slice_bytes(enc, 14, args.b_strides);-- let threads_per_tg = MTLSize::new(32, wn as u64, wm as u64);- let tgs = MTLSize::new(- tiles_n as u64,- tiles_m as u64,- args.batch_size_out as u64,- );- enc.dispatch_thread_groups(tgs, threads_per_tg);- Ok(())- }-}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm/kernel.rsnew file mode 100644index 000000000..9742f81ae--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs@@ -0,0 +1,336 @@+use std::collections::HashMap;++use metal::{+ ComputeCommandEncoderRef, ComputePipelineState as MTLComputePipelineState,+ MTLSize,+};++use super::{+ pipeline_configuration::PipelineConfiguration,+ tile_configuration::TileConfiguration,+};+use crate::{+ DataType,+ backends::metal::{+ DeviceClass, MTLContext, MTLError,+ kernel::matmul::common::{+ GEMMAddMMParams, GEMMParams, MatmulArguments,+ transpose_configuration,+ },+ },+};++pub struct Kernel {+ data_type: DataType,+ transpose_a: bool,+ transpose_b: bool,+ pipelines: HashMap<PipelineConfiguration, MTLComputePipelineState>,+}++impl Kernel {+ pub fn new(+ data_type: DataType,+ transpose_a: bool,+ transpose_b: bool,+ ) -> Result<Self, MTLError> {+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)+ {+ return Err(MTLError::Generic(format!(+ "Unsupported dtype for GEMM: {data_type:?}"+ )));+ }+ Ok(Self {+ data_type,+ transpose_a,+ transpose_b,+ pipelines: HashMap::new(),+ })+ }++ fn type_name(&self) -> &'static str {+ match self.data_type {+ DataType::F16 => "float16",+ DataType::BF16 => "bfloat16",+ DataType::F32 => "float32",+ _ => unreachable!(),+ }+ }++ fn kernel_name(+ &self,+ tile: &TileConfiguration,+ ) -> String {+ let type_name = self.type_name();+ let transpose_suffix =+ transpose_configuration(self.transpose_a, self.transpose_b);+ let prefix = if tile.is_nax() {+ "steel_gemm_nax"+ } else {+ "steel_gemm"+ };+ format!(+ "{}_{}_{}_{}_bm{}_bn{}_bk{}_wm{}_wn{}",+ prefix,+ transpose_suffix.as_str(),+ type_name,+ type_name,+ tile.block_rows,+ tile.block_cols,+ tile.block_depth,+ tile.warps_per_row,+ tile.warps_per_col+ )+ }++ pub fn select_tile(+ &self,+ mtl: &MTLContext,+ args: &MatmulArguments,+ ) -> TileConfiguration {+ let overall_work_elements = (args.batch_count as i64)+ * (args.batch as i64)+ * (args.output_dim as i64);+ let is_float32 = matches!(self.data_type, DataType::F32);+ let prefer_half_or_tf32 = !is_float32 || mtl.tf32_enabled();++ if mtl.is_nax_available() && prefer_half_or_tf32 {+ let base_tile = TileConfiguration::new(128, 128, 512, 4, 4, 0);+ let tile_rows =+ (args.batch + base_tile.block_rows - 1) / base_tile.block_rows;+ let swizzle_log2 = if tile_rows <= 3 {+ 0+ } else {+ 1+ };+ return TileConfiguration {+ swizzle_log2,+ ..base_tile+ };+ }++ let device_class_code = match mtl.architecture.device_class {+ DeviceClass::Desktop => 'd',+ DeviceClass::Integrated => 'g',+ DeviceClass::Phone => 'p',+ DeviceClass::Unknown(_) => 'g',+ };++ match device_class_code {+ 'g' | 'p' => {+ if prefer_half_or_tf32 {+ if !self.transpose_a && self.transpose_b {+ TileConfiguration::new(64, 32, 32, 2, 2, 0)+ } else {+ TileConfiguration::new(64, 64, 16, 1, 2, 0)+ }+ } else if !self.transpose_a && self.transpose_b {+ TileConfiguration::new(32, 64, 16, 1, 2, 0)+ } else {+ TileConfiguration::new(64, 32, 32, 2, 2, 0)+ }+ },+ 'd' => {+ if overall_work_elements >= (1_i64 << 20) {+ if prefer_half_or_tf32 {+ if 2 * std::cmp::max(args.batch, args.output_dim)+ > args.input_dim+ {+ TileConfiguration::new(64, 64, 16, 2, 2, 0)+ } else if !self.transpose_a && self.transpose_b {+ TileConfiguration::new(64, 32, 32, 2, 2, 0)+ } else {+ TileConfiguration::new(32, 64, 16, 1, 2, 0)+ }+ } else if !self.transpose_a && self.transpose_b {+ TileConfiguration::new(32, 64, 16, 1, 2, 0)+ } else {+ TileConfiguration::new(64, 32, 32, 2, 2, 0)+ }+ } else if prefer_half_or_tf32 {+ if !self.transpose_a && self.transpose_b {+ TileConfiguration::new(64, 32, 32, 2, 2, 0)+ } else {+ TileConfiguration::new(64, 64, 16, 1, 2, 0)+ }+ } else if !self.transpose_a && self.transpose_b {+ TileConfiguration::new(32, 64, 16, 1, 2, 0)+ } else {+ TileConfiguration::new(64, 32, 32, 2, 2, 0)+ }+ },+ _ => TileConfiguration::new(64, 64, 16, 2, 2, 0),+ }+ }++ fn get_or_compile_pipeline(+ &mut self,+ mtl: &MTLContext,+ config: &PipelineConfiguration,+ ) -> Result<&MTLComputePipelineState, MTLError> {+ if !self.pipelines.contains_key(config) {+ let name = self.kernel_name(&config.tile);+ let fcv = metal::FunctionConstantValues::new();+ fcv.set_constant_value_at_index(+ &config.has_batch as *const bool as *const _,+ metal::MTLDataType::Bool,+ 10,+ );+ fcv.set_constant_value_at_index(+ &config.use_out_source as *const bool as *const _,+ metal::MTLDataType::Bool,+ 100,+ );+ fcv.set_constant_value_at_index(+ &config.do_axpby as *const bool as *const _,+ metal::MTLDataType::Bool,+ 110,+ );+ fcv.set_constant_value_at_index(+ &config.align_m as *const bool as *const _,+ metal::MTLDataType::Bool,+ 200,+ );+ fcv.set_constant_value_at_index(+ &config.align_n as *const bool as *const _,+ metal::MTLDataType::Bool,+ 201,+ );+ fcv.set_constant_value_at_index(+ &config.align_k as *const bool as *const _,+ metal::MTLDataType::Bool,+ 202,+ );++ let cache_key = format!(+ "{}_am{}_an{}_ak{}_hb{}_uo{}_ax{}",+ name,+ config.align_m as u8,+ config.align_n as u8,+ config.align_k as u8,+ config.has_batch as u8,+ config.use_out_source as u8,+ config.do_axpby as u8+ );+ let (ps, _) = mtl.compute_pipeline_state_with_reflection_cached(+ &cache_key,+ &name,+ Some(&fcv),+ )?;+ self.pipelines.insert(config.clone(), ps);+ }+ Ok(self.pipelines.get(config).unwrap())+ }++ pub fn encode(+ &mut self,+ mtl: &MTLContext,+ enc: &ComputeCommandEncoderRef,+ args: &MatmulArguments,+ ) -> Result<(), MTLError> {+ let tile = self.select_tile(mtl, args);+ self.encode_with_tile(mtl, enc, args, &tile)+ }++ pub fn encode_with_tile(+ &mut self,+ mtl: &MTLContext,+ enc: &ComputeCommandEncoderRef,+ args: &MatmulArguments,+ tile: &TileConfiguration,+ ) -> Result<(), MTLError> {+ let m = args.batch;+ let n = args.output_dim;+ let k = args.input_dim;++ let config = PipelineConfiguration {+ tile: *tile,+ align_m: (m % tile.block_rows) == 0,+ align_n: (n % tile.block_cols) == 0,+ align_k: (k % tile.block_depth) == 0,+ has_batch: args.batch_count > 1,+ use_out_source: args.c.is_some(),+ do_axpby: args.c.is_some()+ && (args.alpha != 1.0 || args.beta != 0.0),+ };++ let ps = self.get_or_compile_pipeline(mtl, &config)?;+ enc.set_compute_pipeline_state(ps);++ enc.set_buffer(0, Some(args.a), args.a_offset);+ enc.set_buffer(1, Some(args.b), 0);+ if config.use_out_source {+ if let Some(c_buf) = args.c {+ enc.set_buffer(2, Some(c_buf), 0);+ }+ }+ enc.set_buffer(3, Some(args.d), 0);++ let tiles_n = (n + tile.block_cols - 1) / tile.block_cols;+ let tiles_m = (m + tile.block_rows - 1) / tile.block_rows;+ let swizzle_log = tile.swizzle_log2;++ let tile_swizzle = 1 << swizzle_log;+ let tm_swizzled = (tiles_m + tile_swizzle - 1) / tile_swizzle;+ let tn_swizzled = tiles_n * tile_swizzle;++ let elements_per_matrix_a = (args.batch as i64) * (args.lda as i64);+ let elements_per_matrix_b = if self.transpose_b {+ (args.output_dim as i64) * (args.ldb as i64)+ } else {+ (args.input_dim as i64) * (args.ldb as i64)+ };+ let elements_per_matrix_d = (args.batch as i64) * (args.ldd as i64);++ let params = GEMMParams {+ M: m,+ N: n,+ K: k,+ lda: args.lda,+ ldb: args.ldb,+ ldd: args.ldd,+ tiles_n,+ tiles_m,+ batch_stride_a: elements_per_matrix_a,+ batch_stride_b: elements_per_matrix_b,+ batch_stride_d: elements_per_matrix_d,+ swizzle_log,+ gemm_k_iterations_aligned: k / tile.block_depth,+ batch_ndim: 1,+ };+ enc.set_bytes(+ 4,+ std::mem::size_of::<GEMMParams>() as u64,+ ¶ms as *const GEMMParams as *const _,+ );++ if config.use_out_source {+ let batch_stride_c = if args.batch_count > 1 {+ (args.ldd as i64) * (args.output_dim as i64)+ } else {+ 0+ };+ let addmm_params = GEMMAddMMParams {+ ldc: args.ldd,+ fdc: 1,+ batch_stride_c,+ alpha: args.alpha,+ beta: args.beta,+ };+ enc.set_bytes(+ 5,+ std::mem::size_of::<GEMMAddMMParams>() as u64,+ &addmm_params as *const GEMMAddMMParams as *const _,+ );+ }++ let threads_per_tg =+ MTLSize::new(32, tile.warps_per_col, tile.warps_per_row);+ let tgs = MTLSize::new(+ tn_swizzled as u64,+ tm_swizzled as u64,+ args.batch_count as u64,+ );+ enc.dispatch_thread_groups(tgs, threads_per_tg);+ Ok(())+ }+}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm/mod.rsnew file mode 100644index 000000000..f78544f8c--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm/mod.rs@@ -0,0 +1,5 @@+mod kernel;+mod pipeline_configuration;+mod tile_configuration;++pub use kernel::Kernel as GemmKernel;diff --git a/crates/uzu/src/backends/metal/kernel/matmul/pipeline.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm/pipeline_configuration.rssimilarity index 62%rename from crates/uzu/src/backends/metal/kernel/matmul/pipeline.rsrename to crates/uzu/src/backends/metal/kernel/matmul/gemm/pipeline_configuration.rsindex 26c0062ad..e92065b03 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/pipeline.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm/pipeline_configuration.rs@@ -1,6 +1,8 @@+use super::tile_configuration::TileConfiguration;+#[derive(Debug, Clone, PartialEq, Eq, Hash)]-pub(super) struct PipelineKey {- pub name: String,+pub struct PipelineConfiguration {+ pub tile: TileConfiguration,pub align_m: bool,pub align_n: bool,pub align_k: bool,diff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_fused.h b/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.hsimilarity index 100%rename from crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_fused.hrename to crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.hdiff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_fused.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.metalsimilarity index 91%rename from crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_fused.metalrename to crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.metalindex 7053470c3..928e3f97f 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_fused.metal+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.metal@@ -1,14 +1,14 @@// clang-format off-#include "../../../../common/utils.h"+#include "../../common/utils.h"-#include "../gemm.h"-#include "steel_gemm_fused.h"+#include "../../common/steel/gemm/gemm.h"+#include "steel_gemm.h"#define instantiate_gemm(tname, trans_a, trans_b, iname, itype, oname, otype, bm, bn, bk, wm, wn) \instantiate_kernel( \- "steel_gemm_fused_" #tname "_" #iname "_" #oname \+ "steel_gemm_" #tname "_" #iname "_" #oname \"_bm" #bm "_bn" #bn "_bk" #bk "_wm" #wm "_wn" #wn, \gemm, itype, bm, bn, bk, wm, wn, trans_a, trans_b, float)diff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_fused_nax.h b/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm_nax.hsimilarity index 100%rename from crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_fused_nax.hrename to crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm_nax.hdiff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_fused_nax.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm_nax.metalsimilarity index 88%rename from crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_fused_nax.metalrename to crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm_nax.metalindex a0c3b9dda..8a3070c0d 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_fused_nax.metal+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm_nax.metal@@ -2,15 +2,15 @@#include <metal_stdlib>-#include "../../../../common/utils.h"+#include "../../common/utils.h"-#include "../gemm_nax.h"-#include "steel_gemm_fused_nax.h"+#include "../../common/steel/gemm/gemm_nax.h"+#include "steel_gemm_nax.h"// clang-format off#define instantiate_gemm(tname, trans_a, trans_b, iname, itype, oname, otype, bm, bn, bk, wm, wn) \instantiate_kernel( \- "steel_gemm_fused_nax_" #tname "_" #iname "_" #oname \+ "steel_gemm_nax_" #tname "_" #iname "_" #oname \"_bm" #bm "_bn" #bn "_bk" #bk "_wm" #wm "_wn" #wn, \gemm, itype, bm, bn, bk, wm, wn, trans_a, trans_b, float)diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/tile_configuration.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm/tile_configuration.rsnew file mode 100644index 000000000..3f857b490--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm/tile_configuration.rs@@ -0,0 +1,33 @@+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+pub struct TileConfiguration {+ pub block_rows: i32,+ pub block_cols: i32,+ pub block_depth: i32,+ pub warps_per_row: u64,+ pub warps_per_col: u64,+ pub swizzle_log2: i32,+}++impl TileConfiguration {+ pub fn new(+ block_rows: i32,+ block_cols: i32,+ block_depth: i32,+ warps_per_row: u64,+ warps_per_col: u64,+ swizzle_log2: i32,+ ) -> Self {+ Self {+ block_rows,+ block_cols,+ block_depth,+ warps_per_row,+ warps_per_col,+ swizzle_log2,+ }+ }++ pub fn is_nax(&self) -> bool {+ self.block_depth >= 256+ }+}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv.rsdeleted file mode 100644index db5ee62da..000000000--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv.rs+++ /dev/null@@ -1,672 +0,0 @@-use std::collections::HashMap;--use metal::{- Buffer as MTLBuffer, ComputeCommandEncoderRef, ComputePipelineState,- MTLSize,-};--use super::{super::mlp::MlpActivationType, arguments::MatmulArguments};-use crate::{- DataType,- backends::metal::{MTLContext, MTLError},-};--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]-struct GemvConfiguration {- transpose_matrix: bool,- threadgroup_rows: u32,- threadgroup_cols: u32,- threads_per_simdgroup_row: u32,- threads_per_simdgroup_col: u32,- elements_per_thread_row: u32,- elements_per_thread_col: u32,- non_contiguous_batch: bool,- do_axpby: bool,-}--impl GemvConfiguration {- fn output_elements_per_threadgroup(&self) -> u32 {- if self.transpose_matrix {- self.threadgroup_cols- * self.threads_per_simdgroup_col- * self.elements_per_thread_col- } else {- self.threadgroup_rows- * self.threads_per_simdgroup_row- * self.elements_per_thread_row- }- }-- fn threads_per_threadgroup(&self) -> MTLSize {- // 32 threads per simdgroup, arranged (x=32, y=BN, z=BM)- MTLSize::new(- 32,- self.threadgroup_cols as u64,- self.threadgroup_rows as u64,- )- }-}--fn gemv_kernel_name(- data_type: DataType,- configuration: &GemvConfiguration,-) -> Result<String, MTLError> {- let dtype_name = match data_type {- DataType::F16 => "float16",- DataType::BF16 => "bfloat16",- DataType::F32 => "float32",- _ => {- return Err(MTLError::Generic(format!(- "Unsupported data type for GEMV: {:?}",- data_type- )));- },- };-- let prefix = if configuration.transpose_matrix {- "gemv_t"- } else {- "gemv"- };-- Ok(format!(- "{prefix}_{dtype_name}_bm{}_bn{}_sm{}_sn{}_tm{}_tn{}_nc{}_axpby{}",- configuration.threadgroup_rows,- configuration.threadgroup_cols,- configuration.threads_per_simdgroup_row,- configuration.threads_per_simdgroup_col,- configuration.elements_per_thread_row,- configuration.elements_per_thread_col,- configuration.non_contiguous_batch as u8,- configuration.do_axpby as u8,- ))-}--fn select_gemv_configuration(- transpose_matrix: bool,- input_dimension: i32,- output_dimension: i32,- non_contiguous_batch: bool,- do_axpby: bool,-) -> GemvConfiguration {- // GEMV tuning surface for axpby kernel selection.- let (threadgroup_rows, threadgroup_cols);- let (threads_per_simdgroup_row, threads_per_simdgroup_col);- let (elements_per_thread_row, elements_per_thread_col);-- if transpose_matrix {- let mut sm = 8;- let mut sn = 4;- if input_dimension >= 8192 && output_dimension >= 2048 {- sm = 4;- sn = 8;- }-- let bn = if output_dimension >= 2048 {- 16- } else if output_dimension >= 512 {- 4- } else {- 2- };-- let tn = if output_dimension < 4 {- 1- } else {- 4- };-- threadgroup_rows = 1;- threadgroup_cols = bn;- threads_per_simdgroup_row = sm;- threads_per_simdgroup_col = sn;- elements_per_thread_row = 4;- elements_per_thread_col = tn;- } else {- let mut bm = if output_dimension >= 4096 {- 8- } else {- 4- };- let mut sm = 1;- let mut sn = 32;- let mut bn = 1;-- if input_dimension <= 64 {- bm = 1;- sm = 8;- sn = 4;- } else if input_dimension >= 16 * output_dimension {- bm = 1;- bn = 8;- }-- let tm = if output_dimension < 4 {- 1- } else {- 4- };-- threadgroup_rows = bm;- threadgroup_cols = bn;- threads_per_simdgroup_row = sm;- threads_per_simdgroup_col = sn;- elements_per_thread_row = tm;- elements_per_thread_col = 4;- }-- GemvConfiguration {- transpose_matrix,- threadgroup_rows,- threadgroup_cols,- threads_per_simdgroup_row,- threads_per_simdgroup_col,- elements_per_thread_row,- elements_per_thread_col,- non_contiguous_batch,- do_axpby,- }-}--pub struct GemvKernel {- data_type: DataType,- lhs_is_transposed: bool,- rhs_is_transposed: bool,- pipelines: HashMap<GemvConfiguration, ComputePipelineState>,-}--impl GemvKernel {- pub fn new(- data_type: DataType,- lhs_is_transposed: bool,- rhs_is_transposed: bool,- ) -> Self {- Self {- data_type,- lhs_is_transposed,- rhs_is_transposed,- pipelines: HashMap::new(),- }- }-- fn get_pipeline(- &mut self,- context: &MTLContext,- configuration: GemvConfiguration,- ) -> Result<&ComputePipelineState, MTLError> {- if !self.pipelines.contains_key(&configuration) {- let kernel_name = gemv_kernel_name(self.data_type, &configuration)?;- let pipeline =- context.compute_pipeline_state(&kernel_name, None)?;- self.pipelines.insert(configuration, pipeline);- }- Ok(self.pipelines.get(&configuration).unwrap())- }-- #[allow(clippy::too_many_arguments)]- fn encode_with_configuration(- &mut self,- context: &MTLContext,- encoder: &ComputeCommandEncoderRef,- arguments: &MatmulArguments,- configuration: GemvConfiguration,- matrix_is_rhs: bool,- bias_buffer: Option<&MTLBuffer>,- alpha: f32,- beta: f32,- bias_stride: i32,- ) -> Result<(), MTLError> {- let pipeline = self.get_pipeline(context, configuration)?;- encoder.set_compute_pipeline_state(pipeline);-- let (buf0, off0) = if matrix_is_rhs {- (arguments.b, 0)- } else {- (arguments.a, arguments.a_offset)- };- encoder.set_buffer(0, Some(buf0), off0);-- let (buf1, off1) = if matrix_is_rhs {- (arguments.a, arguments.a_offset)- } else {- (arguments.b, 0)- };- encoder.set_buffer(1, Some(buf1), off1);-- if configuration.do_axpby {- if let Some(bias) = bias_buffer {- encoder.set_buffer(2, Some(bias), 0);- }- }-- encoder.set_buffer(3, Some(arguments.d), 0);-- let input_dimension = arguments.input_dim;- let output_dimension = if matrix_is_rhs {- arguments.output_dim- } else {- arguments.batch- };- let matrix_leading_dim = if matrix_is_rhs {- arguments.ldb- } else {- arguments.lda- };-- encoder.set_bytes(- 4,- std::mem::size_of::<i32>() as u64,- &input_dimension as *const i32 as *const std::ffi::c_void,- );- encoder.set_bytes(- 5,- std::mem::size_of::<i32>() as u64,- &output_dimension as *const i32 as *const std::ffi::c_void,- );- encoder.set_bytes(- 6,- std::mem::size_of::<i32>() as u64,- &matrix_leading_dim as *const i32 as *const std::ffi::c_void,- );-- encoder.set_bytes(- 7,- std::mem::size_of::<f32>() as u64,- &alpha as *const f32 as *const std::ffi::c_void,- );- encoder.set_bytes(- 8,- std::mem::size_of::<f32>() as u64,- &beta as *const f32 as *const std::ffi::c_void,- );-- let has_batch = arguments.batch_count > 1;- let batch_shape = if has_batch {- vec![arguments.batch_count]- } else {- vec![1]- };- let batch_ndim = 1i32;- let batch_groups = arguments.batch_count.max(1);-- let elements_per_matrix_a =- (arguments.batch as i64) * (arguments.lda as i64);- let elements_per_matrix_b = if self.rhs_is_transposed {- (arguments.output_dim as i64) * (arguments.ldb as i64)- } else {- (arguments.input_dim as i64) * (arguments.ldb as i64)- };-- let vector_batch_stride = if matrix_is_rhs {- vec![elements_per_matrix_a]- } else {- vec![elements_per_matrix_b]- };-- let matrix_batch_stride = if matrix_is_rhs {- vec![elements_per_matrix_b]- } else {- vec![elements_per_matrix_a]- };-- let bias_batch_stride = if arguments.batch_count > 1 {- vec![(output_dimension as i64) * (arguments.ldd as i64)]- } else {- vec![0]- };-- encoder.set_bytes(- 9,- std::mem::size_of::<i32>() as u64,- &batch_ndim as *const i32 as *const std::ffi::c_void,- );- encoder.set_bytes(- 10,- (std::mem::size_of::<i32>() * batch_shape.len()) as u64,- batch_shape.as_ptr() as *const std::ffi::c_void,- );- encoder.set_bytes(- 11,- (std::mem::size_of::<i64>() * vector_batch_stride.len()) as u64,- vector_batch_stride.as_ptr() as *const std::ffi::c_void,- );- encoder.set_bytes(- 12,- (std::mem::size_of::<i64>() * matrix_batch_stride.len()) as u64,- matrix_batch_stride.as_ptr() as *const std::ffi::c_void,- );- encoder.set_bytes(- 13,- (std::mem::size_of::<i64>() * bias_batch_stride.len()) as u64,- bias_batch_stride.as_ptr() as *const std::ffi::c_void,- );- encoder.set_bytes(- 14,- std::mem::size_of::<i32>() as u64,- &bias_stride as *const i32 as *const std::ffi::c_void,- );-- let output_elements_per_threadgroup =- configuration.output_elements_per_threadgroup();- let threadgroup_count_x =- ((output_dimension as u32 + output_elements_per_threadgroup - 1)- / output_elements_per_threadgroup) as u64;- let threadgroup_count_z = batch_groups.max(1) as u64;-- let threadgroup_count =- MTLSize::new(threadgroup_count_x, 1, threadgroup_count_z);- let threads_per_threadgroup = configuration.threads_per_threadgroup();-- encoder- .dispatch_thread_groups(threadgroup_count, threads_per_threadgroup);- Ok(())- }-- pub fn encode(- &mut self,- context: &MTLContext,- encoder: &ComputeCommandEncoderRef,- arguments: &MatmulArguments,- ) -> Result<(), MTLError> {- self.encode_internal(context, encoder, arguments, None)- }-- pub fn encode_with_bias(- &mut self,- context: &MTLContext,- encoder: &ComputeCommandEncoderRef,- arguments: &MatmulArguments,- bias: &MTLBuffer,- ) -> Result<(), MTLError> {- self.encode_internal(context, encoder, arguments, Some(bias))- }-- fn encode_internal(- &mut self,- context: &MTLContext,- encoder: &ComputeCommandEncoderRef,- arguments: &MatmulArguments,- bias: Option<&MTLBuffer>,- ) -> Result<(), MTLError> {- let m = arguments.batch;- let n = arguments.output_dim;- if m != 1 && n != 1 {- return Ok(());- }-- let matrix_is_rhs = n != 1;- let transpose_matrix = if matrix_is_rhs {- !self.rhs_is_transposed- } else {- self.lhs_is_transposed- };-- let has_non_contiguous_batch = false;-- let (do_axpby, bias_buffer, alpha, beta, bias_stride) =- if let Some(bias_buf) = bias {- (true, Some(bias_buf), 1.0f32, 1.0f32, 1)- } else if let Some(c_buf) = arguments.c {- (- true,- Some(c_buf),- arguments.alpha,- arguments.beta,- arguments.ldd,- )- } else {- (false, None, 1.0f32, 0.0f32, 0)- };-- let configuration = select_gemv_configuration(- transpose_matrix,- arguments.input_dim,- if matrix_is_rhs {- arguments.output_dim- } else {- arguments.batch- },- has_non_contiguous_batch,- do_axpby,- );-- if crate::utils::env_utils::debug_matmul_enabled() {- let kernel_name = gemv_kernel_name(self.data_type, &configuration)- .unwrap_or_default();- eprintln!(- "[matmul] GEMV m={} k={} n={} batch={} dtype={:?} transpose={} kernel={}",- arguments.batch,- arguments.input_dim,- arguments.output_dim,- arguments.batch_count,- self.data_type,- configuration.transpose_matrix,- kernel_name- );- }-- self.encode_with_configuration(- context,- encoder,- arguments,- configuration,- matrix_is_rhs,- bias_buffer,- alpha,- beta,- bias_stride,- )- }-}--/// MLP Fused GEMV configuration-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]-struct MlpFusedGemvConfiguration {- threadgroup_rows: u32,- threadgroup_cols: u32,- threads_per_simdgroup_row: u32,- threads_per_simdgroup_col: u32,- elements_per_thread_row: u32,- elements_per_thread_col: u32,- activation: MlpActivationType,-}--impl MlpFusedGemvConfiguration {- fn output_elements_per_threadgroup(&self) -> u32 {- self.threadgroup_rows- * self.threads_per_simdgroup_row- * self.elements_per_thread_row- }-- fn threads_per_threadgroup(&self) -> MTLSize {- MTLSize::new(- 32,- self.threadgroup_cols as u64,- self.threadgroup_rows as u64,- )- }-}--fn mlp_fused_gemv_kernel_name(- data_type: DataType,- config: &MlpFusedGemvConfiguration,-) -> Result<String, MTLError> {- let dtype_name = match data_type {- DataType::F16 => "float16",- DataType::BF16 => "bfloat16",- DataType::F32 => "float32",- _ => {- return Err(MTLError::Generic(format!(- "Unsupported data type for MLP fused GEMV: {:?}",- data_type- )));- },- };-- Ok(format!(- "gemv_mlp_fused_{}_bm{}_bn{}_sm{}_sn{}_tm{}_tn{}",- dtype_name,- config.threadgroup_rows,- config.threadgroup_cols,- config.threads_per_simdgroup_row,- config.threads_per_simdgroup_col,- config.elements_per_thread_row,- config.elements_per_thread_col,- ))-}--fn select_mlp_fused_gemv_configuration(- hidden_dim: i32,- activation: MlpActivationType,-) -> MlpFusedGemvConfiguration {- // Select configuration based on hidden dimension- let threadgroup_rows = if hidden_dim >= 4096 {- 8- } else {- 4- };-- MlpFusedGemvConfiguration {- threadgroup_rows,- threadgroup_cols: 1,- threads_per_simdgroup_row: 1,- threads_per_simdgroup_col: 32,- elements_per_thread_row: 4,- elements_per_thread_col: 4,- activation,- }-}--/// Arguments for MLP fused GEMV-#[derive(Debug)]-pub struct MlpFusedGemvArguments<'a> {- /// Weight matrix [2*hidden_dim, input_dim] (up and gate concatenated)- pub weights: &'a MTLBuffer,- /// Input vector [input_dim]- pub input: &'a MTLBuffer,- /// Input byte offset- pub input_offset: u64,- /// Output vector [hidden_dim]- pub output: &'a MTLBuffer,- /// Input dimension (K)- pub input_dim: i32,- /// Hidden dimension (output size, half of weight rows)- pub hidden_dim: i32,- /// Weight matrix leading dimension- pub weights_ld: i32,- /// Batch count- pub batch_count: i32,- /// Activation type- pub activation: MlpActivationType,-}--/// MLP Fused GEMV Kernel-/// Computes paired up/gate projections with fused activation: out = up * activation(gate)-pub struct MlpFusedGemvKernel {- data_type: DataType,- pipelines: HashMap<MlpFusedGemvConfiguration, ComputePipelineState>,-}--impl MlpFusedGemvKernel {- pub fn new(data_type: DataType) -> Result<Self, MTLError> {- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)- {- return Err(MTLError::Generic(format!(- "Unsupported dtype for MLP fused GEMV: {:?}",- data_type- )));- }- Ok(Self {- data_type,- pipelines: HashMap::new(),- })- }-- fn get_pipeline(- &mut self,- context: &MTLContext,- config: MlpFusedGemvConfiguration,- ) -> Result<&ComputePipelineState, MTLError> {- if !self.pipelines.contains_key(&config) {- let kernel_name =- mlp_fused_gemv_kernel_name(self.data_type, &config)?;-- // Create function constant values for MLP activation- let fcv = metal::FunctionConstantValues::new();- let activation_val = config.activation as u32;- fcv.set_constant_value_at_index(- &activation_val as *const u32 as *const _,- metal::MTLDataType::UInt,- 52, // MLP_ACTIVATION index- );-- let pipeline =- context.compute_pipeline_state(&kernel_name, Some(&fcv))?;- self.pipelines.insert(config, pipeline);- }- Ok(self.pipelines.get(&config).unwrap())- }-- pub fn encode(- &mut self,- context: &MTLContext,- encoder: &ComputeCommandEncoderRef,- args: &MlpFusedGemvArguments,- ) -> Result<(), MTLError> {- let config = select_mlp_fused_gemv_configuration(- args.hidden_dim,- args.activation,- );- let pipeline = self.get_pipeline(context, config)?;-- encoder.set_compute_pipeline_state(pipeline);- encoder.set_buffer(0, Some(args.weights), 0);- encoder.set_buffer(1, Some(args.input), args.input_offset);- encoder.set_buffer(3, Some(args.output), 0);-- encoder.set_bytes(- 4,- std::mem::size_of::<i32>() as u64,- &args.input_dim as *const i32 as *const std::ffi::c_void,- );- encoder.set_bytes(- 5,- std::mem::size_of::<i32>() as u64,- &args.hidden_dim as *const i32 as *const std::ffi::c_void,- );- encoder.set_bytes(- 6,- std::mem::size_of::<i32>() as u64,- &args.weights_ld as *const i32 as *const std::ffi::c_void,- );-- // Batch strides- let vector_batch_stride = args.input_dim as i64;- let matrix_batch_stride = 0i64; // Weights not batched- encoder.set_bytes(- 11,- std::mem::size_of::<i64>() as u64,- &vector_batch_stride as *const i64 as *const std::ffi::c_void,- );- encoder.set_bytes(- 12,- std::mem::size_of::<i64>() as u64,- &matrix_batch_stride as *const i64 as *const std::ffi::c_void,- );-- let output_elements_per_threadgroup =- config.output_elements_per_threadgroup();- let threadgroup_count_x =- ((args.hidden_dim as u32 + output_elements_per_threadgroup - 1)- / output_elements_per_threadgroup) as u64;- let threadgroup_count_z = args.batch_count.max(1) as u64;-- let threadgroup_count =- MTLSize::new(threadgroup_count_x, 1, threadgroup_count_z);- let threads_per_threadgroup = config.threads_per_threadgroup();-- encoder- .dispatch_thread_groups(threadgroup_count, threads_per_threadgroup);- Ok(())- }-}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/configuration.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv/configuration.rsnew file mode 100644index 000000000..0b11354f0--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv/configuration.rs@@ -0,0 +1,121 @@+use metal::MTLSize;++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+pub struct Configuration {+ pub transpose_matrix: bool,+ pub threadgroup_rows: u32,+ pub threadgroup_cols: u32,+ pub threads_per_simdgroup_row: u32,+ pub threads_per_simdgroup_col: u32,+ pub elements_per_thread_row: u32,+ pub elements_per_thread_col: u32,+ pub non_contiguous_batch: bool,+ pub do_axpby: bool,+}++impl Configuration {+ pub fn output_elements_per_threadgroup(&self) -> u32 {+ if self.transpose_matrix {+ self.threadgroup_cols+ * self.threads_per_simdgroup_col+ * self.elements_per_thread_col+ } else {+ self.threadgroup_rows+ * self.threads_per_simdgroup_row+ * self.elements_per_thread_row+ }+ }++ pub fn threads_per_threadgroup(&self) -> MTLSize {+ MTLSize::new(+ 32,+ self.threadgroup_cols as u64,+ self.threadgroup_rows as u64,+ )+ }+}++pub fn select_configuration(+ transpose_matrix: bool,+ input_dimension: i32,+ output_dimension: i32,+ non_contiguous_batch: bool,+ do_axpby: bool,+) -> Configuration {+ let (threadgroup_rows, threadgroup_cols);+ let (threads_per_simdgroup_row, threads_per_simdgroup_col);+ let (elements_per_thread_row, elements_per_thread_col);++ if transpose_matrix {+ let mut sm = 8;+ let mut sn = 4;+ if input_dimension >= 8192 && output_dimension >= 2048 {+ sm = 4;+ sn = 8;+ }++ let bn = if output_dimension >= 2048 {+ 16+ } else if output_dimension >= 512 {+ 4+ } else {+ 2+ };++ let tn = if output_dimension < 4 {+ 1+ } else {+ 4+ };++ threadgroup_rows = 1;+ threadgroup_cols = bn;+ threads_per_simdgroup_row = sm;+ threads_per_simdgroup_col = sn;+ elements_per_thread_row = 4;+ elements_per_thread_col = tn;+ } else {+ let mut bm = if output_dimension >= 4096 {+ 8+ } else {+ 4+ };+ let mut sm = 1;+ let mut sn = 32;+ let mut bn = 1;++ if input_dimension <= 64 {+ bm = 1;+ sm = 8;+ sn = 4;+ } else if input_dimension >= 16 * output_dimension {+ bm = 1;+ bn = 8;+ }++ let tm = if output_dimension < 4 {+ 1+ } else {+ 4+ };++ threadgroup_rows = bm;+ threadgroup_cols = bn;+ threads_per_simdgroup_row = sm;+ threads_per_simdgroup_col = sn;+ elements_per_thread_row = tm;+ elements_per_thread_col = 4;+ }++ Configuration {+ transpose_matrix,+ threadgroup_rows,+ threadgroup_cols,+ threads_per_simdgroup_row,+ threads_per_simdgroup_col,+ elements_per_thread_row,+ elements_per_thread_col,+ non_contiguous_batch,+ do_axpby,+ }+}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv/kernel.rsnew file mode 100644index 000000000..53b50be63--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs@@ -0,0 +1,339 @@+use std::collections::HashMap;++use metal::{+ Buffer as MTLBuffer, ComputeCommandEncoderRef, ComputePipelineState,+ MTLSize,+};++use super::configuration::{Configuration, select_configuration};+use crate::{+ DataType,+ backends::metal::{+ MTLContext, MTLError, kernel::matmul::common::MatmulArguments,+ },+};++fn gemv_kernel_name(+ data_type: DataType,+ configuration: &Configuration,+) -> Result<String, MTLError> {+ let dtype_name = match data_type {+ DataType::F16 => "float16",+ DataType::BF16 => "bfloat16",+ DataType::F32 => "float32",+ _ => {+ return Err(MTLError::Generic(format!(+ "Unsupported data type for GEMV: {:?}",+ data_type+ )));+ },+ };++ let prefix = if configuration.transpose_matrix {+ "gemv_t"+ } else {+ "gemv"+ };++ Ok(format!(+ "{prefix}_{dtype_name}_bm{}_bn{}_sm{}_sn{}_tm{}_tn{}_nc{}_axpby{}",+ configuration.threadgroup_rows,+ configuration.threadgroup_cols,+ configuration.threads_per_simdgroup_row,+ configuration.threads_per_simdgroup_col,+ configuration.elements_per_thread_row,+ configuration.elements_per_thread_col,+ configuration.non_contiguous_batch as u8,+ configuration.do_axpby as u8,+ ))+}++pub struct Kernel {+ data_type: DataType,+ lhs_is_transposed: bool,+ rhs_is_transposed: bool,+ pipelines: HashMap<Configuration, ComputePipelineState>,+}++impl Kernel {+ pub fn new(+ data_type: DataType,+ lhs_is_transposed: bool,+ rhs_is_transposed: bool,+ ) -> Self {+ Self {+ data_type,+ lhs_is_transposed,+ rhs_is_transposed,+ pipelines: HashMap::new(),+ }+ }++ fn get_pipeline(+ &mut self,+ context: &MTLContext,+ configuration: Configuration,+ ) -> Result<&ComputePipelineState, MTLError> {+ if !self.pipelines.contains_key(&configuration) {+ let kernel_name = gemv_kernel_name(self.data_type, &configuration)?;+ let pipeline =+ context.compute_pipeline_state(&kernel_name, None)?;+ self.pipelines.insert(configuration, pipeline);+ }+ Ok(self.pipelines.get(&configuration).unwrap())+ }++ #[allow(clippy::too_many_arguments)]+ fn encode_with_configuration(+ &mut self,+ context: &MTLContext,+ encoder: &ComputeCommandEncoderRef,+ arguments: &MatmulArguments,+ configuration: Configuration,+ matrix_is_rhs: bool,+ bias_buffer: Option<&MTLBuffer>,+ alpha: f32,+ beta: f32,+ bias_stride: i32,+ ) -> Result<(), MTLError> {+ let pipeline = self.get_pipeline(context, configuration)?;+ encoder.set_compute_pipeline_state(pipeline);++ let (buf0, off0) = if matrix_is_rhs {+ (arguments.b, 0)+ } else {+ (arguments.a, arguments.a_offset)+ };+ encoder.set_buffer(0, Some(buf0), off0);++ let (buf1, off1) = if matrix_is_rhs {+ (arguments.a, arguments.a_offset)+ } else {+ (arguments.b, 0)+ };+ encoder.set_buffer(1, Some(buf1), off1);++ if configuration.do_axpby {+ if let Some(bias) = bias_buffer {+ encoder.set_buffer(2, Some(bias), 0);+ }+ }++ encoder.set_buffer(3, Some(arguments.d), 0);++ let input_dimension = arguments.input_dim;+ let output_dimension = if matrix_is_rhs {+ arguments.output_dim+ } else {+ arguments.batch+ };+ let matrix_leading_dim = if matrix_is_rhs {+ arguments.ldb+ } else {+ arguments.lda+ };++ encoder.set_bytes(+ 4,+ std::mem::size_of::<i32>() as u64,+ &input_dimension as *const i32 as *const std::ffi::c_void,+ );+ encoder.set_bytes(+ 5,+ std::mem::size_of::<i32>() as u64,+ &output_dimension as *const i32 as *const std::ffi::c_void,+ );+ encoder.set_bytes(+ 6,+ std::mem::size_of::<i32>() as u64,+ &matrix_leading_dim as *const i32 as *const std::ffi::c_void,+ );++ encoder.set_bytes(+ 7,+ std::mem::size_of::<f32>() as u64,+ &alpha as *const f32 as *const std::ffi::c_void,+ );+ encoder.set_bytes(+ 8,+ std::mem::size_of::<f32>() as u64,+ &beta as *const f32 as *const std::ffi::c_void,+ );++ let batch_shape = if arguments.batch_count > 1 {+ vec![arguments.batch_count]+ } else {+ vec![1]+ };+ let batch_ndim = 1i32;+ let batch_groups = arguments.batch_count.max(1);++ let elements_per_matrix_a =+ (arguments.batch as i64) * (arguments.lda as i64);+ let elements_per_matrix_b = if self.rhs_is_transposed {+ (arguments.output_dim as i64) * (arguments.ldb as i64)+ } else {+ (arguments.input_dim as i64) * (arguments.ldb as i64)+ };++ let vector_batch_stride = if matrix_is_rhs {+ vec![elements_per_matrix_a]+ } else {+ vec![elements_per_matrix_b]+ };++ let matrix_batch_stride = if matrix_is_rhs {+ vec![elements_per_matrix_b]+ } else {+ vec![elements_per_matrix_a]+ };++ let bias_batch_stride = if arguments.batch_count > 1 {+ vec![(output_dimension as i64) * (arguments.ldd as i64)]+ } else {+ vec![0]+ };++ encoder.set_bytes(+ 9,+ std::mem::size_of::<i32>() as u64,+ &batch_ndim as *const i32 as *const std::ffi::c_void,+ );+ encoder.set_bytes(+ 10,+ (std::mem::size_of::<i32>() * batch_shape.len()) as u64,+ batch_shape.as_ptr() as *const std::ffi::c_void,+ );+ encoder.set_bytes(+ 11,+ (std::mem::size_of::<i64>() * vector_batch_stride.len()) as u64,+ vector_batch_stride.as_ptr() as *const std::ffi::c_void,+ );+ encoder.set_bytes(+ 12,+ (std::mem::size_of::<i64>() * matrix_batch_stride.len()) as u64,+ matrix_batch_stride.as_ptr() as *const std::ffi::c_void,+ );+ encoder.set_bytes(+ 13,+ (std::mem::size_of::<i64>() * bias_batch_stride.len()) as u64,+ bias_batch_stride.as_ptr() as *const std::ffi::c_void,+ );+ encoder.set_bytes(+ 14,+ std::mem::size_of::<i32>() as u64,+ &bias_stride as *const i32 as *const std::ffi::c_void,+ );++ let output_elements_per_threadgroup =+ configuration.output_elements_per_threadgroup();+ let threadgroup_count_x =+ ((output_dimension as u32 + output_elements_per_threadgroup - 1)+ / output_elements_per_threadgroup) as u64;+ let threadgroup_count_z = batch_groups.max(1) as u64;++ let threadgroup_count =+ MTLSize::new(threadgroup_count_x, 1, threadgroup_count_z);+ let threads_per_threadgroup = configuration.threads_per_threadgroup();++ encoder+ .dispatch_thread_groups(threadgroup_count, threads_per_threadgroup);+ Ok(())+ }++ pub fn encode(+ &mut self,+ context: &MTLContext,+ encoder: &ComputeCommandEncoderRef,+ arguments: &MatmulArguments,+ ) -> Result<(), MTLError> {+ self.encode_internal(context, encoder, arguments, None)+ }++ pub fn encode_with_bias(+ &mut self,+ context: &MTLContext,+ encoder: &ComputeCommandEncoderRef,+ arguments: &MatmulArguments,+ bias: &MTLBuffer,+ ) -> Result<(), MTLError> {+ self.encode_internal(context, encoder, arguments, Some(bias))+ }++ fn encode_internal(+ &mut self,+ context: &MTLContext,+ encoder: &ComputeCommandEncoderRef,+ arguments: &MatmulArguments,+ bias: Option<&MTLBuffer>,+ ) -> Result<(), MTLError> {+ let m = arguments.batch;+ let n = arguments.output_dim;+ if m != 1 && n != 1 {+ return Ok(());+ }++ let matrix_is_rhs = n != 1;+ let transpose_matrix = if matrix_is_rhs {+ !self.rhs_is_transposed+ } else {+ self.lhs_is_transposed+ };++ let has_non_contiguous_batch = false;++ let (do_axpby, bias_buffer, alpha, beta, bias_stride) =+ if let Some(bias_buf) = bias {+ (true, Some(bias_buf), 1.0f32, 1.0f32, 1)+ } else if let Some(c_buf) = arguments.c {+ (+ true,+ Some(c_buf),+ arguments.alpha,+ arguments.beta,+ arguments.ldd,+ )+ } else {+ (false, None, 1.0f32, 0.0f32, 0)+ };++ let configuration = select_configuration(+ transpose_matrix,+ arguments.input_dim,+ if matrix_is_rhs {+ arguments.output_dim+ } else {+ arguments.batch+ },+ has_non_contiguous_batch,+ do_axpby,+ );++ if crate::utils::env_utils::debug_matmul_enabled() {+ let kernel_name = gemv_kernel_name(self.data_type, &configuration)+ .unwrap_or_default();+ eprintln!(+ "[matmul] GEMV m={} k={} n={} batch={} dtype={:?} transpose={} kernel={}",+ arguments.batch,+ arguments.input_dim,+ arguments.output_dim,+ arguments.batch_count,+ self.data_type,+ configuration.transpose_matrix,+ kernel_name+ );+ }++ self.encode_with_configuration(+ context,+ encoder,+ arguments,+ configuration,+ matrix_is_rhs,+ bias_buffer,+ alpha,+ beta,+ bias_stride,+ )+ }+}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv/mod.rsnew file mode 100644index 000000000..bbd057f95--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv/mod.rs@@ -0,0 +1,4 @@+mod configuration;+mod kernel;++pub use kernel::Kernel as GemvKernel;diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemv/shaders/gemv.metalsimilarity index 76%rename from crates/uzu/src/backends/metal/kernel/matmul/gemv/gemv.metalrename to crates/uzu/src/backends/metal/kernel/matmul/gemv/shaders/gemv.metalindex d79659f90..ff714458a 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv/shaders/gemv.metal@@ -2,9 +2,8 @@#include <metal_stdlib>#include "../../common/utils.h"-#include "../../common/mlp_epilogue.h"-#include "../steel/utils.h"+#include "../../common/steel/utils.h"using namespace metal;@@ -865,264 +864,3 @@ instantiate_gemv_t_bs_blocks(float32, float);instantiate_gemv_t_bs_blocks(float16, half);instantiate_gemv_t_bs_blocks(bfloat16, bfloat16_t);instantiate_gemv_t_bs_blocks(complex64, complex64_t); // clang-format on--///////////////////////////////////////////////////////////////////////////////-/// MLP Fused GEMV - Paired Up/Gate Computation with Activation Fusion-///////////////////////////////////////////////////////////////////////////////--// MLP fused GEMV kernel for decode path (M=1).-// Computes paired up and gate projections, then applies: out = up * activation(gate)-// Output size is hidden_dim (half of the weight matrix columns).-template <- typename T,- const int BM, /* Threadgroup rows (in simdgroups) */- const int BN, /* Threadgroup cols (in simdgroups) */- const int SM, /* Simdgroup rows (in threads) */- const int SN, /* Simdgroup cols (in threads) */- const int TM, /* Thread rows (in elements) */- const int TN> /* Thread cols (in elements) */-struct GEMVMlpFusedKernel {- using AccT = float;-- MTL_CONST int threadsM = BM * SM;- MTL_CONST int threadsN = BN * SN;- MTL_CONST int blockM = threadsM * TM;- MTL_CONST int blockN = threadsN * TN;-- static_assert(SM * SN == 32, "simdgroup can only have 32 threads");-- MTL_CONST short tgp_mem_size = BN > 1 ? 2 * BN * (blockM + TM) : 0;- MTL_CONST bool needs_tgp_reduction = BN > 1;-- template <typename U = T>- static METAL_FUNC void- load_unsafe(const device T* src, thread U dst[TN], const int src_offset = 0) {- MTL_PRAGMA_UNROLL- for (int tn = 0; tn < TN; tn++) {- dst[tn] = static_cast<U>(src[src_offset + tn]);- }- }-- template <typename U = T>- static METAL_FUNC void load_safe(- const device T* src,- thread U dst[TN],- const int src_offset = 0,- const int src_size = TN) {- if (src_offset + TN <= src_size) {- MTL_PRAGMA_UNROLL- for (int tn = 0; tn < TN; tn++) {- dst[tn] = static_cast<U>(src[src_offset + tn]);- }- } else {- MTL_PRAGMA_UNROLL- for (int tn = 0; tn < TN; tn++) {- dst[tn] = src_offset + tn < src_size- ? static_cast<U>(src[src_offset + tn])- : U(0);- }- }- }-- static METAL_FUNC void run(- const device T* mat [[buffer(0)]],- const device T* in_vec [[buffer(1)]],- device T* out_vec [[buffer(3)]],- const int in_vec_size,- const int hidden_dim, // Output size (half of weight matrix rows)- const int matrix_ld,- threadgroup AccT* tgp_memory,- uint3 tid [[threadgroup_position_in_grid]],- uint3 lid [[thread_position_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]) {- (void)lid;-- // Thread local accumulation results for up and gate- thread AccT result_up[TM] = {0};- thread AccT result_gate[TM] = {0};- thread T inter_up[TN];- thread T inter_gate[TN];- thread AccT v_coeff[TN];-- const int thrM = SN != 32 ? simd_lid / SN : 0;- const int thrN = SN != 32 ? simd_lid % SN : int(simd_lid);-- const int sgN = BN != 1 ? (simd_gid % BN) : 0;- const int simdM = BN != 1 ? SM * (simd_gid / BN) : int(SM * simd_gid);- const int simdN = BN != 1 ? SN * (simd_gid % BN) : 0;-- int bm = (simdM + thrM) * TM;- int bn = (simdN + thrN) * TN;-- // Block position - outputs to hidden_dim elements- int out_row = tid.x * blockM + bm;-- if (out_row >= hidden_dim)- return;-- out_row = out_row + TM <= hidden_dim ? out_row : hidden_dim - TM;-- // Pointers to up and gate rows in the weight matrix- // Weight layout: [up_0, up_1, ..., up_H-1, gate_0, gate_1, ..., gate_H-1]- const device T* mat_up = mat + out_row * matrix_ld;- const device T* mat_gate = mat + (out_row + hidden_dim) * matrix_ld;-- constexpr const uniform<int> loop_stride = make_uniform(blockN);- const uniform<int> in_size = make_uniform(in_vec_size);- const uniform<int> n_iter = in_size / loop_stride;- const uniform<int> last_iter = loop_stride * n_iter;- const uniform<int> leftover = in_size - last_iter;-- // Main loop - compute both up and gate dot products- for (int i = 0; i < n_iter; ++i) {- load_unsafe<AccT>(in_vec, v_coeff, bn);-- int mat_offset = 0;- MTL_PRAGMA_UNROLL- for (int tm = 0; tm < TM; tm++) {- load_unsafe(mat_up, inter_up, mat_offset + bn);- load_unsafe(mat_gate, inter_gate, mat_offset + bn);-- MTL_PRAGMA_UNROLL- for (int tn = 0; tn < TN; tn++) {- result_up[tm] += inter_up[tn] * v_coeff[tn];- result_gate[tm] += inter_gate[tn] * v_coeff[tn];- }-- mat_offset += matrix_ld;- }-- bn += blockN;- }-- // Handle leftover- if (leftover > 0) {- load_safe<AccT>(in_vec, v_coeff, bn, in_vec_size);-- MTL_PRAGMA_UNROLL- for (int tm = 0; tm < TM; tm++) {- load_safe(&mat_up[tm * matrix_ld], inter_up, bn, in_vec_size);- load_safe(&mat_gate[tm * matrix_ld], inter_gate, bn, in_vec_size);-- MTL_PRAGMA_UNROLL- for (int tn = 0; tn < TN; tn++) {- result_up[tm] += inter_up[tn] * v_coeff[tn];- result_gate[tm] += inter_gate[tn] * v_coeff[tn];- }- }- }-- // Simdgroup reduction for both up and gate- MTL_PRAGMA_UNROLL- for (int tm = 0; tm < TM; tm++) {- MTL_PRAGMA_UNROLL- for (ushort sn = (SN / 2); sn >= 1; sn >>= 1) {- result_up[tm] += simd_shuffle_down(result_up[tm], sn);- result_gate[tm] += simd_shuffle_down(result_gate[tm], sn);- }- }-- // Threadgroup reduction if needed- if (needs_tgp_reduction) {- threadgroup AccT* tgp_results_up = tgp_memory + sgN * (blockM + TM) + bm;- threadgroup AccT* tgp_results_gate = tgp_memory + BN * (blockM + TM) + sgN * (blockM + TM) + bm;-- if (thrN == 0) {- MTL_PRAGMA_UNROLL- for (int tm = 0; tm < TM; tm++) {- tgp_results_up[tm] = result_up[tm];- tgp_results_gate[tm] = result_gate[tm];- }-- threadgroup_barrier(mem_flags::mem_none);-- if (sgN == 0) {- MTL_PRAGMA_UNROLL- for (int sgn = 1; sgn < BN; sgn++) {- MTL_PRAGMA_UNROLL- for (int tm = 0; tm < TM; tm++) {- result_up[tm] += tgp_results_up[sgn * (blockM + TM) + tm];- result_gate[tm] += tgp_results_gate[sgn * (blockM + TM) + tm];- }- }- }- }- }-- // Apply MLP fused epilogue and write output- if (simdN == 0 && thrN == 0) {- MTL_PRAGMA_UNROLL- for (int tm = 0; tm < TM; tm++) {- float fused_result = mlp_fused_epilogue_f32(result_up[tm], result_gate[tm]);- out_vec[out_row + tm] = static_cast<T>(fused_result);- }- }- }-};--template <- typename T,- const int BM,- const int BN,- const int SM,- const int SN,- const int TM,- const int TN>-[[kernel, max_total_threads_per_threadgroup(BM* BN * 32)]] void gemv_mlp_fused(- const device T* mat [[buffer(0)]],- const device T* in_vec [[buffer(1)]],- device T* out_vec [[buffer(3)]],- const constant int& in_vec_size [[buffer(4)]],- const constant int& hidden_dim [[buffer(5)]], // Output dimension (half of weight rows)- const constant int& matrix_ld [[buffer(6)]],- const constant int64_t& vector_batch_stride [[buffer(11)]],- const constant int64_t& matrix_batch_stride [[buffer(12)]],- uint3 tid [[threadgroup_position_in_grid]],- uint3 lid [[thread_position_in_threadgroup]],- uint simd_gid [[simdgroup_index_in_threadgroup]],- uint simd_lid [[thread_index_in_simdgroup]]) {- using gemv_kernel = GEMVMlpFusedKernel<T, BM, BN, SM, SN, TM, TN>;- threadgroup typename gemv_kernel::AccT tgp_memory- [gemv_kernel::tgp_mem_size == 0 ? 1 : gemv_kernel::tgp_mem_size];-- // Batch offset- in_vec += tid.z * vector_batch_stride;- mat += tid.z * matrix_batch_stride;- out_vec += tid.z * hidden_dim;-- gemv_kernel::run(- mat,- in_vec,- out_vec,- in_vec_size,- hidden_dim,- matrix_ld,- gemv_kernel::tgp_mem_size == 0 ? nullptr : tgp_memory,- tid,- lid,- simd_gid,- simd_lid);-}--#define instantiate_gemv_mlp_fused_helper(name, itype, bm, bn, sm, sn, tm, tn) \- instantiate_kernel( \- "gemv_mlp_fused_" #name "_bm" #bm "_bn" #bn "_sm" #sm "_sn" #sn "_tm" #tm \- "_tn" #tn, \- gemv_mlp_fused, \- itype, \- bm, \- bn, \- sm, \- sn, \- tm, \- tn)--// clang-format off-#define instantiate_gemv_mlp_fused_blocks(name, itype) \- instantiate_gemv_mlp_fused_helper(name, itype, 4, 1, 1, 32, 4, 4) \- instantiate_gemv_mlp_fused_helper(name, itype, 8, 1, 1, 32, 4, 4) // clang-format on--instantiate_gemv_mlp_fused_blocks(float32, float);-instantiate_gemv_mlp_fused_blocks(float16, half);-instantiate_gemv_mlp_fused_blocks(bfloat16, bfloat16_t);diff --git a/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/kernel.rsindex 1c257c8c9..fedf3cfe2 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs@@ -1,18 +1,11 @@-use std::collections::HashMap;--use metal::{- Buffer as MTLBuffer, ComputeCommandEncoderRef,- ComputePipelineState as MTLComputePipelineState, MTLSize,-};+use metal::{Buffer as MTLBuffer, ComputeCommandEncoderRef};use super::{- super::{KernelDataType, TensorAddBias, mlp::MlpActivationType},- arguments::MatmulArguments,+ super::{KernelDataType, TensorAddBias},+ common::MatmulArguments,+ gemm,gemv::GemvKernel,- pipeline::PipelineKey,- shared_types::{GEMMAddMMParams, GEMMParams},- splitk::SplitKGemm,- transpose::transpose_configuration,+ split_k::SplitKGemm,};use crate::{DataType,@@ -20,84 +13,17 @@ use crate::{utils::env_utils::debug_matmul_enabled,};-#[derive(Debug, Clone, Copy)]-struct TileSelection {- block_rows: i32,- block_cols: i32,- block_depth: i32,- warps_per_row: u64,- warps_per_col: u64,- swizzle_log2: i32,-}--impl TileSelection {- fn new(- block_rows: i32,- block_cols: i32,- block_depth: i32,- warps_per_row: u64,- warps_per_col: u64,- swizzle_log2: i32,- ) -> Self {- Self {- block_rows,- block_cols,- block_depth,- warps_per_row,- warps_per_col,- swizzle_log2,- }- }-}-pub struct MatmulKernel {data_type: DataType,lhs_is_transposed: bool,rhs_is_transposed: bool,+ gemm: Option<gemm::GemmKernel>,gemv: Option<GemvKernel>,splitk: Option<SplitKGemm>,bias_add: Option<TensorAddBias>,- pipelines: HashMap<PipelineKey, MTLComputePipelineState>,}impl MatmulKernel {- fn steel_type_name(dt: DataType) -> &'static str {- match dt {- DataType::F16 => "float16",- DataType::BF16 => "bfloat16",- DataType::F32 => "float32",- _ => unreachable!(),- }- }-- fn kernel_name(- &self,- tile: &TileSelection,- ) -> String {- let type_name = Self::steel_type_name(self.data_type);- let transpose_suffix = transpose_configuration(- self.lhs_is_transposed,- self.rhs_is_transposed,- );- let prefix = if tile.block_depth >= 256 {- "steel_gemm_fused_nax"- } else {- "steel_gemm_fused"- };- format!(- "{}_{}_{}_{}_bm{}_bn{}_bk{}_wm{}_wn{}",- prefix,- transpose_suffix.as_str(),- type_name,- type_name,- tile.block_rows,- tile.block_cols,- tile.block_depth,- tile.warps_per_row,- tile.warps_per_col- )- }-pub fn new(_mtl: &MTLContext,data_type: DataType,@@ -115,10 +41,10 @@ impl MatmulKernel {data_type,lhs_is_transposed,rhs_is_transposed,+ gemm: None,gemv: None,splitk: None,bias_add: None,- pipelines: HashMap::new(),})}@@ -129,10 +55,6 @@ impl MatmulKernel {args: &MatmulArguments,bias: Option<&MTLBuffer>,) -> Result<bool, MTLError> {- // GEMV kernel requires:- // - A not transposed (input vector is contiguous)- // - B transposed (weight matrix is [N, K] row-major, each row contiguous)- // Without transpose_b, B is [K, N] and columns are not contiguouslet m = args.batch;let n = args.output_dim;if self.lhs_is_transposed || !self.rhs_is_transposed {@@ -202,169 +124,17 @@ impl MatmulKernel {Ok(true)}- fn select_tile_configuration(- &self,- mtl: &MTLContext,- args: &MatmulArguments,- ) -> TileSelection {- use crate::backends::metal::DeviceClass;-- let overall_work_elements = (args.batch_count as i64)- * (args.batch as i64)- * (args.output_dim as i64);- let is_float32 = matches!(self.data_type, DataType::F32);- let prefer_half_or_tf32 = !is_float32 || mtl.tf32_enabled();-- // NAX path: prefer deeper K tiles on NAX-capable devices.- if mtl.is_nax_available() && prefer_half_or_tf32 {- let base_tile = TileSelection::new(- 128, 128, 512, 4, 4, /*swizzle_log2=*/ 0,- );- let tile_rows =- (args.batch + base_tile.block_rows - 1) / base_tile.block_rows;- let swizzle_log2 = if tile_rows <= 3 {- 0- } else {- 1- };- return TileSelection {- swizzle_log2,- ..base_tile- };- }-- let device_class_code = match mtl.architecture.device_class {- DeviceClass::Desktop => 'd',- DeviceClass::Integrated => 'g',- DeviceClass::Phone => 'p',- DeviceClass::Unknown(_) => 'g',- };-- // Default gemm matmul heuristics (non-NAX), matching observed- // choices on M2 Max: prefer 64x64x16 wm2/wn2 for large shapes and- // fall back to 64x32x32 when transpose_b=true and not N-aligned.- match device_class_code {- 'g' | 'p' => {- if prefer_half_or_tf32 {- if !self.lhs_is_transposed && self.rhs_is_transposed {- TileSelection::new(64, 32, 32, 2, 2, 0)- } else {- TileSelection::new(64, 64, 16, 1, 2, 0)- }- } else if !self.lhs_is_transposed && self.rhs_is_transposed {- TileSelection::new(32, 64, 16, 1, 2, 0)- } else {- TileSelection::new(64, 32, 32, 2, 2, 0)- }- },- 'd' => {- if overall_work_elements >= (1_i64 << 20) {- if prefer_half_or_tf32 {- if 2 * std::cmp::max(args.batch, args.output_dim)- > args.input_dim- {- TileSelection::new(64, 64, 16, 2, 2, 0)- } else if !self.lhs_is_transposed- && self.rhs_is_transposed- {- TileSelection::new(64, 32, 32, 2, 2, 0)- } else {- TileSelection::new(32, 64, 16, 1, 2, 0)- }- } else if !self.lhs_is_transposed && self.rhs_is_transposed- {- TileSelection::new(32, 64, 16, 1, 2, 0)- } else {- TileSelection::new(64, 32, 32, 2, 2, 0)- }- } else if prefer_half_or_tf32 {- if !self.lhs_is_transposed && self.rhs_is_transposed {- TileSelection::new(64, 32, 32, 2, 2, 0)- } else {- TileSelection::new(64, 64, 16, 1, 2, 0)- }- } else if !self.lhs_is_transposed && self.rhs_is_transposed {- TileSelection::new(32, 64, 16, 1, 2, 0)- } else {- TileSelection::new(64, 32, 32, 2, 2, 0)- }- },- _ => TileSelection::new(64, 64, 16, 2, 2, 0),- }- }-- fn get_or_compile_pipeline(- &mut self,- mtl: &MTLContext,- name: &str,- align_m: bool,- align_n: bool,- align_k: bool,- has_batch: bool,- use_out_source: bool,- do_axpby: bool,- ) -> Result<&MTLComputePipelineState, MTLError> {- let key = PipelineKey {- name: name.to_string(),- align_m,- align_n,- align_k,- has_batch,- use_out_source,- do_axpby,- };- if !self.pipelines.contains_key(&key) {- let fcv = metal::FunctionConstantValues::new();- // Base specializations- fcv.set_constant_value_at_index(- &has_batch as *const bool as *const _,- metal::MTLDataType::Bool,- 10,- );- fcv.set_constant_value_at_index(- &use_out_source as *const bool as *const _,- metal::MTLDataType::Bool,- 100,- );- fcv.set_constant_value_at_index(- &do_axpby as *const bool as *const _,- metal::MTLDataType::Bool,- 110,- );- fcv.set_constant_value_at_index(- &align_m as *const bool as *const _,- metal::MTLDataType::Bool,- 200,- );- fcv.set_constant_value_at_index(- &align_n as *const bool as *const _,- metal::MTLDataType::Bool,- 201,- );- fcv.set_constant_value_at_index(- &align_k as *const bool as *const _,- metal::MTLDataType::Bool,- 202,- );-- let cache_key = format!(- "{}_am{}_an{}_ak{}_hb{}_uo{}_ax{}",- name,- align_m as u8,- align_n as u8,- align_k as u8,- has_batch as u8,- use_out_source as u8,- do_axpby as u8- );- let (ps, _) = mtl.compute_pipeline_state_with_reflection_cached(- &cache_key,- name,- Some(&fcv),- )?;- self.pipelines.insert(key.clone(), ps);+ fn get_or_create_gemm(+ &mut self+ ) -> Result<&mut gemm::GemmKernel, MTLError> {+ if self.gemm.is_none() {+ self.gemm = Some(gemm::GemmKernel::new(+ self.data_type,+ self.lhs_is_transposed,+ self.rhs_is_transposed,+ )?);}- Ok(self.pipelines.get(&key).unwrap())+ Ok(self.gemm.as_mut().unwrap())}fn encode_gemm(@@ -373,116 +143,8 @@ impl MatmulKernel {enc: &ComputeCommandEncoderRef,args: &MatmulArguments,) -> Result<(), MTLError> {- let tile = self.select_tile_configuration(mtl, &args);- let kname = self.kernel_name(&tile);-- // M = batch, N = output_dim, K = input_dim- let m = args.batch;- let n = args.output_dim;- let k = args.input_dim;-- let align_m = (m % tile.block_rows) == 0;- let align_n = (n % tile.block_cols) == 0;- let align_k = (k % tile.block_depth) == 0;- let has_batch = args.batch_count > 1;- let use_out_source = args.c.is_some();- let do_axpby =- use_out_source && (args.alpha != 1.0 || args.beta != 0.0);- let ps = self.get_or_compile_pipeline(- mtl,- &kname,- align_m,- align_n,- align_k,- has_batch,- use_out_source,- do_axpby,- )?;-- enc.set_compute_pipeline_state(ps);-- // Set buffers- enc.set_buffer(0, Some(args.a), args.a_offset);- enc.set_buffer(1, Some(args.b), 0);- if use_out_source {- if let Some(c_buf) = args.c {- enc.set_buffer(2, Some(c_buf), 0);- }- }- enc.set_buffer(3, Some(args.d), 0);-- // Params- let tiles_n = (n + tile.block_cols - 1) / tile.block_cols;- let tiles_m = (m + tile.block_rows - 1) / tile.block_rows;- let swizzle_log = tile.swizzle_log2;-- let tile_swizzle = 1 << swizzle_log;- let tm_swizzled = (tiles_m + tile_swizzle - 1) / tile_swizzle;- let tn_swizzled = tiles_n * tile_swizzle;- let elements_per_matrix_a = (args.batch as i64) * (args.lda as i64);- let elements_per_matrix_b = if self.rhs_is_transposed {- (args.output_dim as i64) * (args.ldb as i64)- } else {- (args.input_dim as i64) * (args.ldb as i64)- };- let elements_per_matrix_d = (args.batch as i64) * (args.ldd as i64);- let batch_ndim = 1;- let params = GEMMParams {- M: m,- N: n,- K: k,- lda: args.lda,- ldb: args.ldb,- ldd: args.ldd,- // NOTE: tiles_{n,m} are the *unswizzled* tile counts used for bounds- // checks inside the GEMM kernel. The dispatched grid dimensions are- // swizzled separately via tn_swizzled/tm_swizzled.- tiles_n,- tiles_m,- batch_stride_a: elements_per_matrix_a,- batch_stride_b: elements_per_matrix_b,- batch_stride_d: elements_per_matrix_d,- swizzle_log,- gemm_k_iterations_aligned: k / tile.block_depth,- batch_ndim,- };- enc.set_bytes(- 4,- std::mem::size_of::<GEMMParams>() as u64,- ¶ms as *const GEMMParams as *const _,- );-- if use_out_source {- let ldc = args.ldd;- let fdc = 1;- let batch_stride_c = if args.batch_count > 1 {- (args.ldd as i64) * (args.output_dim as i64)- } else {- 0- };- let addmm_params = GEMMAddMMParams {- ldc,- fdc,- batch_stride_c,- alpha: args.alpha,- beta: args.beta,- };- enc.set_bytes(- 5,- std::mem::size_of::<GEMMAddMMParams>() as u64,- &addmm_params as *const GEMMAddMMParams as *const _,- );- }-- // Threadgroup sizing- let threads_per_tg =- MTLSize::new(32, tile.warps_per_col, tile.warps_per_row);- let tg_x = tn_swizzled as u64;- let tg_y = tm_swizzled as u64;- let tg_z = args.batch_count as u64;- let tgs = MTLSize::new(tg_x, tg_y, tg_z);- enc.dispatch_thread_groups(tgs, threads_per_tg);- Ok(())+ let gemm = self.get_or_create_gemm()?;+ gemm.encode(mtl, enc, args)}pub fn encode(@@ -493,7 +155,6 @@ impl MatmulKernel {) -> Result<(), MTLError> {self.apply_batch_collapse(&mut args);- // Try GEMV fast path for decode shapes.if self.maybe_use_gemv(mtl, enc, &args)? {if debug_matmul_enabled() {self.log_gemv(&args);@@ -501,7 +162,6 @@ impl MatmulKernel {return Ok(());}- // Try Split-K for small M*N and large K.if self.maybe_use_splitk(mtl, enc, &args)? {if debug_matmul_enabled() {self.log_splitk(&args);@@ -510,8 +170,7 @@ impl MatmulKernel {}if debug_matmul_enabled() {- let tile = self.select_tile_configuration(mtl, &args);- self.log_gemm(&args, &tile);+ self.log_gemm(&args, mtl);}self.encode_gemm(mtl, enc, &args)@@ -542,8 +201,7 @@ impl MatmulKernel {}if debug_matmul_enabled() {- let tile = self.select_tile_configuration(mtl, &args);- self.log_gemm(&args, &tile);+ self.log_gemm(&args, mtl);}self.encode_gemm(mtl, enc, &args)?;@@ -588,7 +246,6 @@ impl MatmulKernel {if args.batch_count <= 1 {return;}- // Collapse batch_count into M when A is contiguous and B is broadcast.if args.lda == args.input_dim && self.rhs_is_transposed {args.batch *= args.batch_count;args.batch_count = 1;@@ -626,229 +283,23 @@ impl MatmulKernel {fn log_gemm(&self,args: &MatmulArguments,- tile: &TileSelection,+ mtl: &MTLContext,) {- let kernel_name = self.kernel_name(tile);- eprintln!(- "[matmul] GEMM m={} k={} n={} batch={} dtype={:?} tile={}x{}x{} kernel={}",- args.batch,- args.input_dim,- args.output_dim,- args.batch_count,- self.data_type,- tile.block_rows,- tile.block_cols,- tile.block_depth,- kernel_name- );- }-}--/// Arguments for MLP fused GEMM-#[derive(Debug)]-pub struct MlpFusedGemmArguments<'a> {- /// Input activations [M, K]- pub input: &'a MTLBuffer,- /// Input byte offset- pub input_offset: u64,- /// Weight matrix [K, 2*hidden_dim] or [2*hidden_dim, K] depending on transpose- pub weights: &'a MTLBuffer,- /// Output [M, hidden_dim]- pub output: &'a MTLBuffer,- /// Batch size (M)- pub batch: i32,- /// Input dimension (K)- pub input_dim: i32,- /// Hidden dimension (output size, half of weight columns/rows)- pub hidden_dim: i32,- /// Leading dimension of input- pub lda: i32,- /// Leading dimension of weights- pub ldb: i32,- /// Leading dimension of output- pub ldd: i32,- /// Activation type for fused epilogue- pub activation: MlpActivationType,-}--/// MLP Fused GEMM Kernel for prefill path-/// Computes paired up/gate projections with fused activation: out = up * activation(gate)-pub struct MlpFusedGemmKernel {- data_type: DataType,- weights_transposed: bool,- pipelines: HashMap<- (i32, i32, bool, bool, MlpActivationType),- MTLComputePipelineState,- >,-}--impl MlpFusedGemmKernel {- pub fn new(- data_type: DataType,- weights_transposed: bool,- ) -> Result<Self, MTLError> {- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)- {- return Err(MTLError::Generic(format!(- "Unsupported dtype for MLP fused GEMM: {:?}",- data_type- )));- }- Ok(Self {- data_type,- weights_transposed,- pipelines: HashMap::new(),- })- }-- fn kernel_name(- &self,- bm: i32,- bn: i32,- mn_aligned: bool,- k_aligned: bool,- ) -> String {- let type_name = match self.data_type {- DataType::F16 => "half",- DataType::BF16 => "bfloat",- DataType::F32 => "float",- _ => unreachable!(),- };- let transpose_char = if self.weights_transposed {- "t"- } else {- "n"- };- let mn_str = if mn_aligned {- "true"- } else {- "false"- };- let k_str = if k_aligned {- "true"- } else {- "false"- };-- format!(- "steel_gemm_mlp_fused_{}_{}_{}_bm{}_bn{}_bk16_wm2_wn2_align_MN_{}_K_{}",- transpose_char, type_name, type_name, bm, bn, mn_str, k_str- )- }-- fn get_pipeline(- &mut self,- context: &MTLContext,- bm: i32,- bn: i32,- mn_aligned: bool,- k_aligned: bool,- activation: MlpActivationType,- ) -> Result<&MTLComputePipelineState, MTLError> {- let key = (bm, bn, mn_aligned, k_aligned, activation);- if !self.pipelines.contains_key(&key) {- let kernel_name = self.kernel_name(bm, bn, mn_aligned, k_aligned);-- // Create function constant values for MLP activation- let fcv = metal::FunctionConstantValues::new();- let activation_val = activation as u32;- fcv.set_constant_value_at_index(- &activation_val as *const u32 as *const _,- metal::MTLDataType::UInt,- 52, // MLP_ACTIVATION index+ let gemm_ref = self.gemm.as_ref();+ if let Some(gemm) = gemm_ref {+ let tile = gemm.select_tile(mtl, args);+ eprintln!(+ "[matmul] GEMM m={} k={} n={} batch={} dtype={:?} tile={}x{}x{} nax={}",+ args.batch,+ args.input_dim,+ args.output_dim,+ args.batch_count,+ self.data_type,+ tile.block_rows,+ tile.block_cols,+ tile.block_depth,+ tile.is_nax());-- let pipeline =- context.compute_pipeline_state(&kernel_name, Some(&fcv))?;- self.pipelines.insert(key, pipeline);}- Ok(self.pipelines.get(&key).unwrap())- }-- fn select_tile_size(- &self,- m: i32,- hidden_dim: i32,- k: i32,- ) -> (i32, i32) {- // Select tile size based on problem dimensions- let _ = k; // K doesn't affect tile selection directly- if m >= 64 && hidden_dim >= 64 {- (64, 64)- } else {- (32, 32)- }- }-- pub fn encode(- &mut self,- context: &MTLContext,- encoder: &ComputeCommandEncoderRef,- args: &MlpFusedGemmArguments,- ) -> Result<(), MTLError> {- let (bm, bn) =- self.select_tile_size(args.batch, args.hidden_dim, args.input_dim);- let bk = 16;-- let mn_aligned = args.batch % bm == 0 && args.hidden_dim % bn == 0;- let k_aligned = args.input_dim % bk == 0;-- let pipeline = self.get_pipeline(- context,- bm,- bn,- mn_aligned,- k_aligned,- args.activation,- )?;- encoder.set_compute_pipeline_state(pipeline);-- encoder.set_buffer(0, Some(args.input), args.input_offset);- encoder.set_buffer(1, Some(args.weights), 0);- encoder.set_buffer(2, Some(args.output), 0);-- // Set GEMM params- let tiles_m = (args.batch + bm - 1) / bm;- let tiles_n = (args.hidden_dim + bn - 1) / bn;- let gemm_k_iterations = (args.input_dim + bk - 1) / bk;-- let params = GEMMParams {- M: args.batch,- N: args.hidden_dim * 2, // Full weight width for pointer arithmetic- K: args.input_dim,- lda: args.lda,- ldb: args.ldb,- ldd: args.ldd,- tiles_n,- tiles_m,- batch_stride_a: 0,- batch_stride_b: 0,- batch_stride_d: 0,- swizzle_log: 0,- gemm_k_iterations_aligned: if k_aligned {- gemm_k_iterations- } else {- gemm_k_iterations - 1- },- batch_ndim: 0,- };-- encoder.set_bytes(- 3,- std::mem::size_of::<GEMMParams>() as u64,- ¶ms as *const GEMMParams as *const std::ffi::c_void,- );- encoder.set_bytes(- 10,- std::mem::size_of::<i32>() as u64,- &args.hidden_dim as *const i32 as *const std::ffi::c_void,- );-- let threadgroup_count = MTLSize::new(tiles_n as u64, tiles_m as u64, 1);- let threads_per_threadgroup = MTLSize::new(32, 2, 2); // WM=2, WN=2, 32 threads per simdgroup-- encoder- .dispatch_thread_groups(threadgroup_count, threads_per_threadgroup);- Ok(())}}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/mod.rsindex 81a690ef5..2ef5b3948 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/mod.rs@@ -1,17 +1,10 @@-mod arguments;-mod gather;+pub mod common;+mod gemm;mod gemv;mod kernel;-mod pipeline;-mod segmented;-#[allow(dead_code, clippy::unnecessary_operation, clippy::identity_op)]-mod shared_types;-mod splitk;-mod transpose;+mod split_k;-pub use arguments::MatmulArguments;-pub use gather::{GatherGemm, GatherMmArguments, GatherMmRhsArguments};-pub use gemv::{GemvKernel, MlpFusedGemvArguments, MlpFusedGemvKernel};-pub use kernel::{MatmulKernel, MlpFusedGemmArguments, MlpFusedGemmKernel};-pub use segmented::{SegmentedGemm, SegmentedMmArguments};-pub use splitk::SplitKGemm;+pub use common::MatmulArguments;+pub use gemv::GemvKernel;+pub use kernel::MatmulKernel;+pub use split_k::SplitKGemm;diff --git a/crates/uzu/src/backends/metal/kernel/matmul/segmented.rs b/crates/uzu/src/backends/metal/kernel/matmul/segmented.rsdeleted file mode 100644index 8c75a8fed..000000000--- a/crates/uzu/src/backends/metal/kernel/matmul/segmented.rs+++ /dev/null@@ -1,309 +0,0 @@-use std::collections::HashMap;--use metal::{- Buffer as MTLBuffer, ComputeCommandEncoderRef,- ComputePipelineState as MTLComputePipelineState, MTLSize,-};--use super::shared_types::GEMMParams;-use crate::{- DataType,- backends::metal::{DeviceClass, MTLContext, MTLError},-};--#[derive(Debug, Clone, PartialEq, Eq, Hash)]-struct SegmentedPipelineKey {- name: String,- segments_contiguous: bool,- align_m: bool,- align_n: bool,-}--#[derive(Debug, Clone)]-pub struct SegmentedMmArguments<'a> {- pub a: &'a MTLBuffer,- pub b: &'a MTLBuffer,- pub segments: &'a MTLBuffer,- pub out: &'a MTLBuffer,- pub m: i32,- pub n: i32,- pub k: i32,- pub lda: i32,- pub ldb: i32,- /// Grid Z dimension for the segmented kernel (out.size / (M*N) in the reference backend).- pub batch_size_out: i32,- pub segments_contiguous: bool,- pub transpose_a: bool,- pub transpose_b: bool,-}--pub struct SegmentedGemm {- data_type: DataType,- pipelines: HashMap<SegmentedPipelineKey, MTLComputePipelineState>,-}--impl SegmentedGemm {- pub fn new(data_type: DataType) -> Self {- Self {- data_type,- pipelines: HashMap::new(),- }- }-- fn steel_type_name(&self) -> Result<&'static str, MTLError> {- match self.data_type {- DataType::F16 => Ok("float16"),- DataType::BF16 => Ok("bfloat16"),- DataType::F32 => Ok("float32"),- other => Err(MTLError::Generic(format!(- "Unsupported data type for segmented GEMM: {other:?}"- ))),- }- }-- fn device_class_code(mtl: &MTLContext) -> char {- match mtl.architecture.device_class {- DeviceClass::Desktop => 'd',- DeviceClass::Integrated => 'g',- DeviceClass::Phone => 'p',- DeviceClass::Unknown(_) => 'g',- }- }-- fn select_gemm_tparams(- device_class: char,- data_type: DataType,- transpose_a: bool,- transpose_b: bool,- m: i32,- n: i32,- k: i32,- batch_size_out: i32,- ) -> (i32, i32, i32, i32, i32) {- // Matches the initialization and tparam selection logic used by the- // reference backend for segmented GEMM kernels.- let mut bm = 64;- let mut bn = 64;- let mut bk = 16;- let mut wm = 2;- let mut wn = 2;-- let is_float32 = matches!(data_type, DataType::F32);- let work_elements = (batch_size_out as i64) * (m as i64) * (n as i64);-- match device_class {- 'g' | 'p' => {- if !transpose_a && transpose_b {- bm = 64;- bn = 32;- bk = 32;- wm = 2;- wn = 2;- } else if !is_float32 {- bm = 64;- bn = 64;- bk = 16;- wm = 1;- wn = 2;- }- },- 'd' => {- if work_elements >= (1_i64 << 20) {- if !is_float32 {- if 2 * std::cmp::max(m, n) > k {- bm = 64;- bn = 64;- bk = 16;- wm = 1;- wn = 2;- } else if !transpose_a && transpose_b {- bm = 64;- bn = 32;- bk = 32;- wm = 2;- wn = 2;- } else {- bm = 32;- bn = 64;- bk = 16;- wm = 1;- wn = 2;- }- }- } else if !is_float32 {- if !transpose_a && transpose_b {- bm = 64;- bn = 32;- bk = 32;- wm = 2;- wn = 2;- } else {- bm = 64;- bn = 64;- bk = 16;- wm = 1;- wn = 2;- }- }- },- _ => {},- }-- (bm, bn, bk, wm, wn)- }-- fn segmented_kernel_name(- &self,- transpose_a: bool,- transpose_b: bool,- bm: i32,- bn: i32,- bk: i32,- wm: i32,- wn: i32,- ) -> Result<String, MTLError> {- let tname = format!(- "{}{}",- if transpose_a {- 't'- } else {- 'n'- },- if transpose_b {- 't'- } else {- 'n'- }- );- let dtype = self.steel_type_name()?;- Ok(format!(- "steel_segmented_mm_{tname}_{dtype}_{dtype}_bm{bm}_bn{bn}_bk{bk}_wm{wm}_wn{wn}"- ))- }-- fn get_or_compile_pipeline(- &mut self,- mtl: &MTLContext,- name: &str,- segments_contiguous: bool,- align_m: bool,- align_n: bool,- ) -> Result<&MTLComputePipelineState, MTLError> {- let key = SegmentedPipelineKey {- name: name.to_string(),- segments_contiguous,- align_m,- align_n,- };-- if !self.pipelines.contains_key(&key) {- let fcv = metal::FunctionConstantValues::new();- fcv.set_constant_value_at_index(- &segments_contiguous as *const bool as *const _,- metal::MTLDataType::Bool,- 199,- );- fcv.set_constant_value_at_index(- &align_m as *const bool as *const _,- metal::MTLDataType::Bool,- 200,- );- fcv.set_constant_value_at_index(- &align_n as *const bool as *const _,- metal::MTLDataType::Bool,- 201,- );-- let cache_key = format!(- "{name}_sc{}_am{}_an{}",- segments_contiguous as u8, align_m as u8, align_n as u8- );- let (ps, _) = mtl.compute_pipeline_state_with_reflection_cached(- &cache_key,- name,- Some(&fcv),- )?;- self.pipelines.insert(key.clone(), ps);- }-- Ok(self.pipelines.get(&key).unwrap())- }-- pub fn encode(- &mut self,- mtl: &MTLContext,- enc: &ComputeCommandEncoderRef,- args: &SegmentedMmArguments,- ) -> Result<(), MTLError> {- let device_class = Self::device_class_code(mtl);- let (bm, bn, bk, wm, wn) = Self::select_gemm_tparams(- device_class,- self.data_type,- args.transpose_a,- args.transpose_b,- args.m,- args.n,- args.k,- args.batch_size_out,- );-- let align_m = (args.m % bm) == 0;- let align_n = (args.n % bn) == 0;-- let kname = self.segmented_kernel_name(- args.transpose_a,- args.transpose_b,- bm,- bn,- bk,- wm,- wn,- )?;- let ps = self.get_or_compile_pipeline(- mtl,- &kname,- args.segments_contiguous,- align_m,- align_n,- )?;-- enc.set_compute_pipeline_state(ps);- enc.set_buffer(0, Some(args.a), 0);- enc.set_buffer(1, Some(args.b), 0);- enc.set_buffer(2, Some(args.segments), 0);- enc.set_buffer(3, Some(args.out), 0);-- let tiles_n = (args.n + bn - 1) / bn;- let tiles_m = (args.m + bm - 1) / bm;- let params = GEMMParams {- M: args.m,- N: args.n,- K: args.k,- lda: args.lda,- ldb: args.ldb,- ldd: args.n,- tiles_n,- tiles_m,- batch_stride_a: 0,- batch_stride_b: 0,- batch_stride_d: (args.m as i64) * (args.n as i64),- swizzle_log: 0,- gemm_k_iterations_aligned: 0,- batch_ndim: 0,- };- enc.set_bytes(- 4,- std::mem::size_of::<GEMMParams>() as u64,- ¶ms as *const GEMMParams as *const _,- );-- let threads_per_tg = MTLSize::new(32, wn as u64, wm as u64);- let tgs = MTLSize::new(- tiles_n as u64,- tiles_m as u64,- args.batch_size_out as u64,- );- enc.dispatch_thread_groups(tgs, threads_per_tg);- Ok(())- }-}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/splitk.rs b/crates/uzu/src/backends/metal/kernel/matmul/split_k/kernel.rssimilarity index 67%rename from crates/uzu/src/backends/metal/kernel/matmul/splitk.rsrename to crates/uzu/src/backends/metal/kernel/matmul/split_k/kernel.rsindex 8cc7e7bb2..531755fa0 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/splitk.rs+++ b/crates/uzu/src/backends/metal/kernel/matmul/split_k/kernel.rs@@ -6,39 +6,26 @@ use metal::{};use super::{- arguments::MatmulArguments,- shared_types::GEMMSpiltKParams as SplitKGEMMParams,+ pipeline_configuration::PipelineConfiguration,+ tile_configuration::{TileConfiguration, select_tile_configuration},};use crate::{DataType,- backends::metal::{MTLContext, MTLError},+ backends::metal::{+ MTLContext, MTLError,+ kernel::matmul::common::{GEMMSpiltKParams as SplitKGEMMParams, MatmulArguments},+ },};-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]-struct TileConfig {- tile_rows: i32,- tile_cols: i32,- tile_depth: i32,- warps_per_row: i32,- warps_per_col: i32,-}--#[derive(Debug, Clone, PartialEq, Eq, Hash)]-struct PartialKernelKey {- tile_config: TileConfig,- mn_aligned: bool,- k_aligned: bool,-}--pub struct SplitKGemm {+pub struct Kernel {data_type: DataType,transpose_a: bool,transpose_b: bool,- partial_pipelines: HashMap<PartialKernelKey, MTLComputePipelineState>,+ partial_pipelines: HashMap<PipelineConfiguration, MTLComputePipelineState>,accum_pipeline: Option<MTLComputePipelineState>,}-impl SplitKGemm {+impl Kernel {pub fn new(data_type: DataType,transpose_a: bool,@@ -66,8 +53,6 @@ impl SplitKGemm {}fn splitk_partial_out_name(&self) -> &'static str {- // In steel_gemm_splitk.metal, float16/bfloat16/float32 partial outputs- // are accumulated into float32."float32"}@@ -82,33 +67,25 @@ impl SplitKGemm {fn partial_kernel_name(&self,- config: &TileConfig,+ tile: &TileConfiguration,mn_aligned: bool,k_aligned: bool,) -> Result<String, MTLError> {let in_name = self.steel_type_name()?;let out_name = self.splitk_partial_out_name();let transpose_suffix = self.transpose_suffix();- let mn_tag = if mn_aligned {- "taligned"- } else {- "naligned"- };- let k_tag = if k_aligned {- "taligned"- } else {- "naligned"- };+ let mn_tag = if mn_aligned { "taligned" } else { "naligned" };+ let k_tag = if k_aligned { "taligned" } else { "naligned" };Ok(format!("steel_gemm_splitk_{}_{}_{}_bm{}_bn{}_bk{}_wm{}_wn{}_MN_{}_K_{}",transpose_suffix,in_name,out_name,- config.tile_rows,- config.tile_cols,- config.tile_depth,- config.warps_per_row,- config.warps_per_col,+ tile.tile_rows,+ tile.tile_cols,+ tile.tile_depth,+ tile.warps_per_row,+ tile.warps_per_col,mn_tag,k_tag,))@@ -126,22 +103,15 @@ impl SplitKGemm {fn get_partial_pipeline(&mut self,mtl: &MTLContext,- config: TileConfig,- mn_aligned: bool,- k_aligned: bool,+ config: &PipelineConfiguration,) -> Result<&MTLComputePipelineState, MTLError> {- let key = PartialKernelKey {- tile_config: config,- mn_aligned,- k_aligned,- };- if !self.partial_pipelines.contains_key(&key) {+ if !self.partial_pipelines.contains_key(config) {let name =- self.partial_kernel_name(&config, mn_aligned, k_aligned)?;+ self.partial_kernel_name(&config.tile, config.mn_aligned, config.k_aligned)?;let ps = mtl.compute_pipeline_state(&name, None)?;- self.partial_pipelines.insert(key.clone(), ps);+ self.partial_pipelines.insert(config.clone(), ps);}- Ok(self.partial_pipelines.get(&key).unwrap())+ Ok(self.partial_pipelines.get(config).unwrap())}fn get_accum_pipeline(@@ -156,29 +126,6 @@ impl SplitKGemm {Ok(self.accum_pipeline.as_ref().unwrap())}- fn select_tile_config(- m: i32,- n: i32,- ) -> TileConfig {- let tile_rows = if m < 40 {- 16- } else {- 32- };- let tile_cols = if n < 40 {- 16- } else {- 32- };- TileConfig {- tile_rows,- tile_cols,- tile_depth: 16,- warps_per_row: 2,- warps_per_col: 2,- }- }-fn compute_partition_count(k: i32) -> i32 {let k_tiles = k / 16;if k_tiles < 16 {@@ -220,20 +167,23 @@ impl SplitKGemm {let n = args.output_dim;let k = args.input_dim;- let tile_config = Self::select_tile_config(m, n);+ let tile = select_tile_configuration(m, n);let partition_count = Self::compute_partition_count(k);- let mn_aligned = (m % tile_config.tile_rows) == 0- && (n % tile_config.tile_cols) == 0;- let k_aligned = (k % tile_config.tile_depth) == 0;+ let mn_aligned =+ (m % tile.tile_rows) == 0 && (n % tile.tile_cols) == 0;+ let k_aligned = (k % tile.tile_depth) == 0;- let tile_count_m =- (m + tile_config.tile_rows - 1) / tile_config.tile_rows;- let tile_count_n =- (n + tile_config.tile_cols - 1) / tile_config.tile_cols;+ let config = PipelineConfiguration {+ tile,+ mn_aligned,+ k_aligned,+ };++ let tile_count_m = (m + tile.tile_rows - 1) / tile.tile_rows;+ let tile_count_n = (n + tile.tile_cols - 1) / tile.tile_cols;- let gemm_k_iterations = (k / tile_config.tile_depth) / partition_count;- let k_elements_per_partition =- gemm_k_iterations * tile_config.tile_depth;+ let gemm_k_iterations = (k / tile.tile_depth) / partition_count;+ let k_elements_per_partition = gemm_k_iterations * tile.tile_depth;let output_elements_per_partition = m * n;let accumulator_element_count =@@ -260,8 +210,7 @@ impl SplitKGemm {gemm_k_iterations_aligned: gemm_k_iterations,};- let partial_ps =- self.get_partial_pipeline(mtl, tile_config, mn_aligned, k_aligned)?;+ let partial_ps = self.get_partial_pipeline(mtl, &config)?;enc.set_compute_pipeline_state(partial_ps);enc.set_buffer(0, Some(args.a), args.a_offset);@@ -275,8 +224,8 @@ impl SplitKGemm {let threads_per_threadgroup = MTLSize::new(32,- tile_config.warps_per_col as u64,- tile_config.warps_per_row as u64,+ tile.warps_per_col as u64,+ tile.warps_per_row as u64,);let threadgroups = MTLSize::new(tile_count_n as u64,@@ -297,8 +246,7 @@ impl SplitKGemm {enc.set_bytes(3,4,- &output_elements_per_partition as *const i32- as *const std::ffi::c_void,+ &output_elements_per_partition as *const i32 as *const std::ffi::c_void,);enc.set_bytes(4,@@ -309,10 +257,7 @@ impl SplitKGemm {let accum_total_threads = MTLSize::new(n as u64, m as u64, 1);let accum_threads_per_threadgroup =MTLSize::new(16.min(n as u64), 16.min(m as u64), 1);- enc.dispatch_threads(- accum_total_threads,- accum_threads_per_threadgroup,- );+ enc.dispatch_threads(accum_total_threads, accum_threads_per_threadgroup);Ok(())}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/split_k/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/split_k/mod.rsnew file mode 100644index 000000000..54a1730c2--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/split_k/mod.rs@@ -0,0 +1,5 @@+mod kernel;+mod pipeline_configuration;+mod tile_configuration;++pub use kernel::Kernel as SplitKGemm;diff --git a/crates/uzu/src/backends/metal/kernel/matmul/split_k/pipeline_configuration.rs b/crates/uzu/src/backends/metal/kernel/matmul/split_k/pipeline_configuration.rsnew file mode 100644index 000000000..b745b347a--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/split_k/pipeline_configuration.rs@@ -0,0 +1,8 @@+use super::tile_configuration::TileConfiguration;++#[derive(Debug, Clone, PartialEq, Eq, Hash)]+pub struct PipelineConfiguration {+ pub tile: TileConfiguration,+ pub mn_aligned: bool,+ pub k_aligned: bool,+}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_splitk.h b/crates/uzu/src/backends/metal/kernel/matmul/split_k/shaders/steel_gemm_splitk.hsimilarity index 100%rename from crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_splitk.hrename to crates/uzu/src/backends/metal/kernel/matmul/split_k/shaders/steel_gemm_splitk.hdiff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_splitk.metal b/crates/uzu/src/backends/metal/kernel/matmul/split_k/shaders/steel_gemm_splitk.metalsimilarity index 98%rename from crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_splitk.metalrename to crates/uzu/src/backends/metal/kernel/matmul/split_k/shaders/steel_gemm_splitk.metalindex c573c9ee1..07cf0b530 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_splitk.metal+++ b/crates/uzu/src/backends/metal/kernel/matmul/split_k/shaders/steel_gemm_splitk.metal@@ -1,8 +1,8 @@// clang-format off-#include "../../../../common/utils.h"-#include "../gemm.h"+#include "../../common/utils.h"+#include "../../common/steel/gemm/gemm.h"#include "steel_gemm_splitk.h"#define instantiate_gemm( \diff --git a/crates/uzu/src/backends/metal/kernel/matmul/split_k/tile_configuration.rs b/crates/uzu/src/backends/metal/kernel/matmul/split_k/tile_configuration.rsnew file mode 100644index 000000000..d2abe25d8--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/matmul/split_k/tile_configuration.rs@@ -0,0 +1,20 @@+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+pub struct TileConfiguration {+ pub tile_rows: i32,+ pub tile_cols: i32,+ pub tile_depth: i32,+ pub warps_per_row: i32,+ pub warps_per_col: i32,+}++pub fn select_tile_configuration(m: i32, n: i32) -> TileConfiguration {+ let tile_rows = if m < 40 { 16 } else { 32 };+ let tile_cols = if n < 40 { 16 } else { 32 };+ TileConfiguration {+ tile_rows,+ tile_cols,+ tile_depth: 16,+ warps_per_row: 2,+ warps_per_col: 2,+ }+}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_gather.h b/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_gather.hdeleted file mode 100644index b9c6d3c83..000000000--- a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_gather.h+++ /dev/null@@ -1,459 +0,0 @@---using namespace steel;--constant bool has_batch [[function_constant(10)]];-constant bool align_M [[function_constant(200)]];-constant bool align_N [[function_constant(201)]];-constant bool align_K [[function_constant(202)]];--template <- typename T,- int BM,- int BN,- int BK,- int WM,- int WN,- bool transpose_a,- bool transpose_b,- typename AccumType = float>-[[kernel, max_total_threads_per_threadgroup(WM* WN * 32)]] void gather_mm_rhs(- const device T* A [[buffer(0)]],- const device T* B [[buffer(1)]],- const device uint32_t* rhs_indices [[buffer(2)]],- device T* C [[buffer(3)]],- const constant GEMMParams* params [[buffer(4)]],- uint simd_lane_id [[thread_index_in_simdgroup]],- uint simd_group_id [[simdgroup_index_in_threadgroup]],- uint3 tid [[threadgroup_position_in_grid]]) {- using gemm_kernel = GEMMKernel<- T,- T,- BM,- BN,- BK,- WM,- WN,- transpose_a,- transpose_b,- true,- true,- AccumType>;-- using loader_a_t = typename gemm_kernel::loader_a_t;- using loader_b_t = typename gemm_kernel::loader_b_t;- using mma_t = typename gemm_kernel::mma_t;-- if (params->tiles_n <= static_cast<int>(tid.x) ||- params->tiles_m <= static_cast<int>(tid.y)) {- return;- }-- // Prepare threadgroup memory- threadgroup T As[gemm_kernel::tgp_mem_size_a];- threadgroup T Bs[gemm_kernel::tgp_mem_size_b];-- // Find the block in A, B, C- const int c_row = tid.y * BM;- const int c_col = tid.x * BN;- const size_t c_row_long = size_t(c_row);- const size_t c_col_long = size_t(c_col);-- // Prepare threadgroup bounds- const short tgp_bm = align_M ? BM : short(min(BM, params->M - c_row));- const short tgp_bn = align_N ? BN : short(min(BN, params->N - c_col));-- A += transpose_a ? c_row_long : c_row_long * params->lda;- B += transpose_b ? c_col_long * params->ldb : c_col_long;- C += c_row_long * params->ldd + c_col_long;-- // Do as many matmuls as necessary- uint32_t index;- short offset;- uint32_t index_next = rhs_indices[c_row];- short offset_next = 0;- int n = 0;- while (n < tgp_bm) {- n++;- offset = offset_next;- index = index_next;- offset_next = tgp_bm;- for (; n < tgp_bm; n++) {- if (rhs_indices[c_row + n] != index) {- offset_next = n;- index_next = rhs_indices[c_row + n];- break;- }- }- threadgroup_barrier(mem_flags::mem_none);-- // Prepare threadgroup mma operation- thread mma_t mma_op(simd_group_id, simd_lane_id);-- // Prepare threadgroup loading operations- thread loader_a_t loader_a(A, params->lda, As, simd_group_id, simd_lane_id);- thread loader_b_t loader_b(- B + index * params->batch_stride_b,- params->ldb,- Bs,- simd_group_id,- simd_lane_id);-- // Prepare iterations- const int gemm_k_iterations = params->gemm_k_iterations_aligned;-- // Do unaligned K iterations first- if (!align_K) {- const int k_last = params->gemm_k_iterations_aligned * BK;- const int k_remain = params->K - k_last;- const size_t k_jump_a =- transpose_a ? params->lda * size_t(k_last) : size_t(k_last);- const size_t k_jump_b =- transpose_b ? size_t(k_last) : params->ldb * size_t(k_last);-- // Move loader source ahead to end- loader_a.src += k_jump_a;- loader_b.src += k_jump_b;-- // Load tile- const short2 tile_dims_A =- transpose_a ? short2(tgp_bm, k_remain) : short2(k_remain, tgp_bm);- const short2 tile_dims_B =- transpose_b ? short2(k_remain, tgp_bn) : short2(tgp_bn, k_remain);-- loader_a.load_safe(tile_dims_A);- loader_b.load_safe(tile_dims_B);-- threadgroup_barrier(mem_flags::mem_threadgroup);-- // Do matmul- mma_op.mma(As, Bs);-- // Reset source back to start- loader_a.src -= k_jump_a;- loader_b.src -= k_jump_b;- }-- // Matrix level aligned never check- if (align_M && align_N) {- for (int k = 0; k < gemm_k_iterations; k++) {- threadgroup_barrier(mem_flags::mem_threadgroup);-- // Load elements into threadgroup- loader_a.load_unsafe();- loader_b.load_unsafe();-- threadgroup_barrier(mem_flags::mem_threadgroup);-- // Multiply and accumulate threadgroup elements- mma_op.mma(As, Bs);-- // Prepare for next iteration- loader_a.next();- loader_b.next();- }-- // Store results to device memory- if (offset_next - offset == BM) {- mma_op.store_result(C, params->ldd);- } else {- mma_op.store_result_slice(- C, params->ldd, short2(0, offset), short2(BN, offset_next));- }- } else {- const short lbk = 0;-- // Tile aligned don't check- if ((align_M || tgp_bm == BM) && (align_N || tgp_bn == BN)) {- gemm_kernel::gemm_loop(- As,- Bs,- gemm_k_iterations,- loader_a,- loader_b,- mma_op,- tgp_bm,- tgp_bn,- lbk,- LoopAlignment<true, true, true>{});- if (offset_next - offset == BM) {- mma_op.store_result(C, params->ldd);- } else {- mma_op.store_result_slice(- C, params->ldd, short2(0, offset), short2(BN, offset_next));- }- }-- // Tile partially aligned check rows- else if (align_N || tgp_bn == BN) {- gemm_kernel::gemm_loop(- As,- Bs,- gemm_k_iterations,- loader_a,- loader_b,- mma_op,- tgp_bm,- tgp_bn,- lbk,- LoopAlignment<false, true, true>{});- mma_op.store_result_slice(- C, params->ldd, short2(0, offset), short2(BN, offset_next));- }-- // Tile partially aligned check cols- else if (align_M || tgp_bm == BM) {- gemm_kernel::gemm_loop(- As,- Bs,- gemm_k_iterations,- loader_a,- loader_b,- mma_op,- tgp_bm,- tgp_bn,- lbk,- LoopAlignment<true, false, true>{});- mma_op.store_result_slice(- C, params->ldd, short2(0, offset), short2(tgp_bn, offset_next));- }-- // Nothing aligned so check both rows and cols- else {- gemm_kernel::gemm_loop(- As,- Bs,- gemm_k_iterations,- loader_a,- loader_b,- mma_op,- tgp_bm,- tgp_bn,- lbk,- LoopAlignment<false, false, true>{});- mma_op.store_result_slice(- C, params->ldd, short2(0, offset), short2(tgp_bn, offset_next));- }- }- }-}--template <- typename T,- int BM,- int BN,- int BK,- int WM,- int WN,- bool transpose_a,- bool transpose_b,- typename AccumType = float>-[[kernel, max_total_threads_per_threadgroup(WM* WN * 32)]] void gather_mm(- const device T* A [[buffer(0)]],- const device T* B [[buffer(1)]],- const device uint32_t* lhs_indices [[buffer(2)]],- const device uint32_t* rhs_indices [[buffer(3)]],- device T* C [[buffer(4)]],- const constant GEMMParams* params [[buffer(5)]],- const constant int* indices_shape [[buffer(6)]],- const constant int64_t* lhs_strides [[buffer(7)]],- const constant int64_t* rhs_strides [[buffer(8)]],- const constant int& batch_ndim_a [[buffer(9)]],- const constant int* batch_shape_a [[buffer(10)]],- const constant int64_t* batch_strides_a [[buffer(11)]],- const constant int& batch_ndim_b [[buffer(12)]],- const constant int* batch_shape_b [[buffer(13)]],- const constant int64_t* batch_strides_b [[buffer(14)]],- uint simd_lane_id [[thread_index_in_simdgroup]],- uint simd_group_id [[simdgroup_index_in_threadgroup]],- uint3 tid [[threadgroup_position_in_grid]]) {- using gemm_kernel = GEMMKernel<- T,- T,- BM,- BN,- BK,- WM,- WN,- transpose_a,- transpose_b,- true,- true,- AccumType>;-- using loader_a_t = typename gemm_kernel::loader_a_t;- using loader_b_t = typename gemm_kernel::loader_b_t;- using mma_t = typename gemm_kernel::mma_t;-- if (params->tiles_n <= static_cast<int>(tid.x) ||- params->tiles_m <= static_cast<int>(tid.y)) {- return;- }-- // Move A and B to the locations pointed by lhs_indices and rhs_indices.- uint32_t indx_A, indx_B;- if (has_batch) {- ulong2 indices_offsets = elem_to_loc_broadcast(- tid.z, indices_shape, lhs_strides, rhs_strides, params->batch_ndim);- indx_A = lhs_indices[indices_offsets.x];- indx_B = rhs_indices[indices_offsets.y];- } else {- indx_A = lhs_indices[params->batch_stride_a * tid.z];- indx_B = rhs_indices[params->batch_stride_b * tid.z];- }- A += elem_to_loc(indx_A, batch_shape_a, batch_strides_a, batch_ndim_a);- B += elem_to_loc(indx_B, batch_shape_b, batch_strides_b, batch_ndim_b);- C += params->batch_stride_d * tid.z;-- // Prepare threadgroup memory- threadgroup T As[gemm_kernel::tgp_mem_size_a];- threadgroup T Bs[gemm_kernel::tgp_mem_size_b];-- // Just make sure everybody's finished with the indexing math above.- threadgroup_barrier(mem_flags::mem_none);-- // Find block in A, B, C- const int c_row = tid.y * BM;- const int c_col = tid.x * BN;- const size_t c_row_long = size_t(c_row);- const size_t c_col_long = size_t(c_col);-- A += transpose_a ? c_row_long : c_row_long * params->lda;- B += transpose_b ? c_col_long * params->ldb : c_col_long;- C += c_row_long * params->ldd + c_col_long;-- // Prepare threadgroup mma operation- thread mma_t mma_op(simd_group_id, simd_lane_id);-- // Prepare threadgroup loading operations- thread loader_a_t loader_a(A, params->lda, As, simd_group_id, simd_lane_id);- thread loader_b_t loader_b(B, params->ldb, Bs, simd_group_id, simd_lane_id);-- // Prepare threadgroup bounds- const short tgp_bm = align_M ? BM : short(min(BM, params->M - c_row));- const short tgp_bn = align_N ? BN : short(min(BN, params->N - c_col));-- // Prepare iterations- int gemm_k_iterations = params->gemm_k_iterations_aligned;-- // Do unaligned K iterations first- if (!align_K) {- const int k_last = params->gemm_k_iterations_aligned * BK;- const int k_remain = params->K - k_last;- const size_t k_jump_a =- transpose_a ? params->lda * size_t(k_last) : size_t(k_last);- const size_t k_jump_b =- transpose_b ? size_t(k_last) : params->ldb * size_t(k_last);-- // Move loader source ahead to end- loader_a.src += k_jump_a;- loader_b.src += k_jump_b;-- // Load tile- const short2 tile_dims_A =- transpose_a ? short2(tgp_bm, k_remain) : short2(k_remain, tgp_bm);- const short2 tile_dims_B =- transpose_b ? short2(k_remain, tgp_bn) : short2(tgp_bn, k_remain);-- loader_a.load_safe(tile_dims_A);- loader_b.load_safe(tile_dims_B);-- threadgroup_barrier(mem_flags::mem_threadgroup);-- // Do matmul- mma_op.mma(As, Bs);-- // Reset source back to start- loader_a.src -= k_jump_a;- loader_b.src -= k_jump_b;- }-- // Matrix level aligned never check- if (align_M && align_N) {- for (int k = 0; k < gemm_k_iterations; k++) {- threadgroup_barrier(mem_flags::mem_threadgroup);-- // Load elements into threadgroup- loader_a.load_unsafe();- loader_b.load_unsafe();-- threadgroup_barrier(mem_flags::mem_threadgroup);-- // Multiply and accumulate threadgroup elements- mma_op.mma(As, Bs);-- // Prepare for next iteration- loader_a.next();- loader_b.next();- }-- // Store results to device memory- mma_op.store_result(C, params->ldd);- } else {- const short lbk = 0;-- // Tile aligned don't check- if ((align_M || tgp_bm == BM) && (align_N || tgp_bn == BN)) {- gemm_kernel::gemm_loop(- As,- Bs,- gemm_k_iterations,- loader_a,- loader_b,- mma_op,- tgp_bm,- tgp_bn,- lbk,- LoopAlignment<true, true, true>{});- mma_op.store_result(C, params->ldd);- }-- // Tile partially aligned check rows- else if (align_N || tgp_bn == BN) {- gemm_kernel::gemm_loop(- As,- Bs,- gemm_k_iterations,- loader_a,- loader_b,- mma_op,- tgp_bm,- tgp_bn,- lbk,- LoopAlignment<false, true, true>{});- mma_op.store_result_safe(C, params->ldd, short2(tgp_bn, tgp_bm));- }-- // Tile partially aligned check cols- else if (align_M || tgp_bm == BM) {- gemm_kernel::gemm_loop(- As,- Bs,- gemm_k_iterations,- loader_a,- loader_b,- mma_op,- tgp_bm,- tgp_bn,- lbk,- LoopAlignment<true, false, true>{});- mma_op.store_result_safe(C, params->ldd, short2(tgp_bn, tgp_bm));- }-- // Nothing aligned so check both rows and cols- else {- gemm_kernel::gemm_loop(- As,- Bs,- gemm_k_iterations,- loader_a,- loader_b,- mma_op,- tgp_bm,- tgp_bn,- lbk,- LoopAlignment<false, false, true>{});- mma_op.store_result_safe(C, params->ldd, short2(tgp_bn, tgp_bm));- }- }-}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_gather.metal b/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_gather.metaldeleted file mode 100644index 89372091c..000000000--- a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_gather.metal+++ /dev/null@@ -1,59 +0,0 @@---// clang-format off-#include "../../../../common/utils.h"-#include "../gemm.h"-#include "steel_gemm_gather.h"--#define instantiate_gather_mm_rhs(tname, trans_a, trans_b, iname, itype, oname, otype, bm, bn, bk, wm, wn) \- instantiate_kernel( \- "steel_gather_mm_rhs_" #tname "_" #iname "_" #oname "_bm" #bm "_bn" #bn \- "_bk" #bk "_wm" #wm "_wn" #wn, \- gather_mm_rhs, \- itype, \- bm, \- bn, \- bk, \- wm, \- wn, \- trans_a, \- trans_b, \- float)--#define instantiate_gather_mm(tname, trans_a, trans_b, iname, itype, oname, otype, bm, bn, bk, wm, wn) \- instantiate_kernel( \- "steel_gather_mm_" #tname "_" #iname "_" #oname "_bm" #bm "_bn" #bn \- "_bk" #bk "_wm" #wm "_wn" #wn, \- gather_mm, \- itype, \- bm, \- bn, \- bk, \- wm, \- wn, \- trans_a, \- trans_b, \- float)--#define instantiate_gather_mm_rhs_transpose_helper(iname, itype, oname, otype, bm, bn, bk, wm, wn) \- instantiate_gather_mm_rhs(nn, false, false, iname, itype, oname, otype, bm, bn, bk, wm, wn) \- instantiate_gather_mm_rhs(nt, false, true, iname, itype, oname, otype, bm, bn, bk, wm, wn)--#define instantiate_gather_mm_transpose_helper(iname, itype, oname, otype, bm, bn, bk, wm, wn) \- instantiate_gather_mm(nn, false, false, iname, itype, oname, otype, bm, bn, bk, wm, wn) \- instantiate_gather_mm(nt, false, true , iname, itype, oname, otype, bm, bn, bk, wm, wn) \- instantiate_gather_mm(tn, true , false, iname, itype, oname, otype, bm, bn, bk, wm, wn) \- instantiate_gather_mm(tt, true , true , iname, itype, oname, otype, bm, bn, bk, wm, wn)--#define instantiate_gather_mm_shapes_helper(iname, itype, oname, otype) \- instantiate_gather_mm_rhs_transpose_helper(iname, itype, oname, otype, 16, 64, 16, 1, 2) \- instantiate_gather_mm_transpose_helper(iname, itype, oname, otype, 64, 64, 16, 2, 2) \- instantiate_gather_mm_transpose_helper(iname, itype, oname, otype, 64, 64, 16, 1, 2) \- instantiate_gather_mm_transpose_helper(iname, itype, oname, otype, 64, 32, 32, 2, 2) \- instantiate_gather_mm_transpose_helper(iname, itype, oname, otype, 32, 64, 16, 1, 2) \- instantiate_gather_mm_transpose_helper(iname, itype, oname, otype, 32, 32, 16, 2, 2)-// clang-format on--instantiate_gather_mm_shapes_helper(float16, half, float16, half);-instantiate_gather_mm_shapes_helper(bfloat16, bfloat16_t, bfloat16, bfloat16_t);-instantiate_gather_mm_shapes_helper(float32, float, float32, float);diff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_gather_nax.h b/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_gather_nax.hdeleted file mode 100644index 05f567356..000000000--- a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_gather_nax.h+++ /dev/null@@ -1,132 +0,0 @@---using namespace steel;--constant bool align_M [[function_constant(200)]];-constant bool align_N [[function_constant(201)]];-constant bool align_K [[function_constant(202)]];--template <- typename T,- int BM,- int BN,- int BK,- int WM,- int WN,- bool transpose_a,- bool transpose_b,- typename AccumType = float>-[[kernel, max_total_threads_per_threadgroup(WM* WN * 32)]] void-gather_mm_rhs_nax(- const device T* A [[buffer(0)]],- const device T* B [[buffer(1)]],- const device uint32_t* rhs_indices [[buffer(2)]],- device T* C [[buffer(3)]],- const constant GEMMParams* params [[buffer(4)]],- uint simd_group_id [[simdgroup_index_in_threadgroup]],- uint3 tid [[threadgroup_position_in_grid]]) {- constexpr short UM = 16;- constexpr short UN = 32;- constexpr short UK = 16;- constexpr short SM = BM / WM;- constexpr short SN = BN / WN;- constexpr short SK = 32;- constexpr short TM = SM / UM;- constexpr short TN = SN / UN;-- if (params->tiles_n <= static_cast<int>(tid.x) ||- params->tiles_m <= static_cast<int>(tid.y)) {- return;- }-- // Find the block in A, B, C- const int c_row = tid.y * BM;- const int c_col = tid.x * BN;- const size_t c_row_long = size_t(c_row);- const size_t c_col_long = size_t(c_col);-- A += transpose_a ? c_row_long : c_row_long * params->lda;- B += transpose_b ? c_col_long * params->ldb : c_col_long;- C += c_row_long * params->ldd + c_col_long;- rhs_indices += c_row;-- const short tm = SM * (simd_group_id / WN);- const short tn = SN * (simd_group_id % WN);-- const short sgp_sm = align_M ? SM : min(SM, short(params->M - (c_row + tm)));- const bool is_unaligned_sm = align_M ? false : (sgp_sm != SM);-- const short sgp_sn = align_N ? SN : min(SN, short(params->N - (c_col + tn)));- const bool is_unaligned_sn = align_N ? false : (sgp_sn != SN);-- A += transpose_a ? tm : (tm * params->lda);- B += transpose_b ? (tn * params->ldb) : tn;- C += tm * params->ldd + tn;- rhs_indices += tm;-- // Do as many matmuls as necessary- uint32_t index;- short offset;- uint32_t index_next = rhs_indices[0];- short offset_next = 0;- int n = 0;- while (n < sgp_sm) {- n++;- offset = offset_next;- index = index_next;- offset_next = sgp_sm;- for (; n < sgp_sm; n++) {- if (rhs_indices[n] != index) {- offset_next = n;- index_next = rhs_indices[n];- break;- }- }- threadgroup_barrier(mem_flags::mem_none);-- using DSubTile = NAXSubTile<AccumType, UM, UN>;- NAXTile<AccumType, TM, TN, DSubTile> Ctile;-- dispatch_bool(align_K, [&](auto kAlignedK) {- dispatch_bool(align_M || !is_unaligned_sm, [&](auto kAlignedM) {- dispatch_bool(align_N || !is_unaligned_sn, [&](auto kAlignedN) {- auto do_gemm = gemm_loop<- T,- SM,- SN,- SK,- BK,- transpose_a,- transpose_b,- kAlignedM.value,- kAlignedN.value,- kAlignedK.value,- UM,- UN,- UK,- AccumType>;- Ctile = do_gemm(- A, B + index * params->batch_stride_b, params, sgp_sm, sgp_sn);-- if constexpr (kAlignedN.value) {- if (offset_next - offset == SM) {- Ctile.store(C, int(params->ldd));- } else {- Ctile.store_slice(- C,- int(params->ldd),- short2(0, offset),- short2(SN, offset_next));- }- } else {- Ctile.store_slice(- C,- int(params->ldd),- short2(0, offset),- short2(sgp_sn, offset_next));- }- });- });- });- }-}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_gather_nax.metal b/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_gather_nax.metaldeleted file mode 100644index 47585a9c6..000000000--- a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_gather_nax.metal+++ /dev/null@@ -1,37 +0,0 @@---#include <metal_stdlib>--#include "../../../../common/utils.h"--#include "../gemm_nax.h"-#include "steel_gemm_gather_nax.h"--// clang-format off-#define instantiate_gather_mm_rhs(tname, trans_a, trans_b, iname, itype, oname, otype, bm, bn, bk, wm, wn) \- instantiate_kernel( \- "steel_gather_mm_rhs_nax_" #tname "_" #iname "_" #oname "_bm" #bm "_bn" #bn \- "_bk" #bk "_wm" #wm "_wn" #wn, \- gather_mm_rhs_nax, \- itype, \- bm, \- bn, \- bk, \- wm, \- wn, \- trans_a, \- trans_b, \- float)--#define instantiate_gather_mm_rhs_transpose_helper(iname, itype, oname, otype, bm, bn, bk, wm, wn) \- instantiate_gather_mm_rhs(nn, false, false, iname, itype, oname, otype, bm, bn, bk, wm, wn) \- instantiate_gather_mm_rhs(nt, false, true, iname, itype, oname, otype, bm, bn, bk, wm, wn)--#define instantiate_gather_mm_shapes_helper(iname, itype, oname, otype) \- instantiate_gather_mm_rhs_transpose_helper(iname, itype, oname, otype, 16, 128, 128, 1, 4) \- instantiate_gather_mm_rhs_transpose_helper(iname, itype, oname, otype, 32, 128, 128, 1, 4) \- instantiate_gather_mm_rhs_transpose_helper(iname, itype, oname, otype, 64, 128, 128, 2, 4)-// clang-format on--instantiate_gather_mm_shapes_helper(float16, half, float16, half);-instantiate_gather_mm_shapes_helper(bfloat16, bfloat, bfloat16, bfloat);diff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_segmented.h b/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_segmented.hdeleted file mode 100644index e7238b74e..000000000--- a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_segmented.h+++ /dev/null@@ -1,266 +0,0 @@---using namespace steel;--constant bool segments_contiguous [[function_constant(199)]];-constant bool align_M [[function_constant(200)]];-constant bool align_N [[function_constant(201)]];--template <- typename T,- int BM,- int BN,- int BK,- int WM,- int WN,- bool transpose_a,- bool transpose_b,- typename AccumType = float>-[[kernel, max_total_threads_per_threadgroup(WM* WN * 32)]] void segmented_mm(- const device T* A [[buffer(0)]],- const device T* B [[buffer(1)]],- const device uint32_t* segments [[buffer(2)]],- device T* C [[buffer(3)]],- const constant GEMMParams* params [[buffer(4)]],- uint simd_lane_id [[thread_index_in_simdgroup]],- uint simd_group_id [[simdgroup_index_in_threadgroup]],- uint3 tid [[threadgroup_position_in_grid]]) {- using gemm_kernel = GEMMKernel<- T,- T,- BM,- BN,- BK,- WM,- WN,- transpose_a,- transpose_b,- true,- true,- AccumType>;-- using loader_a_t = typename gemm_kernel::loader_a_t;- using loader_b_t = typename gemm_kernel::loader_b_t;- using mma_t = typename gemm_kernel::mma_t;-- if (params->tiles_n <= static_cast<int>(tid.x) ||- params->tiles_m <= static_cast<int>(tid.y)) {- return;- }-- // Prepare threadgroup memory- threadgroup T As[gemm_kernel::tgp_mem_size_a];- threadgroup T Bs[gemm_kernel::tgp_mem_size_b];-- // Find the block in A, B, C- const int c_row = tid.y * BM;- const int c_col = tid.x * BN;- const size_t c_row_long = size_t(c_row);- const size_t c_col_long = size_t(c_col);-- // Prepare threadgroup bounds- const short tgp_bm = align_M ? BM : short(min(BM, params->M - c_row));- const short tgp_bn = align_N ? BN : short(min(BN, params->N - c_col));-- // Move the pointers to the output tile- A += transpose_a ? c_row_long : c_row_long * params->lda;- B += transpose_b ? c_col_long * params->ldb : c_col_long;- C += c_row_long * params->ldd + c_col_long;-- // Move the pointers to the start of the segment- uint32_t k_start, k_end;- if (segments_contiguous) {- k_start = segments[2 * tid.z];- k_end = segments[2 * tid.z + 1];- } else {- // We accept either contiguous (above) or weird strides where the beginning- // of the next one is the previous one. Basically the last two strides are- // both 1!- k_start = segments[tid.z];- k_end = segments[tid.z + 1];- }- A += transpose_a ? k_start * params->lda : k_start;- B += transpose_b ? k_start : k_start * params->ldb;- C += tid.z * params->batch_stride_d;-- // Prepare threadgroup mma operation- thread mma_t mma_op(simd_group_id, simd_lane_id);-- // Prepare threadgroup loading operations- thread loader_a_t loader_a(A, params->lda, As, simd_group_id, simd_lane_id);- thread loader_b_t loader_b(B, params->ldb, Bs, simd_group_id, simd_lane_id);-- // Matrix level alignment so only check K- if (align_M && align_N) {- uint32_t k = k_start + BK;- for (; k <= k_end; k += BK) {- threadgroup_barrier(mem_flags::mem_threadgroup);-- // Load elements into threadgroup- loader_a.load_unsafe();- loader_b.load_unsafe();-- threadgroup_barrier(mem_flags::mem_threadgroup);-- // Multiply and accumulate threadgroup elements- mma_op.mma(As, Bs);-- // Prepare for next iteration- loader_a.next();- loader_b.next();- }- short k_remain = BK - short(k - k_end);- const short2 tile_dims_A =- transpose_a ? short2(tgp_bm, k_remain) : short2(k_remain, tgp_bm);- const short2 tile_dims_B =- transpose_b ? short2(k_remain, tgp_bn) : short2(tgp_bn, k_remain);- if (k_remain > 0) {- threadgroup_barrier(mem_flags::mem_threadgroup);- loader_a.load_safe(tile_dims_A);- loader_b.load_safe(tile_dims_B);- threadgroup_barrier(mem_flags::mem_threadgroup);- mma_op.mma(As, Bs);- }- mma_op.store_result(C, params->ldd);- } else {- // Tile aligned do the same as above- if ((align_M || tgp_bm == BM) && (align_N || tgp_bn == BN)) {- uint32_t k = k_start + BK;- for (; k <= k_end; k += BK) {- threadgroup_barrier(mem_flags::mem_threadgroup);-- // Load elements into threadgroup- loader_a.load_unsafe();- loader_b.load_unsafe();-- threadgroup_barrier(mem_flags::mem_threadgroup);-- // Multiply and accumulate threadgroup elements- mma_op.mma(As, Bs);-- // Prepare for next iteration- loader_a.next();- loader_b.next();- }- short k_remain = BK - short(k - k_end);- const short2 tile_dims_A =- transpose_a ? short2(tgp_bm, k_remain) : short2(k_remain, tgp_bm);- const short2 tile_dims_B =- transpose_b ? short2(k_remain, tgp_bn) : short2(tgp_bn, k_remain);- if (k_remain > 0) {- threadgroup_barrier(mem_flags::mem_threadgroup);- loader_a.load_safe(tile_dims_A);- loader_b.load_safe(tile_dims_B);- threadgroup_barrier(mem_flags::mem_threadgroup);- mma_op.mma(As, Bs);- }- mma_op.store_result(C, params->ldd);- }-- // Tile partially aligned check rows- else if (align_N || tgp_bn == BN) {- uint32_t k = k_start + BK;- for (; k <= k_end; k += BK) {- threadgroup_barrier(mem_flags::mem_threadgroup);-- // Load elements into threadgroup- loader_a.load_safe(- transpose_a ? short2(tgp_bm, BK) : short2(BK, tgp_bm));- loader_b.load_unsafe();-- threadgroup_barrier(mem_flags::mem_threadgroup);-- // Multiply and accumulate threadgroup elements- mma_op.mma(As, Bs);-- // Prepare for next iteration- loader_a.next();- loader_b.next();- }- short k_remain = BK - short(k - k_end);- const short2 tile_dims_A =- transpose_a ? short2(tgp_bm, k_remain) : short2(k_remain, tgp_bm);- const short2 tile_dims_B =- transpose_b ? short2(k_remain, tgp_bn) : short2(tgp_bn, k_remain);- if (k_remain > 0) {- threadgroup_barrier(mem_flags::mem_threadgroup);- loader_a.load_safe(tile_dims_A);- loader_b.load_safe(tile_dims_B);- threadgroup_barrier(mem_flags::mem_threadgroup);- mma_op.mma(As, Bs);- }- mma_op.store_result_safe(C, params->ldd, short2(tgp_bn, tgp_bm));- }-- // Tile partially aligned check cols- else if (align_M || tgp_bm == BM) {- uint32_t k = k_start + BK;- for (; k <= k_end; k += BK) {- threadgroup_barrier(mem_flags::mem_threadgroup);-- // Load elements into threadgroup- loader_a.load_unsafe();- loader_b.load_safe(- transpose_b ? short2(BK, tgp_bn) : short2(tgp_bn, BK));-- threadgroup_barrier(mem_flags::mem_threadgroup);-- // Multiply and accumulate threadgroup elements- mma_op.mma(As, Bs);-- // Prepare for next iteration- loader_a.next();- loader_b.next();- }- short k_remain = BK - short(k - k_end);- const short2 tile_dims_A =- transpose_a ? short2(tgp_bm, k_remain) : short2(k_remain, tgp_bm);- const short2 tile_dims_B =- transpose_b ? short2(k_remain, tgp_bn) : short2(tgp_bn, k_remain);- if (k_remain > 0) {- threadgroup_barrier(mem_flags::mem_threadgroup);- loader_a.load_safe(tile_dims_A);- loader_b.load_safe(tile_dims_B);- threadgroup_barrier(mem_flags::mem_threadgroup);- mma_op.mma(As, Bs);- }- mma_op.store_result_safe(C, params->ldd, short2(tgp_bn, tgp_bm));- }-- // Nothing aligned so check both rows and cols- else {- uint32_t k = k_start + BK;- for (; k <= k_end; k += BK) {- threadgroup_barrier(mem_flags::mem_threadgroup);-- // Load elements into threadgroup- loader_a.load_safe(- transpose_a ? short2(tgp_bm, BK) : short2(BK, tgp_bm));- loader_b.load_safe(- transpose_b ? short2(BK, tgp_bn) : short2(tgp_bn, BK));-- threadgroup_barrier(mem_flags::mem_threadgroup);-- // Multiply and accumulate threadgroup elements- mma_op.mma(As, Bs);-- // Prepare for next iteration- loader_a.next();- loader_b.next();- }- short k_remain = BK - short(k - k_end);- const short2 tile_dims_A =- transpose_a ? short2(tgp_bm, k_remain) : short2(k_remain, tgp_bm);- const short2 tile_dims_B =- transpose_b ? short2(k_remain, tgp_bn) : short2(tgp_bn, k_remain);- if (k_remain > 0) {- threadgroup_barrier(mem_flags::mem_threadgroup);- loader_a.load_safe(tile_dims_A);- loader_b.load_safe(tile_dims_B);- threadgroup_barrier(mem_flags::mem_threadgroup);- mma_op.mma(As, Bs);- }- mma_op.store_result_safe(C, params->ldd, short2(tgp_bn, tgp_bm));- }- }-}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_segmented.metal b/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_segmented.metaldeleted file mode 100644index fd05e307e..000000000--- a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_segmented.metal+++ /dev/null@@ -1,43 +0,0 @@---// clang-format off-#include "../../../../common/utils.h"-#include "../gemm.h"-#include "steel_gemm_segmented.h"--#define instantiate_segmented_mm(tname, trans_a, trans_b, iname, itype, oname, otype, bm, bn, bk, wm, wn) \- instantiate_kernel( \- "steel_segmented_mm_" #tname "_" #iname "_" #oname "_bm" #bm "_bn" #bn \- "_bk" #bk "_wm" #wm "_wn" #wn, \- segmented_mm, \- itype, \- bm, \- bn, \- bk, \- wm, \- wn, \- trans_a, \- trans_b, \- float)--#define instantiate_segmented_mm_transpose_helper(iname, itype, oname, otype, bm, bn, bk, wm, wn) \- instantiate_segmented_mm(nn, false, false, iname, itype, oname, otype, bm, bn, bk, wm, wn) \- instantiate_segmented_mm(nt, false, true , iname, itype, oname, otype, bm, bn, bk, wm, wn) \- instantiate_segmented_mm(tn, true , false, iname, itype, oname, otype, bm, bn, bk, wm, wn) \- instantiate_segmented_mm(tt, true , true , iname, itype, oname, otype, bm, bn, bk, wm, wn)--#define instantiate_segmented_mm_shapes_helper(iname, itype, oname, otype) \- instantiate_segmented_mm_transpose_helper(iname, itype, oname, otype, 64, 64, 16, 2, 2) \- instantiate_segmented_mm_transpose_helper(iname, itype, oname, otype, 64, 64, 16, 1, 2) \- instantiate_segmented_mm_transpose_helper(iname, itype, oname, otype, 64, 32, 32, 2, 2) \- instantiate_segmented_mm_transpose_helper(iname, itype, oname, otype, 32, 64, 16, 1, 2) \- instantiate_segmented_mm_transpose_helper(iname, itype, oname, otype, 32, 32, 16, 2, 2)-// clang-format on--instantiate_segmented_mm_shapes_helper(float16, half, float16, half);-instantiate_segmented_mm_shapes_helper(- bfloat16,- bfloat16_t,- bfloat16,- bfloat16_t);-instantiate_segmented_mm_shapes_helper(float32, float, float32, float);diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/mod.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/mod.rsnew file mode 100644index 000000000..05b66ce46--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/mod.rs@@ -0,0 +1,237 @@+use std::collections::HashMap;++use metal::{+ Buffer as MTLBuffer, ComputeCommandEncoderRef,+ ComputePipelineState as MTLComputePipelineState, MTLSize,+};++use super::split_k;+use crate::{+ DataType,+ backends::metal::{+ MTLContext, MTLError,+ kernel::{matmul::common::GEMMParams, mlp::MlpActivationType},+ },+};++#[derive(Debug)]+pub struct Arguments<'a> {+ pub input: &'a MTLBuffer,+ pub input_offset: u64,+ pub weights: &'a MTLBuffer,+ pub output: &'a MTLBuffer,+ pub batch: i32,+ pub input_dim: i32,+ pub hidden_dim: i32,+ pub lda: i32,+ pub ldb: i32,+ pub ldd: i32,+ pub activation: MlpActivationType,+}++pub struct Kernel {+ data_type: DataType,+ weights_transposed: bool,+ pipelines: HashMap<+ (i32, i32, bool, bool, MlpActivationType),+ MTLComputePipelineState,+ >,+ splitk_kernel: Option<split_k::Kernel>,+}++impl Kernel {+ pub fn new(+ data_type: DataType,+ weights_transposed: bool,+ ) -> Result<Self, MTLError> {+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)+ {+ return Err(MTLError::Generic(format!(+ "Unsupported dtype for MLP fused GEMM: {:?}",+ data_type+ )));+ }+ let splitk_kernel = if weights_transposed+ && matches!(data_type, DataType::F16 | DataType::BF16)+ {+ Some(split_k::Kernel::new(data_type))+ } else {+ None+ };+ Ok(Self {+ data_type,+ weights_transposed,+ pipelines: HashMap::new(),+ splitk_kernel,+ })+ }++ fn kernel_name(+ &self,+ bm: i32,+ bn: i32,+ mn_aligned: bool,+ k_aligned: bool,+ ) -> String {+ let type_name = match self.data_type {+ DataType::F16 => "half",+ DataType::BF16 => "bfloat",+ DataType::F32 => "float",+ _ => unreachable!(),+ };+ let transpose_char = if self.weights_transposed {+ "t"+ } else {+ "n"+ };+ let mn_str = if mn_aligned {+ "true"+ } else {+ "false"+ };+ let k_str = if k_aligned {+ "true"+ } else {+ "false"+ };++ format!(+ "steel_gemm_mlp_fused_{}_{}_{}_bm{}_bn{}_bk16_wm2_wn2_align_MN_{}_K_{}",+ transpose_char, type_name, type_name, bm, bn, mn_str, k_str+ )+ }++ fn get_pipeline(+ &mut self,+ context: &MTLContext,+ bm: i32,+ bn: i32,+ mn_aligned: bool,+ k_aligned: bool,+ activation: MlpActivationType,+ ) -> Result<&MTLComputePipelineState, MTLError> {+ let key = (bm, bn, mn_aligned, k_aligned, activation);+ if !self.pipelines.contains_key(&key) {+ let name = self.kernel_name(bm, bn, mn_aligned, k_aligned);++ let fcv = metal::FunctionConstantValues::new();+ let activation_val = activation as u32;+ fcv.set_constant_value_at_index(+ &activation_val as *const u32 as *const _,+ metal::MTLDataType::UInt,+ 52,+ );++ let pipeline = context.compute_pipeline_state(&name, Some(&fcv))?;+ self.pipelines.insert(key, pipeline);+ }+ Ok(self.pipelines.get(&key).unwrap())+ }++ fn select_tile_size(+ &self,+ m: i32,+ hidden_dim: i32,+ _k: i32,+ ) -> (i32, i32) {+ if m >= 64 && hidden_dim >= 64 {+ (64, 64)+ } else {+ (32, 32)+ }+ }++ pub fn encode(+ &mut self,+ context: &MTLContext,+ encoder: &ComputeCommandEncoderRef,+ args: &Arguments,+ ) -> Result<(), MTLError> {+ if let Some(ref mut splitk) = self.splitk_kernel {+ if split_k::Kernel::should_use_splitk(+ args.batch,+ args.hidden_dim,+ args.input_dim,+ ) {+ let splitk_args = split_k::Arguments {+ input: args.input,+ input_offset: args.input_offset,+ weights: args.weights,+ output: args.output,+ batch: args.batch,+ input_dim: args.input_dim,+ hidden_dim: args.hidden_dim,+ lda: args.lda,+ ldb: args.ldb,+ ldd: args.ldd,+ activation: args.activation,+ };+ return splitk.encode(context, encoder, &splitk_args);+ }+ }++ let (bm, bn) =+ self.select_tile_size(args.batch, args.hidden_dim, args.input_dim);+ let bk = 16;++ let mn_aligned = args.batch % bm == 0 && args.hidden_dim % bn == 0;+ let k_aligned = args.input_dim % bk == 0;++ let pipeline = self.get_pipeline(+ context,+ bm,+ bn,+ mn_aligned,+ k_aligned,+ args.activation,+ )?;+ encoder.set_compute_pipeline_state(pipeline);++ encoder.set_buffer(0, Some(args.input), args.input_offset);+ encoder.set_buffer(1, Some(args.weights), 0);+ encoder.set_buffer(2, Some(args.output), 0);++ let tiles_m = (args.batch + bm - 1) / bm;+ let tiles_n = (args.hidden_dim + bn - 1) / bn;+ let gemm_k_iterations = (args.input_dim + bk - 1) / bk;++ let params = GEMMParams {+ M: args.batch,+ N: args.hidden_dim * 2,+ K: args.input_dim,+ lda: args.lda,+ ldb: args.ldb,+ ldd: args.ldd,+ tiles_n,+ tiles_m,+ batch_stride_a: 0,+ batch_stride_b: 0,+ batch_stride_d: 0,+ swizzle_log: 0,+ gemm_k_iterations_aligned: if k_aligned {+ gemm_k_iterations+ } else {+ gemm_k_iterations - 1+ },+ batch_ndim: 0,+ };++ encoder.set_bytes(+ 3,+ std::mem::size_of::<GEMMParams>() as u64,+ ¶ms as *const GEMMParams as *const std::ffi::c_void,+ );+ encoder.set_bytes(+ 10,+ std::mem::size_of::<i32>() as u64,+ &args.hidden_dim as *const i32 as *const std::ffi::c_void,+ );++ let threadgroup_count = MTLSize::new(tiles_n as u64, tiles_m as u64, 1);+ let threads_per_threadgroup = MTLSize::new(32, 2, 2);++ encoder+ .dispatch_thread_groups(threadgroup_count, threads_per_threadgroup);+ Ok(())+ }+}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_mlp_fused.h b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/steel_gemm_mlp_fused.hsimilarity index 98%rename from crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_mlp_fused.hrename to crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/steel_gemm_mlp_fused.hindex 88ebcdbe1..6b945428b 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_mlp_fused.h+++ b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/steel_gemm_mlp_fused.h@@ -4,9 +4,9 @@#include <metal_simdgroup>#include <metal_simdgroup_matrix>-#include "../gemm.h"-#include "../params.h"-#include "../../../../common/mlp_epilogue.h"+#include "../../matmul/common/steel/gemm/gemm.h"+#include "../../matmul/common/steel/gemm/params.h"+#include "../../common/mlp_epilogue.h"using namespace metal;diff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_mlp_fused.metal b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/steel_gemm_mlp_fused.metalsimilarity index 99%rename from crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_mlp_fused.metalrename to crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/steel_gemm_mlp_fused.metalindex 5b0da8472..35ffc4949 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_mlp_fused.metal+++ b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/steel_gemm_mlp_fused.metal@@ -1,6 +1,6 @@#include <metal_stdlib>-#include "../../../../common/utils.h"+#include "../../common/utils.h"#include "steel_gemm_mlp_fused.h"using namespace metal;diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/gemv_mlp_fused.metal b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/gemv_mlp_fused.metalnew file mode 100644index 000000000..789063231--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/gemv_mlp_fused.metal@@ -0,0 +1,272 @@+#include <metal_simdgroup>+#include <metal_stdlib>++#include "../../common/utils.h"+#include "../../common/mlp_epilogue.h"++#include "../matmul/common/steel/utils.h"++using namespace metal;++///////////////////////////////////////////////////////////////////////////////+/// MLP Fused GEMV - Paired Up/Gate Computation with Activation Fusion+///////////////////////////////////////////////////////////////////////////////++#define MTL_CONST static constant constexpr const++// MLP fused GEMV kernel for decode path (M=1).+// Computes paired up and gate projections, then applies: out = up * activation(gate)+// Output size is hidden_dim (half of the weight matrix columns).+template <+ typename T,+ const int BM, /* Threadgroup rows (in simdgroups) */+ const int BN, /* Threadgroup cols (in simdgroups) */+ const int SM, /* Simdgroup rows (in threads) */+ const int SN, /* Simdgroup cols (in threads) */+ const int TM, /* Thread rows (in elements) */+ const int TN> /* Thread cols (in elements) */+struct GEMVMlpFusedKernel {+ using AccT = float;++ MTL_CONST int threadsM = BM * SM;+ MTL_CONST int threadsN = BN * SN;+ MTL_CONST int blockM = threadsM * TM;+ MTL_CONST int blockN = threadsN * TN;++ static_assert(SM * SN == 32, "simdgroup can only have 32 threads");++ MTL_CONST short tgp_mem_size = BN > 1 ? 2 * BN * (blockM + TM) : 0;+ MTL_CONST bool needs_tgp_reduction = BN > 1;++ template <typename U = T>+ static METAL_FUNC void+ load_unsafe(const device T* src, thread U dst[TN], const int src_offset = 0) {+ MTL_PRAGMA_UNROLL+ for (int tn = 0; tn < TN; tn++) {+ dst[tn] = static_cast<U>(src[src_offset + tn]);+ }+ }++ template <typename U = T>+ static METAL_FUNC void load_safe(+ const device T* src,+ thread U dst[TN],+ const int src_offset = 0,+ const int src_size = TN) {+ if (src_offset + TN <= src_size) {+ MTL_PRAGMA_UNROLL+ for (int tn = 0; tn < TN; tn++) {+ dst[tn] = static_cast<U>(src[src_offset + tn]);+ }+ } else {+ MTL_PRAGMA_UNROLL+ for (int tn = 0; tn < TN; tn++) {+ dst[tn] = src_offset + tn < src_size+ ? static_cast<U>(src[src_offset + tn])+ : U(0);+ }+ }+ }++ static METAL_FUNC void run(+ const device T* mat [[buffer(0)]],+ const device T* in_vec [[buffer(1)]],+ device T* out_vec [[buffer(3)]],+ const int in_vec_size,+ const int hidden_dim, // Output size (half of weight matrix rows)+ const int matrix_ld,+ threadgroup AccT* tgp_memory,+ uint3 tid [[threadgroup_position_in_grid]],+ uint3 lid [[thread_position_in_threadgroup]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]) {+ (void)lid;++ // Thread local accumulation results for up and gate+ thread AccT result_up[TM] = {0};+ thread AccT result_gate[TM] = {0};+ thread T inter_up[TN];+ thread T inter_gate[TN];+ thread AccT v_coeff[TN];++ const int thrM = SN != 32 ? simd_lid / SN : 0;+ const int thrN = SN != 32 ? simd_lid % SN : int(simd_lid);++ const int sgN = BN != 1 ? (simd_gid % BN) : 0;+ const int simdM = BN != 1 ? SM * (simd_gid / BN) : int(SM * simd_gid);+ const int simdN = BN != 1 ? SN * (simd_gid % BN) : 0;++ int bm = (simdM + thrM) * TM;+ int bn = (simdN + thrN) * TN;++ // Block position - outputs to hidden_dim elements+ int out_row = tid.x * blockM + bm;++ if (out_row >= hidden_dim)+ return;++ out_row = out_row + TM <= hidden_dim ? out_row : hidden_dim - TM;++ // Pointers to up and gate rows in the weight matrix+ // Weight layout: [up_0, up_1, ..., up_H-1, gate_0, gate_1, ..., gate_H-1]+ const device T* mat_up = mat + out_row * matrix_ld;+ const device T* mat_gate = mat + (out_row + hidden_dim) * matrix_ld;++ constexpr const uniform<int> loop_stride = make_uniform(blockN);+ const uniform<int> in_size = make_uniform(in_vec_size);+ const uniform<int> n_iter = in_size / loop_stride;+ const uniform<int> last_iter = loop_stride * n_iter;+ const uniform<int> leftover = in_size - last_iter;++ // Main loop - compute both up and gate dot products+ for (int i = 0; i < n_iter; ++i) {+ load_unsafe<AccT>(in_vec, v_coeff, bn);++ int mat_offset = 0;+ MTL_PRAGMA_UNROLL+ for (int tm = 0; tm < TM; tm++) {+ load_unsafe(mat_up, inter_up, mat_offset + bn);+ load_unsafe(mat_gate, inter_gate, mat_offset + bn);++ MTL_PRAGMA_UNROLL+ for (int tn = 0; tn < TN; tn++) {+ result_up[tm] += inter_up[tn] * v_coeff[tn];+ result_gate[tm] += inter_gate[tn] * v_coeff[tn];+ }++ mat_offset += matrix_ld;+ }++ bn += blockN;+ }++ // Handle leftover+ if (leftover > 0) {+ load_safe<AccT>(in_vec, v_coeff, bn, in_vec_size);++ MTL_PRAGMA_UNROLL+ for (int tm = 0; tm < TM; tm++) {+ load_safe(&mat_up[tm * matrix_ld], inter_up, bn, in_vec_size);+ load_safe(&mat_gate[tm * matrix_ld], inter_gate, bn, in_vec_size);++ MTL_PRAGMA_UNROLL+ for (int tn = 0; tn < TN; tn++) {+ result_up[tm] += inter_up[tn] * v_coeff[tn];+ result_gate[tm] += inter_gate[tn] * v_coeff[tn];+ }+ }+ }++ // Simdgroup reduction for both up and gate+ MTL_PRAGMA_UNROLL+ for (int tm = 0; tm < TM; tm++) {+ MTL_PRAGMA_UNROLL+ for (ushort sn = (SN / 2); sn >= 1; sn >>= 1) {+ result_up[tm] += simd_shuffle_down(result_up[tm], sn);+ result_gate[tm] += simd_shuffle_down(result_gate[tm], sn);+ }+ }++ // Threadgroup reduction if needed+ if (needs_tgp_reduction) {+ threadgroup AccT* tgp_results_up = tgp_memory + sgN * (blockM + TM) + bm;+ threadgroup AccT* tgp_results_gate = tgp_memory + BN * (blockM + TM) + sgN * (blockM + TM) + bm;++ if (thrN == 0) {+ MTL_PRAGMA_UNROLL+ for (int tm = 0; tm < TM; tm++) {+ tgp_results_up[tm] = result_up[tm];+ tgp_results_gate[tm] = result_gate[tm];+ }++ threadgroup_barrier(mem_flags::mem_none);++ if (sgN == 0) {+ MTL_PRAGMA_UNROLL+ for (int sgn = 1; sgn < BN; sgn++) {+ MTL_PRAGMA_UNROLL+ for (int tm = 0; tm < TM; tm++) {+ result_up[tm] += tgp_results_up[sgn * (blockM + TM) + tm];+ result_gate[tm] += tgp_results_gate[sgn * (blockM + TM) + tm];+ }+ }+ }+ }+ }++ // Apply MLP fused epilogue and write output+ if (simdN == 0 && thrN == 0) {+ MTL_PRAGMA_UNROLL+ for (int tm = 0; tm < TM; tm++) {+ float fused_result = mlp_fused_epilogue_f32(result_up[tm], result_gate[tm]);+ out_vec[out_row + tm] = static_cast<T>(fused_result);+ }+ }+ }+};++template <+ typename T,+ const int BM,+ const int BN,+ const int SM,+ const int SN,+ const int TM,+ const int TN>+[[kernel, max_total_threads_per_threadgroup(BM* BN * 32)]] void gemv_mlp_fused(+ const device T* mat [[buffer(0)]],+ const device T* in_vec [[buffer(1)]],+ device T* out_vec [[buffer(3)]],+ const constant int& in_vec_size [[buffer(4)]],+ const constant int& hidden_dim [[buffer(5)]], // Output dimension (half of weight rows)+ const constant int& matrix_ld [[buffer(6)]],+ const constant int64_t& vector_batch_stride [[buffer(11)]],+ const constant int64_t& matrix_batch_stride [[buffer(12)]],+ uint3 tid [[threadgroup_position_in_grid]],+ uint3 lid [[thread_position_in_threadgroup]],+ uint simd_gid [[simdgroup_index_in_threadgroup]],+ uint simd_lid [[thread_index_in_simdgroup]]) {+ using gemv_kernel = GEMVMlpFusedKernel<T, BM, BN, SM, SN, TM, TN>;+ threadgroup typename gemv_kernel::AccT tgp_memory+ [gemv_kernel::tgp_mem_size == 0 ? 1 : gemv_kernel::tgp_mem_size];++ // Batch offset+ in_vec += tid.z * vector_batch_stride;+ mat += tid.z * matrix_batch_stride;+ out_vec += tid.z * hidden_dim;++ gemv_kernel::run(+ mat,+ in_vec,+ out_vec,+ in_vec_size,+ hidden_dim,+ matrix_ld,+ gemv_kernel::tgp_mem_size == 0 ? nullptr : tgp_memory,+ tid,+ lid,+ simd_gid,+ simd_lid);+}++#define instantiate_gemv_mlp_fused_helper(name, itype, bm, bn, sm, sn, tm, tn) \+ instantiate_kernel( \+ "gemv_mlp_fused_" #name "_bm" #bm "_bn" #bn "_sm" #sm "_sn" #sn "_tm" #tm \+ "_tn" #tn, \+ gemv_mlp_fused, \+ itype, \+ bm, \+ bn, \+ sm, \+ sn, \+ tm, \+ tn)++// clang-format off+#define instantiate_gemv_mlp_fused_blocks(name, itype) \+ instantiate_gemv_mlp_fused_helper(name, itype, 4, 1, 1, 32, 4, 4) \+ instantiate_gemv_mlp_fused_helper(name, itype, 8, 1, 1, 32, 4, 4) // clang-format on++instantiate_gemv_mlp_fused_blocks(float32, float);+instantiate_gemv_mlp_fused_blocks(float16, half);+instantiate_gemv_mlp_fused_blocks(bfloat16, bfloat16_t);diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/mod.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/mod.rsnew file mode 100644index 000000000..0fc92d97b--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/mod.rs@@ -0,0 +1,202 @@+use std::collections::HashMap;++use metal::{+ Buffer as MTLBuffer, ComputeCommandEncoderRef, ComputePipelineState,+ MTLSize,+};++use crate::{+ DataType,+ backends::metal::{MTLContext, MTLError, kernel::mlp::MlpActivationType},+};++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+struct Configuration {+ threadgroup_rows: u32,+ threadgroup_cols: u32,+ threads_per_simdgroup_row: u32,+ threads_per_simdgroup_col: u32,+ elements_per_thread_row: u32,+ elements_per_thread_col: u32,+ activation: MlpActivationType,+}++impl Configuration {+ fn output_elements_per_threadgroup(&self) -> u32 {+ self.threadgroup_rows+ * self.threads_per_simdgroup_row+ * self.elements_per_thread_row+ }++ fn threads_per_threadgroup(&self) -> MTLSize {+ MTLSize::new(+ 32,+ self.threadgroup_cols as u64,+ self.threadgroup_rows as u64,+ )+ }+}++fn kernel_name(+ data_type: DataType,+ config: &Configuration,+) -> Result<String, MTLError> {+ let dtype_name = match data_type {+ DataType::F16 => "float16",+ DataType::BF16 => "bfloat16",+ DataType::F32 => "float32",+ _ => {+ return Err(MTLError::Generic(format!(+ "Unsupported data type for MLP fused GEMV: {:?}",+ data_type+ )));+ },+ };++ Ok(format!(+ "gemv_mlp_fused_{}_bm{}_bn{}_sm{}_sn{}_tm{}_tn{}",+ dtype_name,+ config.threadgroup_rows,+ config.threadgroup_cols,+ config.threads_per_simdgroup_row,+ config.threads_per_simdgroup_col,+ config.elements_per_thread_row,+ config.elements_per_thread_col,+ ))+}++fn select_configuration(+ hidden_dim: i32,+ activation: MlpActivationType,+) -> Configuration {+ let threadgroup_rows = if hidden_dim >= 4096 {+ 8+ } else {+ 4+ };++ Configuration {+ threadgroup_rows,+ threadgroup_cols: 1,+ threads_per_simdgroup_row: 1,+ threads_per_simdgroup_col: 32,+ elements_per_thread_row: 4,+ elements_per_thread_col: 4,+ activation,+ }+}++#[derive(Debug)]+pub struct Arguments<'a> {+ pub weights: &'a MTLBuffer,+ pub input: &'a MTLBuffer,+ pub input_offset: u64,+ pub output: &'a MTLBuffer,+ pub input_dim: i32,+ pub hidden_dim: i32,+ pub weights_ld: i32,+ pub batch_count: i32,+ pub activation: MlpActivationType,+}++pub struct Kernel {+ data_type: DataType,+ pipelines: HashMap<Configuration, ComputePipelineState>,+}++impl Kernel {+ pub fn new(data_type: DataType) -> Result<Self, MTLError> {+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)+ {+ return Err(MTLError::Generic(format!(+ "Unsupported dtype for MLP fused GEMV: {:?}",+ data_type+ )));+ }+ Ok(Self {+ data_type,+ pipelines: HashMap::new(),+ })+ }++ fn get_pipeline(+ &mut self,+ context: &MTLContext,+ config: Configuration,+ ) -> Result<&ComputePipelineState, MTLError> {+ if !self.pipelines.contains_key(&config) {+ let name = kernel_name(self.data_type, &config)?;++ let fcv = metal::FunctionConstantValues::new();+ let activation_val = config.activation as u32;+ fcv.set_constant_value_at_index(+ &activation_val as *const u32 as *const _,+ metal::MTLDataType::UInt,+ 52,+ );++ let pipeline = context.compute_pipeline_state(&name, Some(&fcv))?;+ self.pipelines.insert(config, pipeline);+ }+ Ok(self.pipelines.get(&config).unwrap())+ }++ pub fn encode(+ &mut self,+ context: &MTLContext,+ encoder: &ComputeCommandEncoderRef,+ args: &Arguments,+ ) -> Result<(), MTLError> {+ let config = select_configuration(args.hidden_dim, args.activation);+ let pipeline = self.get_pipeline(context, config)?;++ encoder.set_compute_pipeline_state(pipeline);+ encoder.set_buffer(0, Some(args.weights), 0);+ encoder.set_buffer(1, Some(args.input), args.input_offset);+ encoder.set_buffer(3, Some(args.output), 0);++ encoder.set_bytes(+ 4,+ std::mem::size_of::<i32>() as u64,+ &args.input_dim as *const i32 as *const std::ffi::c_void,+ );+ encoder.set_bytes(+ 5,+ std::mem::size_of::<i32>() as u64,+ &args.hidden_dim as *const i32 as *const std::ffi::c_void,+ );+ encoder.set_bytes(+ 6,+ std::mem::size_of::<i32>() as u64,+ &args.weights_ld as *const i32 as *const std::ffi::c_void,+ );++ let vector_batch_stride = args.input_dim as i64;+ let matrix_batch_stride = 0i64;+ encoder.set_bytes(+ 11,+ std::mem::size_of::<i64>() as u64,+ &vector_batch_stride as *const i64 as *const std::ffi::c_void,+ );+ encoder.set_bytes(+ 12,+ std::mem::size_of::<i64>() as u64,+ &matrix_batch_stride as *const i64 as *const std::ffi::c_void,+ );++ let output_elements_per_threadgroup =+ config.output_elements_per_threadgroup();+ let threadgroup_count_x =+ ((args.hidden_dim as u32 + output_elements_per_threadgroup - 1)+ / output_elements_per_threadgroup) as u64;+ let threadgroup_count_z = args.batch_count.max(1) as u64;++ let threadgroup_count =+ MTLSize::new(threadgroup_count_x, 1, threadgroup_count_z);+ let threads_per_threadgroup = config.threads_per_threadgroup();++ encoder+ .dispatch_thread_groups(threadgroup_count, threads_per_threadgroup);+ Ok(())+ }+}diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/mod.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/mod.rsnew file mode 100644index 000000000..bbbf615c1--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/mlp_fused/mod.rs@@ -0,0 +1,7 @@+pub mod gemm;+pub mod gemv;+pub mod split_k;++pub use gemm::{Arguments as GemmArguments, Kernel as GemmKernel};+pub use gemv::{Arguments as GemvArguments, Kernel as GemvKernel};+pub use split_k::{Arguments as SplitKArguments, Kernel as SplitKKernel};diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/mod.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/mod.rsnew file mode 100644index 000000000..191d42704--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/mod.rs@@ -0,0 +1,313 @@+use std::collections::HashMap;++use metal::{+ Buffer as MTLBuffer, ComputeCommandEncoderRef,+ ComputePipelineState as MTLComputePipelineState, MTLSize,+};++use crate::{+ DataType,+ backends::metal::{+ MTLContext, MTLError,+ kernel::{+ matmul::common::GEMMSpiltKMlpFusedParams, mlp::MlpActivationType,+ },+ },+};++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+struct TileConfig {+ tile_rows: i32,+ tile_cols: i32,+ tile_depth: i32,+ warps_per_row: i32,+ warps_per_col: i32,+}++#[derive(Debug, Clone, PartialEq, Eq, Hash)]+struct PartialKernelKey {+ tile_config: TileConfig,+ mn_aligned: bool,+ k_aligned: bool,+}++#[derive(Debug)]+pub struct Arguments<'a> {+ pub input: &'a MTLBuffer,+ pub input_offset: u64,+ pub weights: &'a MTLBuffer,+ pub output: &'a MTLBuffer,+ pub batch: i32,+ pub input_dim: i32,+ pub hidden_dim: i32,+ pub lda: i32,+ pub ldb: i32,+ pub ldd: i32,+ pub activation: MlpActivationType,+}++pub struct Kernel {+ data_type: DataType,+ partial_pipelines: HashMap<PartialKernelKey, MTLComputePipelineState>,+ accum_pipelines: HashMap<MlpActivationType, MTLComputePipelineState>,+}++impl Kernel {+ pub fn new(data_type: DataType) -> Self {+ Self {+ data_type,+ partial_pipelines: HashMap::new(),+ accum_pipelines: HashMap::new(),+ }+ }++ fn steel_type_name(&self) -> Result<&'static str, MTLError> {+ match self.data_type {+ DataType::F16 => Ok("float16"),+ DataType::BF16 => Ok("bfloat16"),+ _ => Err(MTLError::Generic(format!(+ "Unsupported dtype for MLP Fused Split-K: {:?}",+ self.data_type+ ))),+ }+ }++ fn partial_kernel_name(+ &self,+ config: &TileConfig,+ mn_aligned: bool,+ k_aligned: bool,+ ) -> Result<String, MTLError> {+ let in_name = self.steel_type_name()?;+ let mn_tag = if mn_aligned {+ "taligned"+ } else {+ "naligned"+ };+ let k_tag = if k_aligned {+ "taligned"+ } else {+ "naligned"+ };+ Ok(format!(+ "steel_gemm_splitk_mlp_fused_nt_{}_float32_bm{}_bn{}_bk{}_wm{}_wn{}_MN_{}_K_{}",+ in_name,+ config.tile_rows,+ config.tile_cols,+ config.tile_depth,+ config.warps_per_row,+ config.warps_per_col,+ mn_tag,+ k_tag,+ ))+ }++ fn accum_kernel_name(&self) -> Result<String, MTLError> {+ let out_name = self.steel_type_name()?;+ Ok(format!("steel_gemm_splitk_mlp_fused_accum_{}_float32", out_name))+ }++ fn get_partial_pipeline(+ &mut self,+ mtl: &MTLContext,+ config: TileConfig,+ mn_aligned: bool,+ k_aligned: bool,+ ) -> Result<&MTLComputePipelineState, MTLError> {+ let key = PartialKernelKey {+ tile_config: config,+ mn_aligned,+ k_aligned,+ };+ if !self.partial_pipelines.contains_key(&key) {+ let name =+ self.partial_kernel_name(&config, mn_aligned, k_aligned)?;+ let ps = mtl.compute_pipeline_state(&name, None)?;+ self.partial_pipelines.insert(key.clone(), ps);+ }+ Ok(self.partial_pipelines.get(&key).unwrap())+ }++ fn get_accum_pipeline(+ &mut self,+ mtl: &MTLContext,+ activation: MlpActivationType,+ ) -> Result<&MTLComputePipelineState, MTLError> {+ if !self.accum_pipelines.contains_key(&activation) {+ let name = self.accum_kernel_name()?;+ let fcv = metal::FunctionConstantValues::new();+ let activation_val = activation as u32;+ fcv.set_constant_value_at_index(+ &activation_val as *const u32 as *const _,+ metal::MTLDataType::UInt,+ 52,+ );+ let ps = mtl.compute_pipeline_state(&name, Some(&fcv))?;+ self.accum_pipelines.insert(activation, ps);+ }+ Ok(self.accum_pipelines.get(&activation).unwrap())+ }++ fn select_tile_config(+ m: i32,+ n: i32,+ ) -> TileConfig {+ let tile_rows = if m < 40 {+ 16+ } else {+ 32+ };+ let tile_cols = if n < 40 {+ 16+ } else {+ 32+ };+ TileConfig {+ tile_rows,+ tile_cols,+ tile_depth: 16,+ warps_per_row: 2,+ warps_per_col: 2,+ }+ }++ fn compute_partition_count(k: i32) -> i32 {+ let k_tiles = k / 16;+ if k_tiles < 16 {+ 2+ } else if k_tiles < 32 {+ 4+ } else if k_tiles < 64 {+ 8+ } else {+ 16+ }+ }++ pub fn should_use_splitk(+ m: i32,+ hidden_dim: i32,+ k: i32,+ ) -> bool {+ if m != 1 {+ return false;+ }+ if hidden_dim <= 0 || k <= 0 {+ return false;+ }+ let n_tiles = hidden_dim / 16;+ let k_tiles = k / 16;+ n_tiles <= 32 && k_tiles >= 8+ }++ pub fn encode(+ &mut self,+ mtl: &MTLContext,+ enc: &ComputeCommandEncoderRef,+ args: &Arguments,+ ) -> Result<(), MTLError> {+ let m = args.batch;+ let hidden_dim = args.hidden_dim;+ let k = args.input_dim;++ let tile_config = Self::select_tile_config(m, hidden_dim);+ let partition_count = Self::compute_partition_count(k);+ let mn_aligned = (m % tile_config.tile_rows) == 0+ && (hidden_dim % tile_config.tile_cols) == 0;+ let k_aligned = (k % tile_config.tile_depth) == 0;++ let tile_count_m =+ (m + tile_config.tile_rows - 1) / tile_config.tile_rows;+ let tile_count_n =+ (hidden_dim + tile_config.tile_cols - 1) / tile_config.tile_cols;++ let gemm_k_iterations = (k / tile_config.tile_depth) / partition_count;+ let k_elements_per_partition =+ gemm_k_iterations * tile_config.tile_depth;+ let output_elements_per_partition = m * hidden_dim;++ let accumulator_element_count =+ partition_count * output_elements_per_partition;+ let accumulator_bytes =+ (accumulator_element_count as usize) * std::mem::size_of::<f32>();++ let up_accumulator_buffer = mtl.device.new_buffer(+ accumulator_bytes as u64,+ metal::MTLResourceOptions::StorageModePrivate,+ );+ let gate_accumulator_buffer = mtl.device.new_buffer(+ accumulator_bytes as u64,+ metal::MTLResourceOptions::StorageModePrivate,+ );++ let params = GEMMSpiltKMlpFusedParams {+ M: m,+ N: hidden_dim,+ K: k,+ lda: args.lda,+ ldb: args.ldb,+ ldc: hidden_dim,+ tiles_n: tile_count_n,+ tiles_m: tile_count_m,+ split_k_partitions: partition_count,+ split_k_partition_stride: output_elements_per_partition,+ split_k_partition_size: k_elements_per_partition,+ gemm_k_iterations_aligned: gemm_k_iterations,+ hidden_dim,+ };++ let partial_ps =+ self.get_partial_pipeline(mtl, tile_config, mn_aligned, k_aligned)?;++ enc.set_compute_pipeline_state(partial_ps);+ enc.set_buffer(0, Some(args.input), args.input_offset);+ enc.set_buffer(1, Some(args.weights), 0);+ enc.set_buffer(2, Some(&up_accumulator_buffer), 0);+ enc.set_buffer(3, Some(&gate_accumulator_buffer), 0);+ enc.set_bytes(+ 4,+ std::mem::size_of::<GEMMSpiltKMlpFusedParams>() as u64,+ ¶ms as *const GEMMSpiltKMlpFusedParams as *const _,+ );++ let threads_per_threadgroup = MTLSize::new(+ 32,+ tile_config.warps_per_col as u64,+ tile_config.warps_per_row as u64,+ );+ let threadgroups = MTLSize::new(+ tile_count_n as u64,+ tile_count_m as u64,+ partition_count as u64,+ );+ enc.dispatch_thread_groups(threadgroups, threads_per_threadgroup);++ let accum_ps = self.get_accum_pipeline(mtl, args.activation)?;+ enc.set_compute_pipeline_state(accum_ps);+ enc.set_buffer(0, Some(&up_accumulator_buffer), 0);+ enc.set_buffer(1, Some(&gate_accumulator_buffer), 0);+ enc.set_buffer(2, Some(args.output), 0);+ enc.set_bytes(+ 3,+ 4,+ &partition_count as *const i32 as *const std::ffi::c_void,+ );+ enc.set_bytes(+ 4,+ 4,+ &output_elements_per_partition as *const i32+ as *const std::ffi::c_void,+ );+ enc.set_bytes(5, 4, &args.ldd as *const i32 as *const std::ffi::c_void);++ let accum_total_threads = MTLSize::new(hidden_dim as u64, m as u64, 1);+ let accum_threads_per_threadgroup =+ MTLSize::new(16.min(hidden_dim as u64), 16.min(m as u64), 1);+ enc.dispatch_threads(+ accum_total_threads,+ accum_threads_per_threadgroup,+ );++ Ok(())+ }+}diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/steel_gemm_splitk_mlp_fused.h b/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/steel_gemm_splitk_mlp_fused.hnew file mode 100644index 000000000..15d59cfef--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/steel_gemm_splitk_mlp_fused.h@@ -0,0 +1,216 @@+#pragma once++#include <metal_stdlib>+#include <metal_simdgroup>+#include <metal_simdgroup_matrix>++#include "../../matmul/common/steel/gemm/gemm.h"+#include "../../matmul/common/shared_types.h"+#include "../../common/mlp_epilogue.h"++using namespace metal;+using namespace steel;++///////////////////////////////////////////////////////////////////////////////+// MLP Fused Split-K GEMM Kernel+// Computes paired up/gate projections with fused activation using split-K.+///////////////////////////////////////////////////////////////////////////////++template <+ typename T,+ typename U,+ int BM,+ int BN,+ int BK,+ int WM,+ int WN,+ bool transpose_a,+ bool transpose_b,+ bool MN_aligned,+ bool K_aligned>+[[kernel, max_total_threads_per_threadgroup(WM* WN * 32)]] void gemm_splitk_mlp_fused(+ const device T* A [[buffer(0)]],+ const device T* B [[buffer(1)]],+ device U* C [[buffer(2)]],+ const constant uzu::matmul::GEMMSpiltKMlpFusedParams* params [[buffer(3)]],+ uint simd_lane_id [[thread_index_in_simdgroup]],+ uint simd_group_id [[simdgroup_index_in_threadgroup]],+ uint3 tid [[threadgroup_position_in_grid]],+ uint3 lid [[thread_position_in_threadgroup]]) {+ (void)lid;++ using gemm_kernel = GEMMKernel<+ T,+ U,+ BM,+ BN,+ BK,+ WM,+ WN,+ transpose_a,+ transpose_b,+ MN_aligned,+ K_aligned>;+ using loader_a_t = typename gemm_kernel::loader_a_t;+ using loader_b_t = typename gemm_kernel::loader_b_t;+ using mma_t = typename gemm_kernel::mma_t;++ threadgroup T As[gemm_kernel::tgp_mem_size_a];+ threadgroup T Bs_up[gemm_kernel::tgp_mem_size_b];+ threadgroup T Bs_gate[gemm_kernel::tgp_mem_size_b];++ const int tid_x = tid.x;+ const int tid_y = tid.y;+ const int tid_z = tid.z;++ // Only process tiles within hidden_dim columns+ int hidden_tiles = (params->hidden_dim + BN - 1) / BN;+ if (hidden_tiles <= tid_x || params->tiles_m <= tid_y) {+ return;+ }++ // Find block in A, B, C+ const int c_row = tid_y * BM;+ const int c_col = tid_x * BN;+ const int k_start = params->split_k_partition_size * tid_z;++ const size_t c_row_long = size_t(c_row);+ const size_t c_col_long = size_t(c_col);+ const size_t k_start_long = size_t(k_start);++ // Pointers to A (input)+ const device T* A_ptr = A + (transpose_a ? (c_row_long + k_start_long * params->lda)+ : (k_start_long + c_row_long * params->lda));++ // Pointers to B (up and gate weights)+ // B layout: [up_weights | gate_weights] where each is hidden_dim columns+ const device T* B_up = B + (transpose_b ? (k_start_long + c_col_long * params->ldb)+ : (c_col_long + k_start_long * params->ldb));+ const device T* B_gate = B_up + (transpose_b ? params->hidden_dim * params->ldb : params->hidden_dim);++ // C output for partial sums (up and gate interleaved in split_k dimension)+ // Layout: [up_partitions, gate_partitions] in the partition stride+ device U* C_up = C + (size_t(params->split_k_partition_stride) * tid_z) ++ (c_row_long * params->ldc + c_col_long);+ device U* C_gate = C_up + (size_t(params->split_k_partition_stride) * params->split_k_partitions);++ // Loaders+ thread loader_a_t loader_a(A_ptr, params->lda, As, simd_group_id, simd_lane_id);+ thread loader_b_t loader_b_up(B_up, params->ldb, Bs_up, simd_group_id, simd_lane_id);+ thread loader_b_t loader_b_gate(B_gate, params->ldb, Bs_gate, simd_group_id, simd_lane_id);++ // MMA operations+ thread mma_t mma_up(simd_group_id, simd_lane_id);+ thread mma_t mma_gate(simd_group_id, simd_lane_id);++ int gemm_k_iterations = params->gemm_k_iterations_aligned;++ short tgp_bm = min(BM, params->M - c_row);+ short tgp_bn = min(BN, params->hidden_dim - c_col);+ short leftover_bk = params->K % BK;++ // Main loop+ for (int k = 0; k < gemm_k_iterations; k++) {+ threadgroup_barrier(mem_flags::mem_threadgroup);++ if (MN_aligned) {+ loader_a.load_unsafe();+ loader_b_up.load_unsafe();+ loader_b_gate.load_unsafe();+ } else if (tgp_bm == BM && tgp_bn == BN) {+ loader_a.load_unsafe();+ loader_b_up.load_unsafe();+ loader_b_gate.load_unsafe();+ } else {+ short2 tile_dims_A = transpose_a ? short2(tgp_bm, BK) : short2(BK, tgp_bm);+ short2 tile_dims_B = transpose_b ? short2(BK, tgp_bn) : short2(tgp_bn, BK);+ loader_a.load_safe(tile_dims_A);+ loader_b_up.load_safe(tile_dims_B);+ loader_b_gate.load_safe(tile_dims_B);+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ mma_up.mma(As, Bs_up);+ mma_gate.mma(As, Bs_gate);++ loader_a.next();+ loader_b_up.next();+ loader_b_gate.next();+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ // Handle remaining K for last partition+ if ((tid_z + 1) == (params->split_k_partitions)) {+ int gemm_k_iter_remaining =+ (params->K - (k_start + params->split_k_partition_size)) / BK;+ for (int k = 0; k < gemm_k_iter_remaining; k++) {+ threadgroup_barrier(mem_flags::mem_threadgroup);++ if (!K_aligned) {+ short2 tile_dims_A = transpose_a ? short2(tgp_bm, leftover_bk) : short2(leftover_bk, tgp_bm);+ short2 tile_dims_B = transpose_b ? short2(leftover_bk, tgp_bn) : short2(tgp_bn, leftover_bk);+ loader_a.load_safe(tile_dims_A);+ loader_b_up.load_safe(tile_dims_B);+ loader_b_gate.load_safe(tile_dims_B);+ } else {+ loader_a.load_unsafe();+ loader_b_up.load_unsafe();+ loader_b_gate.load_unsafe();+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ mma_up.mma(As, Bs_up);+ mma_gate.mma(As, Bs_gate);++ loader_a.next();+ loader_b_up.next();+ loader_b_gate.next();+ }+ }++ // Store partial results+ if (MN_aligned || (tgp_bm == BM && tgp_bn == BN)) {+ mma_up.store_result(C_up, params->ldc);+ mma_gate.store_result(C_gate, params->ldc);+ } else {+ mma_up.store_result_safe(C_up, params->ldc, short2(tgp_bn, tgp_bm));+ mma_gate.store_result_safe(C_gate, params->ldc, short2(tgp_bn, tgp_bm));+ }+}++///////////////////////////////////////////////////////////////////////////////+// MLP Fused Split-K accumulation kernel+// Accumulates partial sums and applies MLP fused epilogue+///////////////////////////////////////////////////////////////////////////////++template <typename AccT, typename OutT>+[[kernel]] void gemm_splitk_mlp_fused_accum(+ const device AccT* C_split [[buffer(0)]],+ device OutT* D [[buffer(1)]],+ const constant int& k_partitions [[buffer(2)]],+ const constant int& partition_stride [[buffer(3)]],+ const constant int& ldd [[buffer(4)]],+ uint2 gid [[thread_position_in_grid]]) {+ // Adjust pointers+ D += gid.x + gid.y * size_t(ldd);+ const device AccT* C_up = C_split + gid.x + gid.y * size_t(ldd);+ const device AccT* C_gate = C_up + (size_t(partition_stride) * k_partitions);++ // Accumulate up and gate partial sums+ AccT out_up = 0;+ AccT out_gate = 0;++ size_t offset = 0;+ for (int i = 0; i < k_partitions; i++) {+ out_up += C_up[offset];+ out_gate += C_gate[offset];+ offset += partition_stride;+ }++ // Apply MLP fused epilogue: out = up * activation(gate)+ float fused = mlp_fused_epilogue_f32(static_cast<float>(out_up), static_cast<float>(out_gate));+ D[0] = static_cast<OutT>(fused);+}diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/steel_gemm_splitk_mlp_fused.metal b/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/steel_gemm_splitk_mlp_fused.metalnew file mode 100644index 000000000..e58783794--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/steel_gemm_splitk_mlp_fused.metal@@ -0,0 +1,67 @@+// clang-format off+#include "../../common/utils.h"+#include "steel_gemm_splitk_mlp_fused.h"++#define instantiate_gemm( \+ tname, \+ trans_a, \+ trans_b, \+ iname, \+ itype, \+ oname, \+ otype, \+ bm, \+ bn, \+ bk, \+ wm, \+ wn, \+ aname, \+ mn_aligned, \+ kname, \+ k_aligned) \+ instantiate_kernel( \+ "steel_gemm_splitk_mlp_fused_" #tname "_" #iname "_" #oname \+ "_bm" #bm "_bn" #bn "_bk" #bk "_wm" #wm "_wn" #wn \+ "_MN_" #aname "_K_" #kname, \+ gemm_splitk_mlp_fused, \+ itype, \+ otype, \+ bm, \+ bn, \+ bk, \+ wm, \+ wn, \+ trans_a, \+ trans_b, \+ mn_aligned, \+ k_aligned)++#define instantiate_gemm_aligned_helper(tname, trans_a, trans_b, iname, itype, oname, otype, bm, bn, bk, wm, wn) \+ instantiate_gemm(tname, trans_a, trans_b, iname, itype, oname, otype, bm, bn, bk, wm, wn, taligned, true, taligned, true) \+ instantiate_gemm(tname, trans_a, trans_b, iname, itype, oname, otype, bm, bn, bk, wm, wn, taligned, true, naligned, false) \+ instantiate_gemm(tname, trans_a, trans_b, iname, itype, oname, otype, bm, bn, bk, wm, wn, naligned, false, taligned, true) \+ instantiate_gemm(tname, trans_a, trans_b, iname, itype, oname, otype, bm, bn, bk, wm, wn, naligned, false, naligned, false)++// Only NT transpose (common case for MLP)+#define instantiate_gemm_transpose_helper(iname, itype, oname, otype, bm, bn, bk, wm, wn) \+ instantiate_gemm_aligned_helper(nt, false, true, iname, itype, oname, otype, bm, bn, bk, wm, wn)++#define instantiate_gemm_shapes_helper(iname, itype, oname, otype) \+ instantiate_gemm_transpose_helper(iname, itype, oname, otype, 16, 16, 16, 2, 2) \+ instantiate_gemm_transpose_helper(iname, itype, oname, otype, 16, 32, 16, 2, 2) \+ instantiate_gemm_transpose_helper(iname, itype, oname, otype, 32, 16, 16, 2, 2) \+ instantiate_gemm_transpose_helper(iname, itype, oname, otype, 32, 32, 16, 2, 2)++instantiate_gemm_shapes_helper(float16, half, float32, float);+instantiate_gemm_shapes_helper(bfloat16, bfloat16_t, float32, float);+instantiate_gemm_shapes_helper(float32, float, float32, float);++#define instantiate_accum(oname, otype, aname, atype) \+ instantiate_kernel( \+ "steel_gemm_splitk_mlp_fused_accum_" #oname "_" #aname, \+ gemm_splitk_mlp_fused_accum, atype, otype)++instantiate_accum(bfloat16, bfloat16_t, float32, float);+instantiate_accum(float16, half, float32, float);+instantiate_accum(float32, float, float32, float);+// clang-format ondiff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rsindex ff569683d..3caf3ae9b 100644--- a/crates/uzu/src/backends/metal/kernel/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/mod.rs@@ -12,6 +12,7 @@ pub use mlp::{MLP_ACTIVATION_FC_INDEX, MLP_FUSED_FC_INDEX, MLP_HIDDEN_DIM_FC_INDEX,MlpActivationType, MlpFusedConfig, make_non_fused_function_constants,};+pub mod mlp_fused;pub mod moe;pub mod pooling;pub mod quant_matmul;diff --git a/crates/uzu/tests/mlp_fused_correctness_test.rs b/crates/uzu/tests/mlp_fused_correctness_test.rsindex 3e2d97782..04de27d08 100644--- a/crates/uzu/tests/mlp_fused_correctness_test.rs+++ b/crates/uzu/tests/mlp_fused_correctness_test.rs@@ -9,11 +9,13 @@ use uzu::{backends::metal::{MTLContext,kernel::{- matmul::{- MlpFusedGemmArguments, MlpFusedGemmKernel,- MlpFusedGemvArguments, MlpFusedGemvKernel,- },mlp::MlpActivationType,+ mlp_fused::{+ GemmArguments as MlpFusedGemmArguments,+ GemmKernel as MlpFusedGemmKernel,+ GemvArguments as MlpFusedGemvArguments,+ GemvKernel as MlpFusedGemvKernel,+ },},},};diff --git a/crates/uzu/tests/mlp_fused_performance_test.rs b/crates/uzu/tests/mlp_fused_performance_test.rsindex c4f9a64ac..99cc81695 100644--- a/crates/uzu/tests/mlp_fused_performance_test.rs+++ b/crates/uzu/tests/mlp_fused_performance_test.rs@@ -11,11 +11,14 @@ use uzu::{backends::metal::{MTLContext,kernel::{- matmul::{- MatmulArguments, MatmulKernel, MlpFusedGemmArguments,- MlpFusedGemmKernel, MlpFusedGemvArguments, MlpFusedGemvKernel,- },+ matmul::{MatmulArguments, MatmulKernel},mlp::MlpActivationType,+ mlp_fused::{+ GemmArguments as MlpFusedGemmArguments,+ GemmKernel as MlpFusedGemmKernel,+ GemvArguments as MlpFusedGemvArguments,+ GemvKernel as MlpFusedGemvKernel,+ },},},};