Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

reorganize code

Eugene Bokhan53935ffe6701
diff --git a/crates/uzu/build.rs b/crates/uzu/build.rs
index 564b34121..aa2258fba 100644
--- a/crates/uzu/build.rs
+++ b/crates/uzu/build.rs
@@ -23,8 +23,9 @@ fn main() {
/// These headers are also used by Metal shaders, ensuring type consistency.
fn generate_metal_bindings() {
let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());
- let matmul_dir = manifest_dir.join("src/backends/metal/kernel/matmul");
- let types_header = matmul_dir.join("shared_types.h");
+ let matmul_common_dir =
+ manifest_dir.join("src/backends/metal/kernel/matmul/common");
+ let types_header = matmul_common_dir.join("shared_types.h");
// Only regenerate if header changed
println!("cargo:rerun-if-changed={}", types_header.display());
@@ -43,13 +44,14 @@ fn generate_metal_bindings() {
.allowlist_type("GEMMParams")
.allowlist_type("GEMMAddMMParams")
.allowlist_type("GEMMSpiltKParams")
+ .allowlist_type("GEMMSpiltKMlpFusedParams")
// Use core types
.use_core()
.generate()
.expect("Failed to generate bindings from shared_types.h");
// Write bindings to the source directory so it can be checked in
- let bindings_path = matmul_dir.join("shared_types.rs");
+ let bindings_path = matmul_common_dir.join("shared_types.rs");
bindings
.write_to_file(&bindings_path)
.expect("Failed to write shared_types.rs");
@@ -260,7 +262,10 @@ fn macos_sdk_version(sdk: &str) -> Option<(u32, u32)> {
Some((major, minor))
}
-fn should_enable_nax(target_os: &str, sdk: &str) -> bool {
+fn should_enable_nax(
+ target_os: &str,
+ sdk: &str,
+) -> bool {
if target_os != "macos" && target_os != "ios" {
return false;
}
@@ -269,8 +274,8 @@ fn should_enable_nax(target_os: &str, sdk: &str) -> bool {
return false;
}
// Check host OS version
- let os_ok =
- host_macos_major_version().is_some_and(|major| major >= NAX_MIN_OS_MAJOR);
+ let os_ok = host_macos_major_version()
+ .is_some_and(|major| major >= NAX_MIN_OS_MAJOR);
if !os_ok {
return false;
}
@@ -380,6 +385,11 @@ fn compile_metal_files(
if matmul_dir.exists() {
include_dirs.insert(matmul_dir.clone());
+ let matmul_common_dir = matmul_dir.join("common");
+ if matmul_common_dir.exists() {
+ include_dirs.insert(matmul_common_dir);
+ }
+
let gemv_dir = matmul_dir.join("gemv");
if gemv_dir.exists() {
include_dirs.insert(gemv_dir);
diff --git a/crates/uzu/src/backends/metal/encodable_block/mlp.rs b/crates/uzu/src/backends/metal/encodable_block/mlp.rs
index 76ed0bb31..3393dbde5 100644
--- a/crates/uzu/src/backends/metal/encodable_block/mlp.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/mlp.rs
@@ -11,11 +11,11 @@ use crate::{
MTLContext,
forward_pass::{ArrayId, ForwardPassState},
kernel::{
- matmul::{
- MlpFusedGemmArguments, MlpFusedGemmKernel,
- MlpFusedGemvArguments, MlpFusedGemvKernel,
- },
mlp::{MlpActivationType, MlpGateActMulEncodable},
+ mlp_fused::{
+ GemmArguments as MlpFusedGemmArguments, GemmKernel as MlpFusedGemmKernel,
+ GemvArguments as MlpFusedGemvArguments, GemvKernel as MlpFusedGemvKernel,
+ },
quant_matmul::{
MlpFusedQmmArguments, MlpFusedQmmKernel, MlpFusedQmvArguments,
MlpFusedQmvKernel,
diff --git a/crates/uzu/src/backends/metal/kernel/attention/gemm_attention.metal b/crates/uzu/src/backends/metal/kernel/attention/gemm_attention.metal
index c348112a6..bcff92b15 100644
--- a/crates/uzu/src/backends/metal/kernel/attention/gemm_attention.metal
+++ b/crates/uzu/src/backends/metal/kernel/attention/gemm_attention.metal
@@ -1,7 +1,7 @@
#include <metal_stdlib>
-#include "../matmul/loader.h"
-#include "../matmul/mma.h"
+#include "../matmul/common/loader.h"
+#include "../matmul/common/mma.h"
#include "gemm_types.h"
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/defines.h b/crates/uzu/src/backends/metal/kernel/matmul/common/defines.h
similarity index 100%
rename from crates/uzu/src/backends/metal/kernel/matmul/defines.h
rename to crates/uzu/src/backends/metal/kernel/matmul/common/defines.h
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm.h b/crates/uzu/src/backends/metal/kernel/matmul/common/gemm.h
similarity index 100%
rename from crates/uzu/src/backends/metal/kernel/matmul/gemm.h
rename to crates/uzu/src/backends/metal/kernel/matmul/common/gemm.h
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/loader.h b/crates/uzu/src/backends/metal/kernel/matmul/common/loader.h
similarity index 100%
rename from crates/uzu/src/backends/metal/kernel/matmul/loader.h
rename to crates/uzu/src/backends/metal/kernel/matmul/common/loader.h
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/arguments.rs b/crates/uzu/src/backends/metal/kernel/matmul/common/matmul_arguments.rs
similarity index 100%
rename from crates/uzu/src/backends/metal/kernel/matmul/arguments.rs
rename to crates/uzu/src/backends/metal/kernel/matmul/common/matmul_arguments.rs
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/mma.h b/crates/uzu/src/backends/metal/kernel/matmul/common/mma.h
similarity index 100%
rename from crates/uzu/src/backends/metal/kernel/matmul/mma.h
rename to crates/uzu/src/backends/metal/kernel/matmul/common/mma.h
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/common/mod.rs
new file mode 100644
index 000000000..0ca76a5d9
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/mod.rs
@@ -0,0 +1,10 @@
+mod matmul_arguments;
+#[allow(dead_code, clippy::unnecessary_operation, clippy::identity_op)]
+mod shared_types;
+mod transpose_configuration;
+
+pub use matmul_arguments::MatmulArguments;
+pub use shared_types::{
+ GEMMAddMMParams, GEMMParams, GEMMSpiltKMlpFusedParams, GEMMSpiltKParams,
+};
+pub(crate) use transpose_configuration::transpose_configuration;
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/shared_types.h b/crates/uzu/src/backends/metal/kernel/matmul/common/shared_types.h
similarity index 84%
rename from crates/uzu/src/backends/metal/kernel/matmul/shared_types.h
rename to crates/uzu/src/backends/metal/kernel/matmul/common/shared_types.h
index 9a957631f..ad116a541 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/shared_types.h
+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/shared_types.h
@@ -67,6 +67,27 @@ struct GEMMSpiltKParams {
int gemm_k_iterations_aligned;
};
+// Split-K MLP Fused GEMM parameters
+struct GEMMSpiltKMlpFusedParams {
+ int M;
+ int N;
+ int K;
+
+ int lda;
+ int ldb;
+ int ldc;
+
+ int tiles_n;
+ int tiles_m;
+
+ int split_k_partitions;
+ int split_k_partition_stride;
+ int split_k_partition_size;
+
+ int gemm_k_iterations_aligned;
+ int hidden_dim;
+};
+
// AddMM parameters (alpha * A @ B + beta * C)
struct GEMMAddMMParams {
int ldc;
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/shared_types.rs b/crates/uzu/src/backends/metal/kernel/matmul/common/shared_types.rs
similarity index 67%
rename from crates/uzu/src/backends/metal/kernel/matmul/shared_types.rs
rename to crates/uzu/src/backends/metal/kernel/matmul/common/shared_types.rs
index 97e68a5bf..3e4c612d2 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/shared_types.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/shared_types.rs
@@ -114,6 +114,70 @@ const _: () = {
};
#[repr(C)]
#[derive(Debug, Default, Copy, Clone)]
+pub struct GEMMSpiltKMlpFusedParams {
+ pub M: ::core::ffi::c_int,
+ pub N: ::core::ffi::c_int,
+ pub K: ::core::ffi::c_int,
+ pub lda: ::core::ffi::c_int,
+ pub ldb: ::core::ffi::c_int,
+ pub ldc: ::core::ffi::c_int,
+ pub tiles_n: ::core::ffi::c_int,
+ pub tiles_m: ::core::ffi::c_int,
+ pub split_k_partitions: ::core::ffi::c_int,
+ pub split_k_partition_stride: ::core::ffi::c_int,
+ pub split_k_partition_size: ::core::ffi::c_int,
+ pub gemm_k_iterations_aligned: ::core::ffi::c_int,
+ pub hidden_dim: ::core::ffi::c_int,
+}
+#[allow(clippy::unnecessary_operation, clippy::identity_op)]
+const _: () = {
+ ["Size of GEMMSpiltKMlpFusedParams"]
+ [::core::mem::size_of::<GEMMSpiltKMlpFusedParams>() - 52usize];
+ ["Alignment of GEMMSpiltKMlpFusedParams"]
+ [::core::mem::align_of::<GEMMSpiltKMlpFusedParams>() - 4usize];
+ ["Offset of field: GEMMSpiltKMlpFusedParams::M"]
+ [::core::mem::offset_of!(GEMMSpiltKMlpFusedParams, M) - 0usize];
+ ["Offset of field: GEMMSpiltKMlpFusedParams::N"]
+ [::core::mem::offset_of!(GEMMSpiltKMlpFusedParams, N) - 4usize];
+ ["Offset of field: GEMMSpiltKMlpFusedParams::K"]
+ [::core::mem::offset_of!(GEMMSpiltKMlpFusedParams, K) - 8usize];
+ ["Offset of field: GEMMSpiltKMlpFusedParams::lda"]
+ [::core::mem::offset_of!(GEMMSpiltKMlpFusedParams, lda) - 12usize];
+ ["Offset of field: GEMMSpiltKMlpFusedParams::ldb"]
+ [::core::mem::offset_of!(GEMMSpiltKMlpFusedParams, ldb) - 16usize];
+ ["Offset of field: GEMMSpiltKMlpFusedParams::ldc"]
+ [::core::mem::offset_of!(GEMMSpiltKMlpFusedParams, ldc) - 20usize];
+ ["Offset of field: GEMMSpiltKMlpFusedParams::tiles_n"]
+ [::core::mem::offset_of!(GEMMSpiltKMlpFusedParams, tiles_n) - 24usize];
+ ["Offset of field: GEMMSpiltKMlpFusedParams::tiles_m"]
+ [::core::mem::offset_of!(GEMMSpiltKMlpFusedParams, tiles_m) - 28usize];
+ ["Offset of field: GEMMSpiltKMlpFusedParams::split_k_partitions"][::core::mem::offset_of!(
+ GEMMSpiltKMlpFusedParams,
+ split_k_partitions
+ )
+ - 32usize];
+ ["Offset of field: GEMMSpiltKMlpFusedParams::split_k_partition_stride"][::core::mem::offset_of!(
+ GEMMSpiltKMlpFusedParams,
+ split_k_partition_stride
+ )
+ - 36usize];
+ ["Offset of field: GEMMSpiltKMlpFusedParams::split_k_partition_size"][::core::mem::offset_of!(
+ GEMMSpiltKMlpFusedParams,
+ split_k_partition_size
+ )
+ - 40usize];
+ ["Offset of field: GEMMSpiltKMlpFusedParams::gemm_k_iterations_aligned"][::core::mem::offset_of!(
+ GEMMSpiltKMlpFusedParams,
+ gemm_k_iterations_aligned
+ )
+ - 44usize];
+ ["Offset of field: GEMMSpiltKMlpFusedParams::hidden_dim"][::core::mem::offset_of!(
+ GEMMSpiltKMlpFusedParams,
+ hidden_dim
+ ) - 48usize];
+};
+#[repr(C)]
+#[derive(Debug, Default, Copy, Clone)]
pub struct GEMMAddMMParams {
pub ldc: ::core::ffi::c_int,
pub fdc: ::core::ffi::c_int,
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/defines.h b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/defines.h
similarity index 100%
rename from crates/uzu/src/backends/metal/kernel/matmul/steel/defines.h
rename to crates/uzu/src/backends/metal/kernel/matmul/common/steel/defines.h
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/gemm.h b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/gemm.h
similarity index 100%
rename from crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/gemm.h
rename to crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/gemm.h
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/gemm_nax.h b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/gemm_nax.h
similarity index 100%
rename from crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/gemm_nax.h
rename to crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/gemm_nax.h
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/loader.h b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/loader.h
similarity index 100%
rename from crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/loader.h
rename to crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/loader.h
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/mma.h b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/mma.h
similarity index 100%
rename from crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/mma.h
rename to crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/mma.h
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/nax.h b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/nax.h
similarity index 100%
rename from crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/nax.h
rename to crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/nax.h
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/params.h b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/params.h
similarity index 100%
rename from crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/params.h
rename to crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/params.h
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/transforms.h b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/transforms.h
similarity index 98%
rename from crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/transforms.h
rename to crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/transforms.h
index 76a87e808..8dd550212 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/transforms.h
+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/transforms.h
@@ -67,4 +67,4 @@ struct BlockSwizzle {
}
};
-} // namespace steel
\ No newline at end of file
+} // namespace steel
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/utils.h b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/utils.h
similarity index 100%
rename from crates/uzu/src/backends/metal/kernel/matmul/steel/utils.h
rename to crates/uzu/src/backends/metal/kernel/matmul/common/steel/utils.h
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/utils/integral_constant.h b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/utils/integral_constant.h
similarity index 99%
rename from crates/uzu/src/backends/metal/kernel/matmul/steel/utils/integral_constant.h
rename to crates/uzu/src/backends/metal/kernel/matmul/common/steel/utils/integral_constant.h
index 39007d45a..7fc91a29a 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/steel/utils/integral_constant.h
+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/utils/integral_constant.h
@@ -129,4 +129,4 @@ METAL_FUNC constexpr auto sum(T x, Us... us) {
} // namespace steel
-#pragma METAL internals : disable
\ No newline at end of file
+#pragma METAL internals : disable
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/utils/type_traits.h b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/utils/type_traits.h
similarity index 96%
rename from crates/uzu/src/backends/metal/kernel/matmul/steel/utils/type_traits.h
rename to crates/uzu/src/backends/metal/kernel/matmul/common/steel/utils/type_traits.h
index 3c8aca4b9..6423de3b3 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/steel/utils/type_traits.h
+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/utils/type_traits.h
@@ -52,4 +52,4 @@ using pointer_element_t = typename pointer_element<remove_cv_t<T>>::type;
} // namespace metal
-#pragma METAL internals : disable
\ No newline at end of file
+#pragma METAL internals : disable
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/transpose.rs b/crates/uzu/src/backends/metal/kernel/matmul/common/transpose_configuration.rs
similarity index 89%
rename from crates/uzu/src/backends/metal/kernel/matmul/transpose.rs
rename to crates/uzu/src/backends/metal/kernel/matmul/common/transpose_configuration.rs
index d75b4d2c8..b2785dd25 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/transpose.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/transpose_configuration.rs
@@ -1,5 +1,5 @@
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub(super) enum TransposeConfiguration {
+pub(crate) enum TransposeConfiguration {
NN,
NT,
TN,
@@ -17,7 +17,7 @@ impl TransposeConfiguration {
}
}
-pub(super) fn transpose_configuration(
+pub(crate) fn transpose_configuration(
a_t: bool,
b_t: bool,
) -> TransposeConfiguration {
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gather.rs b/crates/uzu/src/backends/metal/kernel/matmul/gather.rs
deleted file mode 100644
index 3f94225bd..000000000
--- a/crates/uzu/src/backends/metal/kernel/matmul/gather.rs
+++ /dev/null
@@ -1,586 +0,0 @@
-use std::collections::HashMap;
-
-use metal::{
- Buffer as MTLBuffer, ComputeCommandEncoderRef,
- ComputePipelineState as MTLComputePipelineState, MTLSize,
-};
-
-use super::shared_types::GEMMParams;
-use crate::{
- DataType,
- backends::metal::{DeviceClass, MTLContext, MTLError},
-};
-
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-struct RhsOnlyPipelineKey {
- name: String,
- align_m: bool,
- align_n: bool,
- align_k: bool,
-}
-
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-struct GatherPipelineKey {
- name: String,
- has_batch: bool,
- align_m: bool,
- align_n: bool,
- align_k: bool,
-}
-
-#[derive(Debug, Clone)]
-pub struct GatherMmRhsArguments<'a> {
- pub a: &'a MTLBuffer,
- pub b: &'a MTLBuffer,
- pub rhs_indices: &'a MTLBuffer,
- pub out: &'a MTLBuffer,
- pub m: i32,
- pub n: i32,
- pub k: i32,
- pub lda: i32,
- pub ldb: i32,
- pub batch_stride_b: i64,
- pub transpose_b: bool,
- /// Number of matrices available on the RHS (E dimension in the reference backend).
- pub rhs_matrix_count: i32,
-}
-
-#[derive(Debug, Clone)]
-pub struct GatherMmArguments<'a> {
- pub a: &'a MTLBuffer,
- pub b: &'a MTLBuffer,
- pub lhs_indices: &'a MTLBuffer,
- pub rhs_indices: &'a MTLBuffer,
- pub out: &'a MTLBuffer,
- pub m: i32,
- pub n: i32,
- pub k: i32,
- pub lda: i32,
- pub ldb: i32,
- /// Grid Z dimension for the gather kernel (out.size / (M*N) in the reference backend).
- pub batch_size_out: i32,
- /// Number of batch dimensions for the output (out.ndim - 2 in the reference backend).
- pub batch_ndim: i32,
-
- // LHS/RHS indices layout (used when has_batch=true).
- pub lhs_indices_shape: &'a [i32],
- pub lhs_indices_strides: &'a [i64],
- pub rhs_indices_strides: &'a [i64],
-
- // A tensor batch layout (for elem_to_loc on selected matrix index).
- pub batch_ndim_a: i32,
- pub a_shape: &'a [i32],
- pub a_strides: &'a [i64],
-
- // B tensor batch layout (for elem_to_loc on selected matrix index).
- pub batch_ndim_b: i32,
- pub b_shape: &'a [i32],
- pub b_strides: &'a [i64],
-
- pub transpose_a: bool,
- pub transpose_b: bool,
-}
-
-pub struct GatherGemm {
- data_type: DataType,
- rhs_only_pipelines: HashMap<RhsOnlyPipelineKey, MTLComputePipelineState>,
- gather_pipelines: HashMap<GatherPipelineKey, MTLComputePipelineState>,
-}
-
-impl GatherGemm {
- pub fn new(data_type: DataType) -> Self {
- Self {
- data_type,
- rhs_only_pipelines: HashMap::new(),
- gather_pipelines: HashMap::new(),
- }
- }
-
- fn steel_type_name(&self) -> Result<&'static str, MTLError> {
- match self.data_type {
- DataType::F16 => Ok("float16"),
- DataType::BF16 => Ok("bfloat16"),
- DataType::F32 => Ok("float32"),
- other => Err(MTLError::Generic(format!(
- "Unsupported data type for gather GEMM: {other:?}"
- ))),
- }
- }
-
- fn device_class_code(mtl: &MTLContext) -> char {
- match mtl.architecture.device_class {
- DeviceClass::Desktop => 'd',
- DeviceClass::Integrated => 'g',
- DeviceClass::Phone => 'p',
- DeviceClass::Unknown(_) => 'g',
- }
- }
-
- fn select_gemm_tparams(
- device_class: char,
- data_type: DataType,
- transpose_a: bool,
- transpose_b: bool,
- m: i32,
- n: i32,
- k: i32,
- batch_size_out: i32,
- ) -> (i32, i32, i32, i32, i32) {
- // Matches the initialization and tparam selection logic used by the
- // reference backend for gather/segmented GEMM kernels.
- let mut bm = 64;
- let mut bn = 64;
- let mut bk = 16;
- let mut wm = 2;
- let mut wn = 2;
-
- let is_float32 = matches!(data_type, DataType::F32);
- let work_elements = (batch_size_out as i64) * (m as i64) * (n as i64);
-
- match device_class {
- 'g' | 'p' => {
- if !transpose_a && transpose_b {
- // nt
- bm = 64;
- bn = 32;
- bk = 32;
- wm = 2;
- wn = 2;
- } else if !is_float32 {
- // half and bfloat
- bm = 64;
- bn = 64;
- bk = 16;
- wm = 1;
- wn = 2;
- }
- },
- 'd' => {
- if work_elements >= (1_i64 << 20) {
- // large matmul
- if !is_float32 {
- if 2 * std::cmp::max(m, n) > k {
- // Reasonable K
- bm = 64;
- bn = 64;
- bk = 16;
- wm = 1;
- wn = 2;
- } else if !transpose_a && transpose_b {
- // nt with large k
- bm = 64;
- bn = 32;
- bk = 32;
- wm = 2;
- wn = 2;
- } else {
- // nn with large K
- bm = 32;
- bn = 64;
- bk = 16;
- wm = 1;
- wn = 2;
- }
- }
- } else if !is_float32 {
- // smaller matmul, half and bfloat
- if !transpose_a && transpose_b {
- // nt
- bm = 64;
- bn = 32;
- bk = 32;
- wm = 2;
- wn = 2;
- } else {
- // nn
- bm = 64;
- bn = 64;
- bk = 16;
- wm = 1;
- wn = 2;
- }
- }
- },
- _ => {},
- }
-
- (bm, bn, bk, wm, wn)
- }
-
- fn gather_kernel_name(
- &self,
- prefix: &str,
- transpose_a: bool,
- transpose_b: bool,
- bm: i32,
- bn: i32,
- bk: i32,
- wm: i32,
- wn: i32,
- ) -> Result<String, MTLError> {
- let tname = format!(
- "{}{}",
- if transpose_a {
- 't'
- } else {
- 'n'
- },
- if transpose_b {
- 't'
- } else {
- 'n'
- }
- );
- let dtype = self.steel_type_name()?;
- Ok(format!(
- "{prefix}_{tname}_{dtype}_{dtype}_bm{bm}_bn{bn}_bk{bk}_wm{wm}_wn{wn}",
- ))
- }
-
- fn set_i32_slice_bytes(
- enc: &ComputeCommandEncoderRef,
- index: u64,
- data: &[i32],
- ) {
- if data.is_empty() {
- let zero = 0_i32;
- enc.set_bytes(
- index,
- std::mem::size_of::<i32>() as u64,
- &zero as *const i32 as *const _,
- );
- } else {
- enc.set_bytes(
- index,
- (data.len() * std::mem::size_of::<i32>()) as u64,
- data.as_ptr() as *const _,
- );
- }
- }
-
- fn set_i64_slice_bytes(
- enc: &ComputeCommandEncoderRef,
- index: u64,
- data: &[i64],
- ) {
- if data.is_empty() {
- let zero = 0_i64;
- enc.set_bytes(
- index,
- std::mem::size_of::<i64>() as u64,
- &zero as *const i64 as *const _,
- );
- } else {
- enc.set_bytes(
- index,
- (data.len() * std::mem::size_of::<i64>()) as u64,
- data.as_ptr() as *const _,
- );
- }
- }
-
- fn get_or_compile_rhs_only_pipeline(
- &mut self,
- mtl: &MTLContext,
- name: &str,
- align_m: bool,
- align_n: bool,
- align_k: bool,
- ) -> Result<&MTLComputePipelineState, MTLError> {
- let key = RhsOnlyPipelineKey {
- name: name.to_string(),
- align_m,
- align_n,
- align_k,
- };
-
- if !self.rhs_only_pipelines.contains_key(&key) {
- let fcv = metal::FunctionConstantValues::new();
- fcv.set_constant_value_at_index(
- &align_m as *const bool as *const _,
- metal::MTLDataType::Bool,
- 200,
- );
- fcv.set_constant_value_at_index(
- &align_n as *const bool as *const _,
- metal::MTLDataType::Bool,
- 201,
- );
- fcv.set_constant_value_at_index(
- &align_k as *const bool as *const _,
- metal::MTLDataType::Bool,
- 202,
- );
-
- let cache_key = format!(
- "{name}_am{}_an{}_ak{}",
- align_m as u8, align_n as u8, align_k as u8
- );
- let (ps, _) = mtl.compute_pipeline_state_with_reflection_cached(
- &cache_key,
- name,
- Some(&fcv),
- )?;
- self.rhs_only_pipelines.insert(key.clone(), ps);
- }
-
- Ok(self.rhs_only_pipelines.get(&key).unwrap())
- }
-
- fn get_or_compile_gather_pipeline(
- &mut self,
- mtl: &MTLContext,
- name: &str,
- has_batch: bool,
- align_m: bool,
- align_n: bool,
- align_k: bool,
- ) -> Result<&MTLComputePipelineState, MTLError> {
- let key = GatherPipelineKey {
- name: name.to_string(),
- has_batch,
- align_m,
- align_n,
- align_k,
- };
-
- if !self.gather_pipelines.contains_key(&key) {
- let fcv = metal::FunctionConstantValues::new();
- fcv.set_constant_value_at_index(
- &has_batch as *const bool as *const _,
- metal::MTLDataType::Bool,
- 10,
- );
- fcv.set_constant_value_at_index(
- &align_m as *const bool as *const _,
- metal::MTLDataType::Bool,
- 200,
- );
- fcv.set_constant_value_at_index(
- &align_n as *const bool as *const _,
- metal::MTLDataType::Bool,
- 201,
- );
- fcv.set_constant_value_at_index(
- &align_k as *const bool as *const _,
- metal::MTLDataType::Bool,
- 202,
- );
-
- let cache_key = format!(
- "{name}_hb{}_am{}_an{}_ak{}",
- has_batch as u8, align_m as u8, align_n as u8, align_k as u8
- );
- let (ps, _) = mtl.compute_pipeline_state_with_reflection_cached(
- &cache_key,
- name,
- Some(&fcv),
- )?;
- self.gather_pipelines.insert(key.clone(), ps);
- }
-
- Ok(self.gather_pipelines.get(&key).unwrap())
- }
-
- pub fn encode_mm_rhs(
- &mut self,
- mtl: &MTLContext,
- enc: &ComputeCommandEncoderRef,
- args: &GatherMmRhsArguments,
- ) -> Result<(), MTLError> {
- // NAX gather kernels are only instantiated for float16/bfloat16.
- let prefer_nax =
- mtl.is_nax_available() && !matches!(self.data_type, DataType::F32);
-
- let (bm, bn, bk, wm, wn, use_nax) = if prefer_nax {
- // Mirrors rhs-only NAX gather tile selection.
- let e = std::cmp::max(1, args.rhs_matrix_count);
- let ratio = args.m / e;
- let (bm, wm) = if ratio > 48 {
- (64, 2)
- } else if ratio > 24 {
- (32, 1)
- } else {
- (16, 1)
- };
- (bm, 128, 128, wm, 4, true)
- } else {
- (16, 64, 16, 1, 2, false)
- };
-
- let align_m = (args.m % bm) == 0;
- let align_n = (args.n % bn) == 0;
- let align_k = (args.k % bk) == 0;
-
- let prefix = if use_nax {
- "steel_gather_mm_rhs_nax"
- } else {
- "steel_gather_mm_rhs"
- };
-
- // gather_mm_rhs only supports transpose_a=false (nn/nt instantiations).
- let kname = self.gather_kernel_name(
- prefix,
- /*transpose_a=*/ false,
- args.transpose_b,
- bm,
- bn,
- bk,
- wm,
- wn,
- )?;
-
- let ps = self.get_or_compile_rhs_only_pipeline(
- mtl, &kname, align_m, align_n, align_k,
- )?;
-
- enc.set_compute_pipeline_state(ps);
- enc.set_buffer(0, Some(args.a), 0);
- enc.set_buffer(1, Some(args.b), 0);
- enc.set_buffer(2, Some(args.rhs_indices), 0);
- enc.set_buffer(3, Some(args.out), 0);
-
- let tiles_n = (args.n + bn - 1) / bn;
- let tiles_m = (args.m + bm - 1) / bm;
- let params = GEMMParams {
- M: args.m,
- N: args.n,
- K: args.k,
- lda: args.lda,
- ldb: args.ldb,
- ldd: args.n,
- tiles_n,
- tiles_m,
- batch_stride_a: 0,
- batch_stride_b: args.batch_stride_b,
- batch_stride_d: 0,
- swizzle_log: 0,
- gemm_k_iterations_aligned: args.k / bk,
- batch_ndim: 0,
- };
- enc.set_bytes(
- 4,
- std::mem::size_of::<GEMMParams>() as u64,
- &params as *const GEMMParams as *const _,
- );
-
- let threads_per_tg = MTLSize::new(32, wn as u64, wm as u64);
- let tgs = MTLSize::new(tiles_n as u64, tiles_m as u64, 1);
- enc.dispatch_thread_groups(tgs, threads_per_tg);
- Ok(())
- }
-
- pub fn encode_mm(
- &mut self,
- mtl: &MTLContext,
- enc: &ComputeCommandEncoderRef,
- args: &GatherMmArguments,
- ) -> Result<(), MTLError> {
- let device_class = Self::device_class_code(mtl);
- let (bm, bn, bk, wm, wn) = Self::select_gemm_tparams(
- device_class,
- self.data_type,
- args.transpose_a,
- args.transpose_b,
- args.m,
- args.n,
- args.k,
- args.batch_size_out,
- );
-
- let has_batch = args.batch_ndim > 1;
- let align_m = (args.m % bm) == 0;
- let align_n = (args.n % bn) == 0;
- let align_k = (args.k % bk) == 0;
-
- let kname = self.gather_kernel_name(
- "steel_gather_mm",
- args.transpose_a,
- args.transpose_b,
- bm,
- bn,
- bk,
- wm,
- wn,
- )?;
-
- let ps = self.get_or_compile_gather_pipeline(
- mtl, &kname, has_batch, align_m, align_n, align_k,
- )?;
-
- enc.set_compute_pipeline_state(ps);
-
- enc.set_buffer(0, Some(args.a), 0);
- enc.set_buffer(1, Some(args.b), 0);
- enc.set_buffer(2, Some(args.lhs_indices), 0);
- enc.set_buffer(3, Some(args.rhs_indices), 0);
- enc.set_buffer(4, Some(args.out), 0);
-
- let tiles_n = (args.n + bn - 1) / bn;
- let tiles_m = (args.m + bm - 1) / bm;
- let batch_stride_a = if args.batch_ndim > 0 {
- *args.lhs_indices_strides.first().unwrap_or(&0)
- } else {
- 0
- };
- let batch_stride_b = if args.batch_ndim > 0 {
- *args.rhs_indices_strides.first().unwrap_or(&0)
- } else {
- 0
- };
-
- let params = GEMMParams {
- M: args.m,
- N: args.n,
- K: args.k,
- lda: args.lda,
- ldb: args.ldb,
- ldd: args.n,
- tiles_n,
- tiles_m,
- batch_stride_a,
- batch_stride_b,
- batch_stride_d: (args.m as i64) * (args.n as i64),
- swizzle_log: 0,
- gemm_k_iterations_aligned: args.k / bk,
- batch_ndim: args.batch_ndim,
- };
-
- enc.set_bytes(
- 5,
- std::mem::size_of::<GEMMParams>() as u64,
- &params as *const GEMMParams as *const _,
- );
-
- // Note: many of these are only used when has_batch=true, but they are
- // present in the kernel signature unconditionally.
- Self::set_i32_slice_bytes(enc, 6, args.lhs_indices_shape);
- Self::set_i64_slice_bytes(enc, 7, args.lhs_indices_strides);
- Self::set_i64_slice_bytes(enc, 8, args.rhs_indices_strides);
-
- enc.set_bytes(
- 9,
- std::mem::size_of::<i32>() as u64,
- &args.batch_ndim_a as *const i32 as *const _,
- );
- Self::set_i32_slice_bytes(enc, 10, args.a_shape);
- Self::set_i64_slice_bytes(enc, 11, args.a_strides);
-
- enc.set_bytes(
- 12,
- std::mem::size_of::<i32>() as u64,
- &args.batch_ndim_b as *const i32 as *const _,
- );
- Self::set_i32_slice_bytes(enc, 13, args.b_shape);
- Self::set_i64_slice_bytes(enc, 14, args.b_strides);
-
- let threads_per_tg = MTLSize::new(32, wn as u64, wm as u64);
- let tgs = MTLSize::new(
- tiles_n as u64,
- tiles_m as u64,
- args.batch_size_out as u64,
- );
- enc.dispatch_thread_groups(tgs, threads_per_tg);
- Ok(())
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
new file mode 100644
index 000000000..9742f81ae
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
@@ -0,0 +1,336 @@
+use std::collections::HashMap;
+
+use metal::{
+ ComputeCommandEncoderRef, ComputePipelineState as MTLComputePipelineState,
+ MTLSize,
+};
+
+use super::{
+ pipeline_configuration::PipelineConfiguration,
+ tile_configuration::TileConfiguration,
+};
+use crate::{
+ DataType,
+ backends::metal::{
+ DeviceClass, MTLContext, MTLError,
+ kernel::matmul::common::{
+ GEMMAddMMParams, GEMMParams, MatmulArguments,
+ transpose_configuration,
+ },
+ },
+};
+
+pub struct Kernel {
+ data_type: DataType,
+ transpose_a: bool,
+ transpose_b: bool,
+ pipelines: HashMap<PipelineConfiguration, MTLComputePipelineState>,
+}
+
+impl Kernel {
+ pub fn new(
+ data_type: DataType,
+ transpose_a: bool,
+ transpose_b: bool,
+ ) -> Result<Self, MTLError> {
+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)
+ {
+ return Err(MTLError::Generic(format!(
+ "Unsupported dtype for GEMM: {data_type:?}"
+ )));
+ }
+ Ok(Self {
+ data_type,
+ transpose_a,
+ transpose_b,
+ pipelines: HashMap::new(),
+ })
+ }
+
+ fn type_name(&self) -> &'static str {
+ match self.data_type {
+ DataType::F16 => "float16",
+ DataType::BF16 => "bfloat16",
+ DataType::F32 => "float32",
+ _ => unreachable!(),
+ }
+ }
+
+ fn kernel_name(
+ &self,
+ tile: &TileConfiguration,
+ ) -> String {
+ let type_name = self.type_name();
+ let transpose_suffix =
+ transpose_configuration(self.transpose_a, self.transpose_b);
+ let prefix = if tile.is_nax() {
+ "steel_gemm_nax"
+ } else {
+ "steel_gemm"
+ };
+ format!(
+ "{}_{}_{}_{}_bm{}_bn{}_bk{}_wm{}_wn{}",
+ prefix,
+ transpose_suffix.as_str(),
+ type_name,
+ type_name,
+ tile.block_rows,
+ tile.block_cols,
+ tile.block_depth,
+ tile.warps_per_row,
+ tile.warps_per_col
+ )
+ }
+
+ pub fn select_tile(
+ &self,
+ mtl: &MTLContext,
+ args: &MatmulArguments,
+ ) -> TileConfiguration {
+ let overall_work_elements = (args.batch_count as i64)
+ * (args.batch as i64)
+ * (args.output_dim as i64);
+ let is_float32 = matches!(self.data_type, DataType::F32);
+ let prefer_half_or_tf32 = !is_float32 || mtl.tf32_enabled();
+
+ if mtl.is_nax_available() && prefer_half_or_tf32 {
+ let base_tile = TileConfiguration::new(128, 128, 512, 4, 4, 0);
+ let tile_rows =
+ (args.batch + base_tile.block_rows - 1) / base_tile.block_rows;
+ let swizzle_log2 = if tile_rows <= 3 {
+ 0
+ } else {
+ 1
+ };
+ return TileConfiguration {
+ swizzle_log2,
+ ..base_tile
+ };
+ }
+
+ let device_class_code = match mtl.architecture.device_class {
+ DeviceClass::Desktop => 'd',
+ DeviceClass::Integrated => 'g',
+ DeviceClass::Phone => 'p',
+ DeviceClass::Unknown(_) => 'g',
+ };
+
+ match device_class_code {
+ 'g' | 'p' => {
+ if prefer_half_or_tf32 {
+ if !self.transpose_a && self.transpose_b {
+ TileConfiguration::new(64, 32, 32, 2, 2, 0)
+ } else {
+ TileConfiguration::new(64, 64, 16, 1, 2, 0)
+ }
+ } else if !self.transpose_a && self.transpose_b {
+ TileConfiguration::new(32, 64, 16, 1, 2, 0)
+ } else {
+ TileConfiguration::new(64, 32, 32, 2, 2, 0)
+ }
+ },
+ 'd' => {
+ if overall_work_elements >= (1_i64 << 20) {
+ if prefer_half_or_tf32 {
+ if 2 * std::cmp::max(args.batch, args.output_dim)
+ > args.input_dim
+ {
+ TileConfiguration::new(64, 64, 16, 2, 2, 0)
+ } else if !self.transpose_a && self.transpose_b {
+ TileConfiguration::new(64, 32, 32, 2, 2, 0)
+ } else {
+ TileConfiguration::new(32, 64, 16, 1, 2, 0)
+ }
+ } else if !self.transpose_a && self.transpose_b {
+ TileConfiguration::new(32, 64, 16, 1, 2, 0)
+ } else {
+ TileConfiguration::new(64, 32, 32, 2, 2, 0)
+ }
+ } else if prefer_half_or_tf32 {
+ if !self.transpose_a && self.transpose_b {
+ TileConfiguration::new(64, 32, 32, 2, 2, 0)
+ } else {
+ TileConfiguration::new(64, 64, 16, 1, 2, 0)
+ }
+ } else if !self.transpose_a && self.transpose_b {
+ TileConfiguration::new(32, 64, 16, 1, 2, 0)
+ } else {
+ TileConfiguration::new(64, 32, 32, 2, 2, 0)
+ }
+ },
+ _ => TileConfiguration::new(64, 64, 16, 2, 2, 0),
+ }
+ }
+
+ fn get_or_compile_pipeline(
+ &mut self,
+ mtl: &MTLContext,
+ config: &PipelineConfiguration,
+ ) -> Result<&MTLComputePipelineState, MTLError> {
+ if !self.pipelines.contains_key(config) {
+ let name = self.kernel_name(&config.tile);
+ let fcv = metal::FunctionConstantValues::new();
+ fcv.set_constant_value_at_index(
+ &config.has_batch as *const bool as *const _,
+ metal::MTLDataType::Bool,
+ 10,
+ );
+ fcv.set_constant_value_at_index(
+ &config.use_out_source as *const bool as *const _,
+ metal::MTLDataType::Bool,
+ 100,
+ );
+ fcv.set_constant_value_at_index(
+ &config.do_axpby as *const bool as *const _,
+ metal::MTLDataType::Bool,
+ 110,
+ );
+ fcv.set_constant_value_at_index(
+ &config.align_m as *const bool as *const _,
+ metal::MTLDataType::Bool,
+ 200,
+ );
+ fcv.set_constant_value_at_index(
+ &config.align_n as *const bool as *const _,
+ metal::MTLDataType::Bool,
+ 201,
+ );
+ fcv.set_constant_value_at_index(
+ &config.align_k as *const bool as *const _,
+ metal::MTLDataType::Bool,
+ 202,
+ );
+
+ let cache_key = format!(
+ "{}_am{}_an{}_ak{}_hb{}_uo{}_ax{}",
+ name,
+ config.align_m as u8,
+ config.align_n as u8,
+ config.align_k as u8,
+ config.has_batch as u8,
+ config.use_out_source as u8,
+ config.do_axpby as u8
+ );
+ let (ps, _) = mtl.compute_pipeline_state_with_reflection_cached(
+ &cache_key,
+ &name,
+ Some(&fcv),
+ )?;
+ self.pipelines.insert(config.clone(), ps);
+ }
+ Ok(self.pipelines.get(config).unwrap())
+ }
+
+ pub fn encode(
+ &mut self,
+ mtl: &MTLContext,
+ enc: &ComputeCommandEncoderRef,
+ args: &MatmulArguments,
+ ) -> Result<(), MTLError> {
+ let tile = self.select_tile(mtl, args);
+ self.encode_with_tile(mtl, enc, args, &tile)
+ }
+
+ pub fn encode_with_tile(
+ &mut self,
+ mtl: &MTLContext,
+ enc: &ComputeCommandEncoderRef,
+ args: &MatmulArguments,
+ tile: &TileConfiguration,
+ ) -> Result<(), MTLError> {
+ let m = args.batch;
+ let n = args.output_dim;
+ let k = args.input_dim;
+
+ let config = PipelineConfiguration {
+ tile: *tile,
+ align_m: (m % tile.block_rows) == 0,
+ align_n: (n % tile.block_cols) == 0,
+ align_k: (k % tile.block_depth) == 0,
+ has_batch: args.batch_count > 1,
+ use_out_source: args.c.is_some(),
+ do_axpby: args.c.is_some()
+ && (args.alpha != 1.0 || args.beta != 0.0),
+ };
+
+ let ps = self.get_or_compile_pipeline(mtl, &config)?;
+ enc.set_compute_pipeline_state(ps);
+
+ enc.set_buffer(0, Some(args.a), args.a_offset);
+ enc.set_buffer(1, Some(args.b), 0);
+ if config.use_out_source {
+ if let Some(c_buf) = args.c {
+ enc.set_buffer(2, Some(c_buf), 0);
+ }
+ }
+ enc.set_buffer(3, Some(args.d), 0);
+
+ let tiles_n = (n + tile.block_cols - 1) / tile.block_cols;
+ let tiles_m = (m + tile.block_rows - 1) / tile.block_rows;
+ let swizzle_log = tile.swizzle_log2;
+
+ let tile_swizzle = 1 << swizzle_log;
+ let tm_swizzled = (tiles_m + tile_swizzle - 1) / tile_swizzle;
+ let tn_swizzled = tiles_n * tile_swizzle;
+
+ let elements_per_matrix_a = (args.batch as i64) * (args.lda as i64);
+ let elements_per_matrix_b = if self.transpose_b {
+ (args.output_dim as i64) * (args.ldb as i64)
+ } else {
+ (args.input_dim as i64) * (args.ldb as i64)
+ };
+ let elements_per_matrix_d = (args.batch as i64) * (args.ldd as i64);
+
+ let params = GEMMParams {
+ M: m,
+ N: n,
+ K: k,
+ lda: args.lda,
+ ldb: args.ldb,
+ ldd: args.ldd,
+ tiles_n,
+ tiles_m,
+ batch_stride_a: elements_per_matrix_a,
+ batch_stride_b: elements_per_matrix_b,
+ batch_stride_d: elements_per_matrix_d,
+ swizzle_log,
+ gemm_k_iterations_aligned: k / tile.block_depth,
+ batch_ndim: 1,
+ };
+ enc.set_bytes(
+ 4,
+ std::mem::size_of::<GEMMParams>() as u64,
+ &params as *const GEMMParams as *const _,
+ );
+
+ if config.use_out_source {
+ let batch_stride_c = if args.batch_count > 1 {
+ (args.ldd as i64) * (args.output_dim as i64)
+ } else {
+ 0
+ };
+ let addmm_params = GEMMAddMMParams {
+ ldc: args.ldd,
+ fdc: 1,
+ batch_stride_c,
+ alpha: args.alpha,
+ beta: args.beta,
+ };
+ enc.set_bytes(
+ 5,
+ std::mem::size_of::<GEMMAddMMParams>() as u64,
+ &addmm_params as *const GEMMAddMMParams as *const _,
+ );
+ }
+
+ let threads_per_tg =
+ MTLSize::new(32, tile.warps_per_col, tile.warps_per_row);
+ let tgs = MTLSize::new(
+ tn_swizzled as u64,
+ tm_swizzled as u64,
+ args.batch_count as u64,
+ );
+ enc.dispatch_thread_groups(tgs, threads_per_tg);
+ Ok(())
+ }
+}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm/mod.rs
new file mode 100644
index 000000000..f78544f8c
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm/mod.rs
@@ -0,0 +1,5 @@
+mod kernel;
+mod pipeline_configuration;
+mod tile_configuration;
+
+pub use kernel::Kernel as GemmKernel;
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/pipeline.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm/pipeline_configuration.rs
similarity index 62%
rename from crates/uzu/src/backends/metal/kernel/matmul/pipeline.rs
rename to crates/uzu/src/backends/metal/kernel/matmul/gemm/pipeline_configuration.rs
index 26c0062ad..e92065b03 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/pipeline.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm/pipeline_configuration.rs
@@ -1,6 +1,8 @@
+use super::tile_configuration::TileConfiguration;
+
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub(super) struct PipelineKey {
- pub name: String,
+pub struct PipelineConfiguration {
+ pub tile: TileConfiguration,
pub align_m: bool,
pub align_n: bool,
pub align_k: bool,
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_fused.h b/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.h
similarity index 100%
rename from crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_fused.h
rename to crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.h
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_fused.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.metal
similarity index 91%
rename from crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_fused.metal
rename to crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.metal
index 7053470c3..928e3f97f 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_fused.metal
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm.metal
@@ -1,14 +1,14 @@
// clang-format off
-#include "../../../../common/utils.h"
+#include "../../common/utils.h"
-#include "../gemm.h"
-#include "steel_gemm_fused.h"
+#include "../../common/steel/gemm/gemm.h"
+#include "steel_gemm.h"
#define instantiate_gemm(tname, trans_a, trans_b, iname, itype, oname, otype, bm, bn, bk, wm, wn) \
instantiate_kernel( \
- "steel_gemm_fused_" #tname "_" #iname "_" #oname \
+ "steel_gemm_" #tname "_" #iname "_" #oname \
"_bm" #bm "_bn" #bn "_bk" #bk "_wm" #wm "_wn" #wn, \
gemm, itype, bm, bn, bk, wm, wn, trans_a, trans_b, float)
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_fused_nax.h b/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm_nax.h
similarity index 100%
rename from crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_fused_nax.h
rename to crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm_nax.h
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_fused_nax.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm_nax.metal
similarity index 88%
rename from crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_fused_nax.metal
rename to crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm_nax.metal
index a0c3b9dda..8a3070c0d 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_fused_nax.metal
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm/shaders/steel_gemm_nax.metal
@@ -2,15 +2,15 @@
#include <metal_stdlib>
-#include "../../../../common/utils.h"
+#include "../../common/utils.h"
-#include "../gemm_nax.h"
-#include "steel_gemm_fused_nax.h"
+#include "../../common/steel/gemm/gemm_nax.h"
+#include "steel_gemm_nax.h"
// clang-format off
#define instantiate_gemm(tname, trans_a, trans_b, iname, itype, oname, otype, bm, bn, bk, wm, wn) \
instantiate_kernel( \
- "steel_gemm_fused_nax_" #tname "_" #iname "_" #oname \
+ "steel_gemm_nax_" #tname "_" #iname "_" #oname \
"_bm" #bm "_bn" #bn "_bk" #bk "_wm" #wm "_wn" #wn, \
gemm, itype, bm, bn, bk, wm, wn, trans_a, trans_b, float)
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/tile_configuration.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm/tile_configuration.rs
new file mode 100644
index 000000000..3f857b490
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm/tile_configuration.rs
@@ -0,0 +1,33 @@
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct TileConfiguration {
+ pub block_rows: i32,
+ pub block_cols: i32,
+ pub block_depth: i32,
+ pub warps_per_row: u64,
+ pub warps_per_col: u64,
+ pub swizzle_log2: i32,
+}
+
+impl TileConfiguration {
+ pub fn new(
+ block_rows: i32,
+ block_cols: i32,
+ block_depth: i32,
+ warps_per_row: u64,
+ warps_per_col: u64,
+ swizzle_log2: i32,
+ ) -> Self {
+ Self {
+ block_rows,
+ block_cols,
+ block_depth,
+ warps_per_row,
+ warps_per_col,
+ swizzle_log2,
+ }
+ }
+
+ pub fn is_nax(&self) -> bool {
+ self.block_depth >= 256
+ }
+}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv.rs
deleted file mode 100644
index db5ee62da..000000000
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv.rs
+++ /dev/null
@@ -1,672 +0,0 @@
-use std::collections::HashMap;
-
-use metal::{
- Buffer as MTLBuffer, ComputeCommandEncoderRef, ComputePipelineState,
- MTLSize,
-};
-
-use super::{super::mlp::MlpActivationType, arguments::MatmulArguments};
-use crate::{
- DataType,
- backends::metal::{MTLContext, MTLError},
-};
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-struct GemvConfiguration {
- transpose_matrix: bool,
- threadgroup_rows: u32,
- threadgroup_cols: u32,
- threads_per_simdgroup_row: u32,
- threads_per_simdgroup_col: u32,
- elements_per_thread_row: u32,
- elements_per_thread_col: u32,
- non_contiguous_batch: bool,
- do_axpby: bool,
-}
-
-impl GemvConfiguration {
- fn output_elements_per_threadgroup(&self) -> u32 {
- if self.transpose_matrix {
- self.threadgroup_cols
- * self.threads_per_simdgroup_col
- * self.elements_per_thread_col
- } else {
- self.threadgroup_rows
- * self.threads_per_simdgroup_row
- * self.elements_per_thread_row
- }
- }
-
- fn threads_per_threadgroup(&self) -> MTLSize {
- // 32 threads per simdgroup, arranged (x=32, y=BN, z=BM)
- MTLSize::new(
- 32,
- self.threadgroup_cols as u64,
- self.threadgroup_rows as u64,
- )
- }
-}
-
-fn gemv_kernel_name(
- data_type: DataType,
- configuration: &GemvConfiguration,
-) -> Result<String, MTLError> {
- let dtype_name = match data_type {
- DataType::F16 => "float16",
- DataType::BF16 => "bfloat16",
- DataType::F32 => "float32",
- _ => {
- return Err(MTLError::Generic(format!(
- "Unsupported data type for GEMV: {:?}",
- data_type
- )));
- },
- };
-
- let prefix = if configuration.transpose_matrix {
- "gemv_t"
- } else {
- "gemv"
- };
-
- Ok(format!(
- "{prefix}_{dtype_name}_bm{}_bn{}_sm{}_sn{}_tm{}_tn{}_nc{}_axpby{}",
- configuration.threadgroup_rows,
- configuration.threadgroup_cols,
- configuration.threads_per_simdgroup_row,
- configuration.threads_per_simdgroup_col,
- configuration.elements_per_thread_row,
- configuration.elements_per_thread_col,
- configuration.non_contiguous_batch as u8,
- configuration.do_axpby as u8,
- ))
-}
-
-fn select_gemv_configuration(
- transpose_matrix: bool,
- input_dimension: i32,
- output_dimension: i32,
- non_contiguous_batch: bool,
- do_axpby: bool,
-) -> GemvConfiguration {
- // GEMV tuning surface for axpby kernel selection.
- let (threadgroup_rows, threadgroup_cols);
- let (threads_per_simdgroup_row, threads_per_simdgroup_col);
- let (elements_per_thread_row, elements_per_thread_col);
-
- if transpose_matrix {
- let mut sm = 8;
- let mut sn = 4;
- if input_dimension >= 8192 && output_dimension >= 2048 {
- sm = 4;
- sn = 8;
- }
-
- let bn = if output_dimension >= 2048 {
- 16
- } else if output_dimension >= 512 {
- 4
- } else {
- 2
- };
-
- let tn = if output_dimension < 4 {
- 1
- } else {
- 4
- };
-
- threadgroup_rows = 1;
- threadgroup_cols = bn;
- threads_per_simdgroup_row = sm;
- threads_per_simdgroup_col = sn;
- elements_per_thread_row = 4;
- elements_per_thread_col = tn;
- } else {
- let mut bm = if output_dimension >= 4096 {
- 8
- } else {
- 4
- };
- let mut sm = 1;
- let mut sn = 32;
- let mut bn = 1;
-
- if input_dimension <= 64 {
- bm = 1;
- sm = 8;
- sn = 4;
- } else if input_dimension >= 16 * output_dimension {
- bm = 1;
- bn = 8;
- }
-
- let tm = if output_dimension < 4 {
- 1
- } else {
- 4
- };
-
- threadgroup_rows = bm;
- threadgroup_cols = bn;
- threads_per_simdgroup_row = sm;
- threads_per_simdgroup_col = sn;
- elements_per_thread_row = tm;
- elements_per_thread_col = 4;
- }
-
- GemvConfiguration {
- transpose_matrix,
- threadgroup_rows,
- threadgroup_cols,
- threads_per_simdgroup_row,
- threads_per_simdgroup_col,
- elements_per_thread_row,
- elements_per_thread_col,
- non_contiguous_batch,
- do_axpby,
- }
-}
-
-pub struct GemvKernel {
- data_type: DataType,
- lhs_is_transposed: bool,
- rhs_is_transposed: bool,
- pipelines: HashMap<GemvConfiguration, ComputePipelineState>,
-}
-
-impl GemvKernel {
- pub fn new(
- data_type: DataType,
- lhs_is_transposed: bool,
- rhs_is_transposed: bool,
- ) -> Self {
- Self {
- data_type,
- lhs_is_transposed,
- rhs_is_transposed,
- pipelines: HashMap::new(),
- }
- }
-
- fn get_pipeline(
- &mut self,
- context: &MTLContext,
- configuration: GemvConfiguration,
- ) -> Result<&ComputePipelineState, MTLError> {
- if !self.pipelines.contains_key(&configuration) {
- let kernel_name = gemv_kernel_name(self.data_type, &configuration)?;
- let pipeline =
- context.compute_pipeline_state(&kernel_name, None)?;
- self.pipelines.insert(configuration, pipeline);
- }
- Ok(self.pipelines.get(&configuration).unwrap())
- }
-
- #[allow(clippy::too_many_arguments)]
- fn encode_with_configuration(
- &mut self,
- context: &MTLContext,
- encoder: &ComputeCommandEncoderRef,
- arguments: &MatmulArguments,
- configuration: GemvConfiguration,
- matrix_is_rhs: bool,
- bias_buffer: Option<&MTLBuffer>,
- alpha: f32,
- beta: f32,
- bias_stride: i32,
- ) -> Result<(), MTLError> {
- let pipeline = self.get_pipeline(context, configuration)?;
- encoder.set_compute_pipeline_state(pipeline);
-
- let (buf0, off0) = if matrix_is_rhs {
- (arguments.b, 0)
- } else {
- (arguments.a, arguments.a_offset)
- };
- encoder.set_buffer(0, Some(buf0), off0);
-
- let (buf1, off1) = if matrix_is_rhs {
- (arguments.a, arguments.a_offset)
- } else {
- (arguments.b, 0)
- };
- encoder.set_buffer(1, Some(buf1), off1);
-
- if configuration.do_axpby {
- if let Some(bias) = bias_buffer {
- encoder.set_buffer(2, Some(bias), 0);
- }
- }
-
- encoder.set_buffer(3, Some(arguments.d), 0);
-
- let input_dimension = arguments.input_dim;
- let output_dimension = if matrix_is_rhs {
- arguments.output_dim
- } else {
- arguments.batch
- };
- let matrix_leading_dim = if matrix_is_rhs {
- arguments.ldb
- } else {
- arguments.lda
- };
-
- encoder.set_bytes(
- 4,
- std::mem::size_of::<i32>() as u64,
- &input_dimension as *const i32 as *const std::ffi::c_void,
- );
- encoder.set_bytes(
- 5,
- std::mem::size_of::<i32>() as u64,
- &output_dimension as *const i32 as *const std::ffi::c_void,
- );
- encoder.set_bytes(
- 6,
- std::mem::size_of::<i32>() as u64,
- &matrix_leading_dim as *const i32 as *const std::ffi::c_void,
- );
-
- encoder.set_bytes(
- 7,
- std::mem::size_of::<f32>() as u64,
- &alpha as *const f32 as *const std::ffi::c_void,
- );
- encoder.set_bytes(
- 8,
- std::mem::size_of::<f32>() as u64,
- &beta as *const f32 as *const std::ffi::c_void,
- );
-
- let has_batch = arguments.batch_count > 1;
- let batch_shape = if has_batch {
- vec![arguments.batch_count]
- } else {
- vec![1]
- };
- let batch_ndim = 1i32;
- let batch_groups = arguments.batch_count.max(1);
-
- let elements_per_matrix_a =
- (arguments.batch as i64) * (arguments.lda as i64);
- let elements_per_matrix_b = if self.rhs_is_transposed {
- (arguments.output_dim as i64) * (arguments.ldb as i64)
- } else {
- (arguments.input_dim as i64) * (arguments.ldb as i64)
- };
-
- let vector_batch_stride = if matrix_is_rhs {
- vec![elements_per_matrix_a]
- } else {
- vec![elements_per_matrix_b]
- };
-
- let matrix_batch_stride = if matrix_is_rhs {
- vec![elements_per_matrix_b]
- } else {
- vec![elements_per_matrix_a]
- };
-
- let bias_batch_stride = if arguments.batch_count > 1 {
- vec![(output_dimension as i64) * (arguments.ldd as i64)]
- } else {
- vec![0]
- };
-
- encoder.set_bytes(
- 9,
- std::mem::size_of::<i32>() as u64,
- &batch_ndim as *const i32 as *const std::ffi::c_void,
- );
- encoder.set_bytes(
- 10,
- (std::mem::size_of::<i32>() * batch_shape.len()) as u64,
- batch_shape.as_ptr() as *const std::ffi::c_void,
- );
- encoder.set_bytes(
- 11,
- (std::mem::size_of::<i64>() * vector_batch_stride.len()) as u64,
- vector_batch_stride.as_ptr() as *const std::ffi::c_void,
- );
- encoder.set_bytes(
- 12,
- (std::mem::size_of::<i64>() * matrix_batch_stride.len()) as u64,
- matrix_batch_stride.as_ptr() as *const std::ffi::c_void,
- );
- encoder.set_bytes(
- 13,
- (std::mem::size_of::<i64>() * bias_batch_stride.len()) as u64,
- bias_batch_stride.as_ptr() as *const std::ffi::c_void,
- );
- encoder.set_bytes(
- 14,
- std::mem::size_of::<i32>() as u64,
- &bias_stride as *const i32 as *const std::ffi::c_void,
- );
-
- let output_elements_per_threadgroup =
- configuration.output_elements_per_threadgroup();
- let threadgroup_count_x =
- ((output_dimension as u32 + output_elements_per_threadgroup - 1)
- / output_elements_per_threadgroup) as u64;
- let threadgroup_count_z = batch_groups.max(1) as u64;
-
- let threadgroup_count =
- MTLSize::new(threadgroup_count_x, 1, threadgroup_count_z);
- let threads_per_threadgroup = configuration.threads_per_threadgroup();
-
- encoder
- .dispatch_thread_groups(threadgroup_count, threads_per_threadgroup);
- Ok(())
- }
-
- pub fn encode(
- &mut self,
- context: &MTLContext,
- encoder: &ComputeCommandEncoderRef,
- arguments: &MatmulArguments,
- ) -> Result<(), MTLError> {
- self.encode_internal(context, encoder, arguments, None)
- }
-
- pub fn encode_with_bias(
- &mut self,
- context: &MTLContext,
- encoder: &ComputeCommandEncoderRef,
- arguments: &MatmulArguments,
- bias: &MTLBuffer,
- ) -> Result<(), MTLError> {
- self.encode_internal(context, encoder, arguments, Some(bias))
- }
-
- fn encode_internal(
- &mut self,
- context: &MTLContext,
- encoder: &ComputeCommandEncoderRef,
- arguments: &MatmulArguments,
- bias: Option<&MTLBuffer>,
- ) -> Result<(), MTLError> {
- let m = arguments.batch;
- let n = arguments.output_dim;
- if m != 1 && n != 1 {
- return Ok(());
- }
-
- let matrix_is_rhs = n != 1;
- let transpose_matrix = if matrix_is_rhs {
- !self.rhs_is_transposed
- } else {
- self.lhs_is_transposed
- };
-
- let has_non_contiguous_batch = false;
-
- let (do_axpby, bias_buffer, alpha, beta, bias_stride) =
- if let Some(bias_buf) = bias {
- (true, Some(bias_buf), 1.0f32, 1.0f32, 1)
- } else if let Some(c_buf) = arguments.c {
- (
- true,
- Some(c_buf),
- arguments.alpha,
- arguments.beta,
- arguments.ldd,
- )
- } else {
- (false, None, 1.0f32, 0.0f32, 0)
- };
-
- let configuration = select_gemv_configuration(
- transpose_matrix,
- arguments.input_dim,
- if matrix_is_rhs {
- arguments.output_dim
- } else {
- arguments.batch
- },
- has_non_contiguous_batch,
- do_axpby,
- );
-
- if crate::utils::env_utils::debug_matmul_enabled() {
- let kernel_name = gemv_kernel_name(self.data_type, &configuration)
- .unwrap_or_default();
- eprintln!(
- "[matmul] GEMV m={} k={} n={} batch={} dtype={:?} transpose={} kernel={}",
- arguments.batch,
- arguments.input_dim,
- arguments.output_dim,
- arguments.batch_count,
- self.data_type,
- configuration.transpose_matrix,
- kernel_name
- );
- }
-
- self.encode_with_configuration(
- context,
- encoder,
- arguments,
- configuration,
- matrix_is_rhs,
- bias_buffer,
- alpha,
- beta,
- bias_stride,
- )
- }
-}
-
-/// MLP Fused GEMV configuration
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-struct MlpFusedGemvConfiguration {
- threadgroup_rows: u32,
- threadgroup_cols: u32,
- threads_per_simdgroup_row: u32,
- threads_per_simdgroup_col: u32,
- elements_per_thread_row: u32,
- elements_per_thread_col: u32,
- activation: MlpActivationType,
-}
-
-impl MlpFusedGemvConfiguration {
- fn output_elements_per_threadgroup(&self) -> u32 {
- self.threadgroup_rows
- * self.threads_per_simdgroup_row
- * self.elements_per_thread_row
- }
-
- fn threads_per_threadgroup(&self) -> MTLSize {
- MTLSize::new(
- 32,
- self.threadgroup_cols as u64,
- self.threadgroup_rows as u64,
- )
- }
-}
-
-fn mlp_fused_gemv_kernel_name(
- data_type: DataType,
- config: &MlpFusedGemvConfiguration,
-) -> Result<String, MTLError> {
- let dtype_name = match data_type {
- DataType::F16 => "float16",
- DataType::BF16 => "bfloat16",
- DataType::F32 => "float32",
- _ => {
- return Err(MTLError::Generic(format!(
- "Unsupported data type for MLP fused GEMV: {:?}",
- data_type
- )));
- },
- };
-
- Ok(format!(
- "gemv_mlp_fused_{}_bm{}_bn{}_sm{}_sn{}_tm{}_tn{}",
- dtype_name,
- config.threadgroup_rows,
- config.threadgroup_cols,
- config.threads_per_simdgroup_row,
- config.threads_per_simdgroup_col,
- config.elements_per_thread_row,
- config.elements_per_thread_col,
- ))
-}
-
-fn select_mlp_fused_gemv_configuration(
- hidden_dim: i32,
- activation: MlpActivationType,
-) -> MlpFusedGemvConfiguration {
- // Select configuration based on hidden dimension
- let threadgroup_rows = if hidden_dim >= 4096 {
- 8
- } else {
- 4
- };
-
- MlpFusedGemvConfiguration {
- threadgroup_rows,
- threadgroup_cols: 1,
- threads_per_simdgroup_row: 1,
- threads_per_simdgroup_col: 32,
- elements_per_thread_row: 4,
- elements_per_thread_col: 4,
- activation,
- }
-}
-
-/// Arguments for MLP fused GEMV
-#[derive(Debug)]
-pub struct MlpFusedGemvArguments<'a> {
- /// Weight matrix [2*hidden_dim, input_dim] (up and gate concatenated)
- pub weights: &'a MTLBuffer,
- /// Input vector [input_dim]
- pub input: &'a MTLBuffer,
- /// Input byte offset
- pub input_offset: u64,
- /// Output vector [hidden_dim]
- pub output: &'a MTLBuffer,
- /// Input dimension (K)
- pub input_dim: i32,
- /// Hidden dimension (output size, half of weight rows)
- pub hidden_dim: i32,
- /// Weight matrix leading dimension
- pub weights_ld: i32,
- /// Batch count
- pub batch_count: i32,
- /// Activation type
- pub activation: MlpActivationType,
-}
-
-/// MLP Fused GEMV Kernel
-/// Computes paired up/gate projections with fused activation: out = up * activation(gate)
-pub struct MlpFusedGemvKernel {
- data_type: DataType,
- pipelines: HashMap<MlpFusedGemvConfiguration, ComputePipelineState>,
-}
-
-impl MlpFusedGemvKernel {
- pub fn new(data_type: DataType) -> Result<Self, MTLError> {
- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)
- {
- return Err(MTLError::Generic(format!(
- "Unsupported dtype for MLP fused GEMV: {:?}",
- data_type
- )));
- }
- Ok(Self {
- data_type,
- pipelines: HashMap::new(),
- })
- }
-
- fn get_pipeline(
- &mut self,
- context: &MTLContext,
- config: MlpFusedGemvConfiguration,
- ) -> Result<&ComputePipelineState, MTLError> {
- if !self.pipelines.contains_key(&config) {
- let kernel_name =
- mlp_fused_gemv_kernel_name(self.data_type, &config)?;
-
- // Create function constant values for MLP activation
- let fcv = metal::FunctionConstantValues::new();
- let activation_val = config.activation as u32;
- fcv.set_constant_value_at_index(
- &activation_val as *const u32 as *const _,
- metal::MTLDataType::UInt,
- 52, // MLP_ACTIVATION index
- );
-
- let pipeline =
- context.compute_pipeline_state(&kernel_name, Some(&fcv))?;
- self.pipelines.insert(config, pipeline);
- }
- Ok(self.pipelines.get(&config).unwrap())
- }
-
- pub fn encode(
- &mut self,
- context: &MTLContext,
- encoder: &ComputeCommandEncoderRef,
- args: &MlpFusedGemvArguments,
- ) -> Result<(), MTLError> {
- let config = select_mlp_fused_gemv_configuration(
- args.hidden_dim,
- args.activation,
- );
- let pipeline = self.get_pipeline(context, config)?;
-
- encoder.set_compute_pipeline_state(pipeline);
- encoder.set_buffer(0, Some(args.weights), 0);
- encoder.set_buffer(1, Some(args.input), args.input_offset);
- encoder.set_buffer(3, Some(args.output), 0);
-
- encoder.set_bytes(
- 4,
- std::mem::size_of::<i32>() as u64,
- &args.input_dim as *const i32 as *const std::ffi::c_void,
- );
- encoder.set_bytes(
- 5,
- std::mem::size_of::<i32>() as u64,
- &args.hidden_dim as *const i32 as *const std::ffi::c_void,
- );
- encoder.set_bytes(
- 6,
- std::mem::size_of::<i32>() as u64,
- &args.weights_ld as *const i32 as *const std::ffi::c_void,
- );
-
- // Batch strides
- let vector_batch_stride = args.input_dim as i64;
- let matrix_batch_stride = 0i64; // Weights not batched
- encoder.set_bytes(
- 11,
- std::mem::size_of::<i64>() as u64,
- &vector_batch_stride as *const i64 as *const std::ffi::c_void,
- );
- encoder.set_bytes(
- 12,
- std::mem::size_of::<i64>() as u64,
- &matrix_batch_stride as *const i64 as *const std::ffi::c_void,
- );
-
- let output_elements_per_threadgroup =
- config.output_elements_per_threadgroup();
- let threadgroup_count_x =
- ((args.hidden_dim as u32 + output_elements_per_threadgroup - 1)
- / output_elements_per_threadgroup) as u64;
- let threadgroup_count_z = args.batch_count.max(1) as u64;
-
- let threadgroup_count =
- MTLSize::new(threadgroup_count_x, 1, threadgroup_count_z);
- let threads_per_threadgroup = config.threads_per_threadgroup();
-
- encoder
- .dispatch_thread_groups(threadgroup_count, threads_per_threadgroup);
- Ok(())
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/configuration.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv/configuration.rs
new file mode 100644
index 000000000..0b11354f0
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv/configuration.rs
@@ -0,0 +1,121 @@
+use metal::MTLSize;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct Configuration {
+ pub transpose_matrix: bool,
+ pub threadgroup_rows: u32,
+ pub threadgroup_cols: u32,
+ pub threads_per_simdgroup_row: u32,
+ pub threads_per_simdgroup_col: u32,
+ pub elements_per_thread_row: u32,
+ pub elements_per_thread_col: u32,
+ pub non_contiguous_batch: bool,
+ pub do_axpby: bool,
+}
+
+impl Configuration {
+ pub fn output_elements_per_threadgroup(&self) -> u32 {
+ if self.transpose_matrix {
+ self.threadgroup_cols
+ * self.threads_per_simdgroup_col
+ * self.elements_per_thread_col
+ } else {
+ self.threadgroup_rows
+ * self.threads_per_simdgroup_row
+ * self.elements_per_thread_row
+ }
+ }
+
+ pub fn threads_per_threadgroup(&self) -> MTLSize {
+ MTLSize::new(
+ 32,
+ self.threadgroup_cols as u64,
+ self.threadgroup_rows as u64,
+ )
+ }
+}
+
+pub fn select_configuration(
+ transpose_matrix: bool,
+ input_dimension: i32,
+ output_dimension: i32,
+ non_contiguous_batch: bool,
+ do_axpby: bool,
+) -> Configuration {
+ let (threadgroup_rows, threadgroup_cols);
+ let (threads_per_simdgroup_row, threads_per_simdgroup_col);
+ let (elements_per_thread_row, elements_per_thread_col);
+
+ if transpose_matrix {
+ let mut sm = 8;
+ let mut sn = 4;
+ if input_dimension >= 8192 && output_dimension >= 2048 {
+ sm = 4;
+ sn = 8;
+ }
+
+ let bn = if output_dimension >= 2048 {
+ 16
+ } else if output_dimension >= 512 {
+ 4
+ } else {
+ 2
+ };
+
+ let tn = if output_dimension < 4 {
+ 1
+ } else {
+ 4
+ };
+
+ threadgroup_rows = 1;
+ threadgroup_cols = bn;
+ threads_per_simdgroup_row = sm;
+ threads_per_simdgroup_col = sn;
+ elements_per_thread_row = 4;
+ elements_per_thread_col = tn;
+ } else {
+ let mut bm = if output_dimension >= 4096 {
+ 8
+ } else {
+ 4
+ };
+ let mut sm = 1;
+ let mut sn = 32;
+ let mut bn = 1;
+
+ if input_dimension <= 64 {
+ bm = 1;
+ sm = 8;
+ sn = 4;
+ } else if input_dimension >= 16 * output_dimension {
+ bm = 1;
+ bn = 8;
+ }
+
+ let tm = if output_dimension < 4 {
+ 1
+ } else {
+ 4
+ };
+
+ threadgroup_rows = bm;
+ threadgroup_cols = bn;
+ threads_per_simdgroup_row = sm;
+ threads_per_simdgroup_col = sn;
+ elements_per_thread_row = tm;
+ elements_per_thread_col = 4;
+ }
+
+ Configuration {
+ transpose_matrix,
+ threadgroup_rows,
+ threadgroup_cols,
+ threads_per_simdgroup_row,
+ threads_per_simdgroup_col,
+ elements_per_thread_row,
+ elements_per_thread_col,
+ non_contiguous_batch,
+ do_axpby,
+ }
+}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs
new file mode 100644
index 000000000..53b50be63
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs
@@ -0,0 +1,339 @@
+use std::collections::HashMap;
+
+use metal::{
+ Buffer as MTLBuffer, ComputeCommandEncoderRef, ComputePipelineState,
+ MTLSize,
+};
+
+use super::configuration::{Configuration, select_configuration};
+use crate::{
+ DataType,
+ backends::metal::{
+ MTLContext, MTLError, kernel::matmul::common::MatmulArguments,
+ },
+};
+
+fn gemv_kernel_name(
+ data_type: DataType,
+ configuration: &Configuration,
+) -> Result<String, MTLError> {
+ let dtype_name = match data_type {
+ DataType::F16 => "float16",
+ DataType::BF16 => "bfloat16",
+ DataType::F32 => "float32",
+ _ => {
+ return Err(MTLError::Generic(format!(
+ "Unsupported data type for GEMV: {:?}",
+ data_type
+ )));
+ },
+ };
+
+ let prefix = if configuration.transpose_matrix {
+ "gemv_t"
+ } else {
+ "gemv"
+ };
+
+ Ok(format!(
+ "{prefix}_{dtype_name}_bm{}_bn{}_sm{}_sn{}_tm{}_tn{}_nc{}_axpby{}",
+ configuration.threadgroup_rows,
+ configuration.threadgroup_cols,
+ configuration.threads_per_simdgroup_row,
+ configuration.threads_per_simdgroup_col,
+ configuration.elements_per_thread_row,
+ configuration.elements_per_thread_col,
+ configuration.non_contiguous_batch as u8,
+ configuration.do_axpby as u8,
+ ))
+}
+
+pub struct Kernel {
+ data_type: DataType,
+ lhs_is_transposed: bool,
+ rhs_is_transposed: bool,
+ pipelines: HashMap<Configuration, ComputePipelineState>,
+}
+
+impl Kernel {
+ pub fn new(
+ data_type: DataType,
+ lhs_is_transposed: bool,
+ rhs_is_transposed: bool,
+ ) -> Self {
+ Self {
+ data_type,
+ lhs_is_transposed,
+ rhs_is_transposed,
+ pipelines: HashMap::new(),
+ }
+ }
+
+ fn get_pipeline(
+ &mut self,
+ context: &MTLContext,
+ configuration: Configuration,
+ ) -> Result<&ComputePipelineState, MTLError> {
+ if !self.pipelines.contains_key(&configuration) {
+ let kernel_name = gemv_kernel_name(self.data_type, &configuration)?;
+ let pipeline =
+ context.compute_pipeline_state(&kernel_name, None)?;
+ self.pipelines.insert(configuration, pipeline);
+ }
+ Ok(self.pipelines.get(&configuration).unwrap())
+ }
+
+ #[allow(clippy::too_many_arguments)]
+ fn encode_with_configuration(
+ &mut self,
+ context: &MTLContext,
+ encoder: &ComputeCommandEncoderRef,
+ arguments: &MatmulArguments,
+ configuration: Configuration,
+ matrix_is_rhs: bool,
+ bias_buffer: Option<&MTLBuffer>,
+ alpha: f32,
+ beta: f32,
+ bias_stride: i32,
+ ) -> Result<(), MTLError> {
+ let pipeline = self.get_pipeline(context, configuration)?;
+ encoder.set_compute_pipeline_state(pipeline);
+
+ let (buf0, off0) = if matrix_is_rhs {
+ (arguments.b, 0)
+ } else {
+ (arguments.a, arguments.a_offset)
+ };
+ encoder.set_buffer(0, Some(buf0), off0);
+
+ let (buf1, off1) = if matrix_is_rhs {
+ (arguments.a, arguments.a_offset)
+ } else {
+ (arguments.b, 0)
+ };
+ encoder.set_buffer(1, Some(buf1), off1);
+
+ if configuration.do_axpby {
+ if let Some(bias) = bias_buffer {
+ encoder.set_buffer(2, Some(bias), 0);
+ }
+ }
+
+ encoder.set_buffer(3, Some(arguments.d), 0);
+
+ let input_dimension = arguments.input_dim;
+ let output_dimension = if matrix_is_rhs {
+ arguments.output_dim
+ } else {
+ arguments.batch
+ };
+ let matrix_leading_dim = if matrix_is_rhs {
+ arguments.ldb
+ } else {
+ arguments.lda
+ };
+
+ encoder.set_bytes(
+ 4,
+ std::mem::size_of::<i32>() as u64,
+ &input_dimension as *const i32 as *const std::ffi::c_void,
+ );
+ encoder.set_bytes(
+ 5,
+ std::mem::size_of::<i32>() as u64,
+ &output_dimension as *const i32 as *const std::ffi::c_void,
+ );
+ encoder.set_bytes(
+ 6,
+ std::mem::size_of::<i32>() as u64,
+ &matrix_leading_dim as *const i32 as *const std::ffi::c_void,
+ );
+
+ encoder.set_bytes(
+ 7,
+ std::mem::size_of::<f32>() as u64,
+ &alpha as *const f32 as *const std::ffi::c_void,
+ );
+ encoder.set_bytes(
+ 8,
+ std::mem::size_of::<f32>() as u64,
+ &beta as *const f32 as *const std::ffi::c_void,
+ );
+
+ let batch_shape = if arguments.batch_count > 1 {
+ vec![arguments.batch_count]
+ } else {
+ vec![1]
+ };
+ let batch_ndim = 1i32;
+ let batch_groups = arguments.batch_count.max(1);
+
+ let elements_per_matrix_a =
+ (arguments.batch as i64) * (arguments.lda as i64);
+ let elements_per_matrix_b = if self.rhs_is_transposed {
+ (arguments.output_dim as i64) * (arguments.ldb as i64)
+ } else {
+ (arguments.input_dim as i64) * (arguments.ldb as i64)
+ };
+
+ let vector_batch_stride = if matrix_is_rhs {
+ vec![elements_per_matrix_a]
+ } else {
+ vec![elements_per_matrix_b]
+ };
+
+ let matrix_batch_stride = if matrix_is_rhs {
+ vec![elements_per_matrix_b]
+ } else {
+ vec![elements_per_matrix_a]
+ };
+
+ let bias_batch_stride = if arguments.batch_count > 1 {
+ vec![(output_dimension as i64) * (arguments.ldd as i64)]
+ } else {
+ vec![0]
+ };
+
+ encoder.set_bytes(
+ 9,
+ std::mem::size_of::<i32>() as u64,
+ &batch_ndim as *const i32 as *const std::ffi::c_void,
+ );
+ encoder.set_bytes(
+ 10,
+ (std::mem::size_of::<i32>() * batch_shape.len()) as u64,
+ batch_shape.as_ptr() as *const std::ffi::c_void,
+ );
+ encoder.set_bytes(
+ 11,
+ (std::mem::size_of::<i64>() * vector_batch_stride.len()) as u64,
+ vector_batch_stride.as_ptr() as *const std::ffi::c_void,
+ );
+ encoder.set_bytes(
+ 12,
+ (std::mem::size_of::<i64>() * matrix_batch_stride.len()) as u64,
+ matrix_batch_stride.as_ptr() as *const std::ffi::c_void,
+ );
+ encoder.set_bytes(
+ 13,
+ (std::mem::size_of::<i64>() * bias_batch_stride.len()) as u64,
+ bias_batch_stride.as_ptr() as *const std::ffi::c_void,
+ );
+ encoder.set_bytes(
+ 14,
+ std::mem::size_of::<i32>() as u64,
+ &bias_stride as *const i32 as *const std::ffi::c_void,
+ );
+
+ let output_elements_per_threadgroup =
+ configuration.output_elements_per_threadgroup();
+ let threadgroup_count_x =
+ ((output_dimension as u32 + output_elements_per_threadgroup - 1)
+ / output_elements_per_threadgroup) as u64;
+ let threadgroup_count_z = batch_groups.max(1) as u64;
+
+ let threadgroup_count =
+ MTLSize::new(threadgroup_count_x, 1, threadgroup_count_z);
+ let threads_per_threadgroup = configuration.threads_per_threadgroup();
+
+ encoder
+ .dispatch_thread_groups(threadgroup_count, threads_per_threadgroup);
+ Ok(())
+ }
+
+ pub fn encode(
+ &mut self,
+ context: &MTLContext,
+ encoder: &ComputeCommandEncoderRef,
+ arguments: &MatmulArguments,
+ ) -> Result<(), MTLError> {
+ self.encode_internal(context, encoder, arguments, None)
+ }
+
+ pub fn encode_with_bias(
+ &mut self,
+ context: &MTLContext,
+ encoder: &ComputeCommandEncoderRef,
+ arguments: &MatmulArguments,
+ bias: &MTLBuffer,
+ ) -> Result<(), MTLError> {
+ self.encode_internal(context, encoder, arguments, Some(bias))
+ }
+
+ fn encode_internal(
+ &mut self,
+ context: &MTLContext,
+ encoder: &ComputeCommandEncoderRef,
+ arguments: &MatmulArguments,
+ bias: Option<&MTLBuffer>,
+ ) -> Result<(), MTLError> {
+ let m = arguments.batch;
+ let n = arguments.output_dim;
+ if m != 1 && n != 1 {
+ return Ok(());
+ }
+
+ let matrix_is_rhs = n != 1;
+ let transpose_matrix = if matrix_is_rhs {
+ !self.rhs_is_transposed
+ } else {
+ self.lhs_is_transposed
+ };
+
+ let has_non_contiguous_batch = false;
+
+ let (do_axpby, bias_buffer, alpha, beta, bias_stride) =
+ if let Some(bias_buf) = bias {
+ (true, Some(bias_buf), 1.0f32, 1.0f32, 1)
+ } else if let Some(c_buf) = arguments.c {
+ (
+ true,
+ Some(c_buf),
+ arguments.alpha,
+ arguments.beta,
+ arguments.ldd,
+ )
+ } else {
+ (false, None, 1.0f32, 0.0f32, 0)
+ };
+
+ let configuration = select_configuration(
+ transpose_matrix,
+ arguments.input_dim,
+ if matrix_is_rhs {
+ arguments.output_dim
+ } else {
+ arguments.batch
+ },
+ has_non_contiguous_batch,
+ do_axpby,
+ );
+
+ if crate::utils::env_utils::debug_matmul_enabled() {
+ let kernel_name = gemv_kernel_name(self.data_type, &configuration)
+ .unwrap_or_default();
+ eprintln!(
+ "[matmul] GEMV m={} k={} n={} batch={} dtype={:?} transpose={} kernel={}",
+ arguments.batch,
+ arguments.input_dim,
+ arguments.output_dim,
+ arguments.batch_count,
+ self.data_type,
+ configuration.transpose_matrix,
+ kernel_name
+ );
+ }
+
+ self.encode_with_configuration(
+ context,
+ encoder,
+ arguments,
+ configuration,
+ matrix_is_rhs,
+ bias_buffer,
+ alpha,
+ beta,
+ bias_stride,
+ )
+ }
+}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv/mod.rs
new file mode 100644
index 000000000..bbd057f95
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv/mod.rs
@@ -0,0 +1,4 @@
+mod configuration;
+mod kernel;
+
+pub use kernel::Kernel as GemvKernel;
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal b/crates/uzu/src/backends/metal/kernel/matmul/gemv/shaders/gemv.metal
similarity index 76%
rename from crates/uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal
rename to crates/uzu/src/backends/metal/kernel/matmul/gemv/shaders/gemv.metal
index d79659f90..ff714458a 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv/gemv.metal
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv/shaders/gemv.metal
@@ -2,9 +2,8 @@
#include <metal_stdlib>
#include "../../common/utils.h"
-#include "../../common/mlp_epilogue.h"
-#include "../steel/utils.h"
+#include "../../common/steel/utils.h"
using namespace metal;
@@ -865,264 +864,3 @@ instantiate_gemv_t_bs_blocks(float32, float);
instantiate_gemv_t_bs_blocks(float16, half);
instantiate_gemv_t_bs_blocks(bfloat16, bfloat16_t);
instantiate_gemv_t_bs_blocks(complex64, complex64_t); // clang-format on
-
-///////////////////////////////////////////////////////////////////////////////
-/// MLP Fused GEMV - Paired Up/Gate Computation with Activation Fusion
-///////////////////////////////////////////////////////////////////////////////
-
-// MLP fused GEMV kernel for decode path (M=1).
-// Computes paired up and gate projections, then applies: out = up * activation(gate)
-// Output size is hidden_dim (half of the weight matrix columns).
-template <
- typename T,
- const int BM, /* Threadgroup rows (in simdgroups) */
- const int BN, /* Threadgroup cols (in simdgroups) */
- const int SM, /* Simdgroup rows (in threads) */
- const int SN, /* Simdgroup cols (in threads) */
- const int TM, /* Thread rows (in elements) */
- const int TN> /* Thread cols (in elements) */
-struct GEMVMlpFusedKernel {
- using AccT = float;
-
- MTL_CONST int threadsM = BM * SM;
- MTL_CONST int threadsN = BN * SN;
- MTL_CONST int blockM = threadsM * TM;
- MTL_CONST int blockN = threadsN * TN;
-
- static_assert(SM * SN == 32, "simdgroup can only have 32 threads");
-
- MTL_CONST short tgp_mem_size = BN > 1 ? 2 * BN * (blockM + TM) : 0;
- MTL_CONST bool needs_tgp_reduction = BN > 1;
-
- template <typename U = T>
- static METAL_FUNC void
- load_unsafe(const device T* src, thread U dst[TN], const int src_offset = 0) {
- MTL_PRAGMA_UNROLL
- for (int tn = 0; tn < TN; tn++) {
- dst[tn] = static_cast<U>(src[src_offset + tn]);
- }
- }
-
- template <typename U = T>
- static METAL_FUNC void load_safe(
- const device T* src,
- thread U dst[TN],
- const int src_offset = 0,
- const int src_size = TN) {
- if (src_offset + TN <= src_size) {
- MTL_PRAGMA_UNROLL
- for (int tn = 0; tn < TN; tn++) {
- dst[tn] = static_cast<U>(src[src_offset + tn]);
- }
- } else {
- MTL_PRAGMA_UNROLL
- for (int tn = 0; tn < TN; tn++) {
- dst[tn] = src_offset + tn < src_size
- ? static_cast<U>(src[src_offset + tn])
- : U(0);
- }
- }
- }
-
- static METAL_FUNC void run(
- const device T* mat [[buffer(0)]],
- const device T* in_vec [[buffer(1)]],
- device T* out_vec [[buffer(3)]],
- const int in_vec_size,
- const int hidden_dim, // Output size (half of weight matrix rows)
- const int matrix_ld,
- threadgroup AccT* tgp_memory,
- uint3 tid [[threadgroup_position_in_grid]],
- uint3 lid [[thread_position_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]) {
- (void)lid;
-
- // Thread local accumulation results for up and gate
- thread AccT result_up[TM] = {0};
- thread AccT result_gate[TM] = {0};
- thread T inter_up[TN];
- thread T inter_gate[TN];
- thread AccT v_coeff[TN];
-
- const int thrM = SN != 32 ? simd_lid / SN : 0;
- const int thrN = SN != 32 ? simd_lid % SN : int(simd_lid);
-
- const int sgN = BN != 1 ? (simd_gid % BN) : 0;
- const int simdM = BN != 1 ? SM * (simd_gid / BN) : int(SM * simd_gid);
- const int simdN = BN != 1 ? SN * (simd_gid % BN) : 0;
-
- int bm = (simdM + thrM) * TM;
- int bn = (simdN + thrN) * TN;
-
- // Block position - outputs to hidden_dim elements
- int out_row = tid.x * blockM + bm;
-
- if (out_row >= hidden_dim)
- return;
-
- out_row = out_row + TM <= hidden_dim ? out_row : hidden_dim - TM;
-
- // Pointers to up and gate rows in the weight matrix
- // Weight layout: [up_0, up_1, ..., up_H-1, gate_0, gate_1, ..., gate_H-1]
- const device T* mat_up = mat + out_row * matrix_ld;
- const device T* mat_gate = mat + (out_row + hidden_dim) * matrix_ld;
-
- constexpr const uniform<int> loop_stride = make_uniform(blockN);
- const uniform<int> in_size = make_uniform(in_vec_size);
- const uniform<int> n_iter = in_size / loop_stride;
- const uniform<int> last_iter = loop_stride * n_iter;
- const uniform<int> leftover = in_size - last_iter;
-
- // Main loop - compute both up and gate dot products
- for (int i = 0; i < n_iter; ++i) {
- load_unsafe<AccT>(in_vec, v_coeff, bn);
-
- int mat_offset = 0;
- MTL_PRAGMA_UNROLL
- for (int tm = 0; tm < TM; tm++) {
- load_unsafe(mat_up, inter_up, mat_offset + bn);
- load_unsafe(mat_gate, inter_gate, mat_offset + bn);
-
- MTL_PRAGMA_UNROLL
- for (int tn = 0; tn < TN; tn++) {
- result_up[tm] += inter_up[tn] * v_coeff[tn];
- result_gate[tm] += inter_gate[tn] * v_coeff[tn];
- }
-
- mat_offset += matrix_ld;
- }
-
- bn += blockN;
- }
-
- // Handle leftover
- if (leftover > 0) {
- load_safe<AccT>(in_vec, v_coeff, bn, in_vec_size);
-
- MTL_PRAGMA_UNROLL
- for (int tm = 0; tm < TM; tm++) {
- load_safe(&mat_up[tm * matrix_ld], inter_up, bn, in_vec_size);
- load_safe(&mat_gate[tm * matrix_ld], inter_gate, bn, in_vec_size);
-
- MTL_PRAGMA_UNROLL
- for (int tn = 0; tn < TN; tn++) {
- result_up[tm] += inter_up[tn] * v_coeff[tn];
- result_gate[tm] += inter_gate[tn] * v_coeff[tn];
- }
- }
- }
-
- // Simdgroup reduction for both up and gate
- MTL_PRAGMA_UNROLL
- for (int tm = 0; tm < TM; tm++) {
- MTL_PRAGMA_UNROLL
- for (ushort sn = (SN / 2); sn >= 1; sn >>= 1) {
- result_up[tm] += simd_shuffle_down(result_up[tm], sn);
- result_gate[tm] += simd_shuffle_down(result_gate[tm], sn);
- }
- }
-
- // Threadgroup reduction if needed
- if (needs_tgp_reduction) {
- threadgroup AccT* tgp_results_up = tgp_memory + sgN * (blockM + TM) + bm;
- threadgroup AccT* tgp_results_gate = tgp_memory + BN * (blockM + TM) + sgN * (blockM + TM) + bm;
-
- if (thrN == 0) {
- MTL_PRAGMA_UNROLL
- for (int tm = 0; tm < TM; tm++) {
- tgp_results_up[tm] = result_up[tm];
- tgp_results_gate[tm] = result_gate[tm];
- }
-
- threadgroup_barrier(mem_flags::mem_none);
-
- if (sgN == 0) {
- MTL_PRAGMA_UNROLL
- for (int sgn = 1; sgn < BN; sgn++) {
- MTL_PRAGMA_UNROLL
- for (int tm = 0; tm < TM; tm++) {
- result_up[tm] += tgp_results_up[sgn * (blockM + TM) + tm];
- result_gate[tm] += tgp_results_gate[sgn * (blockM + TM) + tm];
- }
- }
- }
- }
- }
-
- // Apply MLP fused epilogue and write output
- if (simdN == 0 && thrN == 0) {
- MTL_PRAGMA_UNROLL
- for (int tm = 0; tm < TM; tm++) {
- float fused_result = mlp_fused_epilogue_f32(result_up[tm], result_gate[tm]);
- out_vec[out_row + tm] = static_cast<T>(fused_result);
- }
- }
- }
-};
-
-template <
- typename T,
- const int BM,
- const int BN,
- const int SM,
- const int SN,
- const int TM,
- const int TN>
-[[kernel, max_total_threads_per_threadgroup(BM* BN * 32)]] void gemv_mlp_fused(
- const device T* mat [[buffer(0)]],
- const device T* in_vec [[buffer(1)]],
- device T* out_vec [[buffer(3)]],
- const constant int& in_vec_size [[buffer(4)]],
- const constant int& hidden_dim [[buffer(5)]], // Output dimension (half of weight rows)
- const constant int& matrix_ld [[buffer(6)]],
- const constant int64_t& vector_batch_stride [[buffer(11)]],
- const constant int64_t& matrix_batch_stride [[buffer(12)]],
- uint3 tid [[threadgroup_position_in_grid]],
- uint3 lid [[thread_position_in_threadgroup]],
- uint simd_gid [[simdgroup_index_in_threadgroup]],
- uint simd_lid [[thread_index_in_simdgroup]]) {
- using gemv_kernel = GEMVMlpFusedKernel<T, BM, BN, SM, SN, TM, TN>;
- threadgroup typename gemv_kernel::AccT tgp_memory
- [gemv_kernel::tgp_mem_size == 0 ? 1 : gemv_kernel::tgp_mem_size];
-
- // Batch offset
- in_vec += tid.z * vector_batch_stride;
- mat += tid.z * matrix_batch_stride;
- out_vec += tid.z * hidden_dim;
-
- gemv_kernel::run(
- mat,
- in_vec,
- out_vec,
- in_vec_size,
- hidden_dim,
- matrix_ld,
- gemv_kernel::tgp_mem_size == 0 ? nullptr : tgp_memory,
- tid,
- lid,
- simd_gid,
- simd_lid);
-}
-
-#define instantiate_gemv_mlp_fused_helper(name, itype, bm, bn, sm, sn, tm, tn) \
- instantiate_kernel( \
- "gemv_mlp_fused_" #name "_bm" #bm "_bn" #bn "_sm" #sm "_sn" #sn "_tm" #tm \
- "_tn" #tn, \
- gemv_mlp_fused, \
- itype, \
- bm, \
- bn, \
- sm, \
- sn, \
- tm, \
- tn)
-
-// clang-format off
-#define instantiate_gemv_mlp_fused_blocks(name, itype) \
- instantiate_gemv_mlp_fused_helper(name, itype, 4, 1, 1, 32, 4, 4) \
- instantiate_gemv_mlp_fused_helper(name, itype, 8, 1, 1, 32, 4, 4) // clang-format on
-
-instantiate_gemv_mlp_fused_blocks(float32, float);
-instantiate_gemv_mlp_fused_blocks(float16, half);
-instantiate_gemv_mlp_fused_blocks(bfloat16, bfloat16_t);
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs
index 1c257c8c9..fedf3cfe2 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs
@@ -1,18 +1,11 @@
-use std::collections::HashMap;
-
-use metal::{
- Buffer as MTLBuffer, ComputeCommandEncoderRef,
- ComputePipelineState as MTLComputePipelineState, MTLSize,
-};
+use metal::{Buffer as MTLBuffer, ComputeCommandEncoderRef};
use super::{
- super::{KernelDataType, TensorAddBias, mlp::MlpActivationType},
- arguments::MatmulArguments,
+ super::{KernelDataType, TensorAddBias},
+ common::MatmulArguments,
+ gemm,
gemv::GemvKernel,
- pipeline::PipelineKey,
- shared_types::{GEMMAddMMParams, GEMMParams},
- splitk::SplitKGemm,
- transpose::transpose_configuration,
+ split_k::SplitKGemm,
};
use crate::{
DataType,
@@ -20,84 +13,17 @@ use crate::{
utils::env_utils::debug_matmul_enabled,
};
-#[derive(Debug, Clone, Copy)]
-struct TileSelection {
- block_rows: i32,
- block_cols: i32,
- block_depth: i32,
- warps_per_row: u64,
- warps_per_col: u64,
- swizzle_log2: i32,
-}
-
-impl TileSelection {
- fn new(
- block_rows: i32,
- block_cols: i32,
- block_depth: i32,
- warps_per_row: u64,
- warps_per_col: u64,
- swizzle_log2: i32,
- ) -> Self {
- Self {
- block_rows,
- block_cols,
- block_depth,
- warps_per_row,
- warps_per_col,
- swizzle_log2,
- }
- }
-}
-
pub struct MatmulKernel {
data_type: DataType,
lhs_is_transposed: bool,
rhs_is_transposed: bool,
+ gemm: Option<gemm::GemmKernel>,
gemv: Option<GemvKernel>,
splitk: Option<SplitKGemm>,
bias_add: Option<TensorAddBias>,
- pipelines: HashMap<PipelineKey, MTLComputePipelineState>,
}
impl MatmulKernel {
- fn steel_type_name(dt: DataType) -> &'static str {
- match dt {
- DataType::F16 => "float16",
- DataType::BF16 => "bfloat16",
- DataType::F32 => "float32",
- _ => unreachable!(),
- }
- }
-
- fn kernel_name(
- &self,
- tile: &TileSelection,
- ) -> String {
- let type_name = Self::steel_type_name(self.data_type);
- let transpose_suffix = transpose_configuration(
- self.lhs_is_transposed,
- self.rhs_is_transposed,
- );
- let prefix = if tile.block_depth >= 256 {
- "steel_gemm_fused_nax"
- } else {
- "steel_gemm_fused"
- };
- format!(
- "{}_{}_{}_{}_bm{}_bn{}_bk{}_wm{}_wn{}",
- prefix,
- transpose_suffix.as_str(),
- type_name,
- type_name,
- tile.block_rows,
- tile.block_cols,
- tile.block_depth,
- tile.warps_per_row,
- tile.warps_per_col
- )
- }
-
pub fn new(
_mtl: &MTLContext,
data_type: DataType,
@@ -115,10 +41,10 @@ impl MatmulKernel {
data_type,
lhs_is_transposed,
rhs_is_transposed,
+ gemm: None,
gemv: None,
splitk: None,
bias_add: None,
- pipelines: HashMap::new(),
})
}
@@ -129,10 +55,6 @@ impl MatmulKernel {
args: &MatmulArguments,
bias: Option<&MTLBuffer>,
) -> Result<bool, MTLError> {
- // GEMV kernel requires:
- // - A not transposed (input vector is contiguous)
- // - B transposed (weight matrix is [N, K] row-major, each row contiguous)
- // Without transpose_b, B is [K, N] and columns are not contiguous
let m = args.batch;
let n = args.output_dim;
if self.lhs_is_transposed || !self.rhs_is_transposed {
@@ -202,169 +124,17 @@ impl MatmulKernel {
Ok(true)
}
- fn select_tile_configuration(
- &self,
- mtl: &MTLContext,
- args: &MatmulArguments,
- ) -> TileSelection {
- use crate::backends::metal::DeviceClass;
-
- let overall_work_elements = (args.batch_count as i64)
- * (args.batch as i64)
- * (args.output_dim as i64);
- let is_float32 = matches!(self.data_type, DataType::F32);
- let prefer_half_or_tf32 = !is_float32 || mtl.tf32_enabled();
-
- // NAX path: prefer deeper K tiles on NAX-capable devices.
- if mtl.is_nax_available() && prefer_half_or_tf32 {
- let base_tile = TileSelection::new(
- 128, 128, 512, 4, 4, /*swizzle_log2=*/ 0,
- );
- let tile_rows =
- (args.batch + base_tile.block_rows - 1) / base_tile.block_rows;
- let swizzle_log2 = if tile_rows <= 3 {
- 0
- } else {
- 1
- };
- return TileSelection {
- swizzle_log2,
- ..base_tile
- };
- }
-
- let device_class_code = match mtl.architecture.device_class {
- DeviceClass::Desktop => 'd',
- DeviceClass::Integrated => 'g',
- DeviceClass::Phone => 'p',
- DeviceClass::Unknown(_) => 'g',
- };
-
- // Default gemm matmul heuristics (non-NAX), matching observed
- // choices on M2 Max: prefer 64x64x16 wm2/wn2 for large shapes and
- // fall back to 64x32x32 when transpose_b=true and not N-aligned.
- match device_class_code {
- 'g' | 'p' => {
- if prefer_half_or_tf32 {
- if !self.lhs_is_transposed && self.rhs_is_transposed {
- TileSelection::new(64, 32, 32, 2, 2, 0)
- } else {
- TileSelection::new(64, 64, 16, 1, 2, 0)
- }
- } else if !self.lhs_is_transposed && self.rhs_is_transposed {
- TileSelection::new(32, 64, 16, 1, 2, 0)
- } else {
- TileSelection::new(64, 32, 32, 2, 2, 0)
- }
- },
- 'd' => {
- if overall_work_elements >= (1_i64 << 20) {
- if prefer_half_or_tf32 {
- if 2 * std::cmp::max(args.batch, args.output_dim)
- > args.input_dim
- {
- TileSelection::new(64, 64, 16, 2, 2, 0)
- } else if !self.lhs_is_transposed
- && self.rhs_is_transposed
- {
- TileSelection::new(64, 32, 32, 2, 2, 0)
- } else {
- TileSelection::new(32, 64, 16, 1, 2, 0)
- }
- } else if !self.lhs_is_transposed && self.rhs_is_transposed
- {
- TileSelection::new(32, 64, 16, 1, 2, 0)
- } else {
- TileSelection::new(64, 32, 32, 2, 2, 0)
- }
- } else if prefer_half_or_tf32 {
- if !self.lhs_is_transposed && self.rhs_is_transposed {
- TileSelection::new(64, 32, 32, 2, 2, 0)
- } else {
- TileSelection::new(64, 64, 16, 1, 2, 0)
- }
- } else if !self.lhs_is_transposed && self.rhs_is_transposed {
- TileSelection::new(32, 64, 16, 1, 2, 0)
- } else {
- TileSelection::new(64, 32, 32, 2, 2, 0)
- }
- },
- _ => TileSelection::new(64, 64, 16, 2, 2, 0),
- }
- }
-
- fn get_or_compile_pipeline(
- &mut self,
- mtl: &MTLContext,
- name: &str,
- align_m: bool,
- align_n: bool,
- align_k: bool,
- has_batch: bool,
- use_out_source: bool,
- do_axpby: bool,
- ) -> Result<&MTLComputePipelineState, MTLError> {
- let key = PipelineKey {
- name: name.to_string(),
- align_m,
- align_n,
- align_k,
- has_batch,
- use_out_source,
- do_axpby,
- };
- if !self.pipelines.contains_key(&key) {
- let fcv = metal::FunctionConstantValues::new();
- // Base specializations
- fcv.set_constant_value_at_index(
- &has_batch as *const bool as *const _,
- metal::MTLDataType::Bool,
- 10,
- );
- fcv.set_constant_value_at_index(
- &use_out_source as *const bool as *const _,
- metal::MTLDataType::Bool,
- 100,
- );
- fcv.set_constant_value_at_index(
- &do_axpby as *const bool as *const _,
- metal::MTLDataType::Bool,
- 110,
- );
- fcv.set_constant_value_at_index(
- &align_m as *const bool as *const _,
- metal::MTLDataType::Bool,
- 200,
- );
- fcv.set_constant_value_at_index(
- &align_n as *const bool as *const _,
- metal::MTLDataType::Bool,
- 201,
- );
- fcv.set_constant_value_at_index(
- &align_k as *const bool as *const _,
- metal::MTLDataType::Bool,
- 202,
- );
-
- let cache_key = format!(
- "{}_am{}_an{}_ak{}_hb{}_uo{}_ax{}",
- name,
- align_m as u8,
- align_n as u8,
- align_k as u8,
- has_batch as u8,
- use_out_source as u8,
- do_axpby as u8
- );
- let (ps, _) = mtl.compute_pipeline_state_with_reflection_cached(
- &cache_key,
- name,
- Some(&fcv),
- )?;
- self.pipelines.insert(key.clone(), ps);
+ fn get_or_create_gemm(
+ &mut self
+ ) -> Result<&mut gemm::GemmKernel, MTLError> {
+ if self.gemm.is_none() {
+ self.gemm = Some(gemm::GemmKernel::new(
+ self.data_type,
+ self.lhs_is_transposed,
+ self.rhs_is_transposed,
+ )?);
}
- Ok(self.pipelines.get(&key).unwrap())
+ Ok(self.gemm.as_mut().unwrap())
}
fn encode_gemm(
@@ -373,116 +143,8 @@ impl MatmulKernel {
enc: &ComputeCommandEncoderRef,
args: &MatmulArguments,
) -> Result<(), MTLError> {
- let tile = self.select_tile_configuration(mtl, &args);
- let kname = self.kernel_name(&tile);
-
- // M = batch, N = output_dim, K = input_dim
- let m = args.batch;
- let n = args.output_dim;
- let k = args.input_dim;
-
- let align_m = (m % tile.block_rows) == 0;
- let align_n = (n % tile.block_cols) == 0;
- let align_k = (k % tile.block_depth) == 0;
- let has_batch = args.batch_count > 1;
- let use_out_source = args.c.is_some();
- let do_axpby =
- use_out_source && (args.alpha != 1.0 || args.beta != 0.0);
- let ps = self.get_or_compile_pipeline(
- mtl,
- &kname,
- align_m,
- align_n,
- align_k,
- has_batch,
- use_out_source,
- do_axpby,
- )?;
-
- enc.set_compute_pipeline_state(ps);
-
- // Set buffers
- enc.set_buffer(0, Some(args.a), args.a_offset);
- enc.set_buffer(1, Some(args.b), 0);
- if use_out_source {
- if let Some(c_buf) = args.c {
- enc.set_buffer(2, Some(c_buf), 0);
- }
- }
- enc.set_buffer(3, Some(args.d), 0);
-
- // Params
- let tiles_n = (n + tile.block_cols - 1) / tile.block_cols;
- let tiles_m = (m + tile.block_rows - 1) / tile.block_rows;
- let swizzle_log = tile.swizzle_log2;
-
- let tile_swizzle = 1 << swizzle_log;
- let tm_swizzled = (tiles_m + tile_swizzle - 1) / tile_swizzle;
- let tn_swizzled = tiles_n * tile_swizzle;
- let elements_per_matrix_a = (args.batch as i64) * (args.lda as i64);
- let elements_per_matrix_b = if self.rhs_is_transposed {
- (args.output_dim as i64) * (args.ldb as i64)
- } else {
- (args.input_dim as i64) * (args.ldb as i64)
- };
- let elements_per_matrix_d = (args.batch as i64) * (args.ldd as i64);
- let batch_ndim = 1;
- let params = GEMMParams {
- M: m,
- N: n,
- K: k,
- lda: args.lda,
- ldb: args.ldb,
- ldd: args.ldd,
- // NOTE: tiles_{n,m} are the *unswizzled* tile counts used for bounds
- // checks inside the GEMM kernel. The dispatched grid dimensions are
- // swizzled separately via tn_swizzled/tm_swizzled.
- tiles_n,
- tiles_m,
- batch_stride_a: elements_per_matrix_a,
- batch_stride_b: elements_per_matrix_b,
- batch_stride_d: elements_per_matrix_d,
- swizzle_log,
- gemm_k_iterations_aligned: k / tile.block_depth,
- batch_ndim,
- };
- enc.set_bytes(
- 4,
- std::mem::size_of::<GEMMParams>() as u64,
- &params as *const GEMMParams as *const _,
- );
-
- if use_out_source {
- let ldc = args.ldd;
- let fdc = 1;
- let batch_stride_c = if args.batch_count > 1 {
- (args.ldd as i64) * (args.output_dim as i64)
- } else {
- 0
- };
- let addmm_params = GEMMAddMMParams {
- ldc,
- fdc,
- batch_stride_c,
- alpha: args.alpha,
- beta: args.beta,
- };
- enc.set_bytes(
- 5,
- std::mem::size_of::<GEMMAddMMParams>() as u64,
- &addmm_params as *const GEMMAddMMParams as *const _,
- );
- }
-
- // Threadgroup sizing
- let threads_per_tg =
- MTLSize::new(32, tile.warps_per_col, tile.warps_per_row);
- let tg_x = tn_swizzled as u64;
- let tg_y = tm_swizzled as u64;
- let tg_z = args.batch_count as u64;
- let tgs = MTLSize::new(tg_x, tg_y, tg_z);
- enc.dispatch_thread_groups(tgs, threads_per_tg);
- Ok(())
+ let gemm = self.get_or_create_gemm()?;
+ gemm.encode(mtl, enc, args)
}
pub fn encode(
@@ -493,7 +155,6 @@ impl MatmulKernel {
) -> Result<(), MTLError> {
self.apply_batch_collapse(&mut args);
- // Try GEMV fast path for decode shapes.
if self.maybe_use_gemv(mtl, enc, &args)? {
if debug_matmul_enabled() {
self.log_gemv(&args);
@@ -501,7 +162,6 @@ impl MatmulKernel {
return Ok(());
}
- // Try Split-K for small M*N and large K.
if self.maybe_use_splitk(mtl, enc, &args)? {
if debug_matmul_enabled() {
self.log_splitk(&args);
@@ -510,8 +170,7 @@ impl MatmulKernel {
}
if debug_matmul_enabled() {
- let tile = self.select_tile_configuration(mtl, &args);
- self.log_gemm(&args, &tile);
+ self.log_gemm(&args, mtl);
}
self.encode_gemm(mtl, enc, &args)
@@ -542,8 +201,7 @@ impl MatmulKernel {
}
if debug_matmul_enabled() {
- let tile = self.select_tile_configuration(mtl, &args);
- self.log_gemm(&args, &tile);
+ self.log_gemm(&args, mtl);
}
self.encode_gemm(mtl, enc, &args)?;
@@ -588,7 +246,6 @@ impl MatmulKernel {
if args.batch_count <= 1 {
return;
}
- // Collapse batch_count into M when A is contiguous and B is broadcast.
if args.lda == args.input_dim && self.rhs_is_transposed {
args.batch *= args.batch_count;
args.batch_count = 1;
@@ -626,229 +283,23 @@ impl MatmulKernel {
fn log_gemm(
&self,
args: &MatmulArguments,
- tile: &TileSelection,
+ mtl: &MTLContext,
) {
- let kernel_name = self.kernel_name(tile);
- eprintln!(
- "[matmul] GEMM m={} k={} n={} batch={} dtype={:?} tile={}x{}x{} kernel={}",
- args.batch,
- args.input_dim,
- args.output_dim,
- args.batch_count,
- self.data_type,
- tile.block_rows,
- tile.block_cols,
- tile.block_depth,
- kernel_name
- );
- }
-}
-
-/// Arguments for MLP fused GEMM
-#[derive(Debug)]
-pub struct MlpFusedGemmArguments<'a> {
- /// Input activations [M, K]
- pub input: &'a MTLBuffer,
- /// Input byte offset
- pub input_offset: u64,
- /// Weight matrix [K, 2*hidden_dim] or [2*hidden_dim, K] depending on transpose
- pub weights: &'a MTLBuffer,
- /// Output [M, hidden_dim]
- pub output: &'a MTLBuffer,
- /// Batch size (M)
- pub batch: i32,
- /// Input dimension (K)
- pub input_dim: i32,
- /// Hidden dimension (output size, half of weight columns/rows)
- pub hidden_dim: i32,
- /// Leading dimension of input
- pub lda: i32,
- /// Leading dimension of weights
- pub ldb: i32,
- /// Leading dimension of output
- pub ldd: i32,
- /// Activation type for fused epilogue
- pub activation: MlpActivationType,
-}
-
-/// MLP Fused GEMM Kernel for prefill path
-/// Computes paired up/gate projections with fused activation: out = up * activation(gate)
-pub struct MlpFusedGemmKernel {
- data_type: DataType,
- weights_transposed: bool,
- pipelines: HashMap<
- (i32, i32, bool, bool, MlpActivationType),
- MTLComputePipelineState,
- >,
-}
-
-impl MlpFusedGemmKernel {
- pub fn new(
- data_type: DataType,
- weights_transposed: bool,
- ) -> Result<Self, MTLError> {
- if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)
- {
- return Err(MTLError::Generic(format!(
- "Unsupported dtype for MLP fused GEMM: {:?}",
- data_type
- )));
- }
- Ok(Self {
- data_type,
- weights_transposed,
- pipelines: HashMap::new(),
- })
- }
-
- fn kernel_name(
- &self,
- bm: i32,
- bn: i32,
- mn_aligned: bool,
- k_aligned: bool,
- ) -> String {
- let type_name = match self.data_type {
- DataType::F16 => "half",
- DataType::BF16 => "bfloat",
- DataType::F32 => "float",
- _ => unreachable!(),
- };
- let transpose_char = if self.weights_transposed {
- "t"
- } else {
- "n"
- };
- let mn_str = if mn_aligned {
- "true"
- } else {
- "false"
- };
- let k_str = if k_aligned {
- "true"
- } else {
- "false"
- };
-
- format!(
- "steel_gemm_mlp_fused_{}_{}_{}_bm{}_bn{}_bk16_wm2_wn2_align_MN_{}_K_{}",
- transpose_char, type_name, type_name, bm, bn, mn_str, k_str
- )
- }
-
- fn get_pipeline(
- &mut self,
- context: &MTLContext,
- bm: i32,
- bn: i32,
- mn_aligned: bool,
- k_aligned: bool,
- activation: MlpActivationType,
- ) -> Result<&MTLComputePipelineState, MTLError> {
- let key = (bm, bn, mn_aligned, k_aligned, activation);
- if !self.pipelines.contains_key(&key) {
- let kernel_name = self.kernel_name(bm, bn, mn_aligned, k_aligned);
-
- // Create function constant values for MLP activation
- let fcv = metal::FunctionConstantValues::new();
- let activation_val = activation as u32;
- fcv.set_constant_value_at_index(
- &activation_val as *const u32 as *const _,
- metal::MTLDataType::UInt,
- 52, // MLP_ACTIVATION index
+ let gemm_ref = self.gemm.as_ref();
+ if let Some(gemm) = gemm_ref {
+ let tile = gemm.select_tile(mtl, args);
+ eprintln!(
+ "[matmul] GEMM m={} k={} n={} batch={} dtype={:?} tile={}x{}x{} nax={}",
+ args.batch,
+ args.input_dim,
+ args.output_dim,
+ args.batch_count,
+ self.data_type,
+ tile.block_rows,
+ tile.block_cols,
+ tile.block_depth,
+ tile.is_nax()
);
-
- let pipeline =
- context.compute_pipeline_state(&kernel_name, Some(&fcv))?;
- self.pipelines.insert(key, pipeline);
}
- Ok(self.pipelines.get(&key).unwrap())
- }
-
- fn select_tile_size(
- &self,
- m: i32,
- hidden_dim: i32,
- k: i32,
- ) -> (i32, i32) {
- // Select tile size based on problem dimensions
- let _ = k; // K doesn't affect tile selection directly
- if m >= 64 && hidden_dim >= 64 {
- (64, 64)
- } else {
- (32, 32)
- }
- }
-
- pub fn encode(
- &mut self,
- context: &MTLContext,
- encoder: &ComputeCommandEncoderRef,
- args: &MlpFusedGemmArguments,
- ) -> Result<(), MTLError> {
- let (bm, bn) =
- self.select_tile_size(args.batch, args.hidden_dim, args.input_dim);
- let bk = 16;
-
- let mn_aligned = args.batch % bm == 0 && args.hidden_dim % bn == 0;
- let k_aligned = args.input_dim % bk == 0;
-
- let pipeline = self.get_pipeline(
- context,
- bm,
- bn,
- mn_aligned,
- k_aligned,
- args.activation,
- )?;
- encoder.set_compute_pipeline_state(pipeline);
-
- encoder.set_buffer(0, Some(args.input), args.input_offset);
- encoder.set_buffer(1, Some(args.weights), 0);
- encoder.set_buffer(2, Some(args.output), 0);
-
- // Set GEMM params
- let tiles_m = (args.batch + bm - 1) / bm;
- let tiles_n = (args.hidden_dim + bn - 1) / bn;
- let gemm_k_iterations = (args.input_dim + bk - 1) / bk;
-
- let params = GEMMParams {
- M: args.batch,
- N: args.hidden_dim * 2, // Full weight width for pointer arithmetic
- K: args.input_dim,
- lda: args.lda,
- ldb: args.ldb,
- ldd: args.ldd,
- tiles_n,
- tiles_m,
- batch_stride_a: 0,
- batch_stride_b: 0,
- batch_stride_d: 0,
- swizzle_log: 0,
- gemm_k_iterations_aligned: if k_aligned {
- gemm_k_iterations
- } else {
- gemm_k_iterations - 1
- },
- batch_ndim: 0,
- };
-
- encoder.set_bytes(
- 3,
- std::mem::size_of::<GEMMParams>() as u64,
- &params as *const GEMMParams as *const std::ffi::c_void,
- );
- encoder.set_bytes(
- 10,
- std::mem::size_of::<i32>() as u64,
- &args.hidden_dim as *const i32 as *const std::ffi::c_void,
- );
-
- let threadgroup_count = MTLSize::new(tiles_n as u64, tiles_m as u64, 1);
- let threads_per_threadgroup = MTLSize::new(32, 2, 2); // WM=2, WN=2, 32 threads per simdgroup
-
- encoder
- .dispatch_thread_groups(threadgroup_count, threads_per_threadgroup);
- Ok(())
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/mod.rs
index 81a690ef5..2ef5b3948 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/mod.rs
@@ -1,17 +1,10 @@
-mod arguments;
-mod gather;
+pub mod common;
+mod gemm;
mod gemv;
mod kernel;
-mod pipeline;
-mod segmented;
-#[allow(dead_code, clippy::unnecessary_operation, clippy::identity_op)]
-mod shared_types;
-mod splitk;
-mod transpose;
+mod split_k;
-pub use arguments::MatmulArguments;
-pub use gather::{GatherGemm, GatherMmArguments, GatherMmRhsArguments};
-pub use gemv::{GemvKernel, MlpFusedGemvArguments, MlpFusedGemvKernel};
-pub use kernel::{MatmulKernel, MlpFusedGemmArguments, MlpFusedGemmKernel};
-pub use segmented::{SegmentedGemm, SegmentedMmArguments};
-pub use splitk::SplitKGemm;
+pub use common::MatmulArguments;
+pub use gemv::GemvKernel;
+pub use kernel::MatmulKernel;
+pub use split_k::SplitKGemm;
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/segmented.rs b/crates/uzu/src/backends/metal/kernel/matmul/segmented.rs
deleted file mode 100644
index 8c75a8fed..000000000
--- a/crates/uzu/src/backends/metal/kernel/matmul/segmented.rs
+++ /dev/null
@@ -1,309 +0,0 @@
-use std::collections::HashMap;
-
-use metal::{
- Buffer as MTLBuffer, ComputeCommandEncoderRef,
- ComputePipelineState as MTLComputePipelineState, MTLSize,
-};
-
-use super::shared_types::GEMMParams;
-use crate::{
- DataType,
- backends::metal::{DeviceClass, MTLContext, MTLError},
-};
-
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-struct SegmentedPipelineKey {
- name: String,
- segments_contiguous: bool,
- align_m: bool,
- align_n: bool,
-}
-
-#[derive(Debug, Clone)]
-pub struct SegmentedMmArguments<'a> {
- pub a: &'a MTLBuffer,
- pub b: &'a MTLBuffer,
- pub segments: &'a MTLBuffer,
- pub out: &'a MTLBuffer,
- pub m: i32,
- pub n: i32,
- pub k: i32,
- pub lda: i32,
- pub ldb: i32,
- /// Grid Z dimension for the segmented kernel (out.size / (M*N) in the reference backend).
- pub batch_size_out: i32,
- pub segments_contiguous: bool,
- pub transpose_a: bool,
- pub transpose_b: bool,
-}
-
-pub struct SegmentedGemm {
- data_type: DataType,
- pipelines: HashMap<SegmentedPipelineKey, MTLComputePipelineState>,
-}
-
-impl SegmentedGemm {
- pub fn new(data_type: DataType) -> Self {
- Self {
- data_type,
- pipelines: HashMap::new(),
- }
- }
-
- fn steel_type_name(&self) -> Result<&'static str, MTLError> {
- match self.data_type {
- DataType::F16 => Ok("float16"),
- DataType::BF16 => Ok("bfloat16"),
- DataType::F32 => Ok("float32"),
- other => Err(MTLError::Generic(format!(
- "Unsupported data type for segmented GEMM: {other:?}"
- ))),
- }
- }
-
- fn device_class_code(mtl: &MTLContext) -> char {
- match mtl.architecture.device_class {
- DeviceClass::Desktop => 'd',
- DeviceClass::Integrated => 'g',
- DeviceClass::Phone => 'p',
- DeviceClass::Unknown(_) => 'g',
- }
- }
-
- fn select_gemm_tparams(
- device_class: char,
- data_type: DataType,
- transpose_a: bool,
- transpose_b: bool,
- m: i32,
- n: i32,
- k: i32,
- batch_size_out: i32,
- ) -> (i32, i32, i32, i32, i32) {
- // Matches the initialization and tparam selection logic used by the
- // reference backend for segmented GEMM kernels.
- let mut bm = 64;
- let mut bn = 64;
- let mut bk = 16;
- let mut wm = 2;
- let mut wn = 2;
-
- let is_float32 = matches!(data_type, DataType::F32);
- let work_elements = (batch_size_out as i64) * (m as i64) * (n as i64);
-
- match device_class {
- 'g' | 'p' => {
- if !transpose_a && transpose_b {
- bm = 64;
- bn = 32;
- bk = 32;
- wm = 2;
- wn = 2;
- } else if !is_float32 {
- bm = 64;
- bn = 64;
- bk = 16;
- wm = 1;
- wn = 2;
- }
- },
- 'd' => {
- if work_elements >= (1_i64 << 20) {
- if !is_float32 {
- if 2 * std::cmp::max(m, n) > k {
- bm = 64;
- bn = 64;
- bk = 16;
- wm = 1;
- wn = 2;
- } else if !transpose_a && transpose_b {
- bm = 64;
- bn = 32;
- bk = 32;
- wm = 2;
- wn = 2;
- } else {
- bm = 32;
- bn = 64;
- bk = 16;
- wm = 1;
- wn = 2;
- }
- }
- } else if !is_float32 {
- if !transpose_a && transpose_b {
- bm = 64;
- bn = 32;
- bk = 32;
- wm = 2;
- wn = 2;
- } else {
- bm = 64;
- bn = 64;
- bk = 16;
- wm = 1;
- wn = 2;
- }
- }
- },
- _ => {},
- }
-
- (bm, bn, bk, wm, wn)
- }
-
- fn segmented_kernel_name(
- &self,
- transpose_a: bool,
- transpose_b: bool,
- bm: i32,
- bn: i32,
- bk: i32,
- wm: i32,
- wn: i32,
- ) -> Result<String, MTLError> {
- let tname = format!(
- "{}{}",
- if transpose_a {
- 't'
- } else {
- 'n'
- },
- if transpose_b {
- 't'
- } else {
- 'n'
- }
- );
- let dtype = self.steel_type_name()?;
- Ok(format!(
- "steel_segmented_mm_{tname}_{dtype}_{dtype}_bm{bm}_bn{bn}_bk{bk}_wm{wm}_wn{wn}"
- ))
- }
-
- fn get_or_compile_pipeline(
- &mut self,
- mtl: &MTLContext,
- name: &str,
- segments_contiguous: bool,
- align_m: bool,
- align_n: bool,
- ) -> Result<&MTLComputePipelineState, MTLError> {
- let key = SegmentedPipelineKey {
- name: name.to_string(),
- segments_contiguous,
- align_m,
- align_n,
- };
-
- if !self.pipelines.contains_key(&key) {
- let fcv = metal::FunctionConstantValues::new();
- fcv.set_constant_value_at_index(
- &segments_contiguous as *const bool as *const _,
- metal::MTLDataType::Bool,
- 199,
- );
- fcv.set_constant_value_at_index(
- &align_m as *const bool as *const _,
- metal::MTLDataType::Bool,
- 200,
- );
- fcv.set_constant_value_at_index(
- &align_n as *const bool as *const _,
- metal::MTLDataType::Bool,
- 201,
- );
-
- let cache_key = format!(
- "{name}_sc{}_am{}_an{}",
- segments_contiguous as u8, align_m as u8, align_n as u8
- );
- let (ps, _) = mtl.compute_pipeline_state_with_reflection_cached(
- &cache_key,
- name,
- Some(&fcv),
- )?;
- self.pipelines.insert(key.clone(), ps);
- }
-
- Ok(self.pipelines.get(&key).unwrap())
- }
-
- pub fn encode(
- &mut self,
- mtl: &MTLContext,
- enc: &ComputeCommandEncoderRef,
- args: &SegmentedMmArguments,
- ) -> Result<(), MTLError> {
- let device_class = Self::device_class_code(mtl);
- let (bm, bn, bk, wm, wn) = Self::select_gemm_tparams(
- device_class,
- self.data_type,
- args.transpose_a,
- args.transpose_b,
- args.m,
- args.n,
- args.k,
- args.batch_size_out,
- );
-
- let align_m = (args.m % bm) == 0;
- let align_n = (args.n % bn) == 0;
-
- let kname = self.segmented_kernel_name(
- args.transpose_a,
- args.transpose_b,
- bm,
- bn,
- bk,
- wm,
- wn,
- )?;
- let ps = self.get_or_compile_pipeline(
- mtl,
- &kname,
- args.segments_contiguous,
- align_m,
- align_n,
- )?;
-
- enc.set_compute_pipeline_state(ps);
- enc.set_buffer(0, Some(args.a), 0);
- enc.set_buffer(1, Some(args.b), 0);
- enc.set_buffer(2, Some(args.segments), 0);
- enc.set_buffer(3, Some(args.out), 0);
-
- let tiles_n = (args.n + bn - 1) / bn;
- let tiles_m = (args.m + bm - 1) / bm;
- let params = GEMMParams {
- M: args.m,
- N: args.n,
- K: args.k,
- lda: args.lda,
- ldb: args.ldb,
- ldd: args.n,
- tiles_n,
- tiles_m,
- batch_stride_a: 0,
- batch_stride_b: 0,
- batch_stride_d: (args.m as i64) * (args.n as i64),
- swizzle_log: 0,
- gemm_k_iterations_aligned: 0,
- batch_ndim: 0,
- };
- enc.set_bytes(
- 4,
- std::mem::size_of::<GEMMParams>() as u64,
- &params as *const GEMMParams as *const _,
- );
-
- let threads_per_tg = MTLSize::new(32, wn as u64, wm as u64);
- let tgs = MTLSize::new(
- tiles_n as u64,
- tiles_m as u64,
- args.batch_size_out as u64,
- );
- enc.dispatch_thread_groups(tgs, threads_per_tg);
- Ok(())
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/splitk.rs b/crates/uzu/src/backends/metal/kernel/matmul/split_k/kernel.rs
similarity index 67%
rename from crates/uzu/src/backends/metal/kernel/matmul/splitk.rs
rename to crates/uzu/src/backends/metal/kernel/matmul/split_k/kernel.rs
index 8cc7e7bb2..531755fa0 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/splitk.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/split_k/kernel.rs
@@ -6,39 +6,26 @@ use metal::{
};
use super::{
- arguments::MatmulArguments,
- shared_types::GEMMSpiltKParams as SplitKGEMMParams,
+ pipeline_configuration::PipelineConfiguration,
+ tile_configuration::{TileConfiguration, select_tile_configuration},
};
use crate::{
DataType,
- backends::metal::{MTLContext, MTLError},
+ backends::metal::{
+ MTLContext, MTLError,
+ kernel::matmul::common::{GEMMSpiltKParams as SplitKGEMMParams, MatmulArguments},
+ },
};
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-struct TileConfig {
- tile_rows: i32,
- tile_cols: i32,
- tile_depth: i32,
- warps_per_row: i32,
- warps_per_col: i32,
-}
-
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-struct PartialKernelKey {
- tile_config: TileConfig,
- mn_aligned: bool,
- k_aligned: bool,
-}
-
-pub struct SplitKGemm {
+pub struct Kernel {
data_type: DataType,
transpose_a: bool,
transpose_b: bool,
- partial_pipelines: HashMap<PartialKernelKey, MTLComputePipelineState>,
+ partial_pipelines: HashMap<PipelineConfiguration, MTLComputePipelineState>,
accum_pipeline: Option<MTLComputePipelineState>,
}
-impl SplitKGemm {
+impl Kernel {
pub fn new(
data_type: DataType,
transpose_a: bool,
@@ -66,8 +53,6 @@ impl SplitKGemm {
}
fn splitk_partial_out_name(&self) -> &'static str {
- // In steel_gemm_splitk.metal, float16/bfloat16/float32 partial outputs
- // are accumulated into float32.
"float32"
}
@@ -82,33 +67,25 @@ impl SplitKGemm {
fn partial_kernel_name(
&self,
- config: &TileConfig,
+ tile: &TileConfiguration,
mn_aligned: bool,
k_aligned: bool,
) -> Result<String, MTLError> {
let in_name = self.steel_type_name()?;
let out_name = self.splitk_partial_out_name();
let transpose_suffix = self.transpose_suffix();
- let mn_tag = if mn_aligned {
- "taligned"
- } else {
- "naligned"
- };
- let k_tag = if k_aligned {
- "taligned"
- } else {
- "naligned"
- };
+ let mn_tag = if mn_aligned { "taligned" } else { "naligned" };
+ let k_tag = if k_aligned { "taligned" } else { "naligned" };
Ok(format!(
"steel_gemm_splitk_{}_{}_{}_bm{}_bn{}_bk{}_wm{}_wn{}_MN_{}_K_{}",
transpose_suffix,
in_name,
out_name,
- config.tile_rows,
- config.tile_cols,
- config.tile_depth,
- config.warps_per_row,
- config.warps_per_col,
+ tile.tile_rows,
+ tile.tile_cols,
+ tile.tile_depth,
+ tile.warps_per_row,
+ tile.warps_per_col,
mn_tag,
k_tag,
))
@@ -126,22 +103,15 @@ impl SplitKGemm {
fn get_partial_pipeline(
&mut self,
mtl: &MTLContext,
- config: TileConfig,
- mn_aligned: bool,
- k_aligned: bool,
+ config: &PipelineConfiguration,
) -> Result<&MTLComputePipelineState, MTLError> {
- let key = PartialKernelKey {
- tile_config: config,
- mn_aligned,
- k_aligned,
- };
- if !self.partial_pipelines.contains_key(&key) {
+ if !self.partial_pipelines.contains_key(config) {
let name =
- self.partial_kernel_name(&config, mn_aligned, k_aligned)?;
+ self.partial_kernel_name(&config.tile, config.mn_aligned, config.k_aligned)?;
let ps = mtl.compute_pipeline_state(&name, None)?;
- self.partial_pipelines.insert(key.clone(), ps);
+ self.partial_pipelines.insert(config.clone(), ps);
}
- Ok(self.partial_pipelines.get(&key).unwrap())
+ Ok(self.partial_pipelines.get(config).unwrap())
}
fn get_accum_pipeline(
@@ -156,29 +126,6 @@ impl SplitKGemm {
Ok(self.accum_pipeline.as_ref().unwrap())
}
- fn select_tile_config(
- m: i32,
- n: i32,
- ) -> TileConfig {
- let tile_rows = if m < 40 {
- 16
- } else {
- 32
- };
- let tile_cols = if n < 40 {
- 16
- } else {
- 32
- };
- TileConfig {
- tile_rows,
- tile_cols,
- tile_depth: 16,
- warps_per_row: 2,
- warps_per_col: 2,
- }
- }
-
fn compute_partition_count(k: i32) -> i32 {
let k_tiles = k / 16;
if k_tiles < 16 {
@@ -220,20 +167,23 @@ impl SplitKGemm {
let n = args.output_dim;
let k = args.input_dim;
- let tile_config = Self::select_tile_config(m, n);
+ let tile = select_tile_configuration(m, n);
let partition_count = Self::compute_partition_count(k);
- let mn_aligned = (m % tile_config.tile_rows) == 0
- && (n % tile_config.tile_cols) == 0;
- let k_aligned = (k % tile_config.tile_depth) == 0;
+ let mn_aligned =
+ (m % tile.tile_rows) == 0 && (n % tile.tile_cols) == 0;
+ let k_aligned = (k % tile.tile_depth) == 0;
- let tile_count_m =
- (m + tile_config.tile_rows - 1) / tile_config.tile_rows;
- let tile_count_n =
- (n + tile_config.tile_cols - 1) / tile_config.tile_cols;
+ let config = PipelineConfiguration {
+ tile,
+ mn_aligned,
+ k_aligned,
+ };
+
+ let tile_count_m = (m + tile.tile_rows - 1) / tile.tile_rows;
+ let tile_count_n = (n + tile.tile_cols - 1) / tile.tile_cols;
- let gemm_k_iterations = (k / tile_config.tile_depth) / partition_count;
- let k_elements_per_partition =
- gemm_k_iterations * tile_config.tile_depth;
+ let gemm_k_iterations = (k / tile.tile_depth) / partition_count;
+ let k_elements_per_partition = gemm_k_iterations * tile.tile_depth;
let output_elements_per_partition = m * n;
let accumulator_element_count =
@@ -260,8 +210,7 @@ impl SplitKGemm {
gemm_k_iterations_aligned: gemm_k_iterations,
};
- let partial_ps =
- self.get_partial_pipeline(mtl, tile_config, mn_aligned, k_aligned)?;
+ let partial_ps = self.get_partial_pipeline(mtl, &config)?;
enc.set_compute_pipeline_state(partial_ps);
enc.set_buffer(0, Some(args.a), args.a_offset);
@@ -275,8 +224,8 @@ impl SplitKGemm {
let threads_per_threadgroup = MTLSize::new(
32,
- tile_config.warps_per_col as u64,
- tile_config.warps_per_row as u64,
+ tile.warps_per_col as u64,
+ tile.warps_per_row as u64,
);
let threadgroups = MTLSize::new(
tile_count_n as u64,
@@ -297,8 +246,7 @@ impl SplitKGemm {
enc.set_bytes(
3,
4,
- &output_elements_per_partition as *const i32
- as *const std::ffi::c_void,
+ &output_elements_per_partition as *const i32 as *const std::ffi::c_void,
);
enc.set_bytes(
4,
@@ -309,10 +257,7 @@ impl SplitKGemm {
let accum_total_threads = MTLSize::new(n as u64, m as u64, 1);
let accum_threads_per_threadgroup =
MTLSize::new(16.min(n as u64), 16.min(m as u64), 1);
- enc.dispatch_threads(
- accum_total_threads,
- accum_threads_per_threadgroup,
- );
+ enc.dispatch_threads(accum_total_threads, accum_threads_per_threadgroup);
Ok(())
}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/split_k/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/split_k/mod.rs
new file mode 100644
index 000000000..54a1730c2
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/split_k/mod.rs
@@ -0,0 +1,5 @@
+mod kernel;
+mod pipeline_configuration;
+mod tile_configuration;
+
+pub use kernel::Kernel as SplitKGemm;
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/split_k/pipeline_configuration.rs b/crates/uzu/src/backends/metal/kernel/matmul/split_k/pipeline_configuration.rs
new file mode 100644
index 000000000..b745b347a
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/split_k/pipeline_configuration.rs
@@ -0,0 +1,8 @@
+use super::tile_configuration::TileConfiguration;
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct PipelineConfiguration {
+ pub tile: TileConfiguration,
+ pub mn_aligned: bool,
+ pub k_aligned: bool,
+}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_splitk.h b/crates/uzu/src/backends/metal/kernel/matmul/split_k/shaders/steel_gemm_splitk.h
similarity index 100%
rename from crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_splitk.h
rename to crates/uzu/src/backends/metal/kernel/matmul/split_k/shaders/steel_gemm_splitk.h
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_splitk.metal b/crates/uzu/src/backends/metal/kernel/matmul/split_k/shaders/steel_gemm_splitk.metal
similarity index 98%
rename from crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_splitk.metal
rename to crates/uzu/src/backends/metal/kernel/matmul/split_k/shaders/steel_gemm_splitk.metal
index c573c9ee1..07cf0b530 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_splitk.metal
+++ b/crates/uzu/src/backends/metal/kernel/matmul/split_k/shaders/steel_gemm_splitk.metal
@@ -1,8 +1,8 @@
// clang-format off
-#include "../../../../common/utils.h"
-#include "../gemm.h"
+#include "../../common/utils.h"
+#include "../../common/steel/gemm/gemm.h"
#include "steel_gemm_splitk.h"
#define instantiate_gemm( \
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/split_k/tile_configuration.rs b/crates/uzu/src/backends/metal/kernel/matmul/split_k/tile_configuration.rs
new file mode 100644
index 000000000..d2abe25d8
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/split_k/tile_configuration.rs
@@ -0,0 +1,20 @@
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct TileConfiguration {
+ pub tile_rows: i32,
+ pub tile_cols: i32,
+ pub tile_depth: i32,
+ pub warps_per_row: i32,
+ pub warps_per_col: i32,
+}
+
+pub fn select_tile_configuration(m: i32, n: i32) -> TileConfiguration {
+ let tile_rows = if m < 40 { 16 } else { 32 };
+ let tile_cols = if n < 40 { 16 } else { 32 };
+ TileConfiguration {
+ tile_rows,
+ tile_cols,
+ tile_depth: 16,
+ warps_per_row: 2,
+ warps_per_col: 2,
+ }
+}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_gather.h b/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_gather.h
deleted file mode 100644
index b9c6d3c83..000000000
--- a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_gather.h
+++ /dev/null
@@ -1,459 +0,0 @@
-
-
-using namespace steel;
-
-constant bool has_batch [[function_constant(10)]];
-constant bool align_M [[function_constant(200)]];
-constant bool align_N [[function_constant(201)]];
-constant bool align_K [[function_constant(202)]];
-
-template <
- typename T,
- int BM,
- int BN,
- int BK,
- int WM,
- int WN,
- bool transpose_a,
- bool transpose_b,
- typename AccumType = float>
-[[kernel, max_total_threads_per_threadgroup(WM* WN * 32)]] void gather_mm_rhs(
- const device T* A [[buffer(0)]],
- const device T* B [[buffer(1)]],
- const device uint32_t* rhs_indices [[buffer(2)]],
- device T* C [[buffer(3)]],
- const constant GEMMParams* params [[buffer(4)]],
- uint simd_lane_id [[thread_index_in_simdgroup]],
- uint simd_group_id [[simdgroup_index_in_threadgroup]],
- uint3 tid [[threadgroup_position_in_grid]]) {
- using gemm_kernel = GEMMKernel<
- T,
- T,
- BM,
- BN,
- BK,
- WM,
- WN,
- transpose_a,
- transpose_b,
- true,
- true,
- AccumType>;
-
- using loader_a_t = typename gemm_kernel::loader_a_t;
- using loader_b_t = typename gemm_kernel::loader_b_t;
- using mma_t = typename gemm_kernel::mma_t;
-
- if (params->tiles_n <= static_cast<int>(tid.x) ||
- params->tiles_m <= static_cast<int>(tid.y)) {
- return;
- }
-
- // Prepare threadgroup memory
- threadgroup T As[gemm_kernel::tgp_mem_size_a];
- threadgroup T Bs[gemm_kernel::tgp_mem_size_b];
-
- // Find the block in A, B, C
- const int c_row = tid.y * BM;
- const int c_col = tid.x * BN;
- const size_t c_row_long = size_t(c_row);
- const size_t c_col_long = size_t(c_col);
-
- // Prepare threadgroup bounds
- const short tgp_bm = align_M ? BM : short(min(BM, params->M - c_row));
- const short tgp_bn = align_N ? BN : short(min(BN, params->N - c_col));
-
- A += transpose_a ? c_row_long : c_row_long * params->lda;
- B += transpose_b ? c_col_long * params->ldb : c_col_long;
- C += c_row_long * params->ldd + c_col_long;
-
- // Do as many matmuls as necessary
- uint32_t index;
- short offset;
- uint32_t index_next = rhs_indices[c_row];
- short offset_next = 0;
- int n = 0;
- while (n < tgp_bm) {
- n++;
- offset = offset_next;
- index = index_next;
- offset_next = tgp_bm;
- for (; n < tgp_bm; n++) {
- if (rhs_indices[c_row + n] != index) {
- offset_next = n;
- index_next = rhs_indices[c_row + n];
- break;
- }
- }
- threadgroup_barrier(mem_flags::mem_none);
-
- // Prepare threadgroup mma operation
- thread mma_t mma_op(simd_group_id, simd_lane_id);
-
- // Prepare threadgroup loading operations
- thread loader_a_t loader_a(A, params->lda, As, simd_group_id, simd_lane_id);
- thread loader_b_t loader_b(
- B + index * params->batch_stride_b,
- params->ldb,
- Bs,
- simd_group_id,
- simd_lane_id);
-
- // Prepare iterations
- const int gemm_k_iterations = params->gemm_k_iterations_aligned;
-
- // Do unaligned K iterations first
- if (!align_K) {
- const int k_last = params->gemm_k_iterations_aligned * BK;
- const int k_remain = params->K - k_last;
- const size_t k_jump_a =
- transpose_a ? params->lda * size_t(k_last) : size_t(k_last);
- const size_t k_jump_b =
- transpose_b ? size_t(k_last) : params->ldb * size_t(k_last);
-
- // Move loader source ahead to end
- loader_a.src += k_jump_a;
- loader_b.src += k_jump_b;
-
- // Load tile
- const short2 tile_dims_A =
- transpose_a ? short2(tgp_bm, k_remain) : short2(k_remain, tgp_bm);
- const short2 tile_dims_B =
- transpose_b ? short2(k_remain, tgp_bn) : short2(tgp_bn, k_remain);
-
- loader_a.load_safe(tile_dims_A);
- loader_b.load_safe(tile_dims_B);
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // Do matmul
- mma_op.mma(As, Bs);
-
- // Reset source back to start
- loader_a.src -= k_jump_a;
- loader_b.src -= k_jump_b;
- }
-
- // Matrix level aligned never check
- if (align_M && align_N) {
- for (int k = 0; k < gemm_k_iterations; k++) {
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // Load elements into threadgroup
- loader_a.load_unsafe();
- loader_b.load_unsafe();
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // Multiply and accumulate threadgroup elements
- mma_op.mma(As, Bs);
-
- // Prepare for next iteration
- loader_a.next();
- loader_b.next();
- }
-
- // Store results to device memory
- if (offset_next - offset == BM) {
- mma_op.store_result(C, params->ldd);
- } else {
- mma_op.store_result_slice(
- C, params->ldd, short2(0, offset), short2(BN, offset_next));
- }
- } else {
- const short lbk = 0;
-
- // Tile aligned don't check
- if ((align_M || tgp_bm == BM) && (align_N || tgp_bn == BN)) {
- gemm_kernel::gemm_loop(
- As,
- Bs,
- gemm_k_iterations,
- loader_a,
- loader_b,
- mma_op,
- tgp_bm,
- tgp_bn,
- lbk,
- LoopAlignment<true, true, true>{});
- if (offset_next - offset == BM) {
- mma_op.store_result(C, params->ldd);
- } else {
- mma_op.store_result_slice(
- C, params->ldd, short2(0, offset), short2(BN, offset_next));
- }
- }
-
- // Tile partially aligned check rows
- else if (align_N || tgp_bn == BN) {
- gemm_kernel::gemm_loop(
- As,
- Bs,
- gemm_k_iterations,
- loader_a,
- loader_b,
- mma_op,
- tgp_bm,
- tgp_bn,
- lbk,
- LoopAlignment<false, true, true>{});
- mma_op.store_result_slice(
- C, params->ldd, short2(0, offset), short2(BN, offset_next));
- }
-
- // Tile partially aligned check cols
- else if (align_M || tgp_bm == BM) {
- gemm_kernel::gemm_loop(
- As,
- Bs,
- gemm_k_iterations,
- loader_a,
- loader_b,
- mma_op,
- tgp_bm,
- tgp_bn,
- lbk,
- LoopAlignment<true, false, true>{});
- mma_op.store_result_slice(
- C, params->ldd, short2(0, offset), short2(tgp_bn, offset_next));
- }
-
- // Nothing aligned so check both rows and cols
- else {
- gemm_kernel::gemm_loop(
- As,
- Bs,
- gemm_k_iterations,
- loader_a,
- loader_b,
- mma_op,
- tgp_bm,
- tgp_bn,
- lbk,
- LoopAlignment<false, false, true>{});
- mma_op.store_result_slice(
- C, params->ldd, short2(0, offset), short2(tgp_bn, offset_next));
- }
- }
- }
-}
-
-template <
- typename T,
- int BM,
- int BN,
- int BK,
- int WM,
- int WN,
- bool transpose_a,
- bool transpose_b,
- typename AccumType = float>
-[[kernel, max_total_threads_per_threadgroup(WM* WN * 32)]] void gather_mm(
- const device T* A [[buffer(0)]],
- const device T* B [[buffer(1)]],
- const device uint32_t* lhs_indices [[buffer(2)]],
- const device uint32_t* rhs_indices [[buffer(3)]],
- device T* C [[buffer(4)]],
- const constant GEMMParams* params [[buffer(5)]],
- const constant int* indices_shape [[buffer(6)]],
- const constant int64_t* lhs_strides [[buffer(7)]],
- const constant int64_t* rhs_strides [[buffer(8)]],
- const constant int& batch_ndim_a [[buffer(9)]],
- const constant int* batch_shape_a [[buffer(10)]],
- const constant int64_t* batch_strides_a [[buffer(11)]],
- const constant int& batch_ndim_b [[buffer(12)]],
- const constant int* batch_shape_b [[buffer(13)]],
- const constant int64_t* batch_strides_b [[buffer(14)]],
- uint simd_lane_id [[thread_index_in_simdgroup]],
- uint simd_group_id [[simdgroup_index_in_threadgroup]],
- uint3 tid [[threadgroup_position_in_grid]]) {
- using gemm_kernel = GEMMKernel<
- T,
- T,
- BM,
- BN,
- BK,
- WM,
- WN,
- transpose_a,
- transpose_b,
- true,
- true,
- AccumType>;
-
- using loader_a_t = typename gemm_kernel::loader_a_t;
- using loader_b_t = typename gemm_kernel::loader_b_t;
- using mma_t = typename gemm_kernel::mma_t;
-
- if (params->tiles_n <= static_cast<int>(tid.x) ||
- params->tiles_m <= static_cast<int>(tid.y)) {
- return;
- }
-
- // Move A and B to the locations pointed by lhs_indices and rhs_indices.
- uint32_t indx_A, indx_B;
- if (has_batch) {
- ulong2 indices_offsets = elem_to_loc_broadcast(
- tid.z, indices_shape, lhs_strides, rhs_strides, params->batch_ndim);
- indx_A = lhs_indices[indices_offsets.x];
- indx_B = rhs_indices[indices_offsets.y];
- } else {
- indx_A = lhs_indices[params->batch_stride_a * tid.z];
- indx_B = rhs_indices[params->batch_stride_b * tid.z];
- }
- A += elem_to_loc(indx_A, batch_shape_a, batch_strides_a, batch_ndim_a);
- B += elem_to_loc(indx_B, batch_shape_b, batch_strides_b, batch_ndim_b);
- C += params->batch_stride_d * tid.z;
-
- // Prepare threadgroup memory
- threadgroup T As[gemm_kernel::tgp_mem_size_a];
- threadgroup T Bs[gemm_kernel::tgp_mem_size_b];
-
- // Just make sure everybody's finished with the indexing math above.
- threadgroup_barrier(mem_flags::mem_none);
-
- // Find block in A, B, C
- const int c_row = tid.y * BM;
- const int c_col = tid.x * BN;
- const size_t c_row_long = size_t(c_row);
- const size_t c_col_long = size_t(c_col);
-
- A += transpose_a ? c_row_long : c_row_long * params->lda;
- B += transpose_b ? c_col_long * params->ldb : c_col_long;
- C += c_row_long * params->ldd + c_col_long;
-
- // Prepare threadgroup mma operation
- thread mma_t mma_op(simd_group_id, simd_lane_id);
-
- // Prepare threadgroup loading operations
- thread loader_a_t loader_a(A, params->lda, As, simd_group_id, simd_lane_id);
- thread loader_b_t loader_b(B, params->ldb, Bs, simd_group_id, simd_lane_id);
-
- // Prepare threadgroup bounds
- const short tgp_bm = align_M ? BM : short(min(BM, params->M - c_row));
- const short tgp_bn = align_N ? BN : short(min(BN, params->N - c_col));
-
- // Prepare iterations
- int gemm_k_iterations = params->gemm_k_iterations_aligned;
-
- // Do unaligned K iterations first
- if (!align_K) {
- const int k_last = params->gemm_k_iterations_aligned * BK;
- const int k_remain = params->K - k_last;
- const size_t k_jump_a =
- transpose_a ? params->lda * size_t(k_last) : size_t(k_last);
- const size_t k_jump_b =
- transpose_b ? size_t(k_last) : params->ldb * size_t(k_last);
-
- // Move loader source ahead to end
- loader_a.src += k_jump_a;
- loader_b.src += k_jump_b;
-
- // Load tile
- const short2 tile_dims_A =
- transpose_a ? short2(tgp_bm, k_remain) : short2(k_remain, tgp_bm);
- const short2 tile_dims_B =
- transpose_b ? short2(k_remain, tgp_bn) : short2(tgp_bn, k_remain);
-
- loader_a.load_safe(tile_dims_A);
- loader_b.load_safe(tile_dims_B);
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // Do matmul
- mma_op.mma(As, Bs);
-
- // Reset source back to start
- loader_a.src -= k_jump_a;
- loader_b.src -= k_jump_b;
- }
-
- // Matrix level aligned never check
- if (align_M && align_N) {
- for (int k = 0; k < gemm_k_iterations; k++) {
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // Load elements into threadgroup
- loader_a.load_unsafe();
- loader_b.load_unsafe();
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // Multiply and accumulate threadgroup elements
- mma_op.mma(As, Bs);
-
- // Prepare for next iteration
- loader_a.next();
- loader_b.next();
- }
-
- // Store results to device memory
- mma_op.store_result(C, params->ldd);
- } else {
- const short lbk = 0;
-
- // Tile aligned don't check
- if ((align_M || tgp_bm == BM) && (align_N || tgp_bn == BN)) {
- gemm_kernel::gemm_loop(
- As,
- Bs,
- gemm_k_iterations,
- loader_a,
- loader_b,
- mma_op,
- tgp_bm,
- tgp_bn,
- lbk,
- LoopAlignment<true, true, true>{});
- mma_op.store_result(C, params->ldd);
- }
-
- // Tile partially aligned check rows
- else if (align_N || tgp_bn == BN) {
- gemm_kernel::gemm_loop(
- As,
- Bs,
- gemm_k_iterations,
- loader_a,
- loader_b,
- mma_op,
- tgp_bm,
- tgp_bn,
- lbk,
- LoopAlignment<false, true, true>{});
- mma_op.store_result_safe(C, params->ldd, short2(tgp_bn, tgp_bm));
- }
-
- // Tile partially aligned check cols
- else if (align_M || tgp_bm == BM) {
- gemm_kernel::gemm_loop(
- As,
- Bs,
- gemm_k_iterations,
- loader_a,
- loader_b,
- mma_op,
- tgp_bm,
- tgp_bn,
- lbk,
- LoopAlignment<true, false, true>{});
- mma_op.store_result_safe(C, params->ldd, short2(tgp_bn, tgp_bm));
- }
-
- // Nothing aligned so check both rows and cols
- else {
- gemm_kernel::gemm_loop(
- As,
- Bs,
- gemm_k_iterations,
- loader_a,
- loader_b,
- mma_op,
- tgp_bm,
- tgp_bn,
- lbk,
- LoopAlignment<false, false, true>{});
- mma_op.store_result_safe(C, params->ldd, short2(tgp_bn, tgp_bm));
- }
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_gather.metal b/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_gather.metal
deleted file mode 100644
index 89372091c..000000000
--- a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_gather.metal
+++ /dev/null
@@ -1,59 +0,0 @@
-
-
-// clang-format off
-#include "../../../../common/utils.h"
-#include "../gemm.h"
-#include "steel_gemm_gather.h"
-
-#define instantiate_gather_mm_rhs(tname, trans_a, trans_b, iname, itype, oname, otype, bm, bn, bk, wm, wn) \
- instantiate_kernel( \
- "steel_gather_mm_rhs_" #tname "_" #iname "_" #oname "_bm" #bm "_bn" #bn \
- "_bk" #bk "_wm" #wm "_wn" #wn, \
- gather_mm_rhs, \
- itype, \
- bm, \
- bn, \
- bk, \
- wm, \
- wn, \
- trans_a, \
- trans_b, \
- float)
-
-#define instantiate_gather_mm(tname, trans_a, trans_b, iname, itype, oname, otype, bm, bn, bk, wm, wn) \
- instantiate_kernel( \
- "steel_gather_mm_" #tname "_" #iname "_" #oname "_bm" #bm "_bn" #bn \
- "_bk" #bk "_wm" #wm "_wn" #wn, \
- gather_mm, \
- itype, \
- bm, \
- bn, \
- bk, \
- wm, \
- wn, \
- trans_a, \
- trans_b, \
- float)
-
-#define instantiate_gather_mm_rhs_transpose_helper(iname, itype, oname, otype, bm, bn, bk, wm, wn) \
- instantiate_gather_mm_rhs(nn, false, false, iname, itype, oname, otype, bm, bn, bk, wm, wn) \
- instantiate_gather_mm_rhs(nt, false, true, iname, itype, oname, otype, bm, bn, bk, wm, wn)
-
-#define instantiate_gather_mm_transpose_helper(iname, itype, oname, otype, bm, bn, bk, wm, wn) \
- instantiate_gather_mm(nn, false, false, iname, itype, oname, otype, bm, bn, bk, wm, wn) \
- instantiate_gather_mm(nt, false, true , iname, itype, oname, otype, bm, bn, bk, wm, wn) \
- instantiate_gather_mm(tn, true , false, iname, itype, oname, otype, bm, bn, bk, wm, wn) \
- instantiate_gather_mm(tt, true , true , iname, itype, oname, otype, bm, bn, bk, wm, wn)
-
-#define instantiate_gather_mm_shapes_helper(iname, itype, oname, otype) \
- instantiate_gather_mm_rhs_transpose_helper(iname, itype, oname, otype, 16, 64, 16, 1, 2) \
- instantiate_gather_mm_transpose_helper(iname, itype, oname, otype, 64, 64, 16, 2, 2) \
- instantiate_gather_mm_transpose_helper(iname, itype, oname, otype, 64, 64, 16, 1, 2) \
- instantiate_gather_mm_transpose_helper(iname, itype, oname, otype, 64, 32, 32, 2, 2) \
- instantiate_gather_mm_transpose_helper(iname, itype, oname, otype, 32, 64, 16, 1, 2) \
- instantiate_gather_mm_transpose_helper(iname, itype, oname, otype, 32, 32, 16, 2, 2)
-// clang-format on
-
-instantiate_gather_mm_shapes_helper(float16, half, float16, half);
-instantiate_gather_mm_shapes_helper(bfloat16, bfloat16_t, bfloat16, bfloat16_t);
-instantiate_gather_mm_shapes_helper(float32, float, float32, float);
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_gather_nax.h b/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_gather_nax.h
deleted file mode 100644
index 05f567356..000000000
--- a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_gather_nax.h
+++ /dev/null
@@ -1,132 +0,0 @@
-
-
-using namespace steel;
-
-constant bool align_M [[function_constant(200)]];
-constant bool align_N [[function_constant(201)]];
-constant bool align_K [[function_constant(202)]];
-
-template <
- typename T,
- int BM,
- int BN,
- int BK,
- int WM,
- int WN,
- bool transpose_a,
- bool transpose_b,
- typename AccumType = float>
-[[kernel, max_total_threads_per_threadgroup(WM* WN * 32)]] void
-gather_mm_rhs_nax(
- const device T* A [[buffer(0)]],
- const device T* B [[buffer(1)]],
- const device uint32_t* rhs_indices [[buffer(2)]],
- device T* C [[buffer(3)]],
- const constant GEMMParams* params [[buffer(4)]],
- uint simd_group_id [[simdgroup_index_in_threadgroup]],
- uint3 tid [[threadgroup_position_in_grid]]) {
- constexpr short UM = 16;
- constexpr short UN = 32;
- constexpr short UK = 16;
- constexpr short SM = BM / WM;
- constexpr short SN = BN / WN;
- constexpr short SK = 32;
- constexpr short TM = SM / UM;
- constexpr short TN = SN / UN;
-
- if (params->tiles_n <= static_cast<int>(tid.x) ||
- params->tiles_m <= static_cast<int>(tid.y)) {
- return;
- }
-
- // Find the block in A, B, C
- const int c_row = tid.y * BM;
- const int c_col = tid.x * BN;
- const size_t c_row_long = size_t(c_row);
- const size_t c_col_long = size_t(c_col);
-
- A += transpose_a ? c_row_long : c_row_long * params->lda;
- B += transpose_b ? c_col_long * params->ldb : c_col_long;
- C += c_row_long * params->ldd + c_col_long;
- rhs_indices += c_row;
-
- const short tm = SM * (simd_group_id / WN);
- const short tn = SN * (simd_group_id % WN);
-
- const short sgp_sm = align_M ? SM : min(SM, short(params->M - (c_row + tm)));
- const bool is_unaligned_sm = align_M ? false : (sgp_sm != SM);
-
- const short sgp_sn = align_N ? SN : min(SN, short(params->N - (c_col + tn)));
- const bool is_unaligned_sn = align_N ? false : (sgp_sn != SN);
-
- A += transpose_a ? tm : (tm * params->lda);
- B += transpose_b ? (tn * params->ldb) : tn;
- C += tm * params->ldd + tn;
- rhs_indices += tm;
-
- // Do as many matmuls as necessary
- uint32_t index;
- short offset;
- uint32_t index_next = rhs_indices[0];
- short offset_next = 0;
- int n = 0;
- while (n < sgp_sm) {
- n++;
- offset = offset_next;
- index = index_next;
- offset_next = sgp_sm;
- for (; n < sgp_sm; n++) {
- if (rhs_indices[n] != index) {
- offset_next = n;
- index_next = rhs_indices[n];
- break;
- }
- }
- threadgroup_barrier(mem_flags::mem_none);
-
- using DSubTile = NAXSubTile<AccumType, UM, UN>;
- NAXTile<AccumType, TM, TN, DSubTile> Ctile;
-
- dispatch_bool(align_K, [&](auto kAlignedK) {
- dispatch_bool(align_M || !is_unaligned_sm, [&](auto kAlignedM) {
- dispatch_bool(align_N || !is_unaligned_sn, [&](auto kAlignedN) {
- auto do_gemm = gemm_loop<
- T,
- SM,
- SN,
- SK,
- BK,
- transpose_a,
- transpose_b,
- kAlignedM.value,
- kAlignedN.value,
- kAlignedK.value,
- UM,
- UN,
- UK,
- AccumType>;
- Ctile = do_gemm(
- A, B + index * params->batch_stride_b, params, sgp_sm, sgp_sn);
-
- if constexpr (kAlignedN.value) {
- if (offset_next - offset == SM) {
- Ctile.store(C, int(params->ldd));
- } else {
- Ctile.store_slice(
- C,
- int(params->ldd),
- short2(0, offset),
- short2(SN, offset_next));
- }
- } else {
- Ctile.store_slice(
- C,
- int(params->ldd),
- short2(0, offset),
- short2(sgp_sn, offset_next));
- }
- });
- });
- });
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_gather_nax.metal b/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_gather_nax.metal
deleted file mode 100644
index 47585a9c6..000000000
--- a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_gather_nax.metal
+++ /dev/null
@@ -1,37 +0,0 @@
-
-
-#include <metal_stdlib>
-
-#include "../../../../common/utils.h"
-
-#include "../gemm_nax.h"
-#include "steel_gemm_gather_nax.h"
-
-// clang-format off
-#define instantiate_gather_mm_rhs(tname, trans_a, trans_b, iname, itype, oname, otype, bm, bn, bk, wm, wn) \
- instantiate_kernel( \
- "steel_gather_mm_rhs_nax_" #tname "_" #iname "_" #oname "_bm" #bm "_bn" #bn \
- "_bk" #bk "_wm" #wm "_wn" #wn, \
- gather_mm_rhs_nax, \
- itype, \
- bm, \
- bn, \
- bk, \
- wm, \
- wn, \
- trans_a, \
- trans_b, \
- float)
-
-#define instantiate_gather_mm_rhs_transpose_helper(iname, itype, oname, otype, bm, bn, bk, wm, wn) \
- instantiate_gather_mm_rhs(nn, false, false, iname, itype, oname, otype, bm, bn, bk, wm, wn) \
- instantiate_gather_mm_rhs(nt, false, true, iname, itype, oname, otype, bm, bn, bk, wm, wn)
-
-#define instantiate_gather_mm_shapes_helper(iname, itype, oname, otype) \
- instantiate_gather_mm_rhs_transpose_helper(iname, itype, oname, otype, 16, 128, 128, 1, 4) \
- instantiate_gather_mm_rhs_transpose_helper(iname, itype, oname, otype, 32, 128, 128, 1, 4) \
- instantiate_gather_mm_rhs_transpose_helper(iname, itype, oname, otype, 64, 128, 128, 2, 4)
-// clang-format on
-
-instantiate_gather_mm_shapes_helper(float16, half, float16, half);
-instantiate_gather_mm_shapes_helper(bfloat16, bfloat, bfloat16, bfloat);
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_segmented.h b/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_segmented.h
deleted file mode 100644
index e7238b74e..000000000
--- a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_segmented.h
+++ /dev/null
@@ -1,266 +0,0 @@
-
-
-using namespace steel;
-
-constant bool segments_contiguous [[function_constant(199)]];
-constant bool align_M [[function_constant(200)]];
-constant bool align_N [[function_constant(201)]];
-
-template <
- typename T,
- int BM,
- int BN,
- int BK,
- int WM,
- int WN,
- bool transpose_a,
- bool transpose_b,
- typename AccumType = float>
-[[kernel, max_total_threads_per_threadgroup(WM* WN * 32)]] void segmented_mm(
- const device T* A [[buffer(0)]],
- const device T* B [[buffer(1)]],
- const device uint32_t* segments [[buffer(2)]],
- device T* C [[buffer(3)]],
- const constant GEMMParams* params [[buffer(4)]],
- uint simd_lane_id [[thread_index_in_simdgroup]],
- uint simd_group_id [[simdgroup_index_in_threadgroup]],
- uint3 tid [[threadgroup_position_in_grid]]) {
- using gemm_kernel = GEMMKernel<
- T,
- T,
- BM,
- BN,
- BK,
- WM,
- WN,
- transpose_a,
- transpose_b,
- true,
- true,
- AccumType>;
-
- using loader_a_t = typename gemm_kernel::loader_a_t;
- using loader_b_t = typename gemm_kernel::loader_b_t;
- using mma_t = typename gemm_kernel::mma_t;
-
- if (params->tiles_n <= static_cast<int>(tid.x) ||
- params->tiles_m <= static_cast<int>(tid.y)) {
- return;
- }
-
- // Prepare threadgroup memory
- threadgroup T As[gemm_kernel::tgp_mem_size_a];
- threadgroup T Bs[gemm_kernel::tgp_mem_size_b];
-
- // Find the block in A, B, C
- const int c_row = tid.y * BM;
- const int c_col = tid.x * BN;
- const size_t c_row_long = size_t(c_row);
- const size_t c_col_long = size_t(c_col);
-
- // Prepare threadgroup bounds
- const short tgp_bm = align_M ? BM : short(min(BM, params->M - c_row));
- const short tgp_bn = align_N ? BN : short(min(BN, params->N - c_col));
-
- // Move the pointers to the output tile
- A += transpose_a ? c_row_long : c_row_long * params->lda;
- B += transpose_b ? c_col_long * params->ldb : c_col_long;
- C += c_row_long * params->ldd + c_col_long;
-
- // Move the pointers to the start of the segment
- uint32_t k_start, k_end;
- if (segments_contiguous) {
- k_start = segments[2 * tid.z];
- k_end = segments[2 * tid.z + 1];
- } else {
- // We accept either contiguous (above) or weird strides where the beginning
- // of the next one is the previous one. Basically the last two strides are
- // both 1!
- k_start = segments[tid.z];
- k_end = segments[tid.z + 1];
- }
- A += transpose_a ? k_start * params->lda : k_start;
- B += transpose_b ? k_start : k_start * params->ldb;
- C += tid.z * params->batch_stride_d;
-
- // Prepare threadgroup mma operation
- thread mma_t mma_op(simd_group_id, simd_lane_id);
-
- // Prepare threadgroup loading operations
- thread loader_a_t loader_a(A, params->lda, As, simd_group_id, simd_lane_id);
- thread loader_b_t loader_b(B, params->ldb, Bs, simd_group_id, simd_lane_id);
-
- // Matrix level alignment so only check K
- if (align_M && align_N) {
- uint32_t k = k_start + BK;
- for (; k <= k_end; k += BK) {
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // Load elements into threadgroup
- loader_a.load_unsafe();
- loader_b.load_unsafe();
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // Multiply and accumulate threadgroup elements
- mma_op.mma(As, Bs);
-
- // Prepare for next iteration
- loader_a.next();
- loader_b.next();
- }
- short k_remain = BK - short(k - k_end);
- const short2 tile_dims_A =
- transpose_a ? short2(tgp_bm, k_remain) : short2(k_remain, tgp_bm);
- const short2 tile_dims_B =
- transpose_b ? short2(k_remain, tgp_bn) : short2(tgp_bn, k_remain);
- if (k_remain > 0) {
- threadgroup_barrier(mem_flags::mem_threadgroup);
- loader_a.load_safe(tile_dims_A);
- loader_b.load_safe(tile_dims_B);
- threadgroup_barrier(mem_flags::mem_threadgroup);
- mma_op.mma(As, Bs);
- }
- mma_op.store_result(C, params->ldd);
- } else {
- // Tile aligned do the same as above
- if ((align_M || tgp_bm == BM) && (align_N || tgp_bn == BN)) {
- uint32_t k = k_start + BK;
- for (; k <= k_end; k += BK) {
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // Load elements into threadgroup
- loader_a.load_unsafe();
- loader_b.load_unsafe();
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // Multiply and accumulate threadgroup elements
- mma_op.mma(As, Bs);
-
- // Prepare for next iteration
- loader_a.next();
- loader_b.next();
- }
- short k_remain = BK - short(k - k_end);
- const short2 tile_dims_A =
- transpose_a ? short2(tgp_bm, k_remain) : short2(k_remain, tgp_bm);
- const short2 tile_dims_B =
- transpose_b ? short2(k_remain, tgp_bn) : short2(tgp_bn, k_remain);
- if (k_remain > 0) {
- threadgroup_barrier(mem_flags::mem_threadgroup);
- loader_a.load_safe(tile_dims_A);
- loader_b.load_safe(tile_dims_B);
- threadgroup_barrier(mem_flags::mem_threadgroup);
- mma_op.mma(As, Bs);
- }
- mma_op.store_result(C, params->ldd);
- }
-
- // Tile partially aligned check rows
- else if (align_N || tgp_bn == BN) {
- uint32_t k = k_start + BK;
- for (; k <= k_end; k += BK) {
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // Load elements into threadgroup
- loader_a.load_safe(
- transpose_a ? short2(tgp_bm, BK) : short2(BK, tgp_bm));
- loader_b.load_unsafe();
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // Multiply and accumulate threadgroup elements
- mma_op.mma(As, Bs);
-
- // Prepare for next iteration
- loader_a.next();
- loader_b.next();
- }
- short k_remain = BK - short(k - k_end);
- const short2 tile_dims_A =
- transpose_a ? short2(tgp_bm, k_remain) : short2(k_remain, tgp_bm);
- const short2 tile_dims_B =
- transpose_b ? short2(k_remain, tgp_bn) : short2(tgp_bn, k_remain);
- if (k_remain > 0) {
- threadgroup_barrier(mem_flags::mem_threadgroup);
- loader_a.load_safe(tile_dims_A);
- loader_b.load_safe(tile_dims_B);
- threadgroup_barrier(mem_flags::mem_threadgroup);
- mma_op.mma(As, Bs);
- }
- mma_op.store_result_safe(C, params->ldd, short2(tgp_bn, tgp_bm));
- }
-
- // Tile partially aligned check cols
- else if (align_M || tgp_bm == BM) {
- uint32_t k = k_start + BK;
- for (; k <= k_end; k += BK) {
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // Load elements into threadgroup
- loader_a.load_unsafe();
- loader_b.load_safe(
- transpose_b ? short2(BK, tgp_bn) : short2(tgp_bn, BK));
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // Multiply and accumulate threadgroup elements
- mma_op.mma(As, Bs);
-
- // Prepare for next iteration
- loader_a.next();
- loader_b.next();
- }
- short k_remain = BK - short(k - k_end);
- const short2 tile_dims_A =
- transpose_a ? short2(tgp_bm, k_remain) : short2(k_remain, tgp_bm);
- const short2 tile_dims_B =
- transpose_b ? short2(k_remain, tgp_bn) : short2(tgp_bn, k_remain);
- if (k_remain > 0) {
- threadgroup_barrier(mem_flags::mem_threadgroup);
- loader_a.load_safe(tile_dims_A);
- loader_b.load_safe(tile_dims_B);
- threadgroup_barrier(mem_flags::mem_threadgroup);
- mma_op.mma(As, Bs);
- }
- mma_op.store_result_safe(C, params->ldd, short2(tgp_bn, tgp_bm));
- }
-
- // Nothing aligned so check both rows and cols
- else {
- uint32_t k = k_start + BK;
- for (; k <= k_end; k += BK) {
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // Load elements into threadgroup
- loader_a.load_safe(
- transpose_a ? short2(tgp_bm, BK) : short2(BK, tgp_bm));
- loader_b.load_safe(
- transpose_b ? short2(BK, tgp_bn) : short2(tgp_bn, BK));
-
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- // Multiply and accumulate threadgroup elements
- mma_op.mma(As, Bs);
-
- // Prepare for next iteration
- loader_a.next();
- loader_b.next();
- }
- short k_remain = BK - short(k - k_end);
- const short2 tile_dims_A =
- transpose_a ? short2(tgp_bm, k_remain) : short2(k_remain, tgp_bm);
- const short2 tile_dims_B =
- transpose_b ? short2(k_remain, tgp_bn) : short2(tgp_bn, k_remain);
- if (k_remain > 0) {
- threadgroup_barrier(mem_flags::mem_threadgroup);
- loader_a.load_safe(tile_dims_A);
- loader_b.load_safe(tile_dims_B);
- threadgroup_barrier(mem_flags::mem_threadgroup);
- mma_op.mma(As, Bs);
- }
- mma_op.store_result_safe(C, params->ldd, short2(tgp_bn, tgp_bm));
- }
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_segmented.metal b/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_segmented.metal
deleted file mode 100644
index fd05e307e..000000000
--- a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_segmented.metal
+++ /dev/null
@@ -1,43 +0,0 @@
-
-
-// clang-format off
-#include "../../../../common/utils.h"
-#include "../gemm.h"
-#include "steel_gemm_segmented.h"
-
-#define instantiate_segmented_mm(tname, trans_a, trans_b, iname, itype, oname, otype, bm, bn, bk, wm, wn) \
- instantiate_kernel( \
- "steel_segmented_mm_" #tname "_" #iname "_" #oname "_bm" #bm "_bn" #bn \
- "_bk" #bk "_wm" #wm "_wn" #wn, \
- segmented_mm, \
- itype, \
- bm, \
- bn, \
- bk, \
- wm, \
- wn, \
- trans_a, \
- trans_b, \
- float)
-
-#define instantiate_segmented_mm_transpose_helper(iname, itype, oname, otype, bm, bn, bk, wm, wn) \
- instantiate_segmented_mm(nn, false, false, iname, itype, oname, otype, bm, bn, bk, wm, wn) \
- instantiate_segmented_mm(nt, false, true , iname, itype, oname, otype, bm, bn, bk, wm, wn) \
- instantiate_segmented_mm(tn, true , false, iname, itype, oname, otype, bm, bn, bk, wm, wn) \
- instantiate_segmented_mm(tt, true , true , iname, itype, oname, otype, bm, bn, bk, wm, wn)
-
-#define instantiate_segmented_mm_shapes_helper(iname, itype, oname, otype) \
- instantiate_segmented_mm_transpose_helper(iname, itype, oname, otype, 64, 64, 16, 2, 2) \
- instantiate_segmented_mm_transpose_helper(iname, itype, oname, otype, 64, 64, 16, 1, 2) \
- instantiate_segmented_mm_transpose_helper(iname, itype, oname, otype, 64, 32, 32, 2, 2) \
- instantiate_segmented_mm_transpose_helper(iname, itype, oname, otype, 32, 64, 16, 1, 2) \
- instantiate_segmented_mm_transpose_helper(iname, itype, oname, otype, 32, 32, 16, 2, 2)
-// clang-format on
-
-instantiate_segmented_mm_shapes_helper(float16, half, float16, half);
-instantiate_segmented_mm_shapes_helper(
- bfloat16,
- bfloat16_t,
- bfloat16,
- bfloat16_t);
-instantiate_segmented_mm_shapes_helper(float32, float, float32, float);
diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/mod.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/mod.rs
new file mode 100644
index 000000000..05b66ce46
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/mod.rs
@@ -0,0 +1,237 @@
+use std::collections::HashMap;
+
+use metal::{
+ Buffer as MTLBuffer, ComputeCommandEncoderRef,
+ ComputePipelineState as MTLComputePipelineState, MTLSize,
+};
+
+use super::split_k;
+use crate::{
+ DataType,
+ backends::metal::{
+ MTLContext, MTLError,
+ kernel::{matmul::common::GEMMParams, mlp::MlpActivationType},
+ },
+};
+
+#[derive(Debug)]
+pub struct Arguments<'a> {
+ pub input: &'a MTLBuffer,
+ pub input_offset: u64,
+ pub weights: &'a MTLBuffer,
+ pub output: &'a MTLBuffer,
+ pub batch: i32,
+ pub input_dim: i32,
+ pub hidden_dim: i32,
+ pub lda: i32,
+ pub ldb: i32,
+ pub ldd: i32,
+ pub activation: MlpActivationType,
+}
+
+pub struct Kernel {
+ data_type: DataType,
+ weights_transposed: bool,
+ pipelines: HashMap<
+ (i32, i32, bool, bool, MlpActivationType),
+ MTLComputePipelineState,
+ >,
+ splitk_kernel: Option<split_k::Kernel>,
+}
+
+impl Kernel {
+ pub fn new(
+ data_type: DataType,
+ weights_transposed: bool,
+ ) -> Result<Self, MTLError> {
+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)
+ {
+ return Err(MTLError::Generic(format!(
+ "Unsupported dtype for MLP fused GEMM: {:?}",
+ data_type
+ )));
+ }
+ let splitk_kernel = if weights_transposed
+ && matches!(data_type, DataType::F16 | DataType::BF16)
+ {
+ Some(split_k::Kernel::new(data_type))
+ } else {
+ None
+ };
+ Ok(Self {
+ data_type,
+ weights_transposed,
+ pipelines: HashMap::new(),
+ splitk_kernel,
+ })
+ }
+
+ fn kernel_name(
+ &self,
+ bm: i32,
+ bn: i32,
+ mn_aligned: bool,
+ k_aligned: bool,
+ ) -> String {
+ let type_name = match self.data_type {
+ DataType::F16 => "half",
+ DataType::BF16 => "bfloat",
+ DataType::F32 => "float",
+ _ => unreachable!(),
+ };
+ let transpose_char = if self.weights_transposed {
+ "t"
+ } else {
+ "n"
+ };
+ let mn_str = if mn_aligned {
+ "true"
+ } else {
+ "false"
+ };
+ let k_str = if k_aligned {
+ "true"
+ } else {
+ "false"
+ };
+
+ format!(
+ "steel_gemm_mlp_fused_{}_{}_{}_bm{}_bn{}_bk16_wm2_wn2_align_MN_{}_K_{}",
+ transpose_char, type_name, type_name, bm, bn, mn_str, k_str
+ )
+ }
+
+ fn get_pipeline(
+ &mut self,
+ context: &MTLContext,
+ bm: i32,
+ bn: i32,
+ mn_aligned: bool,
+ k_aligned: bool,
+ activation: MlpActivationType,
+ ) -> Result<&MTLComputePipelineState, MTLError> {
+ let key = (bm, bn, mn_aligned, k_aligned, activation);
+ if !self.pipelines.contains_key(&key) {
+ let name = self.kernel_name(bm, bn, mn_aligned, k_aligned);
+
+ let fcv = metal::FunctionConstantValues::new();
+ let activation_val = activation as u32;
+ fcv.set_constant_value_at_index(
+ &activation_val as *const u32 as *const _,
+ metal::MTLDataType::UInt,
+ 52,
+ );
+
+ let pipeline = context.compute_pipeline_state(&name, Some(&fcv))?;
+ self.pipelines.insert(key, pipeline);
+ }
+ Ok(self.pipelines.get(&key).unwrap())
+ }
+
+ fn select_tile_size(
+ &self,
+ m: i32,
+ hidden_dim: i32,
+ _k: i32,
+ ) -> (i32, i32) {
+ if m >= 64 && hidden_dim >= 64 {
+ (64, 64)
+ } else {
+ (32, 32)
+ }
+ }
+
+ pub fn encode(
+ &mut self,
+ context: &MTLContext,
+ encoder: &ComputeCommandEncoderRef,
+ args: &Arguments,
+ ) -> Result<(), MTLError> {
+ if let Some(ref mut splitk) = self.splitk_kernel {
+ if split_k::Kernel::should_use_splitk(
+ args.batch,
+ args.hidden_dim,
+ args.input_dim,
+ ) {
+ let splitk_args = split_k::Arguments {
+ input: args.input,
+ input_offset: args.input_offset,
+ weights: args.weights,
+ output: args.output,
+ batch: args.batch,
+ input_dim: args.input_dim,
+ hidden_dim: args.hidden_dim,
+ lda: args.lda,
+ ldb: args.ldb,
+ ldd: args.ldd,
+ activation: args.activation,
+ };
+ return splitk.encode(context, encoder, &splitk_args);
+ }
+ }
+
+ let (bm, bn) =
+ self.select_tile_size(args.batch, args.hidden_dim, args.input_dim);
+ let bk = 16;
+
+ let mn_aligned = args.batch % bm == 0 && args.hidden_dim % bn == 0;
+ let k_aligned = args.input_dim % bk == 0;
+
+ let pipeline = self.get_pipeline(
+ context,
+ bm,
+ bn,
+ mn_aligned,
+ k_aligned,
+ args.activation,
+ )?;
+ encoder.set_compute_pipeline_state(pipeline);
+
+ encoder.set_buffer(0, Some(args.input), args.input_offset);
+ encoder.set_buffer(1, Some(args.weights), 0);
+ encoder.set_buffer(2, Some(args.output), 0);
+
+ let tiles_m = (args.batch + bm - 1) / bm;
+ let tiles_n = (args.hidden_dim + bn - 1) / bn;
+ let gemm_k_iterations = (args.input_dim + bk - 1) / bk;
+
+ let params = GEMMParams {
+ M: args.batch,
+ N: args.hidden_dim * 2,
+ K: args.input_dim,
+ lda: args.lda,
+ ldb: args.ldb,
+ ldd: args.ldd,
+ tiles_n,
+ tiles_m,
+ batch_stride_a: 0,
+ batch_stride_b: 0,
+ batch_stride_d: 0,
+ swizzle_log: 0,
+ gemm_k_iterations_aligned: if k_aligned {
+ gemm_k_iterations
+ } else {
+ gemm_k_iterations - 1
+ },
+ batch_ndim: 0,
+ };
+
+ encoder.set_bytes(
+ 3,
+ std::mem::size_of::<GEMMParams>() as u64,
+ &params as *const GEMMParams as *const std::ffi::c_void,
+ );
+ encoder.set_bytes(
+ 10,
+ std::mem::size_of::<i32>() as u64,
+ &args.hidden_dim as *const i32 as *const std::ffi::c_void,
+ );
+
+ let threadgroup_count = MTLSize::new(tiles_n as u64, tiles_m as u64, 1);
+ let threads_per_threadgroup = MTLSize::new(32, 2, 2);
+
+ encoder
+ .dispatch_thread_groups(threadgroup_count, threads_per_threadgroup);
+ Ok(())
+ }
+}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_mlp_fused.h b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/steel_gemm_mlp_fused.h
similarity index 98%
rename from crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_mlp_fused.h
rename to crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/steel_gemm_mlp_fused.h
index 88ebcdbe1..6b945428b 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_mlp_fused.h
+++ b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/steel_gemm_mlp_fused.h
@@ -4,9 +4,9 @@
#include <metal_simdgroup>
#include <metal_simdgroup_matrix>
-#include "../gemm.h"
-#include "../params.h"
-#include "../../../../common/mlp_epilogue.h"
+#include "../../matmul/common/steel/gemm/gemm.h"
+#include "../../matmul/common/steel/gemm/params.h"
+#include "../../common/mlp_epilogue.h"
using namespace metal;
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_mlp_fused.metal b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/steel_gemm_mlp_fused.metal
similarity index 99%
rename from crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_mlp_fused.metal
rename to crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/steel_gemm_mlp_fused.metal
index 5b0da8472..35ffc4949 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/steel/gemm/kernels/steel_gemm_mlp_fused.metal
+++ b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemm/steel_gemm_mlp_fused.metal
@@ -1,6 +1,6 @@
#include <metal_stdlib>
-#include "../../../../common/utils.h"
+#include "../../common/utils.h"
#include "steel_gemm_mlp_fused.h"
using namespace metal;
diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/gemv_mlp_fused.metal b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/gemv_mlp_fused.metal
new file mode 100644
index 000000000..789063231
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/gemv_mlp_fused.metal
@@ -0,0 +1,272 @@
+#include <metal_simdgroup>
+#include <metal_stdlib>
+
+#include "../../common/utils.h"
+#include "../../common/mlp_epilogue.h"
+
+#include "../matmul/common/steel/utils.h"
+
+using namespace metal;
+
+///////////////////////////////////////////////////////////////////////////////
+/// MLP Fused GEMV - Paired Up/Gate Computation with Activation Fusion
+///////////////////////////////////////////////////////////////////////////////
+
+#define MTL_CONST static constant constexpr const
+
+// MLP fused GEMV kernel for decode path (M=1).
+// Computes paired up and gate projections, then applies: out = up * activation(gate)
+// Output size is hidden_dim (half of the weight matrix columns).
+template <
+ typename T,
+ const int BM, /* Threadgroup rows (in simdgroups) */
+ const int BN, /* Threadgroup cols (in simdgroups) */
+ const int SM, /* Simdgroup rows (in threads) */
+ const int SN, /* Simdgroup cols (in threads) */
+ const int TM, /* Thread rows (in elements) */
+ const int TN> /* Thread cols (in elements) */
+struct GEMVMlpFusedKernel {
+ using AccT = float;
+
+ MTL_CONST int threadsM = BM * SM;
+ MTL_CONST int threadsN = BN * SN;
+ MTL_CONST int blockM = threadsM * TM;
+ MTL_CONST int blockN = threadsN * TN;
+
+ static_assert(SM * SN == 32, "simdgroup can only have 32 threads");
+
+ MTL_CONST short tgp_mem_size = BN > 1 ? 2 * BN * (blockM + TM) : 0;
+ MTL_CONST bool needs_tgp_reduction = BN > 1;
+
+ template <typename U = T>
+ static METAL_FUNC void
+ load_unsafe(const device T* src, thread U dst[TN], const int src_offset = 0) {
+ MTL_PRAGMA_UNROLL
+ for (int tn = 0; tn < TN; tn++) {
+ dst[tn] = static_cast<U>(src[src_offset + tn]);
+ }
+ }
+
+ template <typename U = T>
+ static METAL_FUNC void load_safe(
+ const device T* src,
+ thread U dst[TN],
+ const int src_offset = 0,
+ const int src_size = TN) {
+ if (src_offset + TN <= src_size) {
+ MTL_PRAGMA_UNROLL
+ for (int tn = 0; tn < TN; tn++) {
+ dst[tn] = static_cast<U>(src[src_offset + tn]);
+ }
+ } else {
+ MTL_PRAGMA_UNROLL
+ for (int tn = 0; tn < TN; tn++) {
+ dst[tn] = src_offset + tn < src_size
+ ? static_cast<U>(src[src_offset + tn])
+ : U(0);
+ }
+ }
+ }
+
+ static METAL_FUNC void run(
+ const device T* mat [[buffer(0)]],
+ const device T* in_vec [[buffer(1)]],
+ device T* out_vec [[buffer(3)]],
+ const int in_vec_size,
+ const int hidden_dim, // Output size (half of weight matrix rows)
+ const int matrix_ld,
+ threadgroup AccT* tgp_memory,
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint3 lid [[thread_position_in_threadgroup]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]) {
+ (void)lid;
+
+ // Thread local accumulation results for up and gate
+ thread AccT result_up[TM] = {0};
+ thread AccT result_gate[TM] = {0};
+ thread T inter_up[TN];
+ thread T inter_gate[TN];
+ thread AccT v_coeff[TN];
+
+ const int thrM = SN != 32 ? simd_lid / SN : 0;
+ const int thrN = SN != 32 ? simd_lid % SN : int(simd_lid);
+
+ const int sgN = BN != 1 ? (simd_gid % BN) : 0;
+ const int simdM = BN != 1 ? SM * (simd_gid / BN) : int(SM * simd_gid);
+ const int simdN = BN != 1 ? SN * (simd_gid % BN) : 0;
+
+ int bm = (simdM + thrM) * TM;
+ int bn = (simdN + thrN) * TN;
+
+ // Block position - outputs to hidden_dim elements
+ int out_row = tid.x * blockM + bm;
+
+ if (out_row >= hidden_dim)
+ return;
+
+ out_row = out_row + TM <= hidden_dim ? out_row : hidden_dim - TM;
+
+ // Pointers to up and gate rows in the weight matrix
+ // Weight layout: [up_0, up_1, ..., up_H-1, gate_0, gate_1, ..., gate_H-1]
+ const device T* mat_up = mat + out_row * matrix_ld;
+ const device T* mat_gate = mat + (out_row + hidden_dim) * matrix_ld;
+
+ constexpr const uniform<int> loop_stride = make_uniform(blockN);
+ const uniform<int> in_size = make_uniform(in_vec_size);
+ const uniform<int> n_iter = in_size / loop_stride;
+ const uniform<int> last_iter = loop_stride * n_iter;
+ const uniform<int> leftover = in_size - last_iter;
+
+ // Main loop - compute both up and gate dot products
+ for (int i = 0; i < n_iter; ++i) {
+ load_unsafe<AccT>(in_vec, v_coeff, bn);
+
+ int mat_offset = 0;
+ MTL_PRAGMA_UNROLL
+ for (int tm = 0; tm < TM; tm++) {
+ load_unsafe(mat_up, inter_up, mat_offset + bn);
+ load_unsafe(mat_gate, inter_gate, mat_offset + bn);
+
+ MTL_PRAGMA_UNROLL
+ for (int tn = 0; tn < TN; tn++) {
+ result_up[tm] += inter_up[tn] * v_coeff[tn];
+ result_gate[tm] += inter_gate[tn] * v_coeff[tn];
+ }
+
+ mat_offset += matrix_ld;
+ }
+
+ bn += blockN;
+ }
+
+ // Handle leftover
+ if (leftover > 0) {
+ load_safe<AccT>(in_vec, v_coeff, bn, in_vec_size);
+
+ MTL_PRAGMA_UNROLL
+ for (int tm = 0; tm < TM; tm++) {
+ load_safe(&mat_up[tm * matrix_ld], inter_up, bn, in_vec_size);
+ load_safe(&mat_gate[tm * matrix_ld], inter_gate, bn, in_vec_size);
+
+ MTL_PRAGMA_UNROLL
+ for (int tn = 0; tn < TN; tn++) {
+ result_up[tm] += inter_up[tn] * v_coeff[tn];
+ result_gate[tm] += inter_gate[tn] * v_coeff[tn];
+ }
+ }
+ }
+
+ // Simdgroup reduction for both up and gate
+ MTL_PRAGMA_UNROLL
+ for (int tm = 0; tm < TM; tm++) {
+ MTL_PRAGMA_UNROLL
+ for (ushort sn = (SN / 2); sn >= 1; sn >>= 1) {
+ result_up[tm] += simd_shuffle_down(result_up[tm], sn);
+ result_gate[tm] += simd_shuffle_down(result_gate[tm], sn);
+ }
+ }
+
+ // Threadgroup reduction if needed
+ if (needs_tgp_reduction) {
+ threadgroup AccT* tgp_results_up = tgp_memory + sgN * (blockM + TM) + bm;
+ threadgroup AccT* tgp_results_gate = tgp_memory + BN * (blockM + TM) + sgN * (blockM + TM) + bm;
+
+ if (thrN == 0) {
+ MTL_PRAGMA_UNROLL
+ for (int tm = 0; tm < TM; tm++) {
+ tgp_results_up[tm] = result_up[tm];
+ tgp_results_gate[tm] = result_gate[tm];
+ }
+
+ threadgroup_barrier(mem_flags::mem_none);
+
+ if (sgN == 0) {
+ MTL_PRAGMA_UNROLL
+ for (int sgn = 1; sgn < BN; sgn++) {
+ MTL_PRAGMA_UNROLL
+ for (int tm = 0; tm < TM; tm++) {
+ result_up[tm] += tgp_results_up[sgn * (blockM + TM) + tm];
+ result_gate[tm] += tgp_results_gate[sgn * (blockM + TM) + tm];
+ }
+ }
+ }
+ }
+ }
+
+ // Apply MLP fused epilogue and write output
+ if (simdN == 0 && thrN == 0) {
+ MTL_PRAGMA_UNROLL
+ for (int tm = 0; tm < TM; tm++) {
+ float fused_result = mlp_fused_epilogue_f32(result_up[tm], result_gate[tm]);
+ out_vec[out_row + tm] = static_cast<T>(fused_result);
+ }
+ }
+ }
+};
+
+template <
+ typename T,
+ const int BM,
+ const int BN,
+ const int SM,
+ const int SN,
+ const int TM,
+ const int TN>
+[[kernel, max_total_threads_per_threadgroup(BM* BN * 32)]] void gemv_mlp_fused(
+ const device T* mat [[buffer(0)]],
+ const device T* in_vec [[buffer(1)]],
+ device T* out_vec [[buffer(3)]],
+ const constant int& in_vec_size [[buffer(4)]],
+ const constant int& hidden_dim [[buffer(5)]], // Output dimension (half of weight rows)
+ const constant int& matrix_ld [[buffer(6)]],
+ const constant int64_t& vector_batch_stride [[buffer(11)]],
+ const constant int64_t& matrix_batch_stride [[buffer(12)]],
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint3 lid [[thread_position_in_threadgroup]],
+ uint simd_gid [[simdgroup_index_in_threadgroup]],
+ uint simd_lid [[thread_index_in_simdgroup]]) {
+ using gemv_kernel = GEMVMlpFusedKernel<T, BM, BN, SM, SN, TM, TN>;
+ threadgroup typename gemv_kernel::AccT tgp_memory
+ [gemv_kernel::tgp_mem_size == 0 ? 1 : gemv_kernel::tgp_mem_size];
+
+ // Batch offset
+ in_vec += tid.z * vector_batch_stride;
+ mat += tid.z * matrix_batch_stride;
+ out_vec += tid.z * hidden_dim;
+
+ gemv_kernel::run(
+ mat,
+ in_vec,
+ out_vec,
+ in_vec_size,
+ hidden_dim,
+ matrix_ld,
+ gemv_kernel::tgp_mem_size == 0 ? nullptr : tgp_memory,
+ tid,
+ lid,
+ simd_gid,
+ simd_lid);
+}
+
+#define instantiate_gemv_mlp_fused_helper(name, itype, bm, bn, sm, sn, tm, tn) \
+ instantiate_kernel( \
+ "gemv_mlp_fused_" #name "_bm" #bm "_bn" #bn "_sm" #sm "_sn" #sn "_tm" #tm \
+ "_tn" #tn, \
+ gemv_mlp_fused, \
+ itype, \
+ bm, \
+ bn, \
+ sm, \
+ sn, \
+ tm, \
+ tn)
+
+// clang-format off
+#define instantiate_gemv_mlp_fused_blocks(name, itype) \
+ instantiate_gemv_mlp_fused_helper(name, itype, 4, 1, 1, 32, 4, 4) \
+ instantiate_gemv_mlp_fused_helper(name, itype, 8, 1, 1, 32, 4, 4) // clang-format on
+
+instantiate_gemv_mlp_fused_blocks(float32, float);
+instantiate_gemv_mlp_fused_blocks(float16, half);
+instantiate_gemv_mlp_fused_blocks(bfloat16, bfloat16_t);
diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/mod.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/mod.rs
new file mode 100644
index 000000000..0fc92d97b
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/mlp_fused/gemv/mod.rs
@@ -0,0 +1,202 @@
+use std::collections::HashMap;
+
+use metal::{
+ Buffer as MTLBuffer, ComputeCommandEncoderRef, ComputePipelineState,
+ MTLSize,
+};
+
+use crate::{
+ DataType,
+ backends::metal::{MTLContext, MTLError, kernel::mlp::MlpActivationType},
+};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+struct Configuration {
+ threadgroup_rows: u32,
+ threadgroup_cols: u32,
+ threads_per_simdgroup_row: u32,
+ threads_per_simdgroup_col: u32,
+ elements_per_thread_row: u32,
+ elements_per_thread_col: u32,
+ activation: MlpActivationType,
+}
+
+impl Configuration {
+ fn output_elements_per_threadgroup(&self) -> u32 {
+ self.threadgroup_rows
+ * self.threads_per_simdgroup_row
+ * self.elements_per_thread_row
+ }
+
+ fn threads_per_threadgroup(&self) -> MTLSize {
+ MTLSize::new(
+ 32,
+ self.threadgroup_cols as u64,
+ self.threadgroup_rows as u64,
+ )
+ }
+}
+
+fn kernel_name(
+ data_type: DataType,
+ config: &Configuration,
+) -> Result<String, MTLError> {
+ let dtype_name = match data_type {
+ DataType::F16 => "float16",
+ DataType::BF16 => "bfloat16",
+ DataType::F32 => "float32",
+ _ => {
+ return Err(MTLError::Generic(format!(
+ "Unsupported data type for MLP fused GEMV: {:?}",
+ data_type
+ )));
+ },
+ };
+
+ Ok(format!(
+ "gemv_mlp_fused_{}_bm{}_bn{}_sm{}_sn{}_tm{}_tn{}",
+ dtype_name,
+ config.threadgroup_rows,
+ config.threadgroup_cols,
+ config.threads_per_simdgroup_row,
+ config.threads_per_simdgroup_col,
+ config.elements_per_thread_row,
+ config.elements_per_thread_col,
+ ))
+}
+
+fn select_configuration(
+ hidden_dim: i32,
+ activation: MlpActivationType,
+) -> Configuration {
+ let threadgroup_rows = if hidden_dim >= 4096 {
+ 8
+ } else {
+ 4
+ };
+
+ Configuration {
+ threadgroup_rows,
+ threadgroup_cols: 1,
+ threads_per_simdgroup_row: 1,
+ threads_per_simdgroup_col: 32,
+ elements_per_thread_row: 4,
+ elements_per_thread_col: 4,
+ activation,
+ }
+}
+
+#[derive(Debug)]
+pub struct Arguments<'a> {
+ pub weights: &'a MTLBuffer,
+ pub input: &'a MTLBuffer,
+ pub input_offset: u64,
+ pub output: &'a MTLBuffer,
+ pub input_dim: i32,
+ pub hidden_dim: i32,
+ pub weights_ld: i32,
+ pub batch_count: i32,
+ pub activation: MlpActivationType,
+}
+
+pub struct Kernel {
+ data_type: DataType,
+ pipelines: HashMap<Configuration, ComputePipelineState>,
+}
+
+impl Kernel {
+ pub fn new(data_type: DataType) -> Result<Self, MTLError> {
+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)
+ {
+ return Err(MTLError::Generic(format!(
+ "Unsupported dtype for MLP fused GEMV: {:?}",
+ data_type
+ )));
+ }
+ Ok(Self {
+ data_type,
+ pipelines: HashMap::new(),
+ })
+ }
+
+ fn get_pipeline(
+ &mut self,
+ context: &MTLContext,
+ config: Configuration,
+ ) -> Result<&ComputePipelineState, MTLError> {
+ if !self.pipelines.contains_key(&config) {
+ let name = kernel_name(self.data_type, &config)?;
+
+ let fcv = metal::FunctionConstantValues::new();
+ let activation_val = config.activation as u32;
+ fcv.set_constant_value_at_index(
+ &activation_val as *const u32 as *const _,
+ metal::MTLDataType::UInt,
+ 52,
+ );
+
+ let pipeline = context.compute_pipeline_state(&name, Some(&fcv))?;
+ self.pipelines.insert(config, pipeline);
+ }
+ Ok(self.pipelines.get(&config).unwrap())
+ }
+
+ pub fn encode(
+ &mut self,
+ context: &MTLContext,
+ encoder: &ComputeCommandEncoderRef,
+ args: &Arguments,
+ ) -> Result<(), MTLError> {
+ let config = select_configuration(args.hidden_dim, args.activation);
+ let pipeline = self.get_pipeline(context, config)?;
+
+ encoder.set_compute_pipeline_state(pipeline);
+ encoder.set_buffer(0, Some(args.weights), 0);
+ encoder.set_buffer(1, Some(args.input), args.input_offset);
+ encoder.set_buffer(3, Some(args.output), 0);
+
+ encoder.set_bytes(
+ 4,
+ std::mem::size_of::<i32>() as u64,
+ &args.input_dim as *const i32 as *const std::ffi::c_void,
+ );
+ encoder.set_bytes(
+ 5,
+ std::mem::size_of::<i32>() as u64,
+ &args.hidden_dim as *const i32 as *const std::ffi::c_void,
+ );
+ encoder.set_bytes(
+ 6,
+ std::mem::size_of::<i32>() as u64,
+ &args.weights_ld as *const i32 as *const std::ffi::c_void,
+ );
+
+ let vector_batch_stride = args.input_dim as i64;
+ let matrix_batch_stride = 0i64;
+ encoder.set_bytes(
+ 11,
+ std::mem::size_of::<i64>() as u64,
+ &vector_batch_stride as *const i64 as *const std::ffi::c_void,
+ );
+ encoder.set_bytes(
+ 12,
+ std::mem::size_of::<i64>() as u64,
+ &matrix_batch_stride as *const i64 as *const std::ffi::c_void,
+ );
+
+ let output_elements_per_threadgroup =
+ config.output_elements_per_threadgroup();
+ let threadgroup_count_x =
+ ((args.hidden_dim as u32 + output_elements_per_threadgroup - 1)
+ / output_elements_per_threadgroup) as u64;
+ let threadgroup_count_z = args.batch_count.max(1) as u64;
+
+ let threadgroup_count =
+ MTLSize::new(threadgroup_count_x, 1, threadgroup_count_z);
+ let threads_per_threadgroup = config.threads_per_threadgroup();
+
+ encoder
+ .dispatch_thread_groups(threadgroup_count, threads_per_threadgroup);
+ Ok(())
+ }
+}
diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/mod.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/mod.rs
new file mode 100644
index 000000000..bbbf615c1
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/mlp_fused/mod.rs
@@ -0,0 +1,7 @@
+pub mod gemm;
+pub mod gemv;
+pub mod split_k;
+
+pub use gemm::{Arguments as GemmArguments, Kernel as GemmKernel};
+pub use gemv::{Arguments as GemvArguments, Kernel as GemvKernel};
+pub use split_k::{Arguments as SplitKArguments, Kernel as SplitKKernel};
diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/mod.rs b/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/mod.rs
new file mode 100644
index 000000000..191d42704
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/mod.rs
@@ -0,0 +1,313 @@
+use std::collections::HashMap;
+
+use metal::{
+ Buffer as MTLBuffer, ComputeCommandEncoderRef,
+ ComputePipelineState as MTLComputePipelineState, MTLSize,
+};
+
+use crate::{
+ DataType,
+ backends::metal::{
+ MTLContext, MTLError,
+ kernel::{
+ matmul::common::GEMMSpiltKMlpFusedParams, mlp::MlpActivationType,
+ },
+ },
+};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+struct TileConfig {
+ tile_rows: i32,
+ tile_cols: i32,
+ tile_depth: i32,
+ warps_per_row: i32,
+ warps_per_col: i32,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+struct PartialKernelKey {
+ tile_config: TileConfig,
+ mn_aligned: bool,
+ k_aligned: bool,
+}
+
+#[derive(Debug)]
+pub struct Arguments<'a> {
+ pub input: &'a MTLBuffer,
+ pub input_offset: u64,
+ pub weights: &'a MTLBuffer,
+ pub output: &'a MTLBuffer,
+ pub batch: i32,
+ pub input_dim: i32,
+ pub hidden_dim: i32,
+ pub lda: i32,
+ pub ldb: i32,
+ pub ldd: i32,
+ pub activation: MlpActivationType,
+}
+
+pub struct Kernel {
+ data_type: DataType,
+ partial_pipelines: HashMap<PartialKernelKey, MTLComputePipelineState>,
+ accum_pipelines: HashMap<MlpActivationType, MTLComputePipelineState>,
+}
+
+impl Kernel {
+ pub fn new(data_type: DataType) -> Self {
+ Self {
+ data_type,
+ partial_pipelines: HashMap::new(),
+ accum_pipelines: HashMap::new(),
+ }
+ }
+
+ fn steel_type_name(&self) -> Result<&'static str, MTLError> {
+ match self.data_type {
+ DataType::F16 => Ok("float16"),
+ DataType::BF16 => Ok("bfloat16"),
+ _ => Err(MTLError::Generic(format!(
+ "Unsupported dtype for MLP Fused Split-K: {:?}",
+ self.data_type
+ ))),
+ }
+ }
+
+ fn partial_kernel_name(
+ &self,
+ config: &TileConfig,
+ mn_aligned: bool,
+ k_aligned: bool,
+ ) -> Result<String, MTLError> {
+ let in_name = self.steel_type_name()?;
+ let mn_tag = if mn_aligned {
+ "taligned"
+ } else {
+ "naligned"
+ };
+ let k_tag = if k_aligned {
+ "taligned"
+ } else {
+ "naligned"
+ };
+ Ok(format!(
+ "steel_gemm_splitk_mlp_fused_nt_{}_float32_bm{}_bn{}_bk{}_wm{}_wn{}_MN_{}_K_{}",
+ in_name,
+ config.tile_rows,
+ config.tile_cols,
+ config.tile_depth,
+ config.warps_per_row,
+ config.warps_per_col,
+ mn_tag,
+ k_tag,
+ ))
+ }
+
+ fn accum_kernel_name(&self) -> Result<String, MTLError> {
+ let out_name = self.steel_type_name()?;
+ Ok(format!("steel_gemm_splitk_mlp_fused_accum_{}_float32", out_name))
+ }
+
+ fn get_partial_pipeline(
+ &mut self,
+ mtl: &MTLContext,
+ config: TileConfig,
+ mn_aligned: bool,
+ k_aligned: bool,
+ ) -> Result<&MTLComputePipelineState, MTLError> {
+ let key = PartialKernelKey {
+ tile_config: config,
+ mn_aligned,
+ k_aligned,
+ };
+ if !self.partial_pipelines.contains_key(&key) {
+ let name =
+ self.partial_kernel_name(&config, mn_aligned, k_aligned)?;
+ let ps = mtl.compute_pipeline_state(&name, None)?;
+ self.partial_pipelines.insert(key.clone(), ps);
+ }
+ Ok(self.partial_pipelines.get(&key).unwrap())
+ }
+
+ fn get_accum_pipeline(
+ &mut self,
+ mtl: &MTLContext,
+ activation: MlpActivationType,
+ ) -> Result<&MTLComputePipelineState, MTLError> {
+ if !self.accum_pipelines.contains_key(&activation) {
+ let name = self.accum_kernel_name()?;
+ let fcv = metal::FunctionConstantValues::new();
+ let activation_val = activation as u32;
+ fcv.set_constant_value_at_index(
+ &activation_val as *const u32 as *const _,
+ metal::MTLDataType::UInt,
+ 52,
+ );
+ let ps = mtl.compute_pipeline_state(&name, Some(&fcv))?;
+ self.accum_pipelines.insert(activation, ps);
+ }
+ Ok(self.accum_pipelines.get(&activation).unwrap())
+ }
+
+ fn select_tile_config(
+ m: i32,
+ n: i32,
+ ) -> TileConfig {
+ let tile_rows = if m < 40 {
+ 16
+ } else {
+ 32
+ };
+ let tile_cols = if n < 40 {
+ 16
+ } else {
+ 32
+ };
+ TileConfig {
+ tile_rows,
+ tile_cols,
+ tile_depth: 16,
+ warps_per_row: 2,
+ warps_per_col: 2,
+ }
+ }
+
+ fn compute_partition_count(k: i32) -> i32 {
+ let k_tiles = k / 16;
+ if k_tiles < 16 {
+ 2
+ } else if k_tiles < 32 {
+ 4
+ } else if k_tiles < 64 {
+ 8
+ } else {
+ 16
+ }
+ }
+
+ pub fn should_use_splitk(
+ m: i32,
+ hidden_dim: i32,
+ k: i32,
+ ) -> bool {
+ if m != 1 {
+ return false;
+ }
+ if hidden_dim <= 0 || k <= 0 {
+ return false;
+ }
+ let n_tiles = hidden_dim / 16;
+ let k_tiles = k / 16;
+ n_tiles <= 32 && k_tiles >= 8
+ }
+
+ pub fn encode(
+ &mut self,
+ mtl: &MTLContext,
+ enc: &ComputeCommandEncoderRef,
+ args: &Arguments,
+ ) -> Result<(), MTLError> {
+ let m = args.batch;
+ let hidden_dim = args.hidden_dim;
+ let k = args.input_dim;
+
+ let tile_config = Self::select_tile_config(m, hidden_dim);
+ let partition_count = Self::compute_partition_count(k);
+ let mn_aligned = (m % tile_config.tile_rows) == 0
+ && (hidden_dim % tile_config.tile_cols) == 0;
+ let k_aligned = (k % tile_config.tile_depth) == 0;
+
+ let tile_count_m =
+ (m + tile_config.tile_rows - 1) / tile_config.tile_rows;
+ let tile_count_n =
+ (hidden_dim + tile_config.tile_cols - 1) / tile_config.tile_cols;
+
+ let gemm_k_iterations = (k / tile_config.tile_depth) / partition_count;
+ let k_elements_per_partition =
+ gemm_k_iterations * tile_config.tile_depth;
+ let output_elements_per_partition = m * hidden_dim;
+
+ let accumulator_element_count =
+ partition_count * output_elements_per_partition;
+ let accumulator_bytes =
+ (accumulator_element_count as usize) * std::mem::size_of::<f32>();
+
+ let up_accumulator_buffer = mtl.device.new_buffer(
+ accumulator_bytes as u64,
+ metal::MTLResourceOptions::StorageModePrivate,
+ );
+ let gate_accumulator_buffer = mtl.device.new_buffer(
+ accumulator_bytes as u64,
+ metal::MTLResourceOptions::StorageModePrivate,
+ );
+
+ let params = GEMMSpiltKMlpFusedParams {
+ M: m,
+ N: hidden_dim,
+ K: k,
+ lda: args.lda,
+ ldb: args.ldb,
+ ldc: hidden_dim,
+ tiles_n: tile_count_n,
+ tiles_m: tile_count_m,
+ split_k_partitions: partition_count,
+ split_k_partition_stride: output_elements_per_partition,
+ split_k_partition_size: k_elements_per_partition,
+ gemm_k_iterations_aligned: gemm_k_iterations,
+ hidden_dim,
+ };
+
+ let partial_ps =
+ self.get_partial_pipeline(mtl, tile_config, mn_aligned, k_aligned)?;
+
+ enc.set_compute_pipeline_state(partial_ps);
+ enc.set_buffer(0, Some(args.input), args.input_offset);
+ enc.set_buffer(1, Some(args.weights), 0);
+ enc.set_buffer(2, Some(&up_accumulator_buffer), 0);
+ enc.set_buffer(3, Some(&gate_accumulator_buffer), 0);
+ enc.set_bytes(
+ 4,
+ std::mem::size_of::<GEMMSpiltKMlpFusedParams>() as u64,
+ &params as *const GEMMSpiltKMlpFusedParams as *const _,
+ );
+
+ let threads_per_threadgroup = MTLSize::new(
+ 32,
+ tile_config.warps_per_col as u64,
+ tile_config.warps_per_row as u64,
+ );
+ let threadgroups = MTLSize::new(
+ tile_count_n as u64,
+ tile_count_m as u64,
+ partition_count as u64,
+ );
+ enc.dispatch_thread_groups(threadgroups, threads_per_threadgroup);
+
+ let accum_ps = self.get_accum_pipeline(mtl, args.activation)?;
+ enc.set_compute_pipeline_state(accum_ps);
+ enc.set_buffer(0, Some(&up_accumulator_buffer), 0);
+ enc.set_buffer(1, Some(&gate_accumulator_buffer), 0);
+ enc.set_buffer(2, Some(args.output), 0);
+ enc.set_bytes(
+ 3,
+ 4,
+ &partition_count as *const i32 as *const std::ffi::c_void,
+ );
+ enc.set_bytes(
+ 4,
+ 4,
+ &output_elements_per_partition as *const i32
+ as *const std::ffi::c_void,
+ );
+ enc.set_bytes(5, 4, &args.ldd as *const i32 as *const std::ffi::c_void);
+
+ let accum_total_threads = MTLSize::new(hidden_dim as u64, m as u64, 1);
+ let accum_threads_per_threadgroup =
+ MTLSize::new(16.min(hidden_dim as u64), 16.min(m as u64), 1);
+ enc.dispatch_threads(
+ accum_total_threads,
+ accum_threads_per_threadgroup,
+ );
+
+ Ok(())
+ }
+}
diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/steel_gemm_splitk_mlp_fused.h b/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/steel_gemm_splitk_mlp_fused.h
new file mode 100644
index 000000000..15d59cfef
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/steel_gemm_splitk_mlp_fused.h
@@ -0,0 +1,216 @@
+#pragma once
+
+#include <metal_stdlib>
+#include <metal_simdgroup>
+#include <metal_simdgroup_matrix>
+
+#include "../../matmul/common/steel/gemm/gemm.h"
+#include "../../matmul/common/shared_types.h"
+#include "../../common/mlp_epilogue.h"
+
+using namespace metal;
+using namespace steel;
+
+///////////////////////////////////////////////////////////////////////////////
+// MLP Fused Split-K GEMM Kernel
+// Computes paired up/gate projections with fused activation using split-K.
+///////////////////////////////////////////////////////////////////////////////
+
+template <
+ typename T,
+ typename U,
+ int BM,
+ int BN,
+ int BK,
+ int WM,
+ int WN,
+ bool transpose_a,
+ bool transpose_b,
+ bool MN_aligned,
+ bool K_aligned>
+[[kernel, max_total_threads_per_threadgroup(WM* WN * 32)]] void gemm_splitk_mlp_fused(
+ const device T* A [[buffer(0)]],
+ const device T* B [[buffer(1)]],
+ device U* C [[buffer(2)]],
+ const constant uzu::matmul::GEMMSpiltKMlpFusedParams* params [[buffer(3)]],
+ uint simd_lane_id [[thread_index_in_simdgroup]],
+ uint simd_group_id [[simdgroup_index_in_threadgroup]],
+ uint3 tid [[threadgroup_position_in_grid]],
+ uint3 lid [[thread_position_in_threadgroup]]) {
+ (void)lid;
+
+ using gemm_kernel = GEMMKernel<
+ T,
+ U,
+ BM,
+ BN,
+ BK,
+ WM,
+ WN,
+ transpose_a,
+ transpose_b,
+ MN_aligned,
+ K_aligned>;
+ using loader_a_t = typename gemm_kernel::loader_a_t;
+ using loader_b_t = typename gemm_kernel::loader_b_t;
+ using mma_t = typename gemm_kernel::mma_t;
+
+ threadgroup T As[gemm_kernel::tgp_mem_size_a];
+ threadgroup T Bs_up[gemm_kernel::tgp_mem_size_b];
+ threadgroup T Bs_gate[gemm_kernel::tgp_mem_size_b];
+
+ const int tid_x = tid.x;
+ const int tid_y = tid.y;
+ const int tid_z = tid.z;
+
+ // Only process tiles within hidden_dim columns
+ int hidden_tiles = (params->hidden_dim + BN - 1) / BN;
+ if (hidden_tiles <= tid_x || params->tiles_m <= tid_y) {
+ return;
+ }
+
+ // Find block in A, B, C
+ const int c_row = tid_y * BM;
+ const int c_col = tid_x * BN;
+ const int k_start = params->split_k_partition_size * tid_z;
+
+ const size_t c_row_long = size_t(c_row);
+ const size_t c_col_long = size_t(c_col);
+ const size_t k_start_long = size_t(k_start);
+
+ // Pointers to A (input)
+ const device T* A_ptr = A + (transpose_a ? (c_row_long + k_start_long * params->lda)
+ : (k_start_long + c_row_long * params->lda));
+
+ // Pointers to B (up and gate weights)
+ // B layout: [up_weights | gate_weights] where each is hidden_dim columns
+ const device T* B_up = B + (transpose_b ? (k_start_long + c_col_long * params->ldb)
+ : (c_col_long + k_start_long * params->ldb));
+ const device T* B_gate = B_up + (transpose_b ? params->hidden_dim * params->ldb : params->hidden_dim);
+
+ // C output for partial sums (up and gate interleaved in split_k dimension)
+ // Layout: [up_partitions, gate_partitions] in the partition stride
+ device U* C_up = C + (size_t(params->split_k_partition_stride) * tid_z) +
+ (c_row_long * params->ldc + c_col_long);
+ device U* C_gate = C_up + (size_t(params->split_k_partition_stride) * params->split_k_partitions);
+
+ // Loaders
+ thread loader_a_t loader_a(A_ptr, params->lda, As, simd_group_id, simd_lane_id);
+ thread loader_b_t loader_b_up(B_up, params->ldb, Bs_up, simd_group_id, simd_lane_id);
+ thread loader_b_t loader_b_gate(B_gate, params->ldb, Bs_gate, simd_group_id, simd_lane_id);
+
+ // MMA operations
+ thread mma_t mma_up(simd_group_id, simd_lane_id);
+ thread mma_t mma_gate(simd_group_id, simd_lane_id);
+
+ int gemm_k_iterations = params->gemm_k_iterations_aligned;
+
+ short tgp_bm = min(BM, params->M - c_row);
+ short tgp_bn = min(BN, params->hidden_dim - c_col);
+ short leftover_bk = params->K % BK;
+
+ // Main loop
+ for (int k = 0; k < gemm_k_iterations; k++) {
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ if (MN_aligned) {
+ loader_a.load_unsafe();
+ loader_b_up.load_unsafe();
+ loader_b_gate.load_unsafe();
+ } else if (tgp_bm == BM && tgp_bn == BN) {
+ loader_a.load_unsafe();
+ loader_b_up.load_unsafe();
+ loader_b_gate.load_unsafe();
+ } else {
+ short2 tile_dims_A = transpose_a ? short2(tgp_bm, BK) : short2(BK, tgp_bm);
+ short2 tile_dims_B = transpose_b ? short2(BK, tgp_bn) : short2(tgp_bn, BK);
+ loader_a.load_safe(tile_dims_A);
+ loader_b_up.load_safe(tile_dims_B);
+ loader_b_gate.load_safe(tile_dims_B);
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ mma_up.mma(As, Bs_up);
+ mma_gate.mma(As, Bs_gate);
+
+ loader_a.next();
+ loader_b_up.next();
+ loader_b_gate.next();
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // Handle remaining K for last partition
+ if ((tid_z + 1) == (params->split_k_partitions)) {
+ int gemm_k_iter_remaining =
+ (params->K - (k_start + params->split_k_partition_size)) / BK;
+ for (int k = 0; k < gemm_k_iter_remaining; k++) {
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ if (!K_aligned) {
+ short2 tile_dims_A = transpose_a ? short2(tgp_bm, leftover_bk) : short2(leftover_bk, tgp_bm);
+ short2 tile_dims_B = transpose_b ? short2(leftover_bk, tgp_bn) : short2(tgp_bn, leftover_bk);
+ loader_a.load_safe(tile_dims_A);
+ loader_b_up.load_safe(tile_dims_B);
+ loader_b_gate.load_safe(tile_dims_B);
+ } else {
+ loader_a.load_unsafe();
+ loader_b_up.load_unsafe();
+ loader_b_gate.load_unsafe();
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ mma_up.mma(As, Bs_up);
+ mma_gate.mma(As, Bs_gate);
+
+ loader_a.next();
+ loader_b_up.next();
+ loader_b_gate.next();
+ }
+ }
+
+ // Store partial results
+ if (MN_aligned || (tgp_bm == BM && tgp_bn == BN)) {
+ mma_up.store_result(C_up, params->ldc);
+ mma_gate.store_result(C_gate, params->ldc);
+ } else {
+ mma_up.store_result_safe(C_up, params->ldc, short2(tgp_bn, tgp_bm));
+ mma_gate.store_result_safe(C_gate, params->ldc, short2(tgp_bn, tgp_bm));
+ }
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// MLP Fused Split-K accumulation kernel
+// Accumulates partial sums and applies MLP fused epilogue
+///////////////////////////////////////////////////////////////////////////////
+
+template <typename AccT, typename OutT>
+[[kernel]] void gemm_splitk_mlp_fused_accum(
+ const device AccT* C_split [[buffer(0)]],
+ device OutT* D [[buffer(1)]],
+ const constant int& k_partitions [[buffer(2)]],
+ const constant int& partition_stride [[buffer(3)]],
+ const constant int& ldd [[buffer(4)]],
+ uint2 gid [[thread_position_in_grid]]) {
+ // Adjust pointers
+ D += gid.x + gid.y * size_t(ldd);
+ const device AccT* C_up = C_split + gid.x + gid.y * size_t(ldd);
+ const device AccT* C_gate = C_up + (size_t(partition_stride) * k_partitions);
+
+ // Accumulate up and gate partial sums
+ AccT out_up = 0;
+ AccT out_gate = 0;
+
+ size_t offset = 0;
+ for (int i = 0; i < k_partitions; i++) {
+ out_up += C_up[offset];
+ out_gate += C_gate[offset];
+ offset += partition_stride;
+ }
+
+ // Apply MLP fused epilogue: out = up * activation(gate)
+ float fused = mlp_fused_epilogue_f32(static_cast<float>(out_up), static_cast<float>(out_gate));
+ D[0] = static_cast<OutT>(fused);
+}
diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/steel_gemm_splitk_mlp_fused.metal b/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/steel_gemm_splitk_mlp_fused.metal
new file mode 100644
index 000000000..e58783794
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/steel_gemm_splitk_mlp_fused.metal
@@ -0,0 +1,67 @@
+// clang-format off
+#include "../../common/utils.h"
+#include "steel_gemm_splitk_mlp_fused.h"
+
+#define instantiate_gemm( \
+ tname, \
+ trans_a, \
+ trans_b, \
+ iname, \
+ itype, \
+ oname, \
+ otype, \
+ bm, \
+ bn, \
+ bk, \
+ wm, \
+ wn, \
+ aname, \
+ mn_aligned, \
+ kname, \
+ k_aligned) \
+ instantiate_kernel( \
+ "steel_gemm_splitk_mlp_fused_" #tname "_" #iname "_" #oname \
+ "_bm" #bm "_bn" #bn "_bk" #bk "_wm" #wm "_wn" #wn \
+ "_MN_" #aname "_K_" #kname, \
+ gemm_splitk_mlp_fused, \
+ itype, \
+ otype, \
+ bm, \
+ bn, \
+ bk, \
+ wm, \
+ wn, \
+ trans_a, \
+ trans_b, \
+ mn_aligned, \
+ k_aligned)
+
+#define instantiate_gemm_aligned_helper(tname, trans_a, trans_b, iname, itype, oname, otype, bm, bn, bk, wm, wn) \
+ instantiate_gemm(tname, trans_a, trans_b, iname, itype, oname, otype, bm, bn, bk, wm, wn, taligned, true, taligned, true) \
+ instantiate_gemm(tname, trans_a, trans_b, iname, itype, oname, otype, bm, bn, bk, wm, wn, taligned, true, naligned, false) \
+ instantiate_gemm(tname, trans_a, trans_b, iname, itype, oname, otype, bm, bn, bk, wm, wn, naligned, false, taligned, true) \
+ instantiate_gemm(tname, trans_a, trans_b, iname, itype, oname, otype, bm, bn, bk, wm, wn, naligned, false, naligned, false)
+
+// Only NT transpose (common case for MLP)
+#define instantiate_gemm_transpose_helper(iname, itype, oname, otype, bm, bn, bk, wm, wn) \
+ instantiate_gemm_aligned_helper(nt, false, true, iname, itype, oname, otype, bm, bn, bk, wm, wn)
+
+#define instantiate_gemm_shapes_helper(iname, itype, oname, otype) \
+ instantiate_gemm_transpose_helper(iname, itype, oname, otype, 16, 16, 16, 2, 2) \
+ instantiate_gemm_transpose_helper(iname, itype, oname, otype, 16, 32, 16, 2, 2) \
+ instantiate_gemm_transpose_helper(iname, itype, oname, otype, 32, 16, 16, 2, 2) \
+ instantiate_gemm_transpose_helper(iname, itype, oname, otype, 32, 32, 16, 2, 2)
+
+instantiate_gemm_shapes_helper(float16, half, float32, float);
+instantiate_gemm_shapes_helper(bfloat16, bfloat16_t, float32, float);
+instantiate_gemm_shapes_helper(float32, float, float32, float);
+
+#define instantiate_accum(oname, otype, aname, atype) \
+ instantiate_kernel( \
+ "steel_gemm_splitk_mlp_fused_accum_" #oname "_" #aname, \
+ gemm_splitk_mlp_fused_accum, atype, otype)
+
+instantiate_accum(bfloat16, bfloat16_t, float32, float);
+instantiate_accum(float16, half, float32, float);
+instantiate_accum(float32, float, float32, float);
+// clang-format on
diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rs
index ff569683d..3caf3ae9b 100644
--- a/crates/uzu/src/backends/metal/kernel/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/mod.rs
@@ -12,6 +12,7 @@ pub use mlp::{
MLP_ACTIVATION_FC_INDEX, MLP_FUSED_FC_INDEX, MLP_HIDDEN_DIM_FC_INDEX,
MlpActivationType, MlpFusedConfig, make_non_fused_function_constants,
};
+pub mod mlp_fused;
pub mod moe;
pub mod pooling;
pub mod quant_matmul;
diff --git a/crates/uzu/tests/mlp_fused_correctness_test.rs b/crates/uzu/tests/mlp_fused_correctness_test.rs
index 3e2d97782..04de27d08 100644
--- a/crates/uzu/tests/mlp_fused_correctness_test.rs
+++ b/crates/uzu/tests/mlp_fused_correctness_test.rs
@@ -9,11 +9,13 @@ use uzu::{
backends::metal::{
MTLContext,
kernel::{
- matmul::{
- MlpFusedGemmArguments, MlpFusedGemmKernel,
- MlpFusedGemvArguments, MlpFusedGemvKernel,
- },
mlp::MlpActivationType,
+ mlp_fused::{
+ GemmArguments as MlpFusedGemmArguments,
+ GemmKernel as MlpFusedGemmKernel,
+ GemvArguments as MlpFusedGemvArguments,
+ GemvKernel as MlpFusedGemvKernel,
+ },
},
},
};
diff --git a/crates/uzu/tests/mlp_fused_performance_test.rs b/crates/uzu/tests/mlp_fused_performance_test.rs
index c4f9a64ac..99cc81695 100644
--- a/crates/uzu/tests/mlp_fused_performance_test.rs
+++ b/crates/uzu/tests/mlp_fused_performance_test.rs
@@ -11,11 +11,14 @@ use uzu::{
backends::metal::{
MTLContext,
kernel::{
- matmul::{
- MatmulArguments, MatmulKernel, MlpFusedGemmArguments,
- MlpFusedGemmKernel, MlpFusedGemvArguments, MlpFusedGemvKernel,
- },
+ matmul::{MatmulArguments, MatmulKernel},
mlp::MlpActivationType,
+ mlp_fused::{
+ GemmArguments as MlpFusedGemmArguments,
+ GemmKernel as MlpFusedGemmKernel,
+ GemvArguments as MlpFusedGemvArguments,
+ GemvKernel as MlpFusedGemvKernel,
+ },
},
},
};